diff options
Diffstat (limited to 'gcsdk')
88 files changed, 42254 insertions, 0 deletions
diff --git a/gcsdk/accountdetails.cpp b/gcsdk/accountdetails.cpp new file mode 100644 index 0000000..507b9d1 --- /dev/null +++ b/gcsdk/accountdetails.cpp @@ -0,0 +1,721 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Holds the CAccountDetails class. +// +//============================================================================= + +#include "stdafx.h" +#include "accountdetails.h" +#include "rtime.h" +#include "gcsdk_gcmessages.pb.h" + +#include "memdbgon.h" // needs to be the last include in the file + +namespace GCSDK +{ +GCConVar cv_account_details_cache_time( "account_details_cache_time", "600" ); +GCConVar cv_account_details_failure_cache_time( "account_details_failure_cache_time", "10" ); +GCConVar account_details_timeout( "account_details_timeout", "10" ); +GCConVar cv_persona_name_cache_time( "persona_name_cache_time", "60" ); +GCConVar cv_persona_name_failure_cache_time( "persona_name_failure_cache_time", "10" ); +GCConVar cv_persona_name_batch_size( "persona_name_batch_size", "100" ); +GCConVar persona_name_timeout( "persona_name_timeout", "10" ); + +const char *kszAccountDetailsKey = "AccountDetails-v001"; + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CAccountDetails::CAccountDetails() +: m_rtimeCached( CRTime::RTime32TimeCur() ), + m_bValid( false ), + m_bPublicProfile( false ), + m_bVacBanned( false ), + m_bCyberCafe( false ), + m_bSchoolAccount( false ), + m_bFreeTrialAccount( false ), + m_bSubscribed( false ), + m_bLowViolence( false ), + m_bLimited( false ), + m_bAccountLocked( false ), + m_bCommunityBanned( false ), + m_bTradeBanned( false ), + m_bIsSteamGuardEnabled( false ), + m_bIsPhoneVerified( false ), + m_bIsTwoFactorAuthEnabled( false ), + m_bIsPhoneIdentifying( false ), + m_unPackage( 0 ), + m_rtimeVACBanEnd( 0 ), + m_unSteamLevel( 0 ), + m_unFriendCount( 0 ), + m_rtimeAccountCreated( 0 ), + m_rtimeTwoFactorEnabled( 0 ), + m_rtimePhoneVerified( 0 ), + m_unPhoneID( 0 ) +{ +} + +//----------------------------------------------------------------------------- +// Purpose: Initialize a fresh CAccountDetails with data from Steam +//----------------------------------------------------------------------------- +void CAccountDetails::Init( CGCSystemMsg_GetAccountDetails_Response &msgResponse ) +{ + m_bValid = true; + m_sAccountName = msgResponse.account_name().c_str(); + m_bPublicProfile = msgResponse.is_profile_public(); + m_bPublicInventory = msgResponse.is_inventory_public(); + m_bVacBanned = msgResponse.is_vac_banned(); + m_bCyberCafe = msgResponse.is_cyber_cafe(); + m_bSchoolAccount = msgResponse.is_school_account(); + m_bFreeTrialAccount = msgResponse.is_free_trial_account(); + m_bSubscribed = msgResponse.is_subscribed(); + m_bLowViolence = msgResponse.is_low_violence(); + m_bLimited = msgResponse.is_limited(); + m_bAccountLocked = msgResponse.is_account_locked_down(); + m_bCommunityBanned = msgResponse.is_community_banned(); + m_bTradeBanned = msgResponse.is_trade_banned(); + m_unPackage = msgResponse.package(); + m_rtimeVACBanEnd = msgResponse.suspension_end_time(); + m_sCurrency = msgResponse.currency().c_str(); + m_unSteamLevel = msgResponse.steam_level(); + m_unFriendCount = msgResponse.friend_count(); + m_rtimeAccountCreated = msgResponse.account_creation_time(); + m_bIsSteamGuardEnabled = msgResponse.is_steamguard_enabled(); + m_bIsPhoneVerified = msgResponse.is_phone_verified(); + m_bIsTwoFactorAuthEnabled = msgResponse.is_two_factor_auth_enabled(); + m_bIsPhoneIdentifying = msgResponse.is_phone_identifying(); + m_rtimeTwoFactorEnabled = msgResponse.two_factor_enabled_time(); + m_rtimePhoneVerified = msgResponse.phone_verification_time(); + m_unPhoneID = msgResponse.phone_id(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns true if it's time to remove this entry from the cache +//----------------------------------------------------------------------------- +bool CAccountDetails::BIsExpired() const +{ + int nCacheSeconds = BIsValid() ? cv_account_details_cache_time.GetInt() : cv_account_details_failure_cache_time.GetInt(); + + return m_rtimeCached + nCacheSeconds < CRTime::RTime32TimeCur(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Reverts this to an invalid record +//----------------------------------------------------------------------------- +void CAccountDetails::Reset() +{ + m_bValid = false; + m_rtimeCached = CRTime::RTime32TimeCur(); +} + + +#ifdef DBGFLAG_VALIDATE +//----------------------------------------------------------------------------- +// Purpose: Claims all the memory for the AccountDetails object +//----------------------------------------------------------------------------- +void CAccountDetails::Validate( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE(); + ValidateObj( m_sAccountName ); +} +#endif // DBGFLAG_VALIDATE + + +//----------------------------------------------------------------------------- +// Purpose: Sends a message to Steam to get a CAccountDetails object +//----------------------------------------------------------------------------- +class CGCJobSendGetAccountDetailsRequest : public CGCJob +{ + CAccountDetailsManager *m_pManager; + CSteamID m_SteamID; + +public: + CGCJobSendGetAccountDetailsRequest( CGCBase *pGC, CAccountDetailsManager *pManager, const CSteamID &steamID ) + : CGCJob( pGC ), m_pManager( pManager ), m_SteamID( steamID ) {} + virtual bool BYieldingRunGCJob() + { + // Yield immediately to be sure that the calling job gets in the wakeup list + BYield(); + + // These requests should come back very quickly, so if they don't we shouldn't wait very long + // jamming up the system + SetJobTimeout( account_details_timeout.GetInt() ); + + // Get an empty account details object + CAccountDetails *pAccount = m_pManager->m_hashAccountDetailsCache.PvRecordFind( m_SteamID.GetAccountID() ); + if ( NULL == pAccount ) + { + pAccount = m_pManager->m_hashAccountDetailsCache.PvRecordInsert( m_SteamID.GetAccountID() ); + } + else + { + // If the record isn't expired, why is it there? + Assert( pAccount->BIsExpired() ); + pAccount->Reset(); + } + + CProtoBufMsg< CGCSystemMsg_GetAccountDetails > msgReqeust( k_EGCMsgGetAccountDetails ); + CProtoBufMsg< CGCSystemMsg_GetAccountDetails_Response > msgReply; + msgReqeust.Body().set_steamid( m_SteamID.ConvertToUint64() ); + msgReqeust.Body().set_appid( GGCBase()->GetAppID() ); + msgReqeust.ExpectingReply( GJobCur().GetJobID() ); + + // try to get the account details at most 2 times + const int kMaxTries = 2; + for ( int iTries = 0; iTries < kMaxTries; iTries++ ) + { + if( !m_pGC->BSendSystemMessage( msgReqeust ) ) + { + EmitWarning( SPEW_GC, 2, "Unable to send GetAccountDetails system message\n" ); + continue; + } + + // All of our request messages are identical, so if we get our replies + // mixed up, it's OK. Bypass the system used to protect us against + // mismatched replies. + ClearFailedToReceivedMsgType( k_EGCMsgGetAccountDetailsResponse ); + + // Wait for the reply + if( !BYieldingWaitForMsg( &msgReply, k_EGCMsgGetAccountDetailsResponse ) ) + { + EmitWarning( SPEW_GC, 2, "Timeout waiting for GetAccountDetails reply for SteamID %s\n", m_SteamID.Render() ); + continue; + } + + if ( k_EResultOK != msgReply.GetEResult() ) + { + EmitInfo( SPEW_GC, 4, 4, "GetAccountDetails request failed with result %d for SteamID %s\n", msgReply.GetEResult(), m_SteamID.Render() ); + break; + } + + Assert( msgReply.Body().eresult_deprecated() == k_EResultOK ); + + // Sanity check the response + if ( msgReply.Body().has_accountid() && msgReply.Body().accountid() != m_SteamID.GetAccountID() ) + { + static bool bHasAlerted = false; + if ( !bHasAlerted ) + { + GGCBase()->PostAlert( k_EAlertTypeInfo, true, CFmtStr( "GetAccountDetails got a response for account %d, but we were expecting a response for account %s\n", msgReply.Body().accountid(), m_SteamID.Render() ) ); + bHasAlerted = true; + } + + EmitError( SPEW_GC, "GetAccountDetails got a response for account %d, but we were expecting a response for account %s\n", msgReply.Body().accountid(), m_SteamID.Render() ); + break; + } + + // All responses should have this + if ( !msgReply.Body().has_account_name() ) + { + EmitError( SPEW_GC, "GetAccountDetails got a response with missing fields for SteamID %s\n", m_SteamID.Render() ); + break; + } + + pAccount->Init( msgReply.Body() ); + m_pManager->CachePersonaName( CSteamID( m_SteamID ), msgReply.Body().persona_name().c_str() ); + + // We got a response, so we shouldn't try again + break; + } + + m_pManager->WakeWaitingAccountDetailsJobs( m_SteamID ); + return true; + } +}; + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CCachedPersonaName::CCachedPersonaName() + : m_rtimeCached( 0 ) // This initialization value is important because it makes it expired on creation, + // and the rest of the code will only ask Steam for a name if the cached entry is expired + , m_nLoading( 0 ) + , m_bPreloading( false ) +{ +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CCachedPersonaName::~CCachedPersonaName() +{ + Assert( !BIsLoading() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Sets the newly retrieved persona name +//----------------------------------------------------------------------------- +void CCachedPersonaName::Init( const char *pchPersonaName ) +{ + m_sPersonaName = pchPersonaName; + m_rtimeCached = CRTime::RTime32TimeCur(); + m_bPreloading = false; +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns true if it's time to remove this entry from the cache +//----------------------------------------------------------------------------- +bool CCachedPersonaName::BIsExpired() const +{ + int nCacheSeconds = BIsValid() ? cv_persona_name_cache_time.GetInt() : cv_persona_name_failure_cache_time.GetInt(); + + return m_rtimeCached + nCacheSeconds < CRTime::RTime32TimeCur(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns true if there is an actual cached name +//----------------------------------------------------------------------------- +bool CCachedPersonaName::BIsValid() const +{ + return !m_sPersonaName.IsEmpty(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Reverts this to an invalid record +//----------------------------------------------------------------------------- +void CCachedPersonaName::Reset() +{ + m_sPersonaName.Clear(); + m_rtimeCached = CRTime::RTime32TimeCur(); + m_bPreloading = false; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets the cached string +//----------------------------------------------------------------------------- +const char *CCachedPersonaName::GetPersonaName() const +{ + return BIsValid() ? m_sPersonaName.Get() : "[unknown]"; +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns if this name is loading +//----------------------------------------------------------------------------- +bool CCachedPersonaName::BIsLoading() const +{ + return m_nLoading > 0 || m_bPreloading; +} + + +//----------------------------------------------------------------------------- +// Purpose: Sets that this name has been preloaded +//----------------------------------------------------------------------------- +void CCachedPersonaName::SetPreloading() +{ + m_bPreloading = true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Sets that a job is waiting for this name to be loaded +//----------------------------------------------------------------------------- +void CCachedPersonaName::AddLoadingRef() +{ + m_nLoading++; +} + + +//----------------------------------------------------------------------------- +// Purpose: Releases the loading ref +//----------------------------------------------------------------------------- +void CCachedPersonaName::ReleaseLoadingRef() +{ + DbgVerify( --m_nLoading >= 0 ); +} + + +#ifdef DBGFLAG_VALIDATE +//----------------------------------------------------------------------------- +// Purpose: Claims all the memory for the CCachedPersonaName object +//----------------------------------------------------------------------------- +void CCachedPersonaName::Validate( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE(); + ValidateObj( m_sPersonaName ); +} +#endif // DBGFLAG_VALIDATE + + +//----------------------------------------------------------------------------- +// Purpose: Sends a message to Steam to get a CAccountDetails object +//----------------------------------------------------------------------------- +class CGCJobSendGetPersonaNamesRequest : public CGCJob +{ + CAccountDetailsManager *m_pManager; + CUtlVector<CSteamID> m_vecSteamIDs; + +public: + CGCJobSendGetPersonaNamesRequest( CGCBase *pGC, CAccountDetailsManager *pManager, CUtlVector<CSteamID> &vecSteamIDs ) + : CGCJob( pGC ), m_pManager( pManager ) + { + m_vecSteamIDs.Swap( vecSteamIDs ); + } + + virtual bool BYieldingRunGCJob() + { + // Yield immediately to be sure that the calling job gets in the wakeup list + BYield(); + + // These requests should come back very quickly, so if they don't we shouldn't wait very long + // jamming up the system + SetJobTimeout( persona_name_timeout.GetInt() ); + + CProtoBufMsg< CMsgGCGetPersonaNames > msgReqeust( k_EGCMsgGetPersonaNames ); + msgReqeust.ExpectingReply( GJobCur().GetJobID() ); + + FOR_EACH_VEC( m_vecSteamIDs, i ) + { + msgReqeust.Body().add_steamids( m_vecSteamIDs[i].ConvertToUint64() ); + } + + CProtoBufMsg< CMsgGCGetPersonaNames_Response > msgReply; + if( !m_pGC->BSendSystemMessage( msgReqeust ) || + !BYieldingWaitForMsg( &msgReply, k_EGCMsgGetPersonaNamesResponse ) ) + { + FOR_EACH_VEC( m_vecSteamIDs, i ) + { + m_pManager->CachePersonaNameFailure( m_vecSteamIDs[i] ); + } + + //if we are shutting down, don't bother reporting this issue, we know we won't be able to get persona names + if( !GGCBase()->GetIsShuttingDown() ) + { + EmitWarning( SPEW_GC, 2, "GetPersonaNames request failed for %d IDs:", m_vecSteamIDs.Count() ); + FOR_EACH_VEC( m_vecSteamIDs, nID ) + { + EmitWarning( SPEW_GC, 2, " %s", m_vecSteamIDs[ nID ].Render() ); + } + EmitWarning( SPEW_GC, 2, "\n" ); + } + } + else + { + for ( int i = 0; i < msgReply.Body().succeeded_lookups_size(); i++ ) + { + const CMsgGCGetPersonaNames_Response_PersonaName &result = msgReply.Body().succeeded_lookups( i ); + m_pManager->CachePersonaName( CSteamID( result.steamid() ), result.persona_name().c_str() ); + } + for ( int i = 0; i < msgReply.Body().failed_lookup_steamids_size(); i++ ) + { + m_pManager->CachePersonaNameFailure( CSteamID( msgReply.Body().failed_lookup_steamids( i ) ) ); + } + } + + FOR_EACH_VEC( m_vecSteamIDs, i ) + { + m_pManager->WakeWaitingPersonaNameJobs( m_vecSteamIDs[i] ); + } + return true; + } +}; + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CAccountDetailsManager::CAccountDetailsManager() + : m_hashAccountDetailsCache( k_nAccountDetailsRunInterval / k_cMicroSecPerShellFrame ) + , m_hashPersonaNameCache( k_nAccountDetailsRunInterval / k_cMicroSecPerShellFrame ) +{ + m_hashAccountDetailsCache.Init( k_cAccountDetailsInit, k_cBucketAccountDetails ); + m_hashPersonaNameCache.Init( k_cAccountDetailsInit, k_cBucketAccountDetails ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Work to be done once per frame +//----------------------------------------------------------------------------- +void CAccountDetailsManager::MarkFrame() +{ + m_hashAccountDetailsCache.StartFrameSchedule( true ); + m_hashPersonaNameCache.StartFrameSchedule( true ); + SendBatchedPersonaNamesRequest(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets a CAccountDetails object +//----------------------------------------------------------------------------- +CAccountDetails *CAccountDetailsManager::YieldingGetAccountDetails( const CSteamID &steamID, bool bForceReload ) +{ + AssertRunningJob(); + if( !steamID.IsValid() || !steamID.BIndividualAccount() ) + return NULL; + + // Check the local cache + CAccountDetails *pAccountDetails = NULL; + if ( BFindAccountDetailsInLocalCache( steamID, &pAccountDetails ) ) + { + if ( pAccountDetails && bForceReload ) + { + // Clear it, continue with fresh load + m_hashAccountDetailsCache.Remove( pAccountDetails ); + } + else + { + return pAccountDetails; + } + } + + // Not in the local cache, ask Steam + int iMapIndex = m_mapQueuedAccountDetailsRequests.Find( steamID ); + if ( !m_mapQueuedAccountDetailsRequests.IsValidIndex( iMapIndex ) ) + { + iMapIndex = m_mapQueuedAccountDetailsRequests.Insert( steamID ); + CGCJobSendGetAccountDetailsRequest *pJob = new CGCJobSendGetAccountDetailsRequest( GGCBase(), this, steamID ); + pJob->StartJob( NULL ); + } + + m_mapQueuedAccountDetailsRequests[iMapIndex].AddToTail( GJobCur().GetJobID() ); + GJobCur().BYieldingWaitForWorkItem(); + + // Check again, if it's not there then it's not there + BFindAccountDetailsInLocalCache( steamID, &pAccountDetails ); + return pAccountDetails; +} + + +//----------------------------------------------------------------------------- +// Purpose: Finds an AccountDetails object in the local cache. Returns true +// if it was found, false if it should be checked remotely +//----------------------------------------------------------------------------- +bool CAccountDetailsManager::BFindAccountDetailsInLocalCache( const CSteamID &steamID, CAccountDetails **ppAccount ) +{ + CAccountDetails *pAccountLocal = m_hashAccountDetailsCache.PvRecordFind( steamID.GetAccountID() ); + if( NULL == pAccountLocal || pAccountLocal->BIsExpired() ) + return false; + + *ppAccount = pAccountLocal->BIsValid() ? pAccountLocal : NULL; + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Wakes any jobs waiting on this result +//----------------------------------------------------------------------------- +void CAccountDetailsManager::WakeWaitingAccountDetailsJobs( const CSteamID &steamID ) +{ + int iMapIndex = m_mapQueuedAccountDetailsRequests.Find( steamID ); + if ( !m_mapQueuedAccountDetailsRequests.IsValidIndex( iMapIndex ) ) + return; + + CCopyableUtlVector<JobID_t> &vecJobsWaiting = m_mapQueuedAccountDetailsRequests[iMapIndex]; + FOR_EACH_VEC( vecJobsWaiting, i ) + { + GGCBase()->GetJobMgr().BRouteWorkItemCompletedDelayed( vecJobsWaiting[i], false ); + } + m_mapQueuedAccountDetailsRequests.RemoveAt( iMapIndex ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets a persona name for a user +//----------------------------------------------------------------------------- +const char *CAccountDetailsManager::YieldingGetPersonaName( const CSteamID &steamID ) +{ + VPROF_BUDGET( "CAccountDetailsManager::YieldingGetPersonaName", VPROF_BUDGETGROUP_STEAM ); + + AssertRunningJob(); + + if( !steamID.IsValid() || !steamID.BIndividualAccount() ) + return "[unknown]"; + + // Check the local cache + CCachedPersonaName *pPersonaName = FindOrCreateCachedPersonaName( steamID ); + if ( !pPersonaName->BIsExpired() ) + return pPersonaName->GetPersonaName(); + + // Not in the local cache, ask Steam + pPersonaName->AddLoadingRef(); + + // Queue the request and start a lookup job if we have enough pending + int iMapIndex = m_mapQueuedPersonaNameRequests.Find( steamID ); + if ( !m_mapQueuedPersonaNameRequests.IsValidIndex( iMapIndex ) ) + { + iMapIndex = m_mapQueuedPersonaNameRequests.Insert( steamID ); + m_vecPendingPersonaNameLookups.AddToTail( steamID ); + if ( m_vecPendingPersonaNameLookups.Count() >= cv_persona_name_batch_size.GetInt() ) + { + SendBatchedPersonaNamesRequest(); + } + } + + m_mapQueuedPersonaNameRequests[iMapIndex].AddToTail( GJobCur().GetJobID() ); + GJobCur().BYieldingWaitForWorkItem(); + + // At this point we'll either have a persona name or we won't + pPersonaName->ReleaseLoadingRef(); + return pPersonaName->GetPersonaName(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Let's the system know that we should load the persona name for this +// user, but does not block on it +//----------------------------------------------------------------------------- +void CAccountDetailsManager::PreloadPersonaName( const CSteamID &steamID ) +{ + if( !steamID.IsValid() || !steamID.BIndividualAccount() ) + return; + + // See if we already have it + CCachedPersonaName *pCachedName = FindOrCreateCachedPersonaName( steamID ); + if ( !pCachedName->BIsExpired() || pCachedName->BIsLoading() ) + return; + + // Queue the request and start a lookup job if we have enough pending + pCachedName->SetPreloading(); + m_mapQueuedPersonaNameRequests.Insert( steamID ); + m_vecPendingPersonaNameLookups.AddToTail( steamID ); + if ( m_vecPendingPersonaNameLookups.Count() >= cv_persona_name_batch_size.GetInt() ) + { + SendBatchedPersonaNamesRequest(); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Sends a batch of persona name requests +//----------------------------------------------------------------------------- +void CAccountDetailsManager::SendBatchedPersonaNamesRequest() +{ + if ( 0 == m_vecPendingPersonaNameLookups.Count() ) + return; + + // Start the job. This swaps out our buffer with an empty one + CGCJobSendGetPersonaNamesRequest *pJob = new CGCJobSendGetPersonaNamesRequest( GGCBase(), this, m_vecPendingPersonaNameLookups ); + pJob->StartJob( NULL ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Caches a persona name +//----------------------------------------------------------------------------- +void CAccountDetailsManager::CachePersonaName( const CSteamID &steamID, const char *pchPersonaName ) +{ + CCachedPersonaName *pCachedPersonaName = FindOrCreateCachedPersonaName( steamID ); + pCachedPersonaName->Init( pchPersonaName ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Remembers that we failed to cache a persona name +//----------------------------------------------------------------------------- +void CAccountDetailsManager::CachePersonaNameFailure( const CSteamID &steamID ) +{ + CCachedPersonaName *pCachedPersonaName = FindOrCreateCachedPersonaName( steamID ); + pCachedPersonaName->Reset(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Purges a specific persona name from the cache +//----------------------------------------------------------------------------- +void CAccountDetailsManager::ClearCachedPersonaName( const CSteamID &steamID ) +{ + CCachedPersonaName *pPersonaNameLocal = m_hashPersonaNameCache.PvRecordFind( steamID.GetAccountID() ); + if( NULL != pPersonaNameLocal && !pPersonaNameLocal->BIsLoading() ) + { + m_hashPersonaNameCache.Remove( pPersonaNameLocal ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets a CCachedPersonaName record +//----------------------------------------------------------------------------- +CCachedPersonaName *CAccountDetailsManager::FindOrCreateCachedPersonaName( const CSteamID &steamID ) +{ + CCachedPersonaName *pPersonaName = m_hashPersonaNameCache.PvRecordFind( steamID.GetAccountID() ); + if ( NULL != pPersonaName ) + return pPersonaName; + else + return m_hashPersonaNameCache.PvRecordInsert( steamID.GetAccountID() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Wakes any jobs waiting on this result +//----------------------------------------------------------------------------- +void CAccountDetailsManager::WakeWaitingPersonaNameJobs( const CSteamID &steamID ) +{ + int iMapIndex = m_mapQueuedPersonaNameRequests.Find( steamID ); + if ( !m_mapQueuedPersonaNameRequests.IsValidIndex( iMapIndex ) ) + return; + + CCopyableUtlVector<JobID_t> &vecJobsWaiting = m_mapQueuedPersonaNameRequests[iMapIndex]; + FOR_EACH_VEC( vecJobsWaiting, i ) + { + GGCBase()->GetJobMgr().BRouteWorkItemCompletedDelayed( vecJobsWaiting[i], false ); + } + m_mapQueuedPersonaNameRequests.RemoveAt( iMapIndex ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Purges old data from the cache. Returns true if there is more +// work to do +//----------------------------------------------------------------------------- +bool CAccountDetailsManager::BExpireRecords( CLimitTimer &limitTimer ) +{ + VPROF_BUDGET( "Expire account details", VPROF_BUDGETGROUP_STEAM ); + + for ( CAccountDetails *pDetails = m_hashAccountDetailsCache.PvRecordRun(); NULL != pDetails; pDetails = m_hashAccountDetailsCache.PvRecordRun() ) + { + if ( pDetails->BIsExpired() ) + { + m_hashAccountDetailsCache.Remove( pDetails ); + } + + if ( limitTimer.BLimitReached() ) + return true; + } + + for ( CCachedPersonaName *pPersonaName = m_hashPersonaNameCache.PvRecordRun(); NULL != pPersonaName; pPersonaName = m_hashPersonaNameCache.PvRecordRun() ) + { + if ( pPersonaName->BIsExpired() && !pPersonaName->BIsLoading() ) + { + m_hashPersonaNameCache.Remove( pPersonaName ); + } + + if ( limitTimer.BLimitReached() ) + return true; + } + + return false; +} + + +//----------------------------------------------------------------------------- +// Purpose: Prints status +//----------------------------------------------------------------------------- +void CAccountDetailsManager::Dump() const +{ + int nJobsWaiting = 0; + FOR_EACH_MAP_FAST( m_mapQueuedAccountDetailsRequests, iMap ) + { + nJobsWaiting += m_mapQueuedAccountDetailsRequests[iMap].Count(); + } + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\tAccount Details: %d cached, %d lookups in flight, %d jobs waiting\n", m_hashAccountDetailsCache.Count(), m_mapQueuedAccountDetailsRequests.Count(), nJobsWaiting ); + + nJobsWaiting = 0; + FOR_EACH_MAP_FAST( m_mapQueuedPersonaNameRequests, iMap ) + { + nJobsWaiting += m_mapQueuedPersonaNameRequests[iMap].Count(); + } + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\tPersona Names: %d cached, %d lookups in flight, %d jobs waiting\n", m_hashPersonaNameCache.Count(), m_mapQueuedPersonaNameRequests.Count(), nJobsWaiting ); +} + +} // namespace GCSDK diff --git a/gcsdk/bin/SchemaCompiler.exe b/gcsdk/bin/SchemaCompiler.exe Binary files differnew file mode 100644 index 0000000..f473e0e --- /dev/null +++ b/gcsdk/bin/SchemaCompiler.exe diff --git a/gcsdk/bin/linux/protoc b/gcsdk/bin/linux/protoc Binary files differnew file mode 100644 index 0000000..e33c5ee --- /dev/null +++ b/gcsdk/bin/linux/protoc diff --git a/gcsdk/bin/protoc.exe b/gcsdk/bin/protoc.exe Binary files differnew file mode 100644 index 0000000..df3ef49 --- /dev/null +++ b/gcsdk/bin/protoc.exe diff --git a/gcsdk/bufferpool.cpp b/gcsdk/bufferpool.cpp new file mode 100644 index 0000000..f1c2f82 --- /dev/null +++ b/gcsdk/bufferpool.cpp @@ -0,0 +1,120 @@ +//========= Copyright �, Valve LLC, All rights reserved. ====================== +// +// Purpose: Defines a buffer pool used to group small allocations +// +//============================================================================= + + +#include "stdafx.h" +#include "bufferpool.h" + +using namespace GCSDK; + +CUtlVector<CBufferPool *> CBufferPool::sm_vecBufferPools; + + +//---------------------------------------------------------------------------- +// Purpose: Constructor +//---------------------------------------------------------------------------- +CBufferPool::CBufferPool( const char *pchName, const GCConVar &cvMaxSizeMB, const GCConVar &cvInitBufferSize, int nFlags ) + : m_nBuffersInUse( 0 ) + , m_nBuffersTotal( 0 ) + , m_nHighWatermark( 0 ) + , m_cubFree( 0 ) + , m_sName( pchName ) + , m_cvMaxSizeMB( cvMaxSizeMB ) + , m_cvInitBufferSize( cvInitBufferSize ) + , m_nFlags( nFlags ) +{ + sm_vecBufferPools.AddToTail( this ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Destructor +//---------------------------------------------------------------------------- +CBufferPool::~CBufferPool() +{ + m_vecFreeBuffers.PurgeAndDeleteElements(); + sm_vecBufferPools.FindAndRemove( this ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Gives a bind param buffer to a query +//---------------------------------------------------------------------------- +CUtlBuffer *CBufferPool::GetBuffer() +{ + m_nBuffersInUse++; + + if ( m_vecFreeBuffers.Count() > 0 ) + { + CUtlBuffer *pBuffer = m_vecFreeBuffers.Tail(); + m_vecFreeBuffers.Remove( m_vecFreeBuffers.Count() - 1 ); + m_cubFree -= pBuffer->Size(); + + return pBuffer; + } + else + { + m_nBuffersTotal++; + m_nHighWatermark = max( m_nBuffersTotal, m_nHighWatermark ); + return new CUtlBuffer( 0, m_cvInitBufferSize.GetInt(), m_nFlags ); + } +} + + +//---------------------------------------------------------------------------- +// Purpose: Returns a bind param buffer when a query is done with it +//---------------------------------------------------------------------------- +void CBufferPool::ReturnBuffer( CUtlBuffer *pBuffer ) +{ + m_nBuffersInUse--; + + if ( ( int64 )( m_cubFree + pBuffer->Size() ) <= ( m_cvMaxSizeMB.GetInt() * k_nMegabyte ) ) + { + pBuffer->Clear(); + m_cubFree += pBuffer->Size(); + m_vecFreeBuffers.AddToTail( pBuffer ); + } + else + { + m_nBuffersTotal--; + delete pBuffer; + } +} + + +//---------------------------------------------------------------------------- +// Purpose: Print statistics to the console +//---------------------------------------------------------------------------- +void CBufferPool::DumpPools() +{ + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "Reusable Buffer Pools:\n" ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, + " Pool Buffers Free Max Buffers Total Mem (est) Free Mem\n" ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, + " ------------------------------ ------------ ------------ ------------ ---------------- ----------------\n" ); + + FOR_EACH_VEC( sm_vecBufferPools, i ) + { + CBufferPool *pPool = sm_vecBufferPools[i]; + + int nTotalEst = 0; + if ( pPool->m_vecFreeBuffers.Count() > 0 ) + { + nTotalEst = pPool->m_cubFree / pPool->m_vecFreeBuffers.Count() * pPool->m_nBuffersTotal; + } + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " %-30s %12d %12d %12d %16s %16s\n", + pPool->m_sName.Get(), + pPool->m_nBuffersTotal, + pPool->m_vecFreeBuffers.Count(), + pPool->m_nHighWatermark, + Q_pretifymem( nTotalEst, 0, true ), + Q_pretifymem( pPool->m_cubFree, 0, true ) + ); + } + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\n" ); +} diff --git a/gcsdk/directory.cpp b/gcsdk/directory.cpp new file mode 100644 index 0000000..e4c809b --- /dev/null +++ b/gcsdk/directory.cpp @@ -0,0 +1,526 @@ +//====== Copyright �, Valve Corporation, All rights reserved. ================= +// +// Purpose: Defines a directory for all the GC processes for an app +// +//============================================================================= + + +#include "stdafx.h" +#include "gcsdk/directory.h" + + +namespace GCSDK +{ + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CGCDirTypeInstance::CGCDirTypeInstance( const char* pszTypeName, uint32 nType, uint32 nInstance, const KeyValues *pKVConfig, const CGCDirProcess* pProcess ) + : m_nType( nType ) + , m_sTypeName( pszTypeName ) + , m_nInstance( nInstance ) + , m_pProcess( pProcess ) +{ + m_pConfig = pKVConfig->MakeCopy(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CGCDirTypeInstance::~CGCDirTypeInstance() +{ + m_pConfig->deleteThis(); +} + +//----------------------------------------------------------------------------- +// Purpose: Returns the type for this GC +//----------------------------------------------------------------------------- +uint32 CGCDirTypeInstance::GetType() const +{ + return m_nType; +} + +//----------------------------------------------------------------------------- +const char* CGCDirTypeInstance::GetTypeName() const +{ + return m_sTypeName; +} + +//----------------------------------------------------------------------------- +// Purpose: Returns the instance index for GCs of this type +//----------------------------------------------------------------------------- +uint32 CGCDirTypeInstance::GetInstance() const +{ + return m_nInstance; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets any additional configuration data for this GC +//----------------------------------------------------------------------------- +KeyValues * CGCDirTypeInstance::GetConfig() const +{ + return m_pConfig; +} + +//----------------------------------------------------------------------------- +// Purpose: Gets the box that this GC was associated with +//----------------------------------------------------------------------------- +const CGCDirProcess* CGCDirTypeInstance::GetProcess( ) const +{ + return m_pProcess; +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CGCDirProcess::CGCDirProcess( uint32 nGCDirIndex, const char *pchName, const char* pchProcessType, const KeyValues *pKVConfig ) : + m_iDirGC( nGCDirIndex ), + m_sName( pchName ), + m_sType( pchProcessType ), + m_nTypeMask( 0 ) +{ + m_pConfig = pKVConfig->MakeCopy(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CGCDirProcess::~CGCDirProcess() +{ + // Don't need to delete our pointers. They were allocated by the directory + // and will be cleaned up by the directory. +} + +//----------------------------------------------------------------------------- +// Purpose: Gets any additional configuration data for this GC +//----------------------------------------------------------------------------- +KeyValues * CGCDirProcess::GetConfig() const +{ + return m_pConfig; +} + +//----------------------------------------------------------------------------- +// Purpose: Gets the name for this box +//----------------------------------------------------------------------------- +const char *CGCDirProcess::GetName() const +{ + return m_sName.Get(); +} + +const char *CGCDirProcess::GetProcessType() const +{ + return m_sType.Get(); +} + +//----------------------------------------------------------------------------- +// Purpose: Gets the number of GCs assigned to this box +//----------------------------------------------------------------------------- +uint32 CGCDirProcess::GetTypeInstanceCount() const +{ + return m_vecGCs.Count(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets the specified GC definition +//----------------------------------------------------------------------------- +const CGCDirTypeInstance *CGCDirProcess::GetTypeInstance( uint32 nGCIndex ) const +{ + bool bValidIndex = m_vecGCs.IsValidIndex( nGCIndex ); + Assert( bValidIndex ); + if ( !bValidIndex ) + return NULL; + + return m_vecGCs[ nGCIndex ]; +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a GC to this box +//----------------------------------------------------------------------------- +void CGCDirProcess::AddGC( CGCDirTypeInstance *pGC ) +{ + if ( !pGC ) + { + Assert( false ); + return; + } + + //set this within our type mask + Assert( pGC->GetType() < 64 ); + m_nTypeMask |= ( uint64 )1 << pGC->GetType(); + + m_vecGCs.AddToTail( pGC ); + if( !m_vecUniqueTypeList.HasElement( pGC->GetType() ) ) + { + m_vecUniqueTypeList.Insert( pGC->GetType() ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +uint32 CGCDirProcess::GetGCDirIndex() const +{ + return m_iDirGC; +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +bool CGCDirProcess::HasInstanceOfType( uint32 nType ) const +{ + return ( m_nTypeMask & ( ( uint64 )1 << nType ) ) != 0; +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +const CUtlSortVector< uint32 >& CGCDirProcess::GetUniqueTypeList() const +{ + return m_vecUniqueTypeList; +} + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CDirectory::CDirectory() + : m_bInitialized( false ) + , m_mapGCsByType( DefLessFunc( int32 ) ) + , m_mapRegisteredGCTypes( DefLessFunc( int32 ) ) +{ +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CDirectory::~CDirectory() +{ + m_vecProcesses.PurgeAndDeleteElements(); + m_vecTypeInstances.PurgeAndDeleteElements(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Initializes the directory +//----------------------------------------------------------------------------- +bool CDirectory::BInit( KeyValues *pKVDirectory ) +{ + Assert( !m_bInitialized ); + if ( m_bInitialized ) + return false; + + if ( NULL == pKVDirectory ) + { + AssertMsg( false, "Null KV passed to CDirectory::BInit()" ); + return false; + } + + CUtlSymbolTable processNamesTable( 0, 16, true ); + FOR_EACH_TRUE_SUBKEY( pKVDirectory, pkvProcess ) + { + if( 0 != Q_stricmp( pkvProcess->GetName( ), "process" ) ) + continue; + + //get the name of this process and ensure that it is unique + const char *pchProcessName = pkvProcess->GetString( "name" ); + if( !pchProcessName || !pchProcessName[ 0 ] ) + { + AssertMsg( false, "Process defined in the config with no name" ); + return false; + } + if( processNamesTable.Find( pchProcessName ).IsValid() ) + { + AssertMsg( false, "Duplicate box \"%s\" encountered while parsing the config", pchProcessName ); + return false; + } + processNamesTable.AddString( pchProcessName ); + + //get the type associated with this process + const char *pchProcessType = pkvProcess->GetString( "type" ); + if( !pchProcessType || !pchProcessType[ 0 ] ) + { + AssertMsg( false, "Process %s defined in the config with no process type associated with it", pchProcessName ); + return false; + } + + //create our new process info + CGCDirProcess *pProcess = new CGCDirProcess( m_vecProcesses.Count( ), pchProcessName, pchProcessType, pkvProcess ); + m_vecProcesses.AddToTail( pProcess ); + + FOR_EACH_SUBKEY( pkvProcess, pkvType ) + { + if( 0 != Q_stricmp( pkvType->GetName(), "gc" ) ) + continue; + + const char *pchGCType = NULL; + if( pkvType->GetFirstSubKey() ) + { + pchGCType = pkvType->GetString( "gc" ); + } + else + { + pchGCType = pkvType->GetString( ); + } + + if ( !pchGCType || !pchGCType[0] ) + { + AssertMsg( false, "GC defined on box \"%s\" in the config with no type", pchProcessName ); + return false; + } + + int32 nGCType = GetGCTypeForName( pchGCType ); + if ( s_nInvalidGCType == nGCType ) + { + AssertMsg( false, "GC defined on box \"%s\" with unknown type \"%s\" encountered while parsing the config", pchProcessName, pchGCType ); + return false; + } + + //note that to get the count we can't call GetGCCountForType until after we register since otherwise we don't have a type entry + int nGCTypeIndex = m_mapGCsByType.Find( nGCType ); + if ( !m_mapGCsByType.IsValidIndex( nGCType ) ) + { + nGCTypeIndex = m_mapGCsByType.Insert( nGCType ); + } + + CGCDirTypeInstance *pGC = new CGCDirTypeInstance( pchGCType, nGCType, m_mapGCsByType[ nGCTypeIndex ].Count(), pkvType, pProcess ); + pProcess->AddGC( pGC ); + m_vecTypeInstances.AddToTail( pGC ); + m_mapGCsByType[ nGCTypeIndex ].AddToTail( pGC ); + } + } + + // There must be exactly one master GC defined. Make sure it exists + int32 nMasterType = GetGCTypeForName( "master" ); + if ( s_nInvalidGCType == nMasterType ) + { + AssertMsg( false, "Master GC type is not registered" ); + return false; + } + + if ( 1 != GetGCCountForType( nMasterType ) ) + { + AssertMsg( false, "Incorrect number of master GCs in the config. Expected: 1 Actual: %d", GetGCCountForType( nMasterType ) ); + return false; + } + + const CGCDirTypeInstance *pGC = GetGCInstanceForType( nMasterType, 0 ); + if ( 0 != pGC->GetProcess()->GetGCDirIndex() ) + { + AssertMsg( false, "The master GC must be contained within the first GC defined." ); + return false; + } + + m_bInitialized = true; + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Registers a type of GC +//----------------------------------------------------------------------------- +void CDirectory::RegisterGCType( int32 nTypeID, const char *pchName ) +{ + Assert( s_nInvalidGCType != nTypeID ); + if ( s_nInvalidGCType == nTypeID ) + return; + + bool bHasElement = m_mapRegisteredGCTypes.IsValidIndex( m_mapRegisteredGCTypes.Find( nTypeID ) ); + AssertMsg( !bHasElement, "A GC of type %d has already been registered", nTypeID ); + if ( bHasElement ) + return; + + bHasElement = m_dictRegisteredGCNameToType.HasElement( pchName ); + AssertMsg( !bHasElement, "A GC type with name \"%s\" has already been registered", pchName ); + if ( bHasElement ) + return; + + RegisteredGCType_t &gcReg = m_mapRegisteredGCTypes[ m_mapRegisteredGCTypes.Insert( nTypeID ) ]; + gcReg.m_strName = pchName; + + m_dictRegisteredGCNameToType.Insert( pchName, nTypeID ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets the number of boxes hosting GCs in the universe +//----------------------------------------------------------------------------- +uint32 CDirectory::GetProcessCount() const +{ + return m_vecProcesses.Count(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets the definition for a particular box +//----------------------------------------------------------------------------- +const CGCDirProcess *CDirectory::GetProcess( int32 nIndex ) const +{ + if( !m_vecProcesses.IsValidIndex( nIndex ) ) + { + Assert( false ); + return NULL; + } + + return m_vecProcesses[ nIndex ]; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets the number of GCs in the universe +//----------------------------------------------------------------------------- +uint32 CDirectory::GetGCTypeInstanceCount() const +{ + return m_vecTypeInstances.Count(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +const CGCDirTypeInstance *CDirectory::GetGCTypeInstance( uint32 iDirGC ) const +{ + if( iDirGC >= ( uint32 )m_vecTypeInstances.Count() ) + { + Assert( false ); + return NULL; + } + + return m_vecTypeInstances[ iDirGC ]; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets the number of GCs in the universe of the given type +//----------------------------------------------------------------------------- +int32 CDirectory::GetGCCountForType( int32 nTypeID ) const +{ + int nIndex = m_mapGCsByType.Find( nTypeID ); + if ( !m_mapGCsByType.IsValidIndex( nIndex ) ) + { + EmitWarning( SPEW_GC, 2, "CDirectory::GetGCCountForType() called with unregistered type %d (%s)\n", nTypeID, GetNameForGCType( nTypeID ) ); + return 0; + } + + return m_mapGCsByType[nIndex].Count(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +const CGCDirTypeInstance *CDirectory::GetGCInstanceForType( int32 nTypeID, int32 nInstance ) const +{ + int nIndex = m_mapGCsByType.Find( nTypeID ); + if ( !m_mapGCsByType.IsValidIndex( nIndex ) ) + { + EmitWarning( SPEW_GC, 2, "CDirectory::GetGCInstanceForType() called with unregistered type %d (%s)\n", nTypeID, GetNameForGCType( nTypeID ) ); + return NULL; + } + + const CCopyableUtlVector<CGCDirTypeInstance *> &vecGCs = m_mapGCsByType[ nIndex ]; + bool bValidIndex = vecGCs.IsValidIndex( nInstance ); + Assert( bValidIndex ); + if ( !bValidIndex ) + return NULL; + + return vecGCs[ nInstance ]; +} + + +//----------------------------------------------------------------------------- +// Purpose: Given a type and index, this will hash it to be a valid in-range index for that type and return the directory index +//----------------------------------------------------------------------------- +int32 CDirectory::GetGCDirIndexForInstance( int32 nTypeID, uint32 nInstanceIndex ) const +{ + int nIndex = m_mapGCsByType.Find( nTypeID ); + if ( !m_mapGCsByType.IsValidIndex( nIndex ) ) + { + EmitWarning( SPEW_GC, 2, "CDirectory::GetGCDirIndexForInstance() called with unregistered type %d (%s)\n", nTypeID, GetNameForGCType( nTypeID ) ); + return -1; + } + + const CCopyableUtlVector<CGCDirTypeInstance *> &vecGCs = m_mapGCsByType[ nIndex ]; + uint32 nWrappedIndex = nInstanceIndex % vecGCs.Count(); + return vecGCs[ nWrappedIndex ]->GetProcess()->GetGCDirIndex(); +} + +//----------------------------------------------------------------------------- +// Purpose: given a GC type, this will add all of the GC indices associated with that type onto the provided vector +//----------------------------------------------------------------------------- +void CDirectory::GetGCDirIndicesForType( int32 nTypeID, CUtlVector< uint32 >& vecIndices ) const +{ + int nIndex = m_mapGCsByType.Find( nTypeID ); + if ( m_mapGCsByType.IsValidIndex( nIndex ) ) + { + const CCopyableUtlVector<CGCDirTypeInstance *> &vecGCs = m_mapGCsByType[ nIndex ]; + FOR_EACH_VEC( vecGCs, nGC ) + { + vecIndices.AddToTail( vecGCs[ nGC ]->GetProcess()->GetGCDirIndex() ); + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets a name for a registered GC type +//----------------------------------------------------------------------------- +const char *CDirectory::GetNameForGCType( int32 nTypeID ) const +{ + int nIndex = m_mapRegisteredGCTypes.Find( nTypeID ); + if ( !m_mapRegisteredGCTypes.IsValidIndex( nIndex ) ) + return "unknown"; + + return m_mapRegisteredGCTypes[nIndex].m_strName.Get(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets the creation factory for a registered GC types +//----------------------------------------------------------------------------- +CDirectory::GCFactory_t CDirectory::GetFactoryForProcessType( const char* pszProcessType ) const +{ + int nIndex = m_dictProcessTypeToFactory.Find( pszProcessType ); + if( nIndex == m_dictProcessTypeToFactory.InvalidIndex() ) + return NULL; + return m_dictProcessTypeToFactory[ nIndex ]; +} + +//----------------------------------------------------------------------------- +// Purpose: Registers a factory for a specific process type +//----------------------------------------------------------------------------- +void CDirectory::RegisterProcessTypeFactory( const char* pszProcessType, GCFactory_t pfnFactory ) +{ + m_dictProcessTypeToFactory.Insert( pszProcessType, pfnFactory ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets the type for a GC given a name +//----------------------------------------------------------------------------- +int32 CDirectory::GetGCTypeForName( const char *pchName ) const +{ + int nIndex = m_dictRegisteredGCNameToType.Find( pchName ); + if ( !m_dictRegisteredGCNameToType.IsValidIndex( nIndex ) ) + return s_nInvalidGCType; + + return m_dictRegisteredGCNameToType[nIndex]; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets the global directory singleton +//----------------------------------------------------------------------------- +CDirectory *GDirectory() +{ + static CDirectory g_directory; + return &g_directory; +} + + +} // namespace GCSDK diff --git a/gcsdk/framefunction.cpp b/gcsdk/framefunction.cpp new file mode 100644 index 0000000..3acd7cf --- /dev/null +++ b/gcsdk/framefunction.cpp @@ -0,0 +1,252 @@ +//========= Copyright 1996-2005, Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//=============================================================================// +#include "stdafx.h" +#include "framefunction.h" +#include "gclogger.h" +#include "gcsdk/scheduledfunction.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + +//------------------------------------------------------------------------- +CBaseFrameFunction::CBaseFrameFunction( const char *pchName, EFrameType eFrameType ) : + m_sName( pchName ), + m_EFrameType( eFrameType ), + m_nNumCalls( 0 ), + m_nTrackedTime( 0 ), + m_bRegistered( false ) +{ +} + +//------------------------------------------------------------------------- +CBaseFrameFunction::~CBaseFrameFunction() +{ + Deregister(); +} + +//called to handle registering for updates and unregistering. Not registered by default +void CBaseFrameFunction::Register() +{ + if( !m_bRegistered ) + { + GFrameFunctionMgr().Register( this ); + m_bRegistered = true; + } +} + +void CBaseFrameFunction::Deregister() +{ + if( m_bRegistered ) + { + GFrameFunctionMgr().Deregister( this ); + m_bRegistered = false; + } +} + + +//------------------------------------------------------------------------- +CFrameFunctionMgr::CFrameFunctionMgr() : + m_nNumProfileFrames( 0 ), + m_bCompletedHighPri( false ) +{ +} + +//------------------------------------------------------------------------- +void CFrameFunctionMgr::Register( CBaseFrameFunction* pFrameFunc ) +{ + if( !pFrameFunc ) + return; + + //see which type this frame function is + CBaseFrameFunction::EFrameType eType = pFrameFunc->m_EFrameType; + if( eType >= CBaseFrameFunction::k_EFrameType_Count ) + return; + + //don't allow for duplicates + if( m_MainLoopFrameFuncs[ eType ].Find( pFrameFunc ) == m_MainLoopFrameFuncs[ eType ].InvalidIndex() ) + { + m_MainLoopFrameFuncs[ eType ].AddToTail( pFrameFunc ); + } +} + +//------------------------------------------------------------------------- +void CFrameFunctionMgr::Deregister( CBaseFrameFunction* pFrameFunc ) +{ + if( !pFrameFunc ) + return; + + //see which type this frame function is + CBaseFrameFunction::EFrameType eType = pFrameFunc->m_EFrameType; + if( eType >= CBaseFrameFunction::k_EFrameType_Count ) + return; + + m_MainLoopFrameFuncs[ eType ].FindAndRemove( pFrameFunc ); +} + + +//------------------------------------------------------------------------- +void CFrameFunctionMgr::RunFrame( const CLimitTimer& limitTimer ) +{ + VPROF_BUDGET( "CFrameFunctionMgr::RunFrame", VPROF_BUDGETGROUP_STEAM ); + + //track the number of frames we've profiled + m_nNumProfileFrames++; + m_bCompletedHighPri = false; + + //run through each of our 'once a frame' updates + RunFrameList( CBaseFrameFunction::k_EFrameType_RunOnce, limitTimer ); +} + +//------------------------------------------------------------------------- +bool CFrameFunctionMgr::RunFrameTick( const CLimitTimer& limitTimer ) +{ + VPROF_BUDGET( "CFrameFunctionMgr::RunFrameTick", VPROF_BUDGETGROUP_STEAM ); + + //run high priority if we haven't finished it yet + if( !m_bCompletedHighPri ) + { + if( RunFrameList( CBaseFrameFunction::k_EFrameType_RunUntilCompleted, limitTimer ) ) + { + //we need to update another frame + return true; + } + else + { + //we have completed high priority + m_bCompletedHighPri = true; + } + } + + //if we are still here, we have completed high priority, so run until we are done with low priority + return RunFrameList( CBaseFrameFunction::k_EFrameType_RunUntilCompletedLowPri, limitTimer ); +} + +//------------------------------------------------------------------------- +bool CFrameFunctionMgr::RunFrameList( CBaseFrameFunction::EFrameType eType, const CLimitTimer& limitTimer ) +{ + bool bResult = false; + + //run through each of our 'once a frame' updates + FOR_EACH_VEC( m_MainLoopFrameFuncs[ eType ], nCurrFunc ) + { + CBaseFrameFunction* pFunc = m_MainLoopFrameFuncs[ eType ][ nCurrFunc ]; + + uint64 nTimeStart = limitTimer.CMicroSecLeft(); + + { + VPROF_BUDGET( pFunc->m_sName.Get(), VPROF_BUDGETGROUP_STEAM ); + bResult |= pFunc->BRun( limitTimer ); + } + + //track the time spent in this function + pFunc->m_nNumCalls++; + pFunc->m_nTrackedTime += nTimeStart - limitTimer.CMicroSecLeft(); + } + + return bResult; +} + +//------------------------------------------------------------------------- + +bool CFrameFunctionMgr::SortFrameFuncByTime( const CBaseFrameFunction* pLhs, const CBaseFrameFunction* pRhs ) +{ + //sort by time first, then name if the same time (unlikely) + if( pLhs->m_nTrackedTime != pRhs->m_nTrackedTime ) + return pLhs->m_nTrackedTime > pRhs->m_nTrackedTime; + return stricmp( pLhs->m_sName.Get(), pRhs->m_sName.Get() ) < 0; +} + +void CFrameFunctionMgr::DumpProfile() +{ + //collect all of our functions and sort them based upon time elapsed + CUtlVector< CBaseFrameFunction* > FrameFuncs; + uint64 nTotalTime = 0; + uint32 nTotalCalls = 0; + + for( uint32 nCurrType = 0; nCurrType < CBaseFrameFunction::k_EFrameType_Count; nCurrType++ ) + { + FOR_EACH_VEC (m_MainLoopFrameFuncs[ nCurrType ], nCurrFunc ) + { + CBaseFrameFunction* pFunc = m_MainLoopFrameFuncs[ nCurrType ][ nCurrFunc ]; + FrameFuncs.AddToTail( pFunc ); + nTotalTime += pFunc->m_nTrackedTime; + nTotalCalls += pFunc->m_nNumCalls; + } + } + + double fInvFrame = ( m_nNumProfileFrames > 0 ) ? 1.0 / m_nNumProfileFrames : 1.0; + + std::sort( FrameFuncs.begin(), FrameFuncs.end(), SortFrameFuncByTime ); + + //now dump out the timings in a grid + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%s", "Name Time(ms) Calls Time% Time/F Calls/F\n" ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%s", "---------------------------------------- ------------ ---------- ------ ---------- ----------\n" ); + + //print out each API + FOR_EACH_VEC( FrameFuncs, nFunc ) + { + CBaseFrameFunction* pFunc = FrameFuncs[ nFunc ]; + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%-40s %12.2f %10d %5.1f%% %10.3f %10.1f\n", + pFunc->m_sName.Get(), pFunc->m_nTrackedTime / 1000.0, pFunc->m_nNumCalls, pFunc->m_nTrackedTime / ( double )nTotalTime, pFunc->m_nTrackedTime * fInvFrame / 1000.0, pFunc->m_nNumCalls * fInvFrame ); + } + + //print out the footer and totals + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%s", "---------------------------------------- ------------ ---------- ------ ---------- ----------\n" ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%-40s %12.2f %10d %5.1f%% %10.3f %10.1f\n", + "Totals", nTotalTime / 1000.0, nTotalCalls, 100.0, nTotalTime * fInvFrame / 1000.0, nTotalCalls * fInvFrame ); +} + +//------------------------------------------------------------------------- +void CFrameFunctionMgr::ClearProfile() +{ + //run through all of our functions and clear timings + for( uint32 nCurrType = 0; nCurrType < CBaseFrameFunction::k_EFrameType_Count; nCurrType++ ) + { + FOR_EACH_VEC (m_MainLoopFrameFuncs[ nCurrType ], nCurrFunc ) + { + CBaseFrameFunction* pFunc = m_MainLoopFrameFuncs[ nCurrType ][ nCurrFunc ]; + pFunc->m_nTrackedTime = 0; + pFunc->m_nNumCalls = 0; + } + } + + m_nNumProfileFrames = 0; +} + +//------------------------------------------------------------------------- +CFrameFunctionMgr& GFrameFunctionMgr() +{ + static CFrameFunctionMgr s_Singleton; + return s_Singleton; +} + +//------------------------------------------------------------------------- + +//utility class for dumping out the profile results after time has expired +static void DumpFrameFunctionProfile() +{ + GFrameFunctionMgr().DumpProfile(); +} + +GC_CON_COMMAND( framefunc_profile, "Turns on frame function profiling for N seconds and dumps the results" ) +{ + if( args.ArgC() < 2 ) + { + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "Proper usage is framefunc_profile <n> where n is the number of seconds to profile for\n" ); + return; + } + + float fSeconds = MAX( 1.0f, atof( args[ 1 ] ) ); + GFrameFunctionMgr().ClearProfile(); + static CGlobalScheduledFunction s_DumpProfile; + s_DumpProfile.ScheduleMS( DumpFrameFunctionProfile, fSeconds * 1000.0f ); +} + +} //namespace GCSDK diff --git a/gcsdk/gc_convar.cpp b/gcsdk/gc_convar.cpp new file mode 100644 index 0000000..0376562 --- /dev/null +++ b/gcsdk/gc_convar.cpp @@ -0,0 +1,68 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//=============================================================================// + +#include "stdafx.h" +#include "gcsdk/gclogger.h" + +#include "memdbgon.h" // needs to be the last include in the file + +using namespace GCSDK; + +const char *GCConVar::GetName( void ) const +{ + if ( m_strGCName.Length() == 0 ) + { + m_pchBaseName = ConVar::GetName(); + if( GCSDK::GGCBase() ) + { + m_strGCName.Format( "%s_%u", m_pchBaseName, GCSDK::GGCBase()->GetAppID() ); + } + else + { + m_strGCName.Format( "%s_gc", m_pchBaseName ); + } + } + + return m_strGCName.String(); +} + + +const char *GCConCommand::GetName( void ) const +{ + if ( m_strGCName.Length() == 0 ) + { + m_pchBaseName = ConCommand::GetName(); + if( GCSDK::GGCBase() ) + { + m_strGCName.Format( "%s_%u", m_pchBaseName, GCSDK::GGCBase()->GetAppID() ); + } + else + { + m_strGCName.Format( "%s_gc", m_pchBaseName ); + } + } + + return m_strGCName.String(); +} + +//----------------------------------------------------------------------------- +// Purpose: Checks a command to see if it got enough arguments +// Input : nArgs - The minimum required args on the command +// args - The arguments passed to the command +// command - The command being executed +//----------------------------------------------------------------------------- +bool BCheckArgs( int nArgs, const CCommand &args, const ConCommandBase &command ) +{ + if ( args.ArgC() <= nArgs ) + { + EG_ERROR( SPEW_CONSOLE, "Incorrect number of arguments. %d arguments required, %d were given.\n", nArgs, args.ArgC() - 1 ); + EG_ERROR( SPEW_CONSOLE, "\t%s\n", command.GetHelpText() ); + return false; + } + + return true; +} diff --git a/gcsdk/gc_sharedobjectcache.cpp b/gcsdk/gc_sharedobjectcache.cpp new file mode 100644 index 0000000..cfea6b3 --- /dev/null +++ b/gcsdk/gc_sharedobjectcache.cpp @@ -0,0 +1,1064 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Extra GC-specific code on top of CGCSharedObjectCache +// +//============================================================================= + +#include "stdafx.h" +#include <time.h> +#include "gcsdk_gcmessages.pb.h" + +#include "tier0/memdbgoff.h" + +namespace GCSDK +{ + +//---------------------------------------------------------------------------- +// Purpose: Container for the cached message, so we can track how many of these +// things there are +//---------------------------------------------------------------------------- +class CCachedSubscriptionMessage +{ + DECLARE_CLASS_MEMPOOL( CCachedSubscriptionMessage ); +public: + + CCachedSubscriptionMessage() + : message( k_ESOMsg_CacheSubscribed ) + { + + } + + ~CCachedSubscriptionMessage() + { + + } + + CProtoBufMsg<CMsgSOCacheSubscribed> message; + +}; + +IMPLEMENT_CLASS_MEMPOOL( CCachedSubscriptionMessage, 10 * 1000, UTLMEMORYPOOL_GROW_SLOW ); +} + +#include "tier0/memdbgon.h" // needs to be the last include in the file + +namespace GCSDK +{ + +//---------------------------------------------------------------------------- +// Purpose: Default filter object for subscriber messages whi +//---------------------------------------------------------------------------- +class CDefaultSubscriberMessageFilter : public CISubscriberMessageFilter +{ +public: + CDefaultSubscriberMessageFilter() + { + + } + + virtual bool BShouldSendAnyObjectsInCache( CGCSharedObjectTypeCache *pTypeCache, uint32 unFlags ) const + { + return ( CSharedObject::GetTypeFlags( pTypeCache->GetTypeID() ) & unFlags ) != 0; + } + + virtual bool BShouldSendObject( CSharedObject *pSharedObject, uint32 unFlags ) const + { + return ( CSharedObject::GetTypeFlags( pSharedObject->GetTypeID() ) & unFlags ) != 0; + } +}; + +//---------------------------------------------------------------------------- +// Purpose: constructor +//---------------------------------------------------------------------------- +CSharedObjectContext::CSharedObjectContext( const CSteamID & steamIDOwner ) + : m_steamIDOwner( steamIDOwner ) +{ +} + +//---------------------------------------------------------------------------- +// Purpose: Finds the steam ID within the vector +//---------------------------------------------------------------------------- +int CSharedObjectContext::FindSubscriber( const CSteamID& steamID ) const +{ + FOR_EACH_VEC( m_vecSubscribers, nSubscriber ) + { + if( steamID == m_vecSubscribers[ nSubscriber ].m_steamID ) + return nSubscriber; + } + return m_vecSubscribers.InvalidIndex(); +} + +//---------------------------------------------------------------------------- +// Purpose: Adds a new subscriber to the cache. All objects in the cache will +// be sent as create messages to the new subscriber +// Returns false if it simply incremented the refcount of the subscribers of this cache +//---------------------------------------------------------------------------- +bool CSharedObjectContext::BAddSubscriber( const CSteamID & steamID ) +{ + MEM_ALLOC_CREDIT_CLASS(); + int iIdx = FindSubscriber( steamID ); + if ( iIdx != m_vecSubscribers.InvalidIndex() ) + { + m_vecSubscribers[ iIdx ].m_nRefCount++; + return false; + } + + Subscriber_t Subscriber; + Subscriber.m_steamID = steamID; + Subscriber.m_nRefCount = 1; + m_vecSubscribers.AddToTail( Subscriber ); + return true; +} + + +//---------------------------------------------------------------------------- +// Purpose: Removes a subscriber from the cache. All objects in the cache +// will have destroy messages sent for them to the new subscriber. +//---------------------------------------------------------------------------- +bool CSharedObjectContext::BRemoveSubscriber( const CSteamID & steamID ) +{ + int iIdx = FindSubscriber( steamID ); + if ( iIdx != m_vecSubscribers.InvalidIndex() ) + { + m_vecSubscribers[ iIdx ].m_nRefCount--; + + if ( m_vecSubscribers[ iIdx ].m_nRefCount == 0 ) + { + m_vecSubscribers.FastRemove( iIdx ); + return true; + } + } + + return false; +} + + +//---------------------------------------------------------------------------- +// Purpose: Removes all subscribers from the context +//---------------------------------------------------------------------------- +void CSharedObjectContext::RemoveAllSubscribers() +{ + m_vecSubscribers.RemoveAll(); +} + + +//---------------------------------------------------------------------------- +// Purpose: Constructor +//---------------------------------------------------------------------------- +CGCSharedObjectTypeCache::CGCSharedObjectTypeCache( int nTypeID, const CSharedObjectContext & context ) + : m_context( context ), CSharedObjectTypeCache( nTypeID ) +{ + +} + + +//---------------------------------------------------------------------------- +// Purpose: Destructor +//---------------------------------------------------------------------------- +CGCSharedObjectTypeCache::~CGCSharedObjectTypeCache() +{ + g_SharedObjectStats.TrackSharedObjectLifetime( GetTypeID(), -( ( int32 )GetCount() ) ); +} + +//---------------------------------------------------------------------------- +// Purpose: Adds a shared object of the appropriate type to this type cache. +//---------------------------------------------------------------------------- +void CGCSharedObjectTypeCache::EnsureCapacity( uint32 nItems ) +{ + MEM_ALLOC_CREDIT_CLASS(); + Base::EnsureCapacity( nItems ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Adds a shared object of the appropriate type to this type cache. +//---------------------------------------------------------------------------- +bool CGCSharedObjectTypeCache::AddObject( CSharedObject *pObject ) +{ + MEM_ALLOC_CREDIT_CLASS(); + Assert( GetTypeID() == pObject->GetTypeID() ); + if( CSharedObjectTypeCache::AddObject( pObject ) ) + { + g_SharedObjectStats.TrackSharedObjectLifetime( GetTypeID(), 1 ); + return true; + } + + return false; +} + + +//---------------------------------------------------------------------------- +// Purpose: Destroys the object matching the one passed in. This could be the +// same one or simply one with matching index fields. +//---------------------------------------------------------------------------- +CSharedObject *CGCSharedObjectTypeCache::RemoveObject( const CSharedObject & soIndex ) +{ + CSharedObject *pObj = CSharedObjectTypeCache::RemoveObject( soIndex ); + if( pObj ) + { + g_SharedObjectStats.TrackSharedObjectLifetime( GetTypeID(), -1 ); + } + + return pObj; +} + + +//---------------------------------------------------------------------------- +// Purpose: Populates a message with the data to create every object of this +// type in the cache. +//---------------------------------------------------------------------------- +void CGCSharedObjectTypeCache::BuildCacheSubscribedMsg( CMsgSOCacheSubscribed_SubscribedType *pMsgType, uint32 unFlags, const CISubscriberMessageFilter &filter ) +{ + //MEM_ALLOC_CREDIT_( CFmtStr( "StartPlaying - Type BuildMsg - %s", CSharedObject::PchClassName( GetTypeID() ) ) ); + VPROF_BUDGET( CSharedObject::PchClassBuildCacheNodeName( GetTypeID() ), VPROF_BUDGETGROUP_STEAM ); + + uint32 nNumSubscribed = 0; + for( uint32 i=0; i<GetCount(); i++ ) + { + CSharedObject *pObj = GetObject( i ); + + if ( !filter.BShouldSendObject( pObj, unFlags ) ) + continue; + + if ( !pObj->BAddToMessage( pMsgType->add_object_data() ) ) + { + EmitWarning( SPEW_SHAREDOBJ, SPEW_ALWAYS, "Failed to add create fields to message in create" ); + } + else + { + nNumSubscribed++; + } + } + + //track this send + g_SharedObjectStats.TrackSubscription( GetTypeID(), unFlags, nNumSubscribed ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Constructor +//---------------------------------------------------------------------------- +CGCSharedObjectCache::CGCSharedObjectCache( const CSteamID & steamIDOwner ) + : m_context( steamIDOwner ), + m_bInWriteback( false ), + m_unWritebackTime( 0 ), + m_unLRUHandle( CUtlLinkedList< CSteamID, uint32 >::InvalidIndex() ), + m_unCachedSubscriptionMsgFlags( 0 ), + m_pCachedSubscriptionMsg( NULL ), + m_bDetectVersionChanges( true ) +#if WITH_SOCACHE_LRU_DEBUGGING +, m_unDebugTag( 0 ) +#endif // WITH_SOCACHE_LRU_DEBUGGING +{ + +} + + +//---------------------------------------------------------------------------- +// Purpose: Destructor +//---------------------------------------------------------------------------- +CGCSharedObjectCache::~CGCSharedObjectCache() +{ + delete m_pCachedSubscriptionMsg; +} + + +const CISubscriberMessageFilter &CGCSharedObjectCache::GetSubscriberMessageFilter() +{ + static CDefaultSubscriberMessageFilter sFilter; + return sFilter; +} + +//---------------------------------------------------------------------------- +// Purpose: Adds a shared object to the cache. +//---------------------------------------------------------------------------- +bool CGCSharedObjectCache::AddObject( CSharedObject *pSharedObject ) +{ + Assert( pSharedObject ); + if ( !GGCBase()->BIsSOCacheBeingLoaded( GetOwner() ) ) + { + AssertMsg( GGCBase()->IsSteamIDLockedByCurJob( GetOwner() ), "Attempt to add an object to an unlocked SO cache!" ); + } + + if ( !CSharedObjectCache::AddObject( pSharedObject ) ) + return false; + + uint32 unTypeFlags = CSharedObject::GetTypeFlags( pSharedObject->GetTypeID() ); + if( BShouldSendToAnyClients( unTypeFlags ) ) + { + DirtyNetworkObjectCreate( pSharedObject ); + } + + return true; +} + +//---------------------------------------------------------------------------- +// Purpose: Adds a shared object to the cache without dirtying the object or the cache +//---------------------------------------------------------------------------- +bool CGCSharedObjectCache::AddObjectClean( CSharedObject *pSharedObject ) +{ + CSharedObjectTypeCache *pTypeCache = CreateBaseTypeCache( pSharedObject->GetTypeID() ); + return pTypeCache->AddObjectClean( pSharedObject ); +} + +//---------------------------------------------------------------------------- +// Purpose: Removes the object matching the one passed in from this cache, +// without destroying the actual object. +//---------------------------------------------------------------------------- +CSharedObject *CGCSharedObjectCache::RemoveObject( const CSharedObject & soIndex ) +{ + AssertMsg( GGCBase()->IsSteamIDLockedByCurJob( GetOwner() ), "Attempt to remove an object from an unlocked SO cache!" ); + + CSharedObject *pObject = CSharedObjectCache::RemoveObject( soIndex ); + + if ( pObject ) + { + uint32 unTypeFlags = CSharedObject::GetTypeFlags( pObject->GetTypeID() ); + if( BShouldSendToAnyClients( unTypeFlags ) ) + { + const CISubscriberMessageFilter &filter = GetSubscriberMessageFilter(); + const CUtlVector< CSharedObjectContext::Subscriber_t > &vecSubscribers = m_context.GetSubscribers(); + + CUtlVector<CSteamID> vecRecipients( 0, vecSubscribers.Count() ); + FOR_EACH_VEC( vecSubscribers, nSub ) + { + const CSteamID & steamID = vecSubscribers[nSub].m_steamID; + uint32 unFlags = CalcSendFlags( steamID ); + if( filter.BShouldSendObject( pObject, unFlags ) ) + { + VPROF_BUDGET( CSharedObject::PchClassCreateNodeName( pObject->GetTypeID() ), VPROF_BUDGETGROUP_STEAM ); + vecRecipients.AddToTail( vecSubscribers[nSub].m_steamID ); + } + } + + if ( vecRecipients.Count() > 0 ) + { + VPROF_BUDGET( CSharedObject::PchClassCreateNodeName( pObject->GetTypeID() ), VPROF_BUDGETGROUP_STEAM ); +#if 0 + // Kyle says: KFIXME: this is the more current way of doing things but we're missing some multi-broadcast + // code that hasn't been integrated from Dota yet. + pObject->BSendDestroyToSteamIDs( vecRecipients, m_context.GetOwner(), GetVersion() ); +#else + FOR_EACH_VEC( vecRecipients, i ) + { + pObject->BSendDestroyToSteamID( vecRecipients[i], m_context.GetOwner(), GetVersion() ); + } +#endif + g_SharedObjectStats.TrackSharedObjectSendDestroy( pObject->GetTypeID(), vecRecipients.Count() ); + } + } + + m_networkDirtyObjs.Remove( pObject ); + if ( 0 == m_networkDirtyObjs.Count() ) + { + m_networkDirtyObjs.Purge(); + } + + m_networkDirtyObjsCreate.Remove( pObject ); + if ( 0 == m_networkDirtyObjsCreate.Count() ) + { + m_networkDirtyObjsCreate.Purge(); + } + + m_databaseDirtyList.FindAndRemove( pObject ); + } + + return pObject; +} + + +//---------------------------------------------------------------------------- +// Purpose: Destroys the object matching the one passed in. This could be the +// same one or simply one with matching index fields. +//---------------------------------------------------------------------------- +bool CGCSharedObjectCache::BDestroyObject( const CSharedObject & soIndex, bool bRemoveFromDatabase ) +{ + CSharedObject *pObject = RemoveObject( soIndex ); + + if ( !pObject ) + return false; + + if ( bRemoveFromDatabase ) + { + pObject->BYieldingRemoveFromDatabase(); + } + + delete pObject; + return true; +} + +//---------------------------------------------------------------------------- +// Purpose: Adds a new subscriber to the cache. All objects in the cache will +// be sent as create messages to the new subscriber +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::SetTradingPartner( const CSteamID &steamID ) +{ + if ( m_steamIDTradingPartner.IsValid() ) + { + RemoveSubscriber( m_steamIDTradingPartner ); + } + m_steamIDTradingPartner = steamID; + if ( steamID.IsValid() ) + { + AddSubscriber( steamID ); + } +} + +//---------------------------------------------------------------------------- +// Purpose: Adds a new subscriber to the cache. All objects in the cache will +// be sent as create messages to the new subscriber +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::AddSubscriber( const CSteamID & steamID, bool bForceSendSubscriptionMsg ) +{ + MEM_ALLOC_CREDIT_("StartPlaying - AddSubscriber" ); + VPROF_BUDGET( "CGCSharedObjectCache::AddSubscriber()", VPROF_BUDGETGROUP_STEAM ); + + Assert( steamID.IsValid() ); + bool bAdded = m_context.BAddSubscriber( steamID ); + if ( bAdded || bForceSendSubscriptionMsg ) + { + // ask the client if it really needs the contents of this cache, because we could potentially send a lot of stuff + CProtoBufMsg<CMsgSOCacheSubscriptionCheck> msg( k_ESOMsg_CacheSubscriptionCheck ); + msg.Body().set_owner( GetOwner().ConvertToUint64() ); + msg.Body().set_version( GetVersion() ); + GGCBase()->BSendGCMsgToClient( steamID, msg ); + } +} + +//---------------------------------------------------------------------------- +// Purpose: +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::SendSubscriberMessage( const CSteamID & steamID ) +{ + Assert( steamID.IsValid() ); + if ( !m_context.BIsSubscribed( steamID ) ) + return; + + uint32 unFlags = CalcSendFlags( steamID ); + + if ( unFlags == k_ESOFlag_SendToNobody ) + return; + + // cache off the *last* one + if ( unFlags != m_unCachedSubscriptionMsgFlags ) + { + ClearCachedSubscriptionMessage(); + m_pCachedSubscriptionMsg = BuildSubscriberMessage( unFlags ); + m_unCachedSubscriptionMsgFlags = unFlags; + } + GGCBase()->BSendGCMsgToClient( steamID, m_pCachedSubscriptionMsg->message ); +} + +//---------------------------------------------------------------------------- +// Purpose: +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::ClearCachedSubscriptionMessage() +{ + delete m_pCachedSubscriptionMsg; + m_pCachedSubscriptionMsg = NULL; + m_unCachedSubscriptionMsgFlags = 0; +} + +//---------------------------------------------------------------------------- +// Purpose: Builds a subscriber message that can be sent to multiple clients +//---------------------------------------------------------------------------- +CCachedSubscriptionMessage * CGCSharedObjectCache::BuildSubscriberMessage( uint32 unFlags ) +{ + MEM_ALLOC_CREDIT_( "BuildSubscriberMessage" ); + CCachedSubscriptionMessage *pCachedMsg = new CCachedSubscriptionMessage; + + CProtoBufMsg<CMsgSOCacheSubscribed> &msg = pCachedMsg->message; + msg.Body().set_owner( GetOwner().ConvertToUint64() ); + msg.Body().set_version( GetVersion() ); + + const CISubscriberMessageFilter &filter = GetSubscriberMessageFilter(); + + for( int nTypeIndex = 0; nTypeIndex < GetTypeCacheCount(); nTypeIndex++ ) + { + CGCSharedObjectTypeCache *pTypeCache = (CGCSharedObjectTypeCache *)GetTypeCacheByIndex( nTypeIndex ); + if( !pTypeCache || !pTypeCache->GetCount() ) + continue; + + if ( !filter.BShouldSendAnyObjectsInCache( pTypeCache, unFlags ) ) + continue; + + CMsgSOCacheSubscribed_SubscribedType *pMsgType = msg.Body().add_objects(); + pMsgType->set_type_id( pTypeCache->GetTypeID() ); + pTypeCache->BuildCacheSubscribedMsg( pMsgType, unFlags, filter ); + } + + return pCachedMsg; +} + + +//---------------------------------------------------------------------------- +// Purpose: Sends a message to the client to unsubscribe from this cache +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::SendUnsubscribeMessage( const CSteamID & steamID ) +{ + + // Don't send these while shutting down. We'll use higher-level + // connection-oriented messages instead + if ( GGCBase()->GetIsShuttingDown() ) + return; + + CProtoBufMsg< CMsgSOCacheUnsubscribed > msg( k_ESOMsg_CacheUnsubscribed ); + msg.Body().set_owner( GetOwner().ConvertToUint64() ); + GGCBase()->BSendGCMsgToClient( steamID, msg ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Removes a subscriber from the cache. All objects in the cache +// will have destroy messages sent for them to the new subscriber. +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::RemoveSubscriber( const CSteamID & steamID ) +{ + MEM_ALLOC_CREDIT_("CGCSharedObjectCache::RemoveSubscriber" ); + Assert( steamID.IsValid() ); + if( m_context.BRemoveSubscriber( steamID ) ) + { + SendUnsubscribeMessage( steamID ); + } +} + + +//---------------------------------------------------------------------------- +// Purpose: Removes all subscribers from the cache. +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::RemoveAllSubscribers() +{ + const CUtlVector< CSharedObjectContext::Subscriber_t > &vecSubscribers = m_context.GetSubscribers(); + FOR_EACH_VEC( vecSubscribers, i ) + { + SendUnsubscribeMessage( vecSubscribers[i].m_steamID ); + } + + m_context.RemoveAllSubscribers(); +} + +//---------------------------------------------------------------------------- +// Purpose: Check to see if a given shared object is in this SO cache. +//---------------------------------------------------------------------------- +bool CGCSharedObjectCache::IsObjectCached( const CSharedObject *pObj, uint32 nTypeID ) const +{ + // pObj->GetTypeID() isn't called because the method is virtual, + // and this code gets called from a destructor. + CSharedObjectTypeCache *pTypeCache = FindBaseTypeCache( nTypeID ); + if ( pTypeCache == NULL ) + { + return false; + } + for ( uint i = 0; i < pTypeCache->GetCount(); i++ ) + { + if ( pTypeCache->GetObject( i ) == pObj ) + { + return true; + } + } + return false; +} + +//---------------------------------------------------------------------------- +// Purpose: Check to see if a given shared object this SO cache's dirty lists. +//---------------------------------------------------------------------------- +bool CGCSharedObjectCache::IsObjectDirty( const CSharedObject *pConstObj ) const +{ + //the list types aren't really all that const correct + CSharedObject* pObj = const_cast< CSharedObject* >( pConstObj ); + return m_databaseDirtyList.HasElement( pObj ) + || m_networkDirtyObjsCreate.HasElement( pObj ) + || m_networkDirtyObjs.HasElement( pObj ); +} + +//---------------------------------------------------------------------------- +// Purpose: Remembers that this field and this object are dirty +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::DirtyNetworkObject( CSharedObject *pObj ) +{ +#ifdef DEBUG + // Make sure the object is in the shared object cache. + Assert( IsObjectCached( pObj, pObj->GetTypeID() ) ); +#endif + + MarkDirty(); + + // add it to our dirty list if it wasn't already dirty + // if its the create queue, don't put it in the update queue + UtlHashHandle_t netHandleCreate = m_networkDirtyObjsCreate.Find( pObj ); + if ( netHandleCreate == m_networkDirtyObjsCreate.InvalidHandle() ) + { + m_networkDirtyObjs.Insert( pObj ); + } +} + +//---------------------------------------------------------------------------- +// Purpose: Remembers that this field and this object are dirty +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::DirtyNetworkObjectCreate( CSharedObject *pObj ) +{ +#ifdef DEBUG + // Make sure the object is in the shared object cache. + Assert( IsObjectCached( pObj, pObj->GetTypeID() ) ); +#endif + + MarkDirty(); + // add it to our dirty list if it wasn't already dirty + m_networkDirtyObjsCreate.Insert( pObj ); +} + +//---------------------------------------------------------------------------- +// Purpose: Remembers that this field and this object are dirty +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::DirtyDatabaseObjectField( CSharedObject *pObj, int nFieldIndex ) +{ +#ifdef DEBUG + // Make sure the object is in the shared object cache. + Assert( IsObjectCached( pObj, pObj->GetTypeID() ) ); +#endif + + // add it to our dirty list if it wasn't already dirty + m_databaseDirtyList.DirtyObjectField( pObj, nFieldIndex ); +} + +//---------------------------------------------------------------------------- +// Purpose: Remembers that this field and this object are dirty +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::DirtyObjectField( CSharedObject *pObj, int nFieldIndex ) +{ + if ( pObj->BIsNetworked() ) + { + DirtyNetworkObject( pObj ); + } + if ( pObj->BIsDatabaseBacked() ) + { + DirtyDatabaseObjectField( pObj, nFieldIndex ); + } +} + +//---------------------------------------------------------------------------- +// Purpose: Sends all network updates for this object and forgets it was dirty. +// If the object was not dirty, do nothing. +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::SendNetworkUpdates( CSharedObject *pObj ) +{ + UtlHashHandle_t netHandleCreate = m_networkDirtyObjsCreate.Find( pObj ); + if ( netHandleCreate != m_networkDirtyObjsCreate.InvalidHandle() ) + { + SendNetworkCreateInternal( pObj ); + m_networkDirtyObjsCreate.Remove( pObj ); + } + + UtlHashHandle_t netHandle = m_networkDirtyObjs.Find( pObj ); + if ( netHandle != m_networkDirtyObjs.InvalidHandle() ) + { + SendNetworkUpdateInternal( pObj ); + m_networkDirtyObjs.Remove( pObj ); + } +} + +//---------------------------------------------------------------------------- +// Purpose: Sends network updates for all network-dirty objects +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::SendAllNetworkUpdates() +{ + VPROF_BUDGET( "CGCSharedObjectCache::SendAllNetworkUpdates", VPROF_BUDGETGROUP_STEAM ); + + // Create messages first + if ( m_networkDirtyObjsCreate.Count() ) + { + FOR_EACH_HASHTABLE( m_networkDirtyObjsCreate, i ) + { + SendNetworkCreateInternal( m_networkDirtyObjsCreate.Key( i ) ); + } + + m_networkDirtyObjsCreate.RemoveAll(); + } + + // Check if anything is actually dirty first + if ( !m_networkDirtyObjs.Count() ) + return; + + // build all object messages first + + CUtlVector< CMsgSOMultipleObjects_SingleObject > vecObjectMsgs( 0, m_networkDirtyObjs.Count() ); + CUtlVector< CSharedObject* > vecObjects( 0, m_networkDirtyObjs.Count() ); + + { + VPROF_BUDGET( "CGCSharedObjectCache::SendAllNetworkUpdates - build messages", VPROF_BUDGETGROUP_STEAM ); + FOR_EACH_HASHTABLE( m_networkDirtyObjs, i ) + { + CSharedObject *pObj = m_networkDirtyObjs.Key( i ); + uint32 unFlags = CSharedObject::GetTypeFlags( pObj->GetTypeID() ); + if( !BShouldSendToAnyClients( unFlags ) ) + continue; + + // build single object msg + vecObjects.AddToTail( pObj ); + int idx = vecObjectMsgs.AddToTail(); + CMsgSOMultipleObjects_SingleObject &msg = vecObjectMsgs[idx]; + msg.set_type_id( pObj->GetTypeID() ); + pObj->BAddToMessage( msg.mutable_object_data() ); + } + } + m_networkDirtyObjs.RemoveAll(); + + // Check if anything is still dirty. It's possible that everything dirty are not sent to clients + if ( 0 == vecObjects.Count() ) + return; + + // now build uber msg for each subscriber + { + VPROF_BUDGET( "CGCSharedObjectCache::SendAllNetworkUpdates - send all messages", VPROF_BUDGETGROUP_STEAM ); + + CUtlVector<int> vecObjsToSendToThisUser( 0, vecObjects.Count() ); + + const CUtlVector< CSharedObjectContext::Subscriber_t > &vecSubscribers = m_context.GetSubscribers(); + const CISubscriberMessageFilter &filter = GetSubscriberMessageFilter(); + + FOR_EACH_VEC( vecSubscribers, nSub ) + { + vecObjsToSendToThisUser.RemoveAll(); + + // Figure out which, if any, objects should be sent to this user + uint32 unFlags = CalcSendFlags( vecSubscribers[nSub].m_steamID ); + FOR_EACH_VEC( vecObjectMsgs, idx ) + { + CSharedObject *pObj = vecObjects[idx]; + if( filter.BShouldSendObject( pObj, unFlags ) ) + { + vecObjsToSendToThisUser.AddToTail( idx ); + } + } + + // If we have any objects to send then build the message and send it + if ( vecObjsToSendToThisUser.Count() > 0 ) + { + VPROF_BUDGET( "CGCSharedObjectCache::SendAllNetworkUpdates - Sending Msg", VPROF_BUDGETGROUP_STEAM ); + CProtoBufMsg< CMsgSOMultipleObjects > msg( k_ESOMsg_UpdateMultiple ); + msg.Body().set_owner( GetOwner().ConvertToUint64() ); + msg.Body().set_version( GetVersion() ); + + FOR_EACH_VEC( vecObjsToSendToThisUser, idx ) + { + const CMsgSOMultipleObjects_SingleObject &objectMsg = vecObjectMsgs[vecObjsToSendToThisUser[idx]]; + g_SharedObjectStats.TrackSharedObjectSend( objectMsg.type_id(), 1, objectMsg.object_data().size() ); + // KFIXME: Tracking for when multiplex changes integrated + // g_SharedObjectStats.TrackSharedObjectSend( vecObjsToSendToThisUser[idx]->type_id(), mapFlagsToObjectsToSend[iMap]->m_vecRecipients.Count(), vecObjsToSendToThisUser[idx]->object_data().size() ); + CMsgSOMultipleObjects_SingleObject *pObjMsg = msg.Body().add_objects(); + *pObjMsg = objectMsg; + } + + GGCBase()->BSendGCMsgToClient( vecSubscribers[nSub].m_steamID, msg ); + } + } + } +} + +//---------------------------------------------------------------------------- +// Purpose: Enqueues a flush instruction to Econ service for Web Inventory to update +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::FlushInventoryCache() +{ + const CSteamID& steamID = GetOwner(); + if ( steamID.IsValid() && steamID.BIndividualAccount() ) + { + GGCBase()->FlushInventoryCache( steamID.GetAccountID() ); + } +} + +//---------------------------------------------------------------------------- +// Purpose: Adds the write to the transaction +//---------------------------------------------------------------------------- +bool CGCSharedObjectCache::BYieldingAddWriteToTransaction( CSharedObject *pObj, CSQLAccess & sqlAccess ) +{ + CSteamID cacheOwner = m_context.GetOwner(); + Assert( GGCBase()->IsSteamIDLockedByCurJob( cacheOwner ) ); + + CCopyableUtlVector< int > dirtyFields; + m_databaseDirtyList.GetDirtyFieldSetByObj( pObj, dirtyFields ); + if ( !pObj->BYieldingAddWriteToTransaction( sqlAccess, dirtyFields ) ) + return false; + m_databaseDirtyList.FindAndRemove( pObj ); + + // Add a listener to the transaction to re-add these to the cache should the transaction fail. This stores off a + // copy of the dirty object list, such that the caller can clear theirs now and take other actions - we'll merge + // back in upon rollback. + auto *pSelf = this; + sqlAccess.AddRollbackListener( [pSelf, cacheOwner, pObj, dirtyFields] () \ + { + Assert( GGCBase()->IsSteamIDLockedByCurJob( cacheOwner ) ); + FOR_EACH_VEC( dirtyFields, idx ) + { + pSelf->DirtyDatabaseObjectField( pObj, dirtyFields[idx] ); + } + + // we don't need to worry about re-adding to writeback -- Callers must maintain a lock of the cache for the + // duration, so we can't be processed for writeback unless the caller is writeback itself. + } ); + + return true; +} + +//---------------------------------------------------------------------------- +// Purpose: Sends network updates for all network-dirty objects +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::SendNetworkCreateInternal( CSharedObject * pObj ) +{ + uint32 unFlags = CSharedObject::GetTypeFlags( pObj->GetTypeID() ); + if( !BShouldSendToAnyClients( unFlags ) ) + return; + + VPROF_BUDGET( CSharedObject::PchClassUpdateNodeName( pObj->GetTypeID() ), VPROF_BUDGETGROUP_STEAM ); + + const CISubscriberMessageFilter &filter = GetSubscriberMessageFilter(); + const CUtlVector< CSharedObjectContext::Subscriber_t > &vecSubscribers = m_context.GetSubscribers(); + + CUtlVector<CSteamID> vecRecipients( 0, vecSubscribers.Count() ); + FOR_EACH_VEC( vecSubscribers, nSub ) + { + const CSteamID & steamID = vecSubscribers[nSub].m_steamID; + Assert( steamID.IsValid() ); + uint32 unFlags = CalcSendFlags( steamID ); + if( filter.BShouldSendObject( pObj, unFlags ) ) + { + VPROF_BUDGET( CSharedObject::PchClassCreateNodeName( pObj->GetTypeID() ), VPROF_BUDGETGROUP_STEAM ); + pObj->BSendCreateToSteamID( steamID, m_context.GetOwner(), GetVersion() ); + vecRecipients.AddToTail( steamID ); + } + } + + if ( vecRecipients.Count() > 0 ) + { + g_SharedObjectStats.TrackSharedObjectSendCreate( pObj->GetTypeID(), vecRecipients.Count() ); + } +} + +//---------------------------------------------------------------------------- +// Purpose: Sends network updates for all network-dirty objects +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::SendNetworkUpdateInternal( CSharedObject * pObj ) +{ + uint32 unFlags = CSharedObject::GetTypeFlags( pObj->GetTypeID() ); + if( !BShouldSendToAnyClients( unFlags ) ) + return; + + VPROF_BUDGET( CSharedObject::PchClassUpdateNodeName( pObj->GetTypeID() ), VPROF_BUDGETGROUP_STEAM ); + + CProtoBufMsg< CMsgSOSingleObject > msg( k_ESOMsg_Update ); + msg.Body().set_owner( GetOwner().ConvertToUint64() ); + msg.Body().set_type_id( pObj->GetTypeID() ); + msg.Body().set_version( GetVersion() ); + pObj->BAddToMessage( msg.Body().mutable_object_data() ); + + const CISubscriberMessageFilter &filter = GetSubscriberMessageFilter(); + const CUtlVector< CSharedObjectContext::Subscriber_t > &vecSubscribers = m_context.GetSubscribers(); + + FOR_EACH_VEC( vecSubscribers, nSub ) + { + uint32 unFlags = CalcSendFlags( vecSubscribers[nSub].m_steamID ); + if( filter.BShouldSendObject( pObj, unFlags ) ) + { + GGCBase()->BSendGCMsgToClient( vecSubscribers[nSub].m_steamID, msg ); + } + } +} + + +//---------------------------------------------------------------------------- +// Purpose: Saves the object to the database if it is dirty. +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::YieldingWriteToDatabase( CSharedObject *pObj ) +{ + CUtlVector< int > dirtyFields; + if( m_databaseDirtyList.GetDirtyFieldSetByObj( pObj, dirtyFields ) ) + { + if ( pObj->BYieldingWriteToDatabase( dirtyFields ) ) + { + m_databaseDirtyList.FindAndRemove( pObj ); + } + } +} + +//---------------------------------------------------------------------------- +// Purpose: Adds all dirty objects to a transaction to save to the database +// Returns: The number of objects added to the transaction +//---------------------------------------------------------------------------- +uint32 CGCSharedObjectCache::YieldingStageAllWrites( CSQLAccess & sqlAccess ) +{ + int count = 0; + int nDirtyObjects = m_databaseDirtyList.NumDirtyObjects(); + for( int i = 0; i < nDirtyObjects; ++i ) + { + CUtlVector< int > dirtyFields; + CSharedObject *pObj; + if ( m_databaseDirtyList.GetDirtyFieldSetByIndex( i, &pObj, dirtyFields ) ) + { + if ( pObj->BYieldingAddWriteToTransaction( sqlAccess, dirtyFields ) ) + { + count++; + } + } + } + + // Add a listener to flush our list if this transaction succeeds + auto *pSelf = this; + sqlAccess.AddCommitListener( [pSelf, nDirtyObjects] () \ + { + // Changing the dirty list during a transaction is not allowed + Assert( nDirtyObjects == pSelf->m_databaseDirtyList.NumDirtyObjects() ); + pSelf->m_databaseDirtyList.RemoveAll(); + } ); + + return count; +} + +//---------------------------------------------------------------------------- +// Purpose: Sets the writeback flag and remembers the time it was first set +// if appropriate. +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::SetInWriteback( bool bInWriteback ) +{ + if( !m_bInWriteback && bInWriteback ) + { + m_unWritebackTime = time( NULL ); + } + m_bInWriteback = bInWriteback; +} + +//---------------------------------------------------------------------------- +// Purpose: +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::MarkDirty() +{ + //see if we are detecting a version change when we weren't expecting one. This is often tied to loading or other events that should leave the version in a constant state + //or runs the risk of creating unecessary sends + AssertMsg( !m_bDetectVersionChanges, "Warning: Detected a change to the version of the SO cache when it was not expected. This is often done when an SO cache is being loaded, and can result in costly unnecessary updates of caches" ); + + CSharedObjectCache::MarkDirty(); + ClearCachedSubscriptionMessage(); + m_ulVersion = GGCHost()->GenerateGID(); + GGCBase()->AddCacheToVersionChangedList( this ); +} + +//---------------------------------------------------------------------------- +// Purpose: Determine the send flags for the given steam id +//---------------------------------------------------------------------------- +uint32 CGCSharedObjectCache::CalcSendFlags( const CSteamID & steamID ) const +{ + if( m_context.GetOwner() == steamID ) + return k_ESOFlag_SendToOwner; + + if( steamID.BIndividualAccount() ) + { + return k_ESOFlag_SendToOtherUsers; + } + else if( steamID.BGameServerAccount() ) + { + return k_ESOFlag_SendToOtherGameservers | k_ESOFlag_SendToQuestObjectiveTrackers; + } + + // What sort of an account is this?! + Assert( steamID.IsValid() ); + + return k_ESOFlag_SendToNobody; +} + +//---------------------------------------------------------------------------- +// Purpose: Given the flags, are we supposed to send anything to them +//---------------------------------------------------------------------------- +bool CGCSharedObjectCache::BShouldSendToAnyClients( uint32 unFlags ) const +{ + return 0 != (unFlags & (k_ESOFlag_SendToOwner | k_ESOFlag_SendToOtherGameservers | k_ESOFlag_SendToOtherUsers) ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Dumps all the objects in the type cache +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::Dump() const +{ + CSharedObjectCache::Dump(); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\t Subscribers (%d). Version (%llu):\n", m_context.GetSubscribers().Count(), m_ulVersion ); + FOR_EACH_VEC( m_context.GetSubscribers(), nSub ) + { + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\t\t%s\n", m_context.GetSubscribers()[nSub].m_steamID.Render() ); + } +} + +//---------------------------------------------------------------------------- +// Purpose: +//---------------------------------------------------------------------------- +void CGCSharedObjectCache::DumpDirtyObjects() const +{ + if ( m_networkDirtyObjsCreate.Count() > 0 ) + { + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\t Num network create dirty (%d):\n", m_networkDirtyObjsCreate.Count() ); + FOR_EACH_HASHTABLE( m_networkDirtyObjsCreate, i ) + { + CSharedObject *pObj = m_networkDirtyObjsCreate.Key(i); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\t\t Object type id: %d\n", pObj->GetTypeID() ); + } + } + if ( m_networkDirtyObjs.Count() > 0 ) + { + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\t Num network dirty (%d):\n", m_networkDirtyObjs.Count() ); + FOR_EACH_HASHTABLE( m_networkDirtyObjs, i ) + { + CSharedObject *pObj = m_networkDirtyObjs.Key(i); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\t\t Object type id: %d\n", pObj->GetTypeID() ); + } + } + if ( m_databaseDirtyList.NumDirtyObjects() > 0 ) + { + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\t Num database dirty (%d):\n", m_databaseDirtyList.NumDirtyObjects() ); + for( int i = 0; i < m_databaseDirtyList.NumDirtyObjects(); ++i ) + { + CSharedObject *pObj = NULL; + CUtlVector<int> fieldSet; + m_databaseDirtyList.GetDirtyFieldSetByIndex( i, &pObj, fieldSet ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\t\t Object type id: %d\n", pObj->GetTypeID() ); + } + } +} + + +//---------------------------------------------------------------------------- +// Purpose: Claims all the memory for the cache +//---------------------------------------------------------------------------- +#ifdef DBGFLAG_VALIDATE +void CGCSharedObjectCache::Validate( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE(); + + CGCSharedObjectCache::Validate( validator, pchName ); + ValidateObj( m_networkDirtyObjs ); + ValidateObj( m_networkDirtyObjsCreate ); + ValidateObj( m_databaseDirtyList ); +} +#endif + + +//----------------------------------------------------------------------------- +// Purpose: Interface for profiling SO cache stats +//----------------------------------------------------------------------------- + +GC_CON_COMMAND( so_profile_on, "Starts the profiling of shared object stats" ) +{ + //stop any previous one to make sure that our time is up to date + g_SharedObjectStats.StopCollectingStats(); + g_SharedObjectStats.ResetStats(); + g_SharedObjectStats.StartCollectingStats(); +} + +GC_CON_COMMAND( so_profile_off, "Stops the profiling of shared object stats" ) +{ + g_SharedObjectStats.StopCollectingStats(); +} + +GC_CON_COMMAND( so_profile_dump, "Displays the stats collected from profiling shared object stats" ) +{ + g_SharedObjectStats.ReportCollectedStats(); +} + +} // namespace GCSDK diff --git a/gcsdk/gcbase.cpp b/gcsdk/gcbase.cpp new file mode 100644 index 0000000..134967a --- /dev/null +++ b/gcsdk/gcbase.cpp @@ -0,0 +1,4432 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//=============================================================================// +#include "stdafx.h" +#include "gcbase.h" +#include "tier1/interface.h" +#include "tier0/minidump.h" +#include "tier0/icommandline.h" +#include "gcjob.h" +#include "sqlaccess/schemaupdate.h" +#include "gcsystemmsgs.h" +#include "rtime.h" +#include "msgprotobuf.h" +#include "gcsdk_gcmessages.pb.h" +#include "gcsdk/gcparalleljobfarm.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + +//---------------------------------------------------------------------- +// Emit groups +//---------------------------------------------------------------------- +DECLARE_GC_EMIT_GROUP( g_EGHTTPRequest, http_request ); + +CGCBase *g_pGCBase = NULL; + +// Thread pool size convar +static void OnConVarChangeJobMgrThreadPoolSize( IConVar *pConVar, const char *pOldString, float flOldValue ); +GCConVar jobmgr_threadpool_size( "jobmgr_threadpool_size", "-1", 0, + "Maximum threads in the job manager thread pool. Values <= 0 mean number_logical_cpus - this.", + OnConVarChangeJobMgrThreadPoolSize ); + +static uint32 GetThreadPoolSizeFromConVar() +{ + int nVal = jobmgr_threadpool_size.GetInt(); + int nRet = ( nVal > 0 ) ? nVal : GetCPUInformation()->m_nLogicalProcessors + nVal; + return (uint32)Clamp( nRet, 1, INT_MAX ); +} + +static void OnConVarChangeJobMgrThreadPoolSize( IConVar *pConVar, const char *pOldString, float flOldValue ) +{ + if ( GGCBase()->GetIsShuttingDown() ) + return; + + GGCBase()->GetJobMgr().SetThreadPoolSize( GetThreadPoolSizeFromConVar() ); +} + +GCConVar cv_concurrent_start_playing_limit( "concurrent_start_playing_limit", "1000" ); +GCConVar cv_logon_surge_start_playing_limit( "logon_surge_start_playing_limit", "2000" ); +GCConVar cv_logon_surge_request_session_jobs( "logon_surge_request_session_jobs", "1000" ); +GCConVar cv_webapi_throttle_job_threshold( "webapi_throttle_job_threshold", "2000", 0, "If the job count exceeds this threshold, reject low-priority webapi jobs" ); +GCConVar enable_startplaying_gameserver_creation_spew( "enable_startplaying_gameserver_creation_spew", "0" ); +// Enable the restore-version-from-memcache machinery. Disabled because it assumes reloading an SOCache is +// deterministic, which is no longer true for us, resulting in clients with stale versions believing themselves to be in +// sync. +// +// This probably needs a look -- ideally we'd delineate deterministic objects that can be assumed to remain in sync in +// GC reboots, and dynamic objects that cannot. +// +// Note that we already removed hacks for this in player groups and started using lazy-loaded objects in SOCaches that +// violate the assumptions this was making, so re-enabling it requires work. We probably really want to split type +// caches into deterministic-between-GC-reboots and not, and resend based on said flag. +GCConVar socache_persist_version_via_memcached( "socache_persist_version_via_memcached", "0" ); + +static GCConVar cv_assert_minidump_window( "assert_minidump_window", "28800", 0, "Size of the minidump window in seconds. Each unique assert will dump at most assert_max_minidumps_in_window times in this many seconds" ); +static GCConVar cv_assert_max_minidumps_in_window( "assert_max_minidumps_in_window", "5", 0, "The amount of times each unique assert will write a dump in assert_minidump_window seconds" ); + +static GCConVar cv_debug_steam_startplaying( "cv_debug_steam_startplaying", "0", 0, "Turn this ON to debug the stream of startplaying messages we get from Steam" ); + +static GCConVar temp_list_mismatched_replies( "temp_list_mismatched_replies", "0", "When set to 1, this report all replies that fail because the incoming message didn't expect a response. Temporary to help track down some failed state" ); + +static GCConVar writeback_queue_max_accumulate_time( "writeback_queue_max_accumulate_time", "10", 0, "The maximum amount of time in seconds that the writeback queue will accumulate database writes before performing queries. This is the time *before* the queries are executed, which is unbounded." ); +static GCConVar writeback_queue_max_caches( "writeback_queue_max_caches", "0", 0, "The maximum amount of caches to write back in a single transaction. Set to zero to remove this restriction." ); +static GCConVar geolocation_spewlevel( "geolocation_spewlevel", "4", 0, "Spewlevel to use for geolocation debug spew" ); +static GCConVar geolocation_loglevel( "geolocation_loglevel", "4", 0, "Spewlevel to use for geolocation debug spew" ); + +extern GCConVar max_user_messages_per_second; + +// There is also a GCConVar writeback_delay to control how frequently we do writebacks. + +// !KLUDGE! Temp shim. Will get rid of this when we bring over the real gcinterface stuff from DOTA. +CGCInterface g_GCInterface; +CGCInterface *GGCInterface() { return &g_GCInterface; } +CSteamID CGCInterface::ConstructSteamIDForClient( AccountID_t unAccountID ) const +{ + return CSteamID( unAccountID, GetUniverse(), k_EAccountTypeIndividual ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Overrides the spew func used by Msg and DMsg to print to the console +//----------------------------------------------------------------------------- +SpewRetval_t ConsoleSpewFunc( SpewType_t type, const tchar *pMsg ) +{ + const char *fmt = ( sizeof( tchar ) == sizeof( char ) ) ? "%hs" : "%ls"; + switch (type ) + { + default: + case SPEW_MESSAGE: + case SPEW_LOG: + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, fmt, pMsg ); + break; + case SPEW_WARNING: + EmitWarning( SPEW_CONSOLE, SPEW_ALWAYS, fmt, pMsg ); + break; + case SPEW_ASSERT: + if ( ThreadInMainThread() && ( g_pJobCur != NULL ) ) + { + fmt = ( sizeof( tchar ) == sizeof( char ) ) ? "[Job %s] %hs" : "[Job %s] %ls"; + EmitError( SPEW_CONSOLE, fmt, g_pJobCur->GetName(), pMsg ); + } + else + { + EmitError( SPEW_CONSOLE, fmt, pMsg ); + } + break; + case SPEW_ERROR: + EmitError( SPEW_CONSOLE, fmt, pMsg ); + break; + } + + if ( type == SPEW_ASSERT ) + { +#ifndef WIN32 + // Non-win32 + bool bRaiseOnAssert = getenv( "RAISE_ON_ASSERT" ) || !!CommandLine()->FindParm( "-raiseonassert" ); +#elif defined( _DEBUG ) + // Win32 debug + bool bRaiseOnAssert = true; +#else + // Win32 release + bool bRaiseOnAssert = !!CommandLine()->FindParm( "-raiseonassert" ); +#endif + + return bRaiseOnAssert ? SPEW_DEBUGGER : SPEW_CONTINUE; + } + else if ( type == SPEW_ERROR ) + return SPEW_ABORT; + else + return SPEW_CONTINUE; +} + + +class CGCShutdownJob : public CGCJob +{ +public: + CGCShutdownJob( CGCBase *pGC ) : CGCJob( pGC ) {} + + virtual bool BYieldingRunGCJob() + { + m_pGC->SetIsShuttingDown(); + + // Log off all of the game servers and users, so that if something + // in the log off dirties caches they can be written back + CUtlVector<CSteamID> vecIDsToStop; + for( CGCGSSession **ppSession = m_pGC->GetFirstGSSession(); ppSession != NULL; ppSession = m_pGC->GetNextGSSession( ppSession ) ) + { + vecIDsToStop.AddToTail( (*ppSession)->GetSteamID() ); + } + + FOR_EACH_VEC( vecIDsToStop, i ) + { + m_pGC->YieldingStopGameserver( vecIDsToStop[i] ); + ShouldNotHoldAnyLocks(); + } + + vecIDsToStop.RemoveAll(); + + for( CGCUserSession **ppSession = m_pGC->GetFirstUserSession(); ppSession != NULL; ppSession = m_pGC->GetNextUserSession( ppSession ) ) + { + vecIDsToStop.AddToTail( (*ppSession)->GetSteamID() ); + } + + FOR_EACH_VEC( vecIDsToStop, i ) + { + m_pGC->YieldingStopPlaying( vecIDsToStop[i] ); + ShouldNotHoldAnyLocks(); + } + + // wait for jobs to finish (except this one!) + const int kMaxIterations = 100; + int cIter = 0; + while ( cIter++ < kMaxIterations && m_pGC->GetJobMgr().CountJobs() > 1 ) + { + BYieldingWaitOneFrame(); + } + + m_pGC->YieldingGracefulShutdown(); + GGCHost()->ShutdownComplete(); + + return false; + } + +}; + + +class CPreTestSetupJob : public CGCJob +{ +public: + CPreTestSetupJob( CGCBase *pGC ) : CGCJob( pGC ) {} + + virtual bool BYieldingRunGCJob( GCSDK::CNetPacket *pNetPacket ) + { + CGCMsg<MsgGCEmpty_t> msg( pNetPacket ); + m_pGC->YieldingPreTestSetup(); + + return true; + } +}; + +GC_REG_JOB( CGCBase, CPreTestSetupJob, "CPreTestSetupJob", k_EGCMsgPreTestSetup, k_EServerTypeGC ); + +static void SpewSerializedKeyValues( const byte *pubVarData, uint32 cubVarData ) +{ + if ( pubVarData == NULL || cubVarData == 0 ) + { + EmitInfo( SPEW_GC, 1, 1, " No KV data\n" ); + return; + } + char szLine[512] = ""; + for ( uint32 i = 0 ; i < cubVarData ; ++i ) + { + char szByteVal[32]; + V_sprintf_safe( szByteVal, "%02X", pubVarData[ i ] ); + if ( i % 32 ) + { + V_strcat_safe( szLine, ", " ); + V_strcat_safe( szLine, szByteVal ); + } + else + { + if ( szLine[0] ) + EmitInfo( SPEW_GC, 1, 1, " %s\n", szLine ); + V_strcpy_safe( szLine, szByteVal ); + } + } + if ( szLine[0] ) + EmitInfo( SPEW_GC, 1, 1, " %s\n", szLine ); + KeyValuesAD pkvDetails( "SessionDetails" ); + CUtlBuffer buf; + buf.Put( pubVarData, cubVarData ); + if( pkvDetails->ReadAsBinary( buf ) ) + { + FOR_EACH_VALUE( pkvDetails, v ) + { + EmitInfo( SPEW_GC, 1, 1, " %s = %s\n", v->GetName(), v->GetString( NULL, "??" ) ); + } + } + else + { + EmitInfo( SPEW_GC, 1, 1, " KV data failed parse\n" ); + } +} + +class CStartPlayingJob : public CGCJob +{ +public: + CStartPlayingJob( CGCBase *pGC ) : CGCJob( pGC ) {} + + virtual bool BYieldingRunGCJob( GCSDK::IMsgNetPacket *pNetPacket ) + { + CGCMsg<MsgGCStartPlaying_t> msg( pNetPacket ); + + // @note Tom Bui/Joe Ludwig: This can happen for PS3 Steam accounts + if ( !msg.Body().m_steamID.IsValid() ) + return true; + + if ( cv_debug_steam_startplaying.GetBool() ) + { + netadr_t serverAdr( msg.Body().m_unServerAddr, msg.Body().m_usServerPort ); + EmitInfo( SPEW_GC, 1, 1, "Received StartPlaying( user = %s, GS = %s @ %s )\n", msg.Body().m_steamID.Render(), msg.Body().m_steamIDGS.Render(), serverAdr.ToString() ); + SpewSerializedKeyValues( msg.PubVarData(), msg.CubVarData() ); + } + m_pGC->QueueStartPlaying( msg.Body().m_steamID, msg.Body().m_steamIDGS, msg.Body().m_unServerAddr, msg.Body().m_usServerPort, msg.PubVarData(), msg.CubVarData() ); + + return true; + } +}; + +GC_REG_JOB(CGCBase, CStartPlayingJob, "CStartPlayingJob", k_EGCMsgStartPlaying, k_EServerTypeGC); + +class CExecuteStartPlayingJob : public CGCJob +{ +public: + CExecuteStartPlayingJob( CGCBase *pGC ) : CGCJob( pGC ) {} + + virtual bool BYieldingRunGCJob( ) + { + m_pGC->YieldingExecuteNextStartPlaying(); + return true; + } +}; + + +class CStopPlayingJob : public CGCJob +{ +public: + CStopPlayingJob( CGCBase *pGC ) : CGCJob( pGC ) {} + + virtual bool BYieldingRunGCJob( GCSDK::IMsgNetPacket *pNetPacket ) + { + CGCMsg<MsgGCStopSession_t> msg( pNetPacket ); + + // @note Tom Bui/Joe Ludwig: This can happen for PS3 Steam accounts + if ( !msg.Body().m_steamID.IsValid() ) + return true; + + if ( cv_debug_steam_startplaying.GetBool() ) + { + EmitInfo( SPEW_GC, 1, 1, "Received StopPlaying( user = %s )\n", msg.Body().m_steamID.Render() ); + } + + m_pGC->YieldingStopPlaying( msg.Body().m_steamID ); + + return true; + } +}; + +GC_REG_JOB(CGCBase, CStopPlayingJob, "CStopPlayingJob", k_EGCMsgStopPlaying, k_EServerTypeGC); + +class CStartGameserverJob : public CGCJob +{ +public: + CStartGameserverJob( CGCBase *pGC ) : CGCJob( pGC ) {} + + virtual bool BYieldingRunGCJob( GCSDK::IMsgNetPacket *pNetPacket ) + { + CGCMsg<MsgGCStartGameserver_t> msg( pNetPacket ); + m_pGC->QueueStartPlaying( msg.Body().m_steamID, CSteamID(), msg.Body().m_unServerAddr, msg.Body().m_usServerPort, msg.PubVarData(), msg.CubVarData() ); + + return true; + } +}; + +GC_REG_JOB(CGCBase, CStartGameserverJob, "CStartGameserverJob", k_EGCMsgStartGameserver, k_EServerTypeGC); + +class CStopGameserverJob : public CGCJob +{ +public: + CStopGameserverJob( CGCBase *pGC ) : CGCJob( pGC ) {} + + virtual bool BYieldingRunGCJob( GCSDK::IMsgNetPacket *pNetPacket ) + { + CGCMsg<MsgGCStopSession_t> msg( pNetPacket ); + m_pGC->YieldingStopGameserver( msg.Body().m_steamID ); + + return true; + } +}; + +GC_REG_JOB(CGCBase, CStopGameserverJob, "CStopGameserverJob", k_EGCMsgStopGameserver, k_EServerTypeGC); + +class CGetSystemStatsJob : public CGCJob +{ +public: + CGetSystemStatsJob( CGCBase *pGC ) : CGCJob( pGC ) {} + + virtual bool BYieldingRunGCJob( GCSDK::IMsgNetPacket *pNetPacket ) + { + CProtoBufMsg<CGCMsgGetSystemStats> msg( pNetPacket ); + + CProtoBufMsg<CGCMsgGetSystemStatsResponse> msgResponse( k_EGCMsgGetSystemStatsResponse ); + msgResponse.Body().set_gc_app_id( m_pGC->GetAppID() ); + + // @note Tom Bui: we don't support dynamic stats yet, but once we do, we can use the KV stuff + m_pGC->SystemStats_Update( msgResponse.Body() ); + + // KVPacker packer; + // KeyValuesAD pKVStats( "GCStats" ); + // CUtlBuffer buffer; + // if ( packer.WriteAsBinary( pKVStats, buffer ) ) + // { + // msgResponse.Body().set_stats_kv( buffer.Base(), buffer.TellPut() ); + // } + return m_pGC->BSendSystemMessage( msgResponse ); + } +}; + +GC_REG_JOB(CGCBase, CGetSystemStatsJob, "CGetSystemStatsJob", k_EGCMsgGetSystemStats, k_EServerTypeGC); + +//----------------------------------------------------------------------------- +class CGCJobAccountVacStatusChange : public CGCJob +{ +public: + CGCJobAccountVacStatusChange( CGCBase *pGC ) : CGCJob( pGC ) {} + + bool BYieldingRunJobFromMsg( IMsgNetPacket *pNetPacket ) + { + CProtoBufMsg<CMsgGCHAccountVacStatusChange> msg( pNetPacket ); + + if ( GGCBase()->GetAppID() != msg.Body().app_id() ) + return true; + + CSteamID steamID( msg.Body().steam_id() ); + bool bIsVacBanned = msg.Body().is_banned_now(); + + // Fetch app details, but force them to be re-loaded + bool bForceReload = true; + const CAccountDetails *pAccountDetails = GGCBase()->YieldingGetAccountDetails( steamID, bForceReload ); + + // Account details is up to date so just return + if ( pAccountDetails && bIsVacBanned != pAccountDetails->BIsVacBanned() ) + { + EmitWarning( SPEW_GC, 2, "VAC status didn't update for %s afetr receiving VacStatusChange and the force reloading the account details\n", steamID.Render() ); + } + return true; + } +}; +GC_REG_JOB( CGCBase, CGCJobAccountVacStatusChange, "CGCJobAccountVacStatusChange", k_EGCMsgGCAccountVacStatusChange, k_EServerTypeGC ); + +//----------------------------------------------------------------------------- +class CGCJobAccountPhoneNumberChange : public CGCJob +{ +public: + CGCJobAccountPhoneNumberChange( CGCBase *pGC ) : CGCJob( pGC ) {} + + bool BYieldingRunJobFromMsg( IMsgNetPacket *pNetPacket ) + { + CProtoBufMsg<CMsgGCHAccountPhoneNumberChange> msg( pNetPacket ); + + if ( GGCBase()->GetAppID() != msg.Body().appid() ) + return true; + + CSteamID steamID( msg.Body().steamid() ); + CScopedSteamIDLock scopedLock( steamID ); + if ( !scopedLock.BYieldingPerformLock( __FILE__, __LINE__ ) ) + { + EmitError( SPEW_GC, __FUNCTION__ ": Failed to lock steamid %s\n", steamID.Render() ); + return true; + } + + bool bHasPhoneVerified = msg.Body().is_verified(); + bool bIsPhoneIdentifying = msg.Body().is_identifying(); + + // Fetch app details, but force them to be re-loaded + bool bForceReload = true; + const CAccountDetails *pAccountDetails = GGCBase()->YieldingGetAccountDetails( steamID, bForceReload ); + + // Account details is up to date so just return + if ( pAccountDetails && ( bHasPhoneVerified != pAccountDetails->BIsPhoneVerified() || + bIsPhoneIdentifying != pAccountDetails->BIsPhoneIdentifying() ) ) + { + EmitWarning( SPEW_GC, 2, "Phone status didn't update for %s afetr receiving PhoneNumberChange and force reloading the account details\n", + steamID.Render() ); + } + + GGCBase()->YldOnAccountPhoneVerificationChange( steamID ); + + EmitInfo( SPEW_GC, 5, 5, "AccountPhoneVerificationChange for %s\n", steamID.Render() ); + + return true; + } +}; +GC_REG_JOB( CGCBase, CGCJobAccountPhoneNumberChange, "CGCJobAccountPhoneNumberChange", k_EGCMsgAccountPhoneNumberChange, k_EServerTypeGC ); + +//----------------------------------------------------------------------------- +class CGCJobAccountTwoFactorChange : public CGCJob +{ +public: + CGCJobAccountTwoFactorChange( CGCBase *pGC ) : CGCJob( pGC ) {} + + bool BYieldingRunJobFromMsg( IMsgNetPacket *pNetPacket ) + { + CProtoBufMsg<CMsgGCHAccountTwoFactorChange> msg( pNetPacket ); + + if ( GGCBase()->GetAppID() != msg.Body().appid() ) + return true; + + CSteamID steamID( msg.Body().steamid() ); + bool bHasTwoFactor = msg.Body().twofactor_enabled(); + + // Fetch app details, but force them to be re-loaded + bool bForceReload = true; + const CAccountDetails *pAccountDetails = GGCBase()->YieldingGetAccountDetails( steamID, bForceReload ); + + // Account details is up to date so just return + if ( pAccountDetails && bHasTwoFactor != pAccountDetails->BIsTwoFactorAuthEnabled() ) + { + EmitWarning( SPEW_GC, 2, "VAC status didn't update for %s afetr receiving VacStatusChange and the force reloading the account details\n", steamID.Render() ); + } + + GGCBase()->YldOnAccountTwoFactorChange( steamID ); + + EmitInfo( SPEW_GC, 5, 5, "AccountTwoFactorChange for %s\n", steamID.Render() ); + + return true; + } +}; +GC_REG_JOB( CGCBase, CGCJobAccountTwoFactorChange, "CGCJobAccountTwoFactorChange", k_EGCMsgAccountTwoFactorChange, k_EServerTypeGC ); + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CGCBase::CGCBase( ) +: m_mapSOCache( ), + m_rbtreeSOCachesBeingLoaded( DefLessFunc( CSteamID ) ), + m_rbtreeSOCachesWithDirtyVersions( DefLessFunc( CSteamID ) ), + m_hashUserSessions( k_nUserSessionRunInterval/ k_cMicroSecPerShellFrame ), + m_hashGSSessions( k_nGSSessionRunInterval/ k_cMicroSecPerShellFrame ), + m_hashSteamIDLocks( k_nLocksRunInterval / k_cMicroSecPerShellFrame ), + m_bStartupComplete( false ), + m_bIsShuttingDown( false ), + m_bStartProfiling( false ), + m_bStopProfiling( false ), + m_bDumpVprofImbalances( false ), + m_nStartPlayingJobCount( 0 ), + m_nRequestSessionJobsActive( 0 ), + m_nLogonSurgeFramesRemaining( k_nMillion * 10 / k_cMicroSecPerShellFrame ), // stay in "logon surge" mode for at least 10 seconds after boot. + m_mapStartPlayingQueueIndexBySteamID( DefLessFunc( CSteamID ) ), + m_MsgRateLimit( max_user_messages_per_second ), + m_nStartupCompleteTime( CRTime::RTime32TimeCur() ), + m_nInitTime( CRTime::RTime32TimeCur() ), + m_jobidFlushInventoryCacheAccounts( k_GIDNil ), + m_numFlushInventoryCacheAccountsLastScheduled( 0 ) +{ +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CGCBase::~CGCBase() +{ +} + + +//----------------------------------------------------------------------------- +// Purpose: Remembers the app ID and host +//----------------------------------------------------------------------------- +bool CGCBase::BInit( AppId_t unAppID, const char *pchAppPath, IGameCoordinatorHost *pHost ) +{ + VPROF_BUDGET( "CGCBase::BInit", VPROF_BUDGETGROUP_STEAM ); + +// Make sure we can't deploy debug GCs outside the dev environment +#ifdef _DEBUG + if ( pHost->GetUniverse() != k_EUniverseDev ) + { + //pHost->EmitMessage( SPEW_GC, SPEW_ERROR, SPEW_ALWAYS, LOG_ALWAYS, + // CFmtStr( "The GC for App %u is a debug binary. Shutting down.\n", unAppID ) ); + //return false; + pHost->EmitMessage( SPEW_GC.GetName(), SPEW_WARNING, SPEW_ALWAYS, LOG_ALWAYS, + CFmtStr( "The GC for App %u is a debug binary.\n", unAppID ) ); + } +#endif + + m_JobMgr.SetThreadPoolSize( GetThreadPoolSizeFromConVar() ); + + MsgRegistrationFromEnumDescriptor( EGCSystemMsg_descriptor(), GCSDK::MT_GC_SYSTEM ); + MsgRegistrationFromEnumDescriptor( EGCBaseClientMsg_descriptor(), GCSDK::MT_GC ); + MsgRegistrationFromEnumDescriptor( EGCToGCMsg_descriptor(), GCSDK::MT_GC_SYSTEM ); + + m_unAppID = unAppID; + m_pHost = pHost; + m_sPath = pchAppPath; + SetGCHost( pHost ); + + g_pGCBase = this; + + SetMinidumpFilenamePrefix( CFmtStr("dumps\\gc%d", m_unAppID) ); + + // Make sure the assert dialog doesn't come up and hang the process in production + //SetAssertDialogDisabled( pHost->GetUniverse() != k_EUniverseDev ); + SetAssertFailedNotifyFunc( CGCBase::AssertCallbackFunc ); + + // init the time very early so CRTime::RTime32TimeCur will return the right thing + CRTime::UpdateRealTime(); + + m_hashUserSessions.Init( k_cGCUserSessionInit, k_cBucketGCUserSession ); + m_hashGSSessions.Init( k_cGCGSSessionInit, k_cBucketGCGSSession ); + m_hashSteamIDLocks.Init( k_cGCLocksInit, k_cBucketGCLocks ); + + m_OutputFuncPrev = GetSpewOutputFunc(); + SpewOutputFunc( &ConsoleSpewFunc ); + EmitInfo( SPEW_GC, 1, 1, "CGCBase::BInit( AppID=%d, appPath=%s, sPath=%s )\n", unAppID, pchAppPath, m_sPath.String() ); + + if ( !OnInit() ) + return false; + + DbgVerify( g_theMessageList.BInit( ) ); + + /* + // @note Tom Bui: we don't need dynamic stats...yet. + // when we do, we'll need to specify the how the values are aggregated over all the same GCs + // and how the values should be treated + KeyValuesAD pKVStats( "GCStats" ); + SystemStats_Update( pKVStats ); + CUtlBuffer buffer; + KVPacker packer; + if ( packer.WriteAsBinary( pKVStats, buffer ) ) + { + CProtoBufMsg< CGCMsgSystemStatsSchema > msg( GCSDK::k_EGCMsgSystemStatsSchema ); + msg.Body().set_gc_app_id( GetAppID() ); + msg.Body().set_schema_kv( buffer.Base(), buffer.TellPut() ); + BSendSystemMessage( msg ); + } + */ + + return BSendWebApiRegistration(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Report back to the host that startup is complete +//----------------------------------------------------------------------------- +void CGCBase::SetStartupComplete( bool bSuccess ) +{ + // !KLUDGE! Fatal error messages on startup frequently get lost in the + // mass of messages. Let's spray a big error message box if we fail + // to startup. Ideally, the cause of the failure will be + // spewed just above this box. + if ( !bSuccess ) + { + EmitError( SPEW_GC, "^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^\n" ); + EmitError( SPEW_GC, "GC failed to startup. Error mesage is probably directly above\n" ); + EmitError( SPEW_GC, "**************************************************************\n" ); + } + + m_nStartupCompleteTime = CRTime::RTime32TimeCur(); + m_bStartupComplete = true; + GGCHost()->StartupComplete( bSuccess ); +} + +uint32 CGCBase::GetGCUpTime() const +{ + return CRTime::RTime32TimeCur() - m_nInitTime; +} + +//----------------------------------------------------------------------------- +// Purpose: Starts a job to perform graceful shutdown +//----------------------------------------------------------------------------- +void CGCBase::Shutdown() +{ + VPROF_BUDGET( "CGCBase::Shutdown", VPROF_BUDGETGROUP_STEAM ); + m_DumpHTTPErrorsSchedule.Cancel(); + + CGCShutdownJob *pJob = new CGCShutdownJob( this ); + pJob->StartJob( NULL ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Cleans up the GC to prepare for shutdown +//----------------------------------------------------------------------------- +void CGCBase::Uninit( ) +{ + VPROF_BUDGET( "CGCBase::Uninit", VPROF_BUDGETGROUP_STEAM ); + + OnUninit(); + + // clean up all of the sessions and caches here so we can be sure it happens before the memory pools go away at static destruction time + for( CGCUserSession **ppSession = m_hashUserSessions.PvRecordFirst(); ppSession != NULL; ppSession = m_hashUserSessions.PvRecordNext( ppSession ) ) + { + delete (*ppSession); + } + m_hashUserSessions.RemoveAll(); + for( CGCGSSession **ppSession = m_hashGSSessions.PvRecordFirst(); ppSession != NULL; ppSession = m_hashGSSessions.PvRecordNext( ppSession ) ) + { + delete (*ppSession); + } + m_hashGSSessions.RemoveAll(); + FOR_EACH_MAP_FAST( m_mapSOCache, nIndex ) + { + // Remove from map before deleting, to prevent some debug + // code from getting tangled up + CGCSharedObjectCache *pCache = m_mapSOCache[nIndex]; + m_mapSOCache[nIndex] = NULL; + m_mapSOCache.RemoveAt( nIndex ); + delete pCache; + } + m_mapSOCache.RemoveAll(); + m_rbtreeSOCachesBeingLoaded.RemoveAll(); + m_rbtreeSOCachesWithDirtyVersions.RemoveAll(); + m_hashSteamIDLocks.RemoveAll(); + + GSchemaFull().Uninit(); + SpewOutputFunc( m_OutputFuncPrev ); +} + +GCConVar cv_flush_inventory_cache_jobs( "cv_flush_inventory_cache_jobs", "20", 0, "The maximum number of jobs flushing inventory caches that can be in flight at once, zero to disable flushing" ); +GCConVar cv_flush_inventory_cache_contextid( "cv_flush_inventory_cache_contextid", "2" /* k_EEconContextBackpack */, 0, "Which context id we flush for Steam web user-facing inventory" ); +GCConVar cv_flush_inventory_cache_spew( "cv_flush_inventory_cache_spew", "0", 0, "Controls spew level for jobs flushing inventory cache (0=off; 1=summary; 2=verbose)" ); +class CFlushInventoryCacheAccountsJob : public CGCJob, public IYieldingParallelFarmJobHandler +{ +public: + CFlushInventoryCacheAccountsJob( CGCBase *pGC, CUtlRBTree< AccountID_t, int32, CDefLess< AccountID_t > > &rbAccounts ) : CGCJob( pGC ) + { + m_rbAccounts.Swap( rbAccounts ); + } + + virtual bool BYieldingRunGCJob() OVERRIDE + { + if ( !m_rbAccounts.Count() ) + return false; + if ( cv_flush_inventory_cache_jobs.GetInt() <= 0 ) + return false; + + bool bShouldSpew = ( cv_flush_inventory_cache_spew.GetInt() >= 1 ); + uint32 msTimeStart = 0; + int numAccountsWorkload = m_rbAccounts.Count(); + if ( bShouldSpew ) + { + msTimeStart = Plat_MSTime(); + } + + { // Run parallel processing of the workload + int numJobs = numAccountsWorkload; + numJobs = MIN( cv_flush_inventory_cache_jobs.GetInt(), numJobs ); + numJobs = MAX( 1, numJobs ); + + ( void ) BYieldingExecuteParallel( numJobs, "YieldingFlushInventoryCacheAccountsJob" ); + } + + if ( bShouldSpew ) + { + EmitInfo( SPEW_GC, SPEW_ALWAYS, LOG_ALWAYS, "IEconService/FlushInventoryCache: Batch for %d accounts completed in %u ms\n", + numAccountsWorkload, Plat_MSTime() - msTimeStart ); + } + + return true; + } + + virtual bool BYieldingRunWorkload( int iJobSequenceCounter, bool *pbWorkloadCompleted ) OVERRIDE + { + if ( m_rbAccounts.Count() ) + { + int32 idxElement = m_rbAccounts.FirstInorder(); + AccountID_t unAccountID = m_rbAccounts.Element( idxElement ); + m_rbAccounts.RemoveAt( idxElement ); + + ( void ) BYieldingFlushRequest( unAccountID ); + } + + if ( !m_rbAccounts.Count() ) + { + *pbWorkloadCompleted = true; + } + + return true; + } + + bool BYieldingFlushRequest( AccountID_t unAccountID ) + { + bool bShouldSpew = ( cv_flush_inventory_cache_spew.GetInt() >= 2 ); + uint32 msTimeStart = 0; + if ( bShouldSpew ) + { + msTimeStart = Plat_MSTime(); + } + + CSteamID steamID( GGCInterface()->ConstructSteamIDForClient( unAccountID ) ); + CSteamAPIRequest apiRequest( k_EHTTPMethodPOST, "IEconService", "FlushInventoryCache", 1 ); + apiRequest.SetPOSTParamUInt32( "appid", GGCBase()->GetAppID() ); + apiRequest.SetPOSTParamUInt64( "steamid", steamID.ConvertToUint64() ); + apiRequest.SetPOSTParamUInt32( "contextid", 2 ); + + CHTTPResponse apiResponse; + bool bSucceededQuery = m_pGC->BYieldingSendHTTPRequest( &apiRequest, &apiResponse ); + if ( !bSucceededQuery ) + { + EmitErrorRatelimited( SPEW_GC, "IEconService/FlushInventoryCache: Web call did not get a response for %s.\n", steamID.Render() ); + } + else if ( k_EHTTPStatusCode200OK != apiResponse.GetStatusCode() ) + { + EmitErrorRatelimited( SPEW_GC, "IEconService/FlushInventoryCache: Web call got failure code %d for %s\n", apiResponse.GetStatusCode(), steamID.Render() ); + bSucceededQuery = false; + } + + if ( bSucceededQuery ) + { + // Have a valid response + KeyValuesAD pKVResponse( "response" ); + pKVResponse->UsesEscapeSequences( true ); + if ( !pKVResponse->LoadFromBuffer( "webResponse", *apiResponse.GetBodyBuffer() ) ) + { + EmitErrorRatelimited( SPEW_GC, "IEconService/FlushInventoryCache: Web call got code %d for %s, but failed to parse response\n", apiResponse.GetStatusCode(), steamID.Render() ); + bSucceededQuery = false; + } + else if ( !pKVResponse->GetBool( "success" ) ) + { + // We got a response, and it's not success + EmitErrorRatelimited( SPEW_GC, "IEconService/FlushInventoryCache: Web call got code %d for %s, but not success\n", apiResponse.GetStatusCode(), steamID.Render() ); + bSucceededQuery = false; + } + } + + if ( bShouldSpew ) + { + EmitInfo( SPEW_GC, SPEW_ALWAYS, LOG_ALWAYS, "IEconService/FlushInventoryCache: Web call for %s %s in %u ms\n", + steamID.Render(), bSucceededQuery ? "succeeded" : "failed", Plat_MSTime() - msTimeStart ); + } + + return bSucceededQuery; + } + +public: + CUtlRBTree< AccountID_t, int32, CDefLess< AccountID_t > > m_rbAccounts; +}; + +//----------------------------------------------------------------------------- +// Purpose: Called every frame. Mostly updates times and pulses the job manager +//----------------------------------------------------------------------------- +bool CGCBase::BMainLoopOncePerFrame( uint64 ulLimitMicroseconds ) +{ + // if we don't have a GCHost yet, don't do any work per frame + if( !GGCHost() ) + return false; + +#ifndef STEAM + CRTime::UpdateRealTime(); +#endif + + +#ifdef VPROF_ENABLED + // Make sure we end the frame at the root node + if ( !g_VProfCurrentProfile.AtRoot() && m_bDumpVprofImbalances ) + { + EmitWarning( SPEW_GC, SPEW_ALWAYS, "VProf not at root at end of frame. Stack:\n" ); + } + + for( int i = 0; !g_VProfCurrentProfile.AtRoot() && i < 100; i++ ) + { + if ( m_bDumpVprofImbalances ) + { + EmitWarning( SPEW_GC, SPEW_ALWAYS, " %s\n", g_VProfCurrentProfile.GetCurrentNode()->GetName() ); + } + g_VProfCurrentProfile.ExitScope(); + } + + g_VProfCurrentProfile.MarkFrame(); + + if ( m_bStopProfiling || m_bStartProfiling ) + { + while ( g_VProfCurrentProfile.IsEnabled() ) + { + g_VProfCurrentProfile.Stop(); + } + m_bStopProfiling = false; + + if ( m_bStartProfiling ) + { + g_VProfCurrentProfile.Reset(); + g_VProfCurrentProfile.Start(); + m_bStartProfiling = false; + } + } +#endif + + VPROF_BUDGET( "Main Loop", VPROF_BUDGETGROUP_STEAM ); + + CLimitTimer limitTimer; + limitTimer.SetLimit( ulLimitMicroseconds ); + CJobTime::UpdateJobTime( k_cMicroSecPerShellFrame ); + + bool bWorkRemaining = m_JobMgr.BFrameFuncRunSleepingJobs( limitTimer ); + + //run all of our frame functions + GFrameFunctionMgr().RunFrame( limitTimer ); + + { + VPROF_BUDGET( "Run Sessions", VPROF_BUDGETGROUP_STEAM ); + + m_AccountDetailsManager.MarkFrame(); + m_hashUserSessions.StartFrameSchedule( true ); + m_hashGSSessions.StartFrameSchedule( true ); + m_hashSteamIDLocks.StartFrameSchedule( true ); + bool bUsersFinished = false, bGSFinished = false; + while( !limitTimer.BLimitReached() && ( !bUsersFinished || !bGSFinished ) ) + { + if( !bUsersFinished ) + { + CGCUserSession **ppSession = m_hashUserSessions.PvRecordRun(); + if ( ppSession && *ppSession ) + { + (*ppSession)->Run(); + } + else + { + bUsersFinished = true; + } + if ( m_hashUserSessions.BCompletedPass() ) + { + FinishedMainLoopUserSweep(); + } + } + + if( !bGSFinished ) + { + CGCGSSession **ppSession = m_hashGSSessions.PvRecordRun(); + if ( ppSession && *ppSession ) + { + (*ppSession)->Run(); + } + else + { + bGSFinished = true; + } + } + } + } + + { + VPROF_BUDGET( "UpdateSOCacheVersions", VPROF_BUDGETGROUP_STEAM ); + UpdateSOCacheVersions(); + } + + if( m_llStartPlaying.Count() > 0 ) + { + VPROF_BUDGET( "StartStartPlayingJobs", VPROF_BUDGETGROUP_STEAM ); + + int nJobsNeeded = min( m_llStartPlaying.Count(), cv_concurrent_start_playing_limit.GetInt() - m_nStartPlayingJobCount ); + while( nJobsNeeded > 0 ) + { + nJobsNeeded--; + m_nStartPlayingJobCount++; + + CExecuteStartPlayingJob *pJob = new CExecuteStartPlayingJob( this ); + pJob->StartJob( NULL ); + } + } + + // Decide if we should be in logon surge + bool bShouldBeInlogonSurge = + m_llStartPlaying.Count() >= cv_logon_surge_start_playing_limit.GetInt(); + // This might be a good idea, but let's see what the real numbers are during logon surge. + //|| m_nRequestSessionJobsActive >= cv_logon_surge_request_session_jobs.GetInt(); + + // Check if we're already in logon surge, is it time to check if we should leave, + // and should we dump our status periodically? + const int k_nLogonSurgeFrameInterval = k_nMillion * 10 / k_cMicroSecPerShellFrame; + if ( m_nLogonSurgeFramesRemaining > 0 ) + { + + // Currently in logon surge + --m_nLogonSurgeFramesRemaining; + if ( m_nLogonSurgeFramesRemaining == 0 ) + { + + // Time to check for leaving logon surge mode. + // Should I flip the flag off? + if ( bShouldBeInlogonSurge ) + { + // We're still in logon surge. Schedule another check + // a few frames from now, and dump our status. + m_nLogonSurgeFramesRemaining = k_nLogonSurgeFrameInterval; + Dump(); + } + else + { + // We're over the hump! + EmitInfo( SPEW_GC, SPEW_ALWAYS, LOG_ALWAYS, "** LOGON SURGE COMPLETED **\n" ); + } + } + } + else if ( bShouldBeInlogonSurge ) + { + // We finished logon surge one, but now we are re-entering it. + // This usually doesn't happen. This is suspicious. + EmitWarning( SPEW_GC, SPEW_ALWAYS, "RE-ENTERING logon surge mode!\n" ); + m_nLogonSurgeFramesRemaining = k_nLogonSurgeFrameInterval; + } + else + { + // Not in logon surge. make sure flag is slammed to zero + m_nLogonSurgeFramesRemaining = 0; + } + + // Flush inventory cache for accounts + if ( m_rbFlushInventoryCacheAccounts.Count() && ( ( m_jobidFlushInventoryCacheAccounts == k_GIDNil ) || + !GetJobMgr().BJobExists( m_jobidFlushInventoryCacheAccounts ) ) ) + { + m_numFlushInventoryCacheAccountsLastScheduled = m_rbFlushInventoryCacheAccounts.Count(); + m_jobidFlushInventoryCacheAccounts = StartNewJobDelayed( new CFlushInventoryCacheAccountsJob( this, m_rbFlushInventoryCacheAccounts ) )->GetJobID(); + } + + bool bSubRet = OnMainLoopOncePerFrame( limitTimer ); + return bWorkRemaining || bSubRet; +} + +bool CGCBase::BShouldThrottleLowServiceLevelWebAPIJobs() const +{ + + // Always throttle them during logon surge. + if ( BIsInLogonSurge() ) + return true; + + // Check threshold + if ( m_JobMgr.CountJobs() > cv_webapi_throttle_job_threshold.GetInt() ) + return true; + + // We are not too busy, we can service the request + return false; +} + + +bool CGCBase::BMainLoopUntilFrameCompletion( uint64 ulLimitMicroseconds ) +{ + VPROF_BUDGET( "Main Loop", VPROF_BUDGETGROUP_STEAM ); + + CLimitTimer limitTimer; + limitTimer.SetLimit( ulLimitMicroseconds ); + bool bRet = m_JobMgr.BFrameFuncRunYieldingJobs( limitTimer ); + + bRet |= GSDOCache().BFrameFuncRunJobsUntilCompleted( limitTimer ); + bRet |= GSDOCache().BFrameFuncRunMemcachedQueriesUntilCompleted( limitTimer ); + bRet |= GSDOCache().BFrameFuncRunSQLQueriesUntilCompleted( limitTimer ); + bRet |= m_AccountDetailsManager.BExpireRecords( limitTimer ); + + bool bSubRet = OnMainLoopUntilFrameCompletion( limitTimer ); + + bRet |= GFrameFunctionMgr().RunFrameTick( limitTimer ); + + { + VPROF_BUDGET( "Expire locks", VPROF_BUDGETGROUP_STEAM ); + + for ( CLock *pLock = m_hashSteamIDLocks.PvRecordRun(); NULL != pLock; pLock = m_hashSteamIDLocks.PvRecordRun() ) + { + if ( !pLock->BIsLocked() && pLock->GetMicroSecondsSinceLock() > k_cMicroSecLockLifetime ) + { + m_hashSteamIDLocks.Remove( pLock ); + } + + if ( limitTimer.BLimitReached() ) + return true; + } + } + + return bRet || bSubRet; + } + +//----------------------------------------------------------------------------- +// Purpose: Called when we get to the end of a user session Run() sweep, and +// are about to start over with the first session in the list. +//----------------------------------------------------------------------------- +void CGCBase::FinishedMainLoopUserSweep() +{ + // Base class does nothing +} + + +//----------------------------------------------------------------------------- +// Purpose: Queues up a start playing request that we should process when we +// get a chance. +//----------------------------------------------------------------------------- +void CGCBase::QueueStartPlaying( const CSteamID & steamID, const CSteamID & gsSteamID, uint32 unServerAddr, uint16 usServerPort, const uint8 *pubVarData, uint32 cubVarData ) +{ + MEM_ALLOC_CREDIT_( "QueueStartPlaying" ); + + Assert( steamID.BIndividualAccount() || steamID.BGameServerAccount() ); + Assert( steamID.IsValid() ); + + // Should be one-to-one correspondence in these data structures + Assert( (size_t)m_mapStartPlayingQueueIndexBySteamID.Count() == (size_t)m_llStartPlaying.Count() ); + + // !FIXME! Here we really should check whether they already have a session. + // if so, we've already gone through all the startplaying work and shouldn't + // repeat it. We might just need to kick the communications or make + // sure they are on the right game server. + + // Check if we already have an entry in the queue for this guy. + StartPlayingWork_t *pWork = NULL; + int nMapIndex = m_mapStartPlayingQueueIndexBySteamID.Find( steamID ); + if ( nMapIndex != m_mapStartPlayingQueueIndexBySteamID.InvalidIndex() ) + { + // We already have an entry for this guy, let's update this one, rather than creating a new one + int nQueueIndex = m_mapStartPlayingQueueIndexBySteamID[ nMapIndex ]; + pWork = &m_llStartPlaying[ nQueueIndex ]; + + // Sanity check data structures. I'd use an assert, + // but this is going live in an environment without + // asserts enabled, so I need to use spew. + if ( pWork->m_steamID == steamID ) + { + // Don't leak user data, if we had any + delete pWork->m_pVarData; + pWork->m_pVarData = NULL; + +// // This could definitely happen occasionally, but if it happens with massive frequency, +// // something is wrong +// if ( gsSteamID == pWork->m_gsSteamID ) +// { +// EmitInfo( SPEW_GC, 4, LOG_ALWAYS, "Got StartPlaying message for %s, who was already in the startplaying queue for the same gameserver %s.\n", steamID.Render(), gsSteamID.Render() ); +// } +// else +// { +// EmitInfo( SPEW_GC, 4, LOG_ALWAYS, "Got StartPlaying message for %s, who was already in the startplaying queue; changing gameserver %s -> %s.\n", steamID.Render(), pWork->m_gsSteamID.Render(), gsSteamID.Render() ); +// } + } + else + { + EmitWarning( SPEW_GC, SPEW_ALWAYS, "Start playing queue corruption! Map entry points to wrong queue entry!\n" ); + pWork = NULL; + m_mapStartPlayingQueueIndexBySteamID.RemoveAt( nMapIndex ); + } + } + else + { +// EmitInfo( SPEW_GC, 4, LOG_ALWAYS, "Got StartPlaying message for %s, new queue for gameserver %s.\n", steamID.Render(), gsSteamID.Render() ); + } + + // Need to create a new entry? + if ( pWork == NULL ) + { + // Create a new queue entry + int nQueueIndex = m_llStartPlaying.AddToTail(); + pWork = &m_llStartPlaying[ nQueueIndex ]; + + // Add it to the steam ID map, so we can locate this guy quickly in the future + m_mapStartPlayingQueueIndexBySteamID.Insert( steamID, nQueueIndex ); + } + + // Fill in the queue entry with the latest details + pWork->m_steamID = steamID; + pWork->m_gsSteamID = gsSteamID; + pWork->m_unServerAddr = unServerAddr; + pWork->m_usServerPort = usServerPort; + + if( cubVarData ) + { + pWork->m_pVarData = new CUtlBuffer; + pWork->m_pVarData->Put( pubVarData, cubVarData ); + } + else + { + pWork->m_pVarData = NULL; + } + + // Should be one-to-one correspondence in these data structures + Assert( (size_t)m_mapStartPlayingQueueIndexBySteamID.Count() == (size_t)m_llStartPlaying.Count() ); +} + +//----------------------------------------------------------------------------- +bool CGCBase::BRemoveStartPlayingQueueEntry( const CSteamID & steamID ) +{ + int nMapIndex = m_mapStartPlayingQueueIndexBySteamID.Find( steamID ); + if ( nMapIndex == m_mapStartPlayingQueueIndexBySteamID.InvalidIndex() ) + { + return false; + } + + //EmitInfo( SPEW_GC, 4, LOG_ALWAYS, "Removed startplaying queue entry for %s.\n", steamID.Render() ); + + // Locate queue entry, make sure it matches, and remote it + int nQueueIndex = m_mapStartPlayingQueueIndexBySteamID[ nMapIndex ]; + if ( m_llStartPlaying[ nQueueIndex ].m_steamID == steamID ) + { + delete m_llStartPlaying[ nQueueIndex ].m_pVarData; + m_llStartPlaying.Remove( nQueueIndex ); + } + else + { + EmitWarning( SPEW_GC, SPEW_ALWAYS, "Start playing queue corruption! Map entry doesn't point to matching queue index (found while removing entry in BRemoveStartPlayingQueueEntry)!\n" ); + } + + // Remove from map + m_mapStartPlayingQueueIndexBySteamID.RemoveAt( nQueueIndex ); + + // Found and removed + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Pull the next startplaying job off the queue and executes it +//----------------------------------------------------------------------------- +void CGCBase::YieldingExecuteNextStartPlaying() +{ + // maybe we have nothing to do! + if( m_llStartPlaying.Count() > 0 ) + { + // Execute the entry at the head + YieldingExecuteStartPlayingQueueEntryByIndex( m_llStartPlaying.Head() ); + } + m_nStartPlayingJobCount--; +} + +//----------------------------------------------------------------------------- +// Purpose: Executes a single entry from the start playing queue, given the linked list handle +//----------------------------------------------------------------------------- +void CGCBase::YieldingExecuteStartPlayingQueueEntryByIndex( int idxStartPlayingQueue ) +{ + VPROF_BUDGET( "CGCBase::YieldingExecuteStartPlayingQueueEntryByIndex - LinkedList", VPROF_BUDGETGROUP_STEAM ); + // Remove the entry from the queue + StartPlayingWork_t work = m_llStartPlaying[ idxStartPlayingQueue ]; + m_llStartPlaying.Remove( idxStartPlayingQueue ); + + VPROF_BUDGET( "CGCBase::YieldingExecuteStartPlayingQueueEntryByIndex", VPROF_BUDGETGROUP_STEAM ); + // Remove it from the Steam ID map, too. + int nMapIndex = m_mapStartPlayingQueueIndexBySteamID.Find( work.m_steamID ); + if ( nMapIndex == m_mapStartPlayingQueueIndexBySteamID.InvalidIndex() ) + { + EmitWarning( SPEW_GC, SPEW_ALWAYS, "Start playing queue corruption! Queue entry is not in map!\n" ); + } + else if ( m_mapStartPlayingQueueIndexBySteamID[ nMapIndex ] != idxStartPlayingQueue ) + { + EmitWarning( SPEW_GC, SPEW_ALWAYS, "Start playing queue corruption! Map entry doesn't have proper queue index!\n" ); + } + else + { + m_mapStartPlayingQueueIndexBySteamID.RemoveAt( nMapIndex ); + } + + // Do the work. + if ( work.m_steamID.BIndividualAccount() ) + { + YieldingStartPlaying( work.m_steamID, work.m_gsSteamID, work.m_unServerAddr, work.m_usServerPort, work.m_pVarData ); + } + else if ( work.m_steamID.BGameServerAccount() ) + { + const uint8 *pVarData = NULL; + uint32 cubVarData = 0; + if ( work.m_pVarData != NULL ) + { + pVarData = (const uint8 *)work.m_pVarData->Base(); + cubVarData = work.m_pVarData->TellMaxPut(); + } + YieldingStartGameserver( work.m_steamID, work.m_unServerAddr, work.m_usServerPort, pVarData, cubVarData ); + } + else + { + AssertMsg1( false, "Bogus steam ID %s in start playing queue", work.m_steamID.Render() ); + } + + // Clean up + delete work.m_pVarData; +} + +void CGCBase::SetUserSessionDetails( CGCUserSession *pUserSession, KeyValues *pkvDetails ) +{ + if( pkvDetails ) + { + pUserSession->m_unIPPublic = pkvDetails->GetInt( "ip", 0 ); + pUserSession->m_osType = static_cast<EOSType>( pkvDetails->GetInt( "osType", k_eOSUnknown ) ); + pUserSession->m_bIsTestSession = pkvDetails->GetInt( "isTestSession", 0 ) != 0; + pUserSession->m_bIsSecure = pkvDetails->GetInt( "secure", 0 ) != 0; + } +} + +//----------------------------------------------------------------------------- +// Purpose: Does the real work when a player starts playing (inside a job) +//----------------------------------------------------------------------------- +void CGCBase::YieldingStartPlaying( const CSteamID & steamID, const CSteamID & gsSteamID, uint32 unServerAddr, uint16 usServerPort, CUtlBuffer *pVarData ) +{ + VPROF_BUDGET( "CGCBase::YieldingStartPlaying", VPROF_BUDGETGROUP_STEAM ); + if ( m_bIsShuttingDown ) + return; + + if( !BYieldingLockSteamID( steamID, __FILE__, __LINE__ ) ) + { + EmitError( SPEW_GC, "Failed to lock steamID %s in YieldingStartPlaying\n", steamID.Render() ); + return; + } + + // if var data came with this StartPlaying message, parse it into a KV and stick it on the session + KeyValues *pkvDetails = NULL; + if( pVarData ) + { + MEM_ALLOC_CREDIT_("StartPlaying - SessionDetails" ); + pkvDetails = new KeyValues( "SessionDetails" ); + if( !pkvDetails->ReadAsBinary( *pVarData ) ) + { + EmitError( SPEW_GC, "Unable to parse session details for %s\n", steamID.Render() ); + pkvDetails->deleteThis(); + pkvDetails = NULL; + } + } + + CGCUserSession *pSession = FindUserSession( steamID ); + if( !pSession ) + { + // Load their SO cache. Remember, we already have their steam ID locked. + VPROF_BUDGET( "CGCBase::YieldingStartPlaying - Load SOCache", VPROF_BUDGETGROUP_STEAM ); + CGCSharedObjectCache *pSOCache = YieldingFindOrLoadSOCache( steamID ); + if ( !pSOCache ) + { + EmitError( SPEW_GC, "Failed to get cache for user %s\n", steamID.Render() ); + return; + } + + // Create session of app-specific type + VPROF_BUDGET( "CGCBase::YieldingStartPlaying - CreateUserSession", VPROF_BUDGETGROUP_STEAM ); + pSession = CreateUserSession( steamID, pSOCache ); + if ( !pSession ) + { + EmitError( SPEW_GC, "Failed to create user session for %s\n", steamID.Render() ); + return; + } + + VPROF_BUDGET( "CGCBase::YieldingStartPlaying - LRU Update", VPROF_BUDGETGROUP_STEAM ); + RemoveCacheFromLRU( pSOCache ); + + CGCUserSession **ppSession = m_hashUserSessions.PvRecordInsert( steamID.ConvertToUint64() ); + *ppSession = pSession; + + SetUserSessionDetails( pSession, pkvDetails ); + + // Do game-specific logic here. Note that we're still holding the game server + // lock... + VPROF_BUDGET( "CGCBase::YieldingStartPlaying - Game-specific start playing", VPROF_BUDGETGROUP_STEAM ); + YieldingSessionStartPlaying( pSession ); + } + else if ( pSession->BIsShuttingDown() ) + { + pkvDetails->deleteThis(); + pkvDetails = NULL; + return; + } + else + { + // Update secure flag, etc from KV details, if any + SetUserSessionDetails( pSession, pkvDetails ); + } + + if ( pkvDetails ) + { + pkvDetails->deleteThis(); + pkvDetails = NULL; + } + + VPROF_BUDGET( "CGCBase::YieldingStartPlaying - Game Server binding", VPROF_BUDGETGROUP_STEAM ); + // Make sure the server exists and then try to join it + if ( gsSteamID.IsValid() && gsSteamID.BGameServerAccount() && BYieldingLockSteamID( gsSteamID, __FILE__, __LINE__ ) ) + { + + // First, try to obtain a session through ordinary means, by validating + // the session + if ( YieldingGetLockedGSSession( gsSteamID, __FILE__, __LINE__ ) != NULL ) + { + // Maintain lock balance + UnlockSteamID( gsSteamID ); + } + else + { + // Failed to get a session --- probably an AM is down. + // This is hopefully relatively rare, as it's not ideal. + // log it + if ( enable_startplaying_gameserver_creation_spew.GetBool() ) + { + netadr_t serverAdr( unServerAddr, usServerPort ); + EmitInfo( SPEW_GC, 2, LOG_ALWAYS, "Creating gameserver session %s @ %s as a result of user %s StartPlaying.\n", gsSteamID.Render(), serverAdr.ToString(), steamID.Render() ); + } + YieldingFindOrCreateGSSession( gsSteamID, unServerAddr, usServerPort, NULL, 0 ); + } + + // Mark that we are joined to this server + pSession->BSetServer( gsSteamID ); + + // Done, clean up lock + UnlockSteamID( gsSteamID ); + } + else + { + // Steam was sometimes sending us messages with zero Steam ID, even when we're on a server. + if ( cv_debug_steam_startplaying.GetBool() ) + EmitInfo( SPEW_GC, 1, 1, "YieldingStartPlaying ( user = %s ) with invalid GS steam ID %s, calling LeaveServer\n", steamID.Render(), gsSteamID.Render() ); + + pSession->BLeaveServer(); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Called when a player stops playing our game +//----------------------------------------------------------------------------- +void CGCBase::YieldingStopPlaying( const CSteamID & steamID ) +{ + // Should be one-to-one correspondence in these data structures + Assert( (size_t)m_mapStartPlayingQueueIndexBySteamID.Count() == (size_t)m_llStartPlaying.Count() ); + + // Check if they have an entry in the startplaying queue, then get rid of it! + BRemoveStartPlayingQueueEntry( steamID ); + + if ( !BLockSteamIDImmediate( steamID ) ) + { + CGCUserSession *pSession = FindUserSession( steamID ); + if ( !pSession ) + { + return; + } + + pSession->SetIsShuttingDown( true ); + if( !BYieldingLockSteamID( steamID, __FILE__, __LINE__ ) ) + { + EmitError( SPEW_GC, "Unable to lock steamID %s in YieldingStopPlaying\n", steamID.Render() ); + return; + } + } + + CGCUserSession *pSession = FindUserSession( steamID ); + if( pSession ) + { + pSession->BLeaveServer(); + YieldingSessionStopPlaying( pSession ); + if( pSession->GetSOCache() ) + { + AddCacheToLRU( pSession->GetSOCache() ); + } + m_hashUserSessions.Remove( steamID.ConvertToUint64() ); + delete pSession; + } + + // Clean up lock. Even if the session is gone and there's nothing + // for the lock to protect, we need this to avoid spurious asserts that check + // lock imbalance + UnlockSteamID( steamID ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Called when a gameserver stops running for our game +//----------------------------------------------------------------------------- +void CGCBase::YieldingStartGameserver( const CSteamID & steamID, uint32 unServerAddr, uint16 usServerPort, const uint8 *pubVarData, uint32 cubVarData ) +{ + VPROF_BUDGET( "CGCBase::YieldingStartGameserver", VPROF_BUDGETGROUP_STEAM ); + if ( m_bIsShuttingDown ) + return; + + if( !BYieldingLockSteamID( steamID, __FILE__, __LINE__ ) ) + { + EmitError( SPEW_GC, "Failed to lock steamID %s in YieldingStartGameserver\n", steamID.Render() ); + return; + } + + YieldingFindOrCreateGSSession( steamID, unServerAddr, usServerPort, pubVarData, cubVarData ); + + // Clean up + UnlockSteamID( steamID ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Called when a gameserver stops running for our game +//----------------------------------------------------------------------------- +void CGCBase::YieldingStopGameserver( const CSteamID & steamID ) +{ + // Should be one-to-one correspondence in these data structures + Assert( (size_t)m_mapStartPlayingQueueIndexBySteamID.Count() == (size_t)m_llStartPlaying.Count() ); + + // Check if they have an entry in the startplaying queue, then get rid of it! + BRemoveStartPlayingQueueEntry( steamID ); + + if ( !BLockSteamIDImmediate( steamID ) ) + { + CGCGSSession *pSession = FindGSSession( steamID ); + if ( !pSession ) + { + return; + } + + pSession->SetIsShuttingDown( true ); + if( !BYieldingLockSteamID( steamID, __FILE__, __LINE__ ) ) + { + EmitError( SPEW_GC, "Unable to lock steamID %s in YieldingStopGameserver\n", steamID.Render() ); + return; + } + } + + CGCGSSession *pSession = FindGSSession( steamID ); + if( pSession ) + { + pSession->RemoveAllUsers(); + YieldingSessionStopServer( pSession ); + if( pSession->GetSOCache() ) + { + AddCacheToLRU( pSession->GetSOCache() ); + } + m_hashGSSessions.Remove( steamID.ConvertToUint64() ); + delete pSession; + } + + // Clean up lock. Even if the session is gone and there's nothing + // for the lock to protect, we need this to avoid spurious asserts that check + // lock imbalance + UnlockSteamID( steamID ); +} + +IMsgNetPacket *CreateIMsgNetPacket( GCProtoBufMsgSrc eReplyType, const CSteamID senderID, uint32 nGCDirIndex, uint32 unMsgType, void *pubData, uint32 cubData ) +{ + VPROF_BUDGET( "CreateIMsgNetPacket", VPROF_BUDGETGROUP_STEAM ); + + if( 0 != ( unMsgType & k_EMsgProtoBufFlag ) ) + { + if ( cubData < sizeof( ProtoBufMsgHeader_t ) ) + { + uint32 unMsgTypeNoFlag = unMsgType & (~k_EMsgProtoBufFlag); + AssertMsg3( false, "Received packet %s(%u) from %s less than the minimum protobuf size", PchMsgNameFromEMsg( unMsgTypeNoFlag ), unMsgTypeNoFlag, senderID.Render() ); + return NULL; + } + + // make a new packet for the message so we can dispatch it + // The CNetPacket takes ownership of the buffer allocated above + CNetPacket *pGCPacket = CNetPacketPool::AllocNetPacket(); + pGCPacket->Init( cubData ); + + // copy the bits for the message over to the full size buffer + Q_memcpy( pGCPacket->PubData(), pubData, cubData ); + + CProtoBufNetPacket *pMsgNetPacket = new CProtoBufNetPacket( pGCPacket, eReplyType, senderID, nGCDirIndex, unMsgType & ( ~k_EMsgProtoBufFlag ) ); + + // release the inner packet since the wrapper now has a ref to it + pGCPacket->Release(); + + if ( !pMsgNetPacket->IsValid() ) + { + pMsgNetPacket->Release(); + return NULL; + } + + return pMsgNetPacket; + } + else + { + //note that we do not currently support reply to GC messages through this pipeline + AssertMsg( eReplyType != GCProtoBufMsgSrc_FromGC, "Warning: Encountered a message from GC to GC that was not of protobuff type, will be unable to reply to this message. Message type: %d", unMsgType ); + + if ( cubData < sizeof( GCMsgHdrEx_t ) - sizeof( GCMsgHdr_t ) ) + { + AssertMsg( false, "Received packet %s(%u) from %s less than the minimum struct size", PchMsgNameFromEMsg( unMsgType ), unMsgType, senderID.Render() ); + return NULL; + } + + // Determine the size of the packet. sizeof(GCMsgHdr_t) was not sent as part of the data + uint32 unFullSize = cubData + sizeof( GCMsgHdr_t ); + + // make a new packet for the message so we can dispatch it + // The CNetPacket takes ownership of the buffer allocated above + CNetPacket *pGCPacket = CNetPacketPool::AllocNetPacket(); + pGCPacket->Init( unFullSize ); + + //fill in our header and copy over the body + uint8 *pFullPacket = pGCPacket->PubData(); + + // get the header so we can fix it up + GCMsgHdrEx_t *pHdr = (GCMsgHdrEx_t *)pFullPacket; + //pHdr->m_nSrcGCDirIndex = nGCDirIndex; + pHdr->m_eMsg = unMsgType; + pHdr->m_ulSteamID = senderID.ConvertToUint64(); + + // copy the bits for the message over to the full size buffer + Q_memcpy( pFullPacket+sizeof(GCMsgHdr_t), pubData, cubData ); + + + CStructNetPacket *pMsgNetPacket = new CStructNetPacket( pGCPacket ); + + // release the packet + pGCPacket->Release(); + return pMsgNetPacket; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Processes an incoming message from the client by turning it into a +// CGCMsg and sending it on to a job. +//----------------------------------------------------------------------------- +void CGCBase::MessageFromClient( const CSteamID & senderID, uint32 unMsgType, void *pubData, uint32 cubData ) +{ + VPROF_BUDGET( "CGCBase::MessageFromClient", VPROF_BUDGETGROUP_STEAM ); + + // if we don't have a GCHost yet, we won't be able to do much with this message + if( !GGCHost() ) + return; + + if ( OnMessageFromClient( senderID, unMsgType, pubData, cubData ) ) + return; + + // Rate limit messages from ordinary clients + if ( senderID.IsValid() ) + { + MsgType_t eMsg = unMsgType & ~k_EMsgProtoBufFlag; + if ( m_MsgRateLimit.BIsRateLimited( senderID, eMsg ) ) + { + g_RateLimitTracker.TrackRateLimitedMsg( senderID, eMsg ); + return; + } + } + + // !FIXME! DOTAMERGE + uint32 nGCDirIndex = 0; // GetGCDirIndex() + IMsgNetPacket *pMsgNetPacket = CreateIMsgNetPacket( GCProtoBufMsgSrc_FromSteamID, senderID, nGCDirIndex, unMsgType, pubData, cubData ); + if ( NULL == pMsgNetPacket ) + return; + + // dispatch the packet (some messages require special consideration) + switch( unMsgType ) + { + case k_EGCMsgWGRequest: + m_wgJobMgr.BHandleMsg( pMsgNetPacket ); + g_theMessageList.TallySendMessage( pMsgNetPacket->GetEMsg(), cubData ); + break; + + default: + GetJobMgr().BRouteMsgToJob( this, pMsgNetPacket, JobMsgInfo_t( pMsgNetPacket->GetEMsg(), pMsgNetPacket->GetSourceJobID(), pMsgNetPacket->GetTargetJobID(), k_EServerTypeGC ) ); + g_theMessageList.TallySendMessage( pMsgNetPacket->GetEMsg(), cubData ); + break; + } + + // release the packet + pMsgNetPacket->Release(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Sends a message to the given SteamID +//----------------------------------------------------------------------------- +bool CGCBase::BSendGCMsgToClient( const CSteamID & steamIDTarget, const CGCMsgBase& msg ) +{ + g_theMessageList.TallySendMessage( msg.Hdr().m_eMsg, msg.CubPkt() - sizeof(GCMsgHdr_t) ); + VPROF_BUDGET( "GCHost", VPROF_BUDGETGROUP_STEAM ); + { + VPROF_BUDGET( "GCHost - SendMessageToClient", VPROF_BUDGETGROUP_STEAM ); + return m_pHost->BSendMessageToClient( m_unAppID, steamIDTarget, msg.Hdr().m_eMsg, msg.PubPkt() + sizeof(GCMsgHdr_t), msg.CubPkt() - sizeof(GCMsgHdr_t) ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Used to send protobuf system messages to a client +//----------------------------------------------------------------------------- +class CProtoBufClientSendHandler : public CProtoBufMsgBase::IProtoBufSendHandler +{ +public: + CProtoBufClientSendHandler( const CSteamID & steamIDTarget ) + : m_steamIDTarget( steamIDTarget ), m_cubSent( 0 ) {} + virtual bool BAsyncSend( MsgType_t eMsg, const uint8 *pubMsgBytes, uint32 cubSize ) OVERRIDE + { + m_cubSent = cubSize; + // !FIXME! DOTAMERGE + //return GGCInterface()->BProcessSystemMessage( eMsg | k_EMsgProtoBufFlag, pubMsgBytes, cubSize ); + g_theMessageList.TallySendMessage( eMsg & ~k_EMsgProtoBufFlag, cubSize ); + VPROF_BUDGET( "GCHost", VPROF_BUDGETGROUP_STEAM ); + { + VPROF_BUDGET( "GCHost - SendMessageToClient (ProtoBuf)", VPROF_BUDGETGROUP_STEAM ); + return GGCHost()->BSendMessageToClient( GGCBase()->GetAppID(), m_steamIDTarget, eMsg | k_EMsgProtoBufFlag, pubMsgBytes, cubSize ); + } + } + uint32 GetCubSent() const { return m_cubSent; } +private: + uint32 m_cubSent; + CSteamID m_steamIDTarget; +}; + +//----------------------------------------------------------------------------- +// Purpose: Used to send protobuf system messages into the GC +//----------------------------------------------------------------------------- +class CProtoBufSystemSendHandler : public CProtoBufMsgBase::IProtoBufSendHandler +{ +public: + CProtoBufSystemSendHandler() + : m_cubSent( 0 ) {} + virtual bool BAsyncSend( MsgType_t eMsg, const uint8 *pubMsgBytes, uint32 cubSize ) OVERRIDE + { + m_cubSent = cubSize; + // !FIXME! DOTAMERGE + //return GGCInterface()->BProcessSystemMessage( eMsg | k_EMsgProtoBufFlag, pubMsgBytes, cubSize ); + g_theMessageList.TallySendMessage( eMsg & ~k_EMsgProtoBufFlag, cubSize ); + VPROF_BUDGET( "GCHost", VPROF_BUDGETGROUP_STEAM ); + { + VPROF_BUDGET( "GCHost - SendMessageToSystem (ProtoBuf)", VPROF_BUDGETGROUP_STEAM ); + return GGCHost()->BSendMessageToClient( GGCBase()->GetAppID(), CSteamID(), eMsg | k_EMsgProtoBufFlag, pubMsgBytes, cubSize ); + } + } + uint32 GetCubSent() const { return m_cubSent; } +private: + uint32 m_cubSent; +}; + + +//----------------------------------------------------------------------------- +// Purpose: Sends a message to the given SteamID +//----------------------------------------------------------------------------- +bool CGCBase::BSendGCMsgToClient( const CSteamID & steamIDTarget, const CProtoBufMsgBase& msg ) +{ + CProtoBufClientSendHandler sender( steamIDTarget ); + return msg.BAsyncSend( sender ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Sends a system message to the GC Host +//----------------------------------------------------------------------------- +bool CGCBase::BSendSystemMessage( const CGCMsgBase& msg, uint32 *pcubSent ) +{ + uint32 cubSent = msg.CubPkt() - sizeof(GCMsgHdr_t); + if ( NULL != pcubSent ) + { + *pcubSent = cubSent; + } + + // !FIXME! DOTAMERGE + //return GGCInterface()->BProcessSystemMessage( msg.Hdr().m_eMsg, msg.PubPkt() + sizeof(GCMsgHdr_t), cubSent ); + return BSendGCMsgToClient( CSteamID(), msg ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Sends a system message to the GC Host +//----------------------------------------------------------------------------- +bool CGCBase::BSendSystemMessage( const CProtoBufMsgBase & msg, uint32 *pcubSent ) +{ + CProtoBufSystemSendHandler sender; + bool bRet = msg.BAsyncSend( sender ); + if ( NULL != pcubSent ) + { + *pcubSent = sender.GetCubSent(); + } + return bRet; +} + +bool CGCBase::BSendSystemMessage( const ::google::protobuf::Message &msgOut, MsgType_t eSendMsg ) +{ + CProtoBufSystemSendHandler sender; + CMsgProtoBufHeader hdr; + return CProtoBufMsgBase::BAsyncSendProto( sender, eSendMsg, hdr, msgOut ); +} + +//----------------------------------------------------------------------------- +// Purpose: send msgOut to the place that msgIn came from +//----------------------------------------------------------------------------- +bool CGCBase::BReplyToMessage( CGCMsgBase &msgOut, const CGCMsgBase &msgIn ) +{ + // Don't reply if the source is not expecting it + if ( !msgIn.BIsExpectingReply() ) + return true; + + msgOut.Hdr().m_JobIDTarget = msgIn.Hdr().m_JobIDSource; + return BSendGCMsgToClient( msgIn.Hdr().m_ulSteamID, msgOut ); +} + + +//----------------------------------------------------------------------------- +// Purpose: send msgOut to the place that msgIn came from +//----------------------------------------------------------------------------- +bool CGCBase::BReplyToMessage( CProtoBufMsgBase &msgOut, const CProtoBufMsgBase &msgIn ) +{ + // Don't reply if the source is not expecting it + if ( !msgIn.GetJobIDSource() ) + return true; + + msgOut.SetJobIDTarget( msgIn.GetJobIDSource() ); + return BSendGCMsgToClient( msgIn.GetClientSteamID(), msgOut ); +} + +//----------------------------------------------------------------------------- +// Purpose: Sends a message to the given SteamID +//----------------------------------------------------------------------------- +bool CGCBase::BSendGCMsgToClientWithPreSerializedBody( const CSteamID & steamIDTarget, MsgType_t eMsgType, const CMsgProtoBufHeader& hdr, const byte *pubBody, uint32 cubBody ) const +{ + CProtoBufClientSendHandler sender( steamIDTarget ); + return CProtoBufMsgBase::BAsyncSendWithPreSerializedBody( sender, eMsgType, hdr, pubBody, cubBody ); +} + +//----------------------------------------------------------------------------- +// Purpose: Sends a message that has already been packed to the system handler +//----------------------------------------------------------------------------- +bool CGCBase::BSendGCMsgToSystemWithPreSerializedBody( MsgType_t eMsgType, const CMsgProtoBufHeader& hdr, const byte *pubBody, uint32 cubBody ) const +{ + CProtoBufSystemSendHandler sender; + return CProtoBufMsgBase::BAsyncSendWithPreSerializedBody( sender, eMsgType, hdr, pubBody, cubBody ); +} + +//----------------------------------------------------------------------------- +// Purpose: send msgOut to the place that msgIn came from +//----------------------------------------------------------------------------- +bool CGCBase::BReplyToMessageWithPreSerializedBody( MsgType_t eMsgType, const CProtoBufMsgBase &msgIn, const byte *pubBody, uint32 cubBody ) const +{ + // Don't reply if the source is not expecting it + if ( !msgIn.GetJobIDSource() ) + return true; + + if( temp_list_mismatched_replies.GetBool() && !msgIn.BIsExpectingReply() ) + { + EG_MSG( g_EGMessages, "Message %s was sent to client %s which did not expect a reply\n", PchMsgNameFromEMsg( eMsgType ), msgIn.GetClientSteamID().Render() ); + } + + CMsgProtoBufHeader hdr; + hdr.set_job_id_target( msgIn.GetJobIDSource() ); + + //is this a system message or a client message we are responding to? + bool bSystemReply = ( msgIn.GetClientSteamID() == k_steamIDNil ); + + if( bSystemReply ) + { + return BSendGCMsgToSystemWithPreSerializedBody( eMsgType, hdr, pubBody, cubBody ); + } + else + { + return BSendGCMsgToClientWithPreSerializedBody( msgIn.GetClientSteamID(), eMsgType, hdr, pubBody, cubBody ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: send msgOut to the place that msgIn came from +//----------------------------------------------------------------------------- + +bool CGCBase::BYldSendMessageAndGetReply( const CSteamID &steamIDTarget, CProtoBufMsgBase &msgOut, CProtoBufMsgBase *pMsgIn, MsgType_t eMsg ) +{ + CJob& curJob = GJobCur(); + msgOut.ExpectingReply( curJob.GetJobID() ); + + if ( !BSendGCMsgToClient( steamIDTarget, msgOut ) ) + return false; + + if( !curJob.BYieldingWaitForMsg( pMsgIn, eMsg, steamIDTarget ) ) + return false; + + return true; +} + +//bool CGCBase::BYldSendGCMessageAndGetReply( int32 nGCDirIndex, CProtoBufMsgBase &msgOut, CProtoBufMsgBase *pMsgIn, MsgType_t eMsg ) +//{ +// CJob& curJob = GJobCur(); +// msgOut.ExpectingReply( curJob.GetJobID() ); +// +// if ( !BSendGCMessage( nGCDirIndex, msgOut ) ) +// return false; +// +// if( !curJob.BYieldingWaitForMsg( pMsgIn, eMsg, CSteamID() ) ) +// return false; +// +// return true; +//} + +bool CGCBase::BYldSendSystemMessageAndGetReply( CGCMsgBase &msgOut, CGCMsgBase *pMsgIn, MsgType_t eMsg ) +{ + CJob& curJob = GJobCur(); + msgOut.ExpectingReply( curJob.GetJobID() ); + + if ( !BSendSystemMessage( msgOut ) ) + return false; + + if( !curJob.BYieldingWaitForMsg( pMsgIn, eMsg, CSteamID() ) ) + return false; + + return true; +} + +bool CGCBase::BYldSendSystemMessageAndGetReply( CProtoBufMsgBase &msgOut, CProtoBufMsgBase *pMsgIn, MsgType_t eMsg ) +{ + CJob& curJob = GJobCur(); + msgOut.ExpectingReply( curJob.GetJobID() ); + + if ( !BSendSystemMessage( msgOut ) ) + return false; + + if( !curJob.BYieldingWaitForMsg( pMsgIn, eMsg, CSteamID() ) ) + return false; + + return true; +} + +bool CGCBase::BYldSendSystemMessageAndGetReply( const ::google::protobuf::Message &msgSend, MsgType_t eSendMsg, ::google::protobuf::Message *pMsgResponse, MsgType_t eRespondMsg ) +{ + CJob& curJob = GJobCur(); + + CMsgProtoBufHeader hdr; + hdr.set_job_id_source( curJob.GetJobID() ); + + CProtoBufSystemSendHandler sender; + CProtoBufMsgBase::BAsyncSendProto( sender, eSendMsg, hdr, msgSend ); + + CProtoBufPtrMsg protoMsg( pMsgResponse ); + //return curJob.BYieldingWaitForMsg( &protoMsg, eRespondMsg, CSteamID() ); + return curJob.BYieldingWaitForMsg( &protoMsg, eRespondMsg ); // !FIXME! For some reason system replies are coming back with a universe and instance set (but account ID zero). +} + + +//----------------------------------------------------------------------------- +// Purpose: Creates a new session for the steam ID +//----------------------------------------------------------------------------- +CGCUserSession *CGCBase::CreateUserSession( const CSteamID & steamID, CGCSharedObjectCache *pSOCache ) const +{ + return new CGCUserSession( steamID, pSOCache ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Creates a new session for the steam ID +//----------------------------------------------------------------------------- +CGCGSSession *CGCBase::CreateGSSession( const CSteamID & steamID, CGCSharedObjectCache *pSOCache, uint32 unServerAddr, uint16 usServerPort ) const +{ + return new CGCGSSession( steamID, pSOCache, unServerAddr, usServerPort ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Locks the session for this steam ID and returns it. Returns NULL +// if the lock could not be granted or if the session could not be +// found. +//----------------------------------------------------------------------------- +CGCUserSession *CGCBase::YieldingGetLockedUserSession( const CSteamID & steamID, const char *pszFilename, int nLineNum ) +{ + if( !steamID.BIndividualAccount() ) + return NULL; + + if( !BYieldingLockSteamID( steamID, pszFilename, nLineNum ) ) + return NULL; + + CGCUserSession *pSession = FindUserSession( steamID ); + if( !pSession ) + { + //EmitInfo( SPEW_GC, SPEW_ALWAYS, LOG_ALWAYS, "Unable to find session %s to lock it. Attempting to fetch it from the AM\n", steamID.Render() ); + pSession = (CGCUserSession *)YieldingRequestSession( steamID ); + if( !pSession ) + { + UnlockSteamID( steamID ); + } + } + + return pSession; +} + +//----------------------------------------------------------------------------- +// Purpose: Checks if a user is in the start playing queue +//----------------------------------------------------------------------------- +bool CGCBase::BUserSessionPending( const CSteamID & steamID ) const +{ + int nStartPlayingMapIndex = m_mapStartPlayingQueueIndexBySteamID.Find( steamID ); + return ( nStartPlayingMapIndex != m_mapStartPlayingQueueIndexBySteamID.InvalidIndex() ); +} + +//----------------------------------------------------------------------------- +// Purpose: Returns the session for this steamID or NULL if that session could +// not be found. +//----------------------------------------------------------------------------- +CGCUserSession *CGCBase::FindUserSession( const CSteamID & steamID ) const +{ + // we should only call this on individual ids + if ( !steamID.IsValid() ) + { + AssertMsg1( steamID.IsValid(), "CGCBase::FindUserSession was passed invalid Steam ID %s", steamID.Render() ); + return NULL; + } + if ( !steamID.BIndividualAccount() ) + { + AssertMsg1( steamID.BIndividualAccount(), "CGCBase::FindUserSession was passed non-individual Steam ID %s", steamID.Render() ); + return NULL; + } + + CGCUserSession **ppSession = m_hashUserSessions.PvRecordFind( steamID.ConvertToUint64() ); + if( ppSession ) + { + (*ppSession)->MarkAccess(); + return *ppSession; + } + else + { + return NULL; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns true if the session associated with the steam id is online, false otherwise +//----------------------------------------------------------------------------- +bool CGCBase::BYieldingIsOnline( const CSteamID & steamID ) +{ + CGCMsg< MsgGCValidateSession_t > msg( k_EGCMsgValidateSession ); + msg.Body().m_ulSteamID = steamID.ConvertToUint64(); + msg.ExpectingReply( GJobCur().GetJobID() ); + if( !BSendSystemMessage( msg ) ) + return false; + + CGCMsg< MsgGCValidateSessionResponse_t > msgReply; + if( !GJobCur().BYieldingWaitForMsg( &msgReply, k_EGCMsgValidateSessionResponse ) ) + { + EmitWarning( SPEW_GC, LOG_ALWAYS, "Didn't get reply from AM for %s in YieldingRequestSession\n", steamID.Render() ); + return false; + } + + return msgReply.Body().m_bOnline; +} + + +//----------------------------------------------------------------------------- +// Purpose: Looks up a session from the AM for the provided steam ID. +//----------------------------------------------------------------------------- +template <typename T > +class CScopedIncrement +{ +public: + inline CScopedIncrement( T & counter) : m_counter(counter) { ++m_counter; } + inline ~CScopedIncrement() { --m_counter; } +private: + T &m_counter; +}; + +CGCSession *CGCBase::YieldingRequestSession( const CSteamID & steamID ) +{ + AssertRunningJob(); + if( !steamID.BIndividualAccount() && !steamID.BGameServerAccount() ) + return NULL; + Assert( IsSteamIDUnlockedOrLockedByCurJob( steamID ) ); + + // Check if we already have info in the logon queue for this SteamID + int nStartPlayingMapIndex = m_mapStartPlayingQueueIndexBySteamID.Find( steamID ); + if ( nStartPlayingMapIndex != m_mapStartPlayingQueueIndexBySteamID.InvalidIndex() ) + { + + // Sanity + int idxStartPlayingQueue = m_mapStartPlayingQueueIndexBySteamID[ nStartPlayingMapIndex ]; + Assert( m_llStartPlaying[ idxStartPlayingQueue ].m_steamID == steamID ); + + // Pull the logon out of the queue and execute it NOW + YieldingExecuteStartPlayingQueueEntryByIndex( idxStartPlayingQueue ); + + // Now return the session that was created, if any + return FindUserOrGSSession( steamID ); + } + + CGCMsg< MsgGCValidateSession_t > msg( k_EGCMsgValidateSession ); + msg.Body().m_ulSteamID = steamID.ConvertToUint64(); + msg.ExpectingReply( GJobCur().GetJobID() ); + if( !BSendSystemMessage( msg ) ) + return NULL; + + CScopedIncrement<int> increment( m_nRequestSessionJobsActive ); + + CGCMsg< MsgGCValidateSessionResponse_t > msgReply; + if( !GJobCur().BYieldingWaitForMsg( &msgReply, k_EGCMsgValidateSessionResponse ) ) + { + EmitWarning( SPEW_GC, LOG_ALWAYS, "Didn't get reply from AM for %s in YieldingRequestSession\n", steamID.Render() ); + return NULL; + } + + if( steamID.BIndividualAccount() ) + { + if( msgReply.Body().m_bOnline ) + { + CUtlBuffer bufVarData; + if( msgReply.CubVarData() ) + { + bufVarData.Put( msgReply.PubVarData(), msgReply.CubVarData() ); + } + + // Check if they have an entry in the startplaying queue, then get rid of it! + // They data we just received is the most up-to-date we have. We should + // prefer this data over anything in the queue for sure. + BRemoveStartPlayingQueueEntry( steamID ); + + YieldingStartPlaying( steamID, msgReply.Body().m_ulSteamIDGS, msgReply.Body().m_unServerAddr, msgReply.Body().m_usServerPort, msgReply.CubVarData() ? &bufVarData : NULL ); + return FindUserSession( steamID ); + } + else + { + //EmitWarning( SPEW_GC, LOG_ALWAYS, "Reply from AM is logging off %s in YieldingRequestSession\n", steamID.Render() ); + YieldingStopPlaying( steamID ); + return NULL; + } + } + else + { + if( msgReply.Body().m_bOnline ) + { + YieldingStartGameserver( steamID, msgReply.Body().m_unServerAddr, msgReply.Body().m_usServerPort, msgReply.PubVarData(), msgReply.CubVarData() ); + return FindGSSession( steamID ); + } + else + { + //EmitWarning( SPEW_GC, LOG_ALWAYS, "Reply from AM is stopping %s in YieldingRequestSession\n", steamID.Render() ); + YieldingStopGameserver( steamID ); + return NULL; + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Send outgoing HTTP request to some other server. Probably a WebAPI +// request to steam itself, but it could be a request on a more +// remote server. +//----------------------------------------------------------------------------- +bool CGCBase::BYieldingSendHTTPRequest( const CHTTPRequest *pRequest, CHTTPResponse *pResponse ) +{ + if ( !pRequest || !pResponse ) + { + AssertMsg( false, "Bad parameters for BYieldingSendHTTPRequest" ); + return false; + } + + CMsgHttpResponse msgResponse; + if( !BYldSendSystemMessageAndGetReply( pRequest->GetProtoObj(), k_EGCMsgSendHTTPRequest, &msgResponse, k_EGCMsgSendHTTPRequestResponse ) ) + { + ReportHTTPError( CFmtStr( "No response to HTTP system message for %s", pRequest->GetURL() ), CGCEmitGroup::kMsg_Error ); + return false; + } + + if ( !msgResponse.has_status_code() ) + { + ReportHTTPError( CFmtStr( "No status code on HTTP response for %s", pRequest->GetURL() ), CGCEmitGroup::kMsg_Error ); + return false; + } + + //log the result of this request + if( msgResponse.status_code() != k_EHTTPStatusCode200OK ) + { + ReportHTTPError( CFmtStr( "Invalid status code %u for %s", msgResponse.status_code(), pRequest->GetURL() ), CGCEmitGroup::kMsg_Warning ); + } + else + { + ReportHTTPError( CFmtStr( "Success status code for %s", pRequest->GetURL() ), CGCEmitGroup::kMsg_Verbose ); + } + + pResponse->DeserializeFromProtoBuf( msgResponse ); + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Send an outgoing HTTP request and parse the result into KeyValues. +//----------------------------------------------------------------------------- +EResult CGCBase::YieldingSendHTTPRequestKV( const CHTTPRequest *pRequest, KeyValues *pKVResponse ) +{ + CHTTPResponse apiResponse; + if ( !BYieldingSendHTTPRequest( pRequest, &apiResponse ) ) + { + EmitError( SPEW_GC, __FUNCTION__ ": web call to %s timed out\n", pRequest->GetURL() ); + return k_EResultTimeout; + } + + if ( k_EHTTPStatusCode200OK != apiResponse.GetStatusCode() ) + { + EmitError( SPEW_GC, __FUNCTION__ ": web call to %s got failure code %d\n", pRequest->GetURL(), apiResponse.GetStatusCode() ); + return k_EResultRemoteCallFailed; + } + + pKVResponse->UsesEscapeSequences( true ); + if ( !pKVResponse->LoadFromBuffer( "webResponse", *apiResponse.GetBodyBuffer() ) ) + { + EmitError( SPEW_GC, "Web call to %s could not parse response\n", pRequest->GetURL() ); + return k_EResultRemoteCallFailed; + } + + return k_EResultOK; +} + + +//----------------------------------------------------------------------------- +// Purpose: Locks the session for this steam ID and returns it. Returns NULL +// if the lock could not be granted or if the session could not be +// found. +//----------------------------------------------------------------------------- +CGCGSSession *CGCBase::YieldingGetLockedGSSession( const CSteamID & steamID, const char *pszFilename, int nLineNum ) +{ + if( !steamID.BGameServerAccount() ) + return NULL; + + if( !BYieldingLockSteamID( steamID, pszFilename, nLineNum ) ) + return NULL; + + CGCGSSession *pSession = FindGSSession( steamID ); + if( !pSession ) + { + pSession = (CGCGSSession *)YieldingRequestSession( steamID ); + if( !pSession ) + { + UnlockSteamID( steamID ); + } + } + + return pSession; +} + +void CGCBase::ReportHTTPError( const char* pszError, CGCEmitGroup::EMsgLevel eLevel ) +{ + //see if we can find a match + int nIndex = m_HTTPErrors.Find( pszError ); + if( nIndex != m_HTTPErrors.InvalidIndex() ) + { + //just increment our count + m_HTTPErrors[ nIndex ]->m_nCount++; + m_HTTPErrors[ nIndex ]->m_eSeverity = MIN( eLevel, m_HTTPErrors[ nIndex ]->m_eSeverity ); + } + else + { + //add one + SHTTPError* pError = new SHTTPError; + pError->m_sStr = pszError; + pError->m_nCount = 1; + pError->m_eSeverity = eLevel; + m_HTTPErrors.Insert( pError->m_sStr, pError ); + } + + if( !m_DumpHTTPErrorsSchedule.BIsScheduled() ) + { + m_DumpHTTPErrorsSchedule.ScheduleMS( this, &CGCBase::DumpHTTPErrors, 1000 ); + } +} + +void CGCBase::DumpHTTPErrors() +{ + FOR_EACH_MAP_FAST( m_HTTPErrors, nCurrError ) + { + SHTTPError* pError = m_HTTPErrors[ nCurrError ]; + EG_EMIT( g_EGHTTPRequest, m_HTTPErrors[ nCurrError ]->m_eSeverity, "%s - %d times\n", pError->m_sStr.String(), pError->m_nCount ); + delete pError; + } + m_HTTPErrors.RemoveAll(); +} + +//----------------------------------------------------------------------------- +// Purpose: Returns the session for this steamID or NULL if that session could +// not be found. +//----------------------------------------------------------------------------- +CGCGSSession *CGCBase::YieldingFindOrCreateGSSession( const CSteamID & steamID, uint32 unServerAddr, uint16 usServerPort, const uint8 *pubVarData, uint32 cubVarData ) +{ + Assert( IsSteamIDLockedByJob( steamID, &GJobCur() ) ); + + // If it's not a game server ID, then we shouldn't make a session for it. + if( !steamID.BGameServerAccount() ) + return NULL; + + MEM_ALLOC_CREDIT_( "YieldingFindOrCreateGSSession" ); + + // if var data came with this StartPlaying message, parse it into a KV and stick it on the session + KeyValues *pkvDetails = NULL; + if( pubVarData && cubVarData ) + { + CUtlBuffer bufDetails; + bufDetails.Put( pubVarData, cubVarData ); + pkvDetails = new KeyValues( "SessionDetails" ); + if( !pkvDetails->ReadAsBinary( bufDetails ) ) + { + EmitError( SPEW_GC, "Unable to parse session details for %s\n", steamID.Render() ); + pkvDetails->deleteThis(); + pkvDetails = NULL; + } + } + +// // Since we might have to lock the session in some cases, let's just always grab the lock here, +// // to keep things simpler. +// if ( !BYieldingLockSteamID( steamID, __FILE__, __LINE__ ) ) +// return NULL; + + CGCGSSession *pSession = FindGSSession( steamID ); + CGCSharedObjectCache *pSOCache = NULL; + if( !pSession ) + { + pSOCache = YieldingFindOrLoadSOCache( steamID ); + + // Did anybody create a session while we held the lock? + // We hold the lock, and you must hold the lock to create + // the session, so this race condition should be impossible + pSession = FindGSSession( steamID ); + Assert( pSession == NULL ); + } + if( !pSession ) + { + + // Create session of app-specific type + pSession = CreateGSSession( steamID, pSOCache, unServerAddr, usServerPort ); + Assert( pSession ); + if ( !pSession ) + { + AssertMsg1( false, "Failed creating GC GS session for %llu", steamID.ConvertToUint64() ); + if ( pkvDetails ) + { + pkvDetails->deleteThis(); + } + //UnlockSteamID( steamID ); // I like to clean up after myself + return NULL; + } + RemoveCacheFromLRU( pSOCache ); + + CGCGSSession **ppSession = m_hashGSSessions.PvRecordInsert( steamID.ConvertToUint64() ); + *ppSession = pSession; + + // Do game-specific work + YieldingSessionStartServer( pSession ); + } + else + { + if ( unServerAddr != 0 && usServerPort != 0 && ( unServerAddr != pSession->GetAddr() || usServerPort != pSession->GetPort() ) ) + { + UpdateGSSessionAddress( pSession, unServerAddr, usServerPort ); + } + } + + if( pkvDetails ) + { + uint32 ip = pkvDetails->GetInt( "ip", 0 ); + if ( ip != 0 ) + pSession->m_unIPPublic = ip; + pSession->m_osType = static_cast<EOSType>( pkvDetails->GetInt( "osType", k_eOSUnknown ) ); + pSession->m_bIsTestSession = pkvDetails->GetInt( "isTestSession", 0 ) != 0; + pkvDetails->deleteThis(); + } + + //UnlockSteamID( steamID ); // I like to clean up after myself + return pSession; +} + +//----------------------------------------------------------------------------- +// Purpose: Called when a Session is moved to a different address. +//----------------------------------------------------------------------------- +void CGCBase::UpdateGSSessionAddress( CGCGSSession *pSession, uint32 unServerAddr, uint16 usServerPort ) +{ + pSession->SetIPAndPort( unServerAddr, usServerPort ); +} + +//----------------------------------------------------------------------------- +// Purpose: Returns the session for this steamID or NULL if that session could +// not be found. +//----------------------------------------------------------------------------- +CGCGSSession *CGCBase::FindGSSession( const CSteamID & steamID ) const +{ + // we should only call this on server ids + if ( !steamID.IsValid() || steamID.GetAccountID() == 0 ) + { + AssertMsg1( false, "CGCBase::FindGSSession was passed invalid Steam ID %s", steamID.Render() ); + return NULL; + } + if ( !steamID.BGameServerAccount() ) + { + AssertMsg1( steamID.BGameServerAccount(), "CGCBase::FindGSSession was passed non-gameserver Steam ID %s", steamID.Render() ); + return NULL; + } + + CGCGSSession **ppSession = m_hashGSSessions.PvRecordFind( steamID.ConvertToUint64() ); + if( ppSession ) + { + (*ppSession)->MarkAccess(); + return *ppSession; + } + else + { + return NULL; + } +} + +//----------------------------------------------------------------------------- +// Purpose: Locate session from appropriate table, depending on if it's +// an individual or gameserver ID +//----------------------------------------------------------------------------- +CGCSession *CGCBase::FindUserOrGSSession( const CSteamID & steamID ) const +{ + if ( steamID.BIndividualAccount() ) + return FindUserSession( steamID ); + if ( steamID.BGameServerAccount() ) + return FindGSSession( steamID ); + AssertMsg1( false, "CGCBase::FindUserOrGSSession, steam ID %s isn't an individual or a gameserver ID", steamID.Render() ); + return NULL; +} + + +//----------------------------------------------------------------------------- +// Purpose: Wakes up the job waiting for this SQL result +//----------------------------------------------------------------------------- +void CGCBase::SQLResults( GID_t gidContextID ) +{ + VPROF_BUDGET( "CGCBase::SQLResults", VPROF_BUDGETGROUP_STEAM ); + m_JobMgr.BResumeSQLJob( gidContextID ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Finds the cache in the map for a new session +//----------------------------------------------------------------------------- +CGCSharedObjectCache *CGCBase::FindSOCache( const CSteamID & steamID ) +{ + CUtlMap< CSteamID, CGCSharedObjectCache *, int >::IndexType_t nCache = m_mapSOCache.Find( steamID ); + if( m_mapSOCache.IsValidIndex( nCache ) ) + return m_mapSOCache[nCache]; + else + return NULL; +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +bool CGCBase::BYieldingLoadSOCache( CGCSharedObjectCache *pSOCache ) +{ + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void CGCBase::YieldingSOCacheLoaded( CGCSharedObjectCache *pSOCache ) +{ + // remove it, so we don't stomp the copy in memcached + m_rbtreeSOCachesWithDirtyVersions.Remove( pSOCache->GetOwner() ); + + // stomp the version with the one we set in memcached previously if possible, otherwise, re-add it to the set + if ( !BYieldingRetrieveCacheVersion( pSOCache ) ) + { + m_rbtreeSOCachesWithDirtyVersions.InsertIfNotFound( pSOCache->GetOwner() ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Removes the cache for this steamID +//----------------------------------------------------------------------------- +void CGCBase::RemoveSOCache( const CSteamID & steamID ) +{ + CUtlMap< CSteamID, CGCSharedObjectCache *, int >::IndexType_t nCache = m_mapSOCache.Find( steamID ); + if( m_mapSOCache.IsValidIndex( nCache ) ) + { + CGCSharedObjectCache *pSOCache = m_mapSOCache[nCache]; + pSOCache->RemoveAllSubscribers(); + + if( pSOCache->BIsDatabaseDirty() ) + { + EmitError( SPEW_GC, "Attempting to remove SO Cache %s while it was dirty. Adding to Writeback instead\n", steamID.Render() ); + pSOCache->DumpDirtyObjects(); + AddCacheToWritebackQueue( pSOCache ); + + // adding the cache to the LRU list too, just so it will go away once writeback does its thing + if( !m_listCachesToUnload.IsValidIndex( pSOCache->GetLRUHandle() ) ) + { + AddCacheToLRU( pSOCache ); + } + } + else + { + RemoveCacheFromLRU(pSOCache); + + delete pSOCache; + m_mapSOCache.RemoveAt( nCache ); + } + } +} + +//----------------------------------------------------------------------------- +// Purpose: Enqueues a flush instruction to Econ service for Web Inventory to update +//----------------------------------------------------------------------------- +void CGCBase::FlushInventoryCache( AccountID_t unAccountID ) +{ + VPROF_BUDGET( "FlushInventoryCache - enqueue", VPROF_BUDGETGROUP_STEAM ); + m_rbFlushInventoryCacheAccounts.InsertIfNotFound( unAccountID ); +} + +//----------------------------------------------------------------------------- +// Purpose: Finds the cache in the map for a new session and locks it +//----------------------------------------------------------------------------- +bool CGCBase::UnloadUnusedCaches( uint32 unMaxCacheCount, CLimitTimer *pLimitTimer ) +{ + VPROF_BUDGET( "UnloadUnusedCaches", VPROF_BUDGETGROUP_STEAM ); + + uint32 unCachesUnloaded = 0; + for( uint32 unCache = m_listCachesToUnload.Head(), unNextCache = m_listCachesToUnload.InvalidIndex(); unCache != m_listCachesToUnload.InvalidIndex(); unCache = unNextCache ) + { + unNextCache = m_listCachesToUnload.Next( unCache ); + + // only remove caches until we are under our limit + if( (uint32)m_mapSOCache.Count() <= unMaxCacheCount ) + return false; + + // only loop until we need to stop consuming heartbeat time. We'll finish in later frames + if( pLimitTimer && pLimitTimer->BLimitReached() ) + return true; + + CSteamID ownerID = m_listCachesToUnload[ unCache ]; + CGCSharedObjectCache *pSOCache = FindSOCache( ownerID ); + Assert( pSOCache ); + if( !pSOCache ) + { + EmitError( SPEW_GC, "Cache for %s could not be found even though it is in the LRU list\n", ownerID.Render() ); + m_listCachesToUnload.Remove( unCache ); + continue; + } + + // make sure there's no session using this cache + if( ( ownerID.BIndividualAccount() && FindUserSession( ownerID ) ) + || ( ownerID.BGameServerAccount() && FindGSSession( ownerID ) ) ) + { + EmitError( SPEW_GC, "Cache for %s has a session even though it is in the LRU list\n", ownerID.Render() ); + Assert( pSOCache->GetLRUHandle() == unCache ); + if ( pSOCache->GetLRUHandle() != unCache ) + { + EmitError( SPEW_GC, "Cache for %s has a different LRU handle than the one retrieved from the iterator! 0x%08x vs 0x%08x\n", ownerID.Render(), pSOCache->GetLRUHandle(), unCache ); + } + + RemoveCacheFromLRU( pSOCache ); + continue; + } + + // Locked steam IDs mean someone is using the cache. + // Being in the writeback queue means that you haven't actually been unused for very long. + // Just move on to the next one in those cases. + if( IsSteamIDLocked( ownerID ) || pSOCache->GetInWriteback() ) + continue; + + // either count down by one or still in LRU? + int iPreRemoveCount = m_listCachesToUnload.Count(); + + // remove and delete the cache (which will remove it from the LRU list too.) + RemoveSOCache( ownerID ); + unCachesUnloaded++; + + if ( iPreRemoveCount != m_listCachesToUnload.Count() + 1 && + iPreRemoveCount != m_listCachesToUnload.Count() ) + { + EmitError( SPEW_GC, "CGCBase::UnloadUnusedCaches() sanity check failed! List size changed dramatically removing 0x%08x; delta %i\n", unCache, iPreRemoveCount - m_listCachesToUnload.Count() ); + } + } + + return false; +} + + +//----------------------------------------------------------------------------- +// Purpose: Does some sanity checks on the SO cache LRU +//----------------------------------------------------------------------------- +void CGCBase::VerifySOCacheLRU() +{ + CUtlRBTree<CSteamID, int> rbTreeUsersEncountered( 0, m_listCachesToUnload.Count(), DefLessFunc( CSteamID ) ); + + for( uint32 unCache = m_listCachesToUnload.Head(), unNextCache = m_listCachesToUnload.InvalidIndex(); unCache != m_listCachesToUnload.InvalidIndex(); unCache = unNextCache ) + { + unNextCache = m_listCachesToUnload.Next( unCache ); + CSteamID ownerID = m_listCachesToUnload[ unCache ]; + CGCSharedObjectCache *pSOCache = FindSOCache( ownerID ); + if ( !pSOCache ) + { + EmitError( SPEW_GC, "CGCBase::UnloadUnusedCaches() sanity[0] check failed! Empty cache in list in slot 0x%08x\n", unCache ); + continue; + } + + if ( pSOCache->GetLRUHandle() != unCache ) + { + EmitError( SPEW_GC, "CGCBase::UnloadUnusedCaches() sanity[1] check failed! Cache entry mismatch [ 0x%08x vs 0x%08x ] (owner: %s)\n", pSOCache->GetLRUHandle(), unCache, pSOCache->GetOwner().Render() ); + } + + if ( !rbTreeUsersEncountered.IsValidIndex( rbTreeUsersEncountered.InsertIfNotFound( ownerID ) ) ) + { + EmitError( SPEW_GC, "CGCBase::UnloadUnusedCaches() sanity[2] check failed! Duplicate entry in list for 0x%08x (owner: %s)\n", unCache, pSOCache->GetOwner().Render() ); + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds the cache to the LRU list +//----------------------------------------------------------------------------- +void CGCBase::AddCacheToLRU( CGCSharedObjectCache * pSOCache ) +{ + Assert( pSOCache->GetLRUHandle() == m_listCachesToUnload.InvalidIndex() ); +#if WITH_SOCACHE_LRU_DEBUGGING + if ( pSOCache->GetLRUHandle() != m_listCachesToUnload.InvalidIndex() ) + { + EmitError( SPEW_GC, "CGCBase::AddCacheToLRU() sanity[4] check failed! Adding SO Cache with existing LRU handle: 0x%08x\n", pSOCache->GetLRUHandle() ); + } +#endif + + // remove it just in case. Crashes are bad. + RemoveCacheFromLRU( pSOCache ); + +#if WITH_SOCACHE_LRU_DEBUGGING + Assert( pSOCache->GetLRUHandle() == m_listCachesToUnload.InvalidIndex() ); + if ( pSOCache->GetLRUHandle() != m_listCachesToUnload.InvalidIndex() ) + { + EmitError( SPEW_GC, "CGCBase::AddCacheToLRU() sanity[5] check failed! Adding SO Cache with existing LRU handle: 0x%08x\n", pSOCache->GetLRUHandle() ); + } +#endif + + pSOCache->SetLRUHandle( m_listCachesToUnload.AddToTail( pSOCache->GetOwner() ) ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Removes the cache from the LRU list +//----------------------------------------------------------------------------- +void CGCBase::RemoveCacheFromLRU( CGCSharedObjectCache * pSOCache ) +{ +#if WITH_SOCACHE_LRU_DEBUGGING + if ( m_listCachesToUnload.IsValidIndex( pSOCache->GetLRUHandle() ) == ( pSOCache->GetLRUHandle() == m_listCachesToUnload.InvalidIndex() ) ) + { + EmitError( SPEW_GC, "CGCBase::RemoveCacheFromLRU() sanity[6] check failed! SO Cache has an invalid index, but IsValidIndex() is returning true: 0x%08x\n", pSOCache->GetLRUHandle() ); + } +#endif + if( m_listCachesToUnload.IsValidIndex( pSOCache->GetLRUHandle() ) ) + { + if( m_listCachesToUnload[ pSOCache->GetLRUHandle() ] != pSOCache->GetOwner() ) + { + EmitError( SPEW_GC, "CGCBase::RemoveCacheFromLRU() Attempting to remove SOCache LRU index %d for %s, which really holds %s\n", + pSOCache->GetLRUHandle(), pSOCache->GetOwner().Render(), m_listCachesToUnload[ pSOCache->GetLRUHandle() ].Render() ); + } + else + { + m_listCachesToUnload.Remove( pSOCache->GetLRUHandle() ); + } + } + pSOCache->SetLRUHandle( m_listCachesToUnload.InvalidIndex() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Finds the cache in the map for a new session and locks it +//----------------------------------------------------------------------------- +CGCSharedObjectCache *CGCBase::YieldingGetLockedSOCache( const CSteamID &steamID, const char *pszFilename, int nLineNum ) +{ + if( !BYieldingLockSteamID( steamID, pszFilename, nLineNum ) ) + return NULL; + + return YieldingFindOrLoadSOCache( steamID ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Finds the cache in the map for a new session +//----------------------------------------------------------------------------- +CGCSharedObjectCache *CGCBase::YieldingFindOrLoadSOCache( const CSteamID &steamID ) +{ + AssertRunningJob(); + + if( !steamID.IsValid() ) + { + AssertMsg1( false, "Unable to load SO cache for invalid steam ID %s", steamID.Render() ); + EmitError( SPEW_GC, "Unable to load SO cache for invalid steam ID %s (instance: %d)\n", steamID.Render(), steamID.GetUnAccountInstance() ); + return NULL; + } + + // check to see if the SO cache is being loaded--if so, then we yield until it is done + // the reason we are not just locking the steam id is because the current job may have + // a lock on something else, and jobs can only have one lock active at a time. + CJobTime timeStartedWaiting; + timeStartedWaiting.SetToJobTime(); + while ( m_rbtreeSOCachesBeingLoaded.Find( steamID ) != m_rbtreeSOCachesBeingLoaded.InvalidIndex() ) + { + + // !TEST! Looks like we might have a bug where we're spinning here waiting forever. + // Add a timeout just in case. + if ( timeStartedWaiting.CServerMicroSecsPassed() > 180 * k_nMillion ) + { + AssertMsg1( false, "Timed out waiting for SO cache %s to finish loading", steamID.Render() ); + return false; + } + GJobCur().BYieldingWaitOneFrame(); + } + + CGCSharedObjectCache *pSOCache = FindSOCache( steamID ); + if( !pSOCache ) + { + m_rbtreeSOCachesBeingLoaded.Insert( steamID ); + pSOCache = CreateSOCache( steamID ); + CJobTime timeStartedLoading; + timeStartedLoading.SetToJobTime(); + if( BYieldingLoadSOCache( pSOCache ) ) + { + if ( FindSOCache( steamID ) != NULL ) + { + EmitError( SPEW_GC, "HOLY FUCKING SHIT WE ARE DUPLICATING SO CACHES [%s]\n", steamID.Render() ); + } + m_mapSOCache.Insert( steamID, pSOCache ); + + float flSecondsToLoad = (float)timeStartedLoading.CServerMicroSecsPassed() / (float)k_nMillion; + if ( flSecondsToLoad > 10.0f ) + { + EmitInfo( SPEW_GC, 4, 1, "Loading of SO cache for %s took %.1fs\n", steamID.Render(), flSecondsToLoad ); + } + + //mark this cache as loaded so that it's version can change again + pSOCache->SetDetectVersionChanges( false ); + + CJobTime timeStartedNotify; + timeStartedNotify.SetToJobTime(); + YieldingSOCacheLoaded( pSOCache ); + float flSecondsToNotify = (float)timeStartedNotify.CServerMicroSecsPassed() / (float)k_nMillion; + if ( flSecondsToNotify > 10.0f ) + { + EmitInfo( SPEW_GC, 1, 1, "YieldingSOCacheLoaded for %s took %.1fs\n", steamID.Render(), flSecondsToNotify ); + } + + AddCacheToLRU( pSOCache ); // in case the cache isn't about to be attached to a session + m_rbtreeSOCachesBeingLoaded.Remove( steamID ); + } + else + { + AssertMsg1( false, "Unable to load SO cache for %llu", steamID.ConvertToUint64() ); + EmitError( SPEW_GC, "Unable to load SO cache for %llu\n", steamID.ConvertToUint64() ); + delete pSOCache; + m_rbtreeSOCachesBeingLoaded.Remove( steamID ); + return NULL; + } + } + else + { + // if the cache is in the LRU, move it to the end of the list + if( m_listCachesToUnload.IsValidIndex( pSOCache->GetLRUHandle() ) ) + { + RemoveCacheFromLRU( pSOCache ); + AddCacheToLRU( pSOCache ); + } + } + + return pSOCache; +} + + +//----------------------------------------------------------------------------- +// Purpose: Reloads the SO cache +//----------------------------------------------------------------------------- +void CGCBase::YieldingReloadCache( CGCSharedObjectCache *pSOCache ) +{ + Assert( IsSteamIDLockedByJob( pSOCache->GetOwner(), &GJobCur() ) ); + if( !IsSteamIDLockedByJob( pSOCache->GetOwner(), &GJobCur() ) ) + return; + + // Flush all pending writes + CSQLAccess sqlAccess; + sqlAccess.BBeginTransaction( "CGCBase::YieldingReloadCache - Flush writes" ); + pSOCache->YieldingStageAllWrites( sqlAccess ); + if ( !sqlAccess.BCommitTransaction( true ) ) + { + EmitError( SPEW_SHAREDOBJ, "%s: Unable to flush pending writes for %s, reload failed", + __FUNCTION__, pSOCache->GetOwner().Render() ); + return; + } + + // load the data into a new cache + CGCSharedObjectCache *pNewCache = CreateSOCache( pSOCache->GetOwner() ); + if( !BYieldingLoadSOCache( pNewCache ) ) + { + EmitError( SPEW_SHAREDOBJ, "Unable to reload cache for %s because of a SQL error", pSOCache->GetOwner().Render() ); + return; + } + + // process every object in the new cache and move it to the old one if necessary + FOR_EACH_MAP_FAST( CSharedObject::GetFactories(), nType ) + { + int nTypeID = CSharedObject::GetFactories().Key( nType ); + + // remove all the old items of this type + CSharedObjectTypeCache *pOldTypeCache = pSOCache->FindTypeCache( nTypeID ); + if( pOldTypeCache ) + { + for( uint32 nCurrObj = 0; nCurrObj < pOldTypeCache->GetCount(); ) + { + //not all objects should be deleted (for example lobbies/parties), so for those objects + //don't delete and instead just skip over them + if( pOldTypeCache->GetObject( nCurrObj )->BShouldDeleteByCache() ) + { + pSOCache->RemoveObject( *pOldTypeCache->GetObject( nCurrObj ) ); + } + else + { + nCurrObj++; + } + } + } + + // add all the new objects of this type + CSharedObjectTypeCache *pNewTypeCache = pNewCache->FindTypeCache( nTypeID ); + if( pNewTypeCache ) + { + for( uint unObject = 0; unObject < pNewTypeCache->GetCount(); unObject++ ) + { + pSOCache->AddObject( pNewTypeCache->GetObject( unObject ) ); + } + } + } + + // remove all the objects in the new cache + pNewCache->RemoveAllObjectsWithoutDeleting(); + delete pNewCache; + + // if there's a session for this cache, tell it about the reload + if( pSOCache->GetOwner().BIndividualAccount() ) + { + CGCUserSession *pUserSession = FindUserSession( pSOCache->GetOwner() ); + if( pUserSession ) + pUserSession->YieldingSOCacheReloaded(); + } + else if( pSOCache->GetOwner().BGameServerAccount() ) + { + CGCGSSession *pGSSession = FindGSSession( pSOCache->GetOwner() ); + if( pGSSession ) + pGSSession->YieldingSOCacheReloaded(); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Factory method to create a CGCSharedObjectCache +// Input : &steamID - steamID that will own the CGCSharedObjectCache +// Output : Returns a new CGCSharedObjectCache +//----------------------------------------------------------------------------- +CGCSharedObjectCache *CGCBase::CreateSOCache( const CSteamID &steamID ) +{ + return new CGCSharedObjectCache( steamID ); +} + + +//----------------------------------------------------------------------------- +// Purpose: yields until the lock on the specified steamID is taken +// Input : &steamID - steamID to lock +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CGCBase::BYieldingLockSteamID( const CSteamID &steamID, const char *pszFilename, int nLineNum ) +{ + AssertRunningJob(); + Assert( steamID.GetEAccountType() != k_EAccountTypePending ); + + // lookup + CLock *pLock = m_hashSteamIDLocks.PvRecordFind( steamID ); + if ( !pLock ) + { + // no lock yet, insert one + pLock = m_hashSteamIDLocks.PvRecordInsert( steamID ); + pLock->SetName( steamID ); + pLock->SetLockSubType( steamID.GetAccountID() ); + if ( steamID.BIndividualAccount() ) + { + pLock->SetLockType( k_nLockTypeIndividual ); + } + else if ( steamID.BGameServerAccount() ) + { + pLock->SetLockType( k_nLockTypeGameServer ); + } + else + { + AssertMsg1( false, "Lock taken for unexpected steamID: %s", steamID.Render() ); + } + } + + Assert( pLock ); + if ( !pLock ) + { + EmitInfo( SPEW_GC, SPEW_ALWAYS, LOG_ALWAYS, "Unable to create lock for %s\n", steamID.Render() ); + return false; + } + + return GJobCur()._BYieldingAcquireLock( pLock, pszFilename, nLineNum ); +} + + +//----------------------------------------------------------------------------- +// Purpose: locks a pair of steam IDs, grabbing the highest account ID first +// to satisfy the deadlock-avoidance code in the job system +//----------------------------------------------------------------------------- +bool CGCBase::BYieldingLockSteamIDPair( const CSteamID &steamIDA, const CSteamID &steamIDB, const char *pszFilename, int nLineNum ) +{ + if( steamIDA == steamIDB ) + return BYieldingLockSteamID( steamIDA, pszFilename, nLineNum ); + + // + // !FIXME! This is really not the correct sort criteron to use. The correct + // criteria is to use the full lock priority. For example, + // what if we pass a gameserver ID and a user ID. The whole + // concept of locking two SteamID's is probably broken when we split up + // things on the GC, though, so this might not be worth fixing. + // + + if( steamIDA.GetAccountID() < steamIDB.GetAccountID() ) + { + if( !BYieldingLockSteamID( steamIDB, pszFilename, nLineNum ) ) + return false; + if( !BYieldingLockSteamID( steamIDA, pszFilename, nLineNum ) ) + { + UnlockSteamID( steamIDB ); + return false; + } + } + else + { + if( !BYieldingLockSteamID( steamIDA, pszFilename, nLineNum ) ) + return false; + if( !BYieldingLockSteamID( steamIDB, pszFilename, nLineNum ) ) + { + UnlockSteamID( steamIDA ); + return false; + } + } + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: locks the specified steamID +// Input : &steamID - steamID to unlock +//----------------------------------------------------------------------------- +bool CGCBase::BLockSteamIDImmediate( const CSteamID &steamID ) +{ + AssertRunningJob(); + Assert( steamID.GetEAccountType() != k_EAccountTypePending ); + + // lookup + CLock *pLock = m_hashSteamIDLocks.PvRecordFind( steamID ); + if ( pLock == NULL ) + { + // no lock yet, insert one + pLock = m_hashSteamIDLocks.PvRecordInsert( steamID ); + Assert( pLock != NULL ); + if ( pLock == NULL ) + { + return false; + } + + if ( steamID.BIndividualAccount() ) + { + pLock->SetLockType( k_nLockTypeIndividual ); + } + else if ( steamID.BGameServerAccount() ) + { + pLock->SetLockType( k_nLockTypeGameServer ); + } + else + { + AssertMsg1( false, "Lock taken for unexpected steamID: %s", steamID.Render() ); + } + + pLock->SetName( steamID ); + pLock->SetLockSubType( steamID.GetAccountID() ); + } + + return GJobCur().BAcquireLockImmediate( pLock ); +} + + +//----------------------------------------------------------------------------- +// Purpose: unlocks the specified steamID +// Input : &steamID - steamID to unlock +//----------------------------------------------------------------------------- +void CGCBase::UnlockSteamID( const CSteamID &steamID ) +{ + AssertRunningJob(); + Assert( steamID.GetEAccountType() != k_EAccountTypePending ); + + // lookup + CLock *pLock = m_hashSteamIDLocks.PvRecordFind( steamID ); + Assert( pLock ); + if ( !pLock ) + { + AssertMsg2( false, "UnlockSteamID( '%s' ) called by %s but unable to find lock in map", steamID.Render(), GJobCur().GetName() ); + return; + } + + if ( pLock->GetJobLocking() != &GJobCur() ) + { + AssertMsg2( false, "UnlockSteamID( '%s' ) called when job %s doesn't own the lock", steamID.Render(), GJobCur().GetName() ); + return; + } + + GJobCur().ReleaseLock( pLock ); +} + + +//----------------------------------------------------------------------------- +// Purpose: returns true if the specified steamID is locked +//----------------------------------------------------------------------------- +bool CGCBase::IsSteamIDLocked( const CSteamID &steamID ) +{ + CLock *pLock = m_hashSteamIDLocks.PvRecordFind( steamID ); + if ( pLock ) + return pLock->BIsLocked(); + + return false; +} + + +//----------------------------------------------------------------------------- +// Purpose: returns true if the specified steamID is locked by the specified job +//----------------------------------------------------------------------------- +bool CGCBase::IsSteamIDLockedByJob( const CSteamID &steamID, const CJob *pJob ) const +{ + CLock *pLock = m_hashSteamIDLocks.PvRecordFind( steamID ); + if ( pLock ) + return ( pLock->GetJobLocking() == pJob ); + + return false; +} + + +//----------------------------------------------------------------------------- +// Purpose: returns true if the specified steamID is locked by the current job +//----------------------------------------------------------------------------- +bool CGCBase::IsSteamIDLockedByCurJob( const CSteamID &steamID ) const +{ + AssertRunningJob(); + + return IsSteamIDLockedByJob( steamID, &GJobCur() ); +} + +//----------------------------------------------------------------------------- +// Purpose: returns true if the specified steamID is unlocked, or locked by the current job +//----------------------------------------------------------------------------- +bool CGCBase::IsSteamIDUnlockedOrLockedByCurJob( const CSteamID &steamID ) +{ + AssertRunningJob(); + + // lookup + CLock *pLock = m_hashSteamIDLocks.PvRecordFind( steamID ); + if ( !pLock ) + { + // Unlocked + return true; + } + + // It is in the hash of locks and is locked return true only if it is locked by the current job + if ( pLock->BIsLocked() ) + { + return ( pLock->GetJobLocking() == &GJobCur() ); + } + else + { + return true; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: returns a pointer to the lock for the steamID, or NULL if none +//----------------------------------------------------------------------------- +const CLock *CGCBase::FindSteamIDLock( const CSteamID &steamID ) +{ + // lookup + return m_hashSteamIDLocks.PvRecordFind( steamID ); +} + + +//----------------------------------------------------------------------------- +// Purpose: returns a pointer to the job holding the lock for this steamID, NULL if none +//----------------------------------------------------------------------------- +CJob *CGCBase::PJobHoldingLock( const CSteamID &steamID ) +{ + AssertRunningJob(); + + // lookup + CLock *pLock = m_hashSteamIDLocks.PvRecordFind( steamID ); + if ( !pLock || !pLock->BIsLocked() ) + { + // Unlocked + return NULL; + } + + // Return the job holding the lock + return pLock->GetJobLocking(); +} + + +//----------------------------------------------------------------------------- +// Purpose: returns a pointer to the job holding the lock for this steamID, NULL if none +//----------------------------------------------------------------------------- +bool CGCBase::YieldingWritebackDirtyCaches( uint32 unSecondToDelayWrite ) +{ + CSQLAccess sqlAccess; + CUtlVector< CGCSharedObjectCache * > vecCachesWritten; + uint32 unWrittenCount = 0; + sqlAccess.BBeginTransaction( "CGCBase::YieldingWritebackDirtyCaches()" ); + RTime32 unFirstTimeToWrite = time( NULL ) - unSecondToDelayWrite; + FOR_EACH_VEC( m_vecCacheWritebacks, nCache ) + { + CGCSharedObjectCache *pSOCache = m_vecCacheWritebacks[ nCache ]; + + // if this cache entered the writeback list too frequently, skip it for now + if( unSecondToDelayWrite > 0 && pSOCache->GetWritebackTime() > unFirstTimeToWrite ) + { + continue; + } + + // if we can't get the lock for ourselves, catch it on the next time around + if( !BLockSteamIDImmediate( pSOCache->GetOwner() ) ) + { + continue; + } + + unWrittenCount += pSOCache->YieldingStageAllWrites( sqlAccess ); + vecCachesWritten.AddToTail( pSOCache ); + m_vecCacheWritebacks.Remove( nCache ); + nCache--; + + // don't hog all the CPU. Yield and wait for the next frame if + // we've been running for too long. Go ahead and write these + // caches so we don't hold their locks forever though. + if( GJobCur().GetMicrosecondsRun() > (uint64)(writeback_queue_max_accumulate_time.GetInt() * k_nThousand) || + ( writeback_queue_max_caches.GetInt() > 0 && vecCachesWritten.Count() > writeback_queue_max_caches.GetInt() ) ) + { + // We've spent enough time accumulating work. Time to run some SQL + // queries. + break; + } + } + + // Commit the transaction + if( !sqlAccess.BCommitTransaction( true ) ) + { + // the transaction failed. Put those caches back on the TODO list + EmitError( SPEW_GC, "CGCBase::YieldingWritebackDirtyCaches() - Writeback failed\n" ); + + m_vecCacheWritebacks.AddMultipleToTail( vecCachesWritten.Count(), vecCachesWritten.Base() ); + FOR_EACH_VEC( vecCachesWritten, nCache ) + { + CGCSharedObjectCache *pSOCache = vecCachesWritten[nCache]; + UnlockSteamID( pSOCache->GetOwner() ); + } + return false; + } + else + { + // the transaction was successful. Tell those caches to forget their dirtiness + FOR_EACH_VEC( vecCachesWritten, nCache ) + { + CGCSharedObjectCache *pSOCache = vecCachesWritten[nCache]; + pSOCache->SetInWriteback( false ); + UnlockSteamID( pSOCache->GetOwner() ); + } + return true; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void CGCBase::AddCacheToWritebackQueue( CGCSharedObjectCache *pSOCache ) +{ + Assert( pSOCache ); + if ( ( g_pJobCur != NULL ) && PJobHoldingLock( pSOCache->GetOwner() ) != g_pJobCur && !GGCBase()->BIsSOCacheBeingLoaded( pSOCache->GetOwner() ) ) + { + AssertMsg2( false, "CGCBase::AddCacheToWritebackQueue called by job %s for %s, but job does not own lock", g_pJobCur->GetName(), pSOCache->GetOwner().Render() ); + } + if( !pSOCache->GetInWriteback() ) + { + m_vecCacheWritebacks.AddToTail( pSOCache ); + pSOCache->SetInWriteback( true ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +bool CGCBase::BYieldingRetrieveCacheVersion( CGCSharedObjectCache *pSOCache ) +{ + if ( !socache_persist_version_via_memcached.GetBool() ) + { + // We'll keep doing the updates, but fail to restore it if not requested. + return false; + } + + CFmtStr1024 key( "SOCacheVersionV2_%llu", pSOCache->GetOwner().ConvertToUint64() ); + + GCMemcachedGetResult_t data; + if ( !BYieldingMemcachedGet( key.Access(), data ) || !data.m_bKeyFound || sizeof( uint64 ) != data.m_bufValue.Count() ) + { +#ifdef _DEBUG + EmitInfo( SPEW_CONSOLE, 3, 3, "SOCacheVersion - Failed to retrieve SO Cache version for: %s\n", pSOCache->GetOwner().Render() ); +#endif + return false; + } + + //we have a memcached version, so make sure that our version matches what was stored in memcache + uint64 unVersion = *( (uint64 *)data.m_bufValue.Base() ); + pSOCache->SetVersion( unVersion ); +#ifdef _DEBUG + EmitInfo( SPEW_CONSOLE, 3, 3, "SOCacheVersion::Load - Loaded version from memcached for %s (%llu)\n", pSOCache->GetOwner().Render(), pSOCache->GetVersion() ); +#endif + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void CGCBase::AddCacheToVersionChangedList( CGCSharedObjectCache *pSOCache ) +{ + m_rbtreeSOCachesWithDirtyVersions.InsertIfNotFound( pSOCache->GetOwner() ); +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void CGCBase::UpdateSOCacheVersions() +{ + CUtlVector<CUtlString> vecSetKeys( 0, m_rbtreeSOCachesWithDirtyVersions.Count() ); + CUtlVector<GCMemcachedBuffer_t> vecSetValues( 0, m_rbtreeSOCachesWithDirtyVersions.Count() ); + CUtlBuffer bufData( 0, ( sizeof( uint64 ) * m_rbtreeSOCachesWithDirtyVersions.Count() ) + 1 ); + + CUtlVector<CUtlString> vecDeleteKeys( 0, m_rbtreeSOCachesWithDirtyVersions.Count() ); + + for ( int idx = 0; idx < m_rbtreeSOCachesWithDirtyVersions.MaxElement(); ++idx ) + { + if ( !m_rbtreeSOCachesWithDirtyVersions.IsValidIndex( idx ) ) + continue; + + const CSteamID &steamID = m_rbtreeSOCachesWithDirtyVersions[idx]; + + // if the SO Cache is being loaded, ignore + if ( m_rbtreeSOCachesBeingLoaded.Find( steamID ) != m_rbtreeSOCachesBeingLoaded.InvalidIndex() ) + continue; + + CSharedObjectCache *pSOCache = FindSOCache( steamID ); + if ( pSOCache ) + { + CUtlString &strKey = vecSetKeys[ vecSetKeys.AddToTail() ]; + strKey.Format( "SOCacheVersionV2_%llu", steamID.ConvertToUint64() ); + + GCMemcachedBuffer_t &bufVal = vecSetValues[ vecSetValues.AddToTail() ]; + bufVal.m_pubData = (byte *)bufData.Base() + bufData.TellPut(); + bufVal.m_cubData = sizeof( uint64 ); + + bufData.PutInt64( pSOCache->GetVersion() ); + +#ifdef _DEBUG + EmitInfo( SPEW_CONSOLE, 3, 3, "SOCacheVersion - storing version in memcached for %s (%llu).\n", steamID.Render(), pSOCache->GetVersion() ); +#endif + } + else + { + // SO Cache is gone, so to be safe, remove the cached version number from memcached + CUtlString &strKey = vecDeleteKeys[ vecDeleteKeys.AddToTail() ]; + strKey.Format( "SOCacheVersionV2_%llu", steamID.ConvertToUint64() ); + +#ifdef _DEBUG + EmitInfo( SPEW_CONSOLE, 3, 3, "SOCacheVersion - no SO Cache, removing version in memcached for %s.\n", steamID.Render() ); +#endif + } + } + + if ( vecSetKeys.Count() > 0 ) + { + BMemcachedSet( vecSetKeys, vecSetValues ); + } + + if ( vecDeleteKeys.Count() > 0 ) + { + BMemcachedDelete( vecDeleteKeys ); + } + + m_rbtreeSOCachesWithDirtyVersions.RemoveAll(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns the publisher access key for Steam Web APIs. This is just +// a stub and must be implimented by a child class if they want this +// funtionality. +//----------------------------------------------------------------------------- +const char *CGCBase::GetSteamAPIKey() +{ + AssertMsg( false, "GetWebAPIKey(): Implement me!" ); + EmitError( SPEW_CONSOLE, "GetWebAPIKey(): Implement me!\n" ); + + return "InvalidKey"; +} + +//----------------------------------------------------------------------------- +// Purpose: Returns true if the protobuf object was stored successfully, false otherwise +//----------------------------------------------------------------------------- +bool CGCBase::BMemcachedSet( const char *pKey, const ::google::protobuf::Message &protoBufObj ) +{ + // build key + CUtlVector< CUtlString > vecKeys; + int idx = vecKeys.AddToTail(); + vecKeys[idx].Set( pKey ); + + // allocate buffer we will use to stuff into the memcached buffer + CUtlVector< CGCBase::GCMemcachedBuffer_t > vecValues; + uint32 unSize = protoBufObj.ByteSize(); + void *pvBuf = stackalloc( unSize ); + protoBufObj.SerializeWithCachedSizesToArray( (uint8*)pvBuf ); + + // stuff the data into the memcached buffer + CGCBase::GCMemcachedBuffer_t buffer; + buffer.m_pubData = pvBuf; + buffer.m_cubData = unSize; + vecValues.AddToTail( buffer ); + + return BMemcachedSet( vecKeys, vecValues ); +} + +//----------------------------------------------------------------------------- +// Purpose: Returns true if the memcached value stored via pKey was removed succesfully, false otherwise +//----------------------------------------------------------------------------- +bool CGCBase::BMemcachedDelete( const char *pKey ) +{ + CUtlVector< CUtlString > vecKeys; + int idx = vecKeys.AddToTail(); + vecKeys[idx].Set( pKey ); + return BMemcachedDelete( vecKeys ); +} + +//----------------------------------------------------------------------------- +// Purpose: Returns true if the protobuf object was retrieved from memcached successfully, false otherwise +//----------------------------------------------------------------------------- +bool CGCBase::BYieldingMemcachedGet( const char *pKey, ::google::protobuf::Message &protoBufMsg ) +{ + // build key + CUtlVector< CUtlString > vecKeys; + int idx = vecKeys.AddToTail(); + vecKeys[idx].Set( pKey ); + + // get results + CUtlVector< CGCBase::GCMemcachedGetResult_t > vecResults; + if ( !BYieldingMemcachedGet( vecKeys, vecResults ) || vecResults.Count() != 1 || vecResults[0].m_bKeyFound == false ) + { + return false; + } + if ( !protoBufMsg.ParseFromArray( vecResults[0].m_bufValue.Base(), vecResults[0].m_bufValue.Count() ) ) + { + return false; + } + if ( !protoBufMsg.IsInitialized() ) + { + return false; + } + + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Set the keys and values into memcached +//----------------------------------------------------------------------------- +bool CGCBase::BMemcachedSet( const CUtlVector<CUtlString> &vecKeys, const CUtlVector<GCMemcachedBuffer_t> &vecValues ) +{ + Assert( vecKeys.Count() == vecValues.Count() ); + if ( vecKeys.Count() != vecValues.Count() ) + return false; + + CProtoBufMsg<CGCMsgMemCachedSet> msgRequest( k_EGCMsgMemCachedSet ); + for ( int i = 0; i < vecKeys.Count(); ++i ) + { + CGCMsgMemCachedSet_KeyPair *keypair = msgRequest.Body().add_keys(); + keypair->set_name( vecKeys[i].String() ); + keypair->set_value( vecValues[i].m_pubData, vecValues[i].m_cubData ); + } + + if( !BSendSystemMessage( msgRequest ) ) + return false; + + // There is no reply to setting in memcached + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Overload for a single key/value +//----------------------------------------------------------------------------- +bool CGCBase::BMemcachedSet( const CUtlString &strKey, const CUtlBuffer &buf ) +{ + CUtlVector<CUtlString> memcachedMemberKeys( 0, 1 ); + CUtlVector<CGCBase::GCMemcachedBuffer_t> memcachedMemberValues( 0, 1 ); + + memcachedMemberKeys.AddToTail( strKey ); + + CGCBase::GCMemcachedBuffer_t &memcachedBuffer = memcachedMemberValues[ memcachedMemberValues.AddToTail() ]; + memcachedBuffer.m_pubData = buf.Base(); + memcachedBuffer.m_cubData = buf.TellPut(); + + return BMemcachedSet( memcachedMemberKeys, memcachedMemberValues ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Delete the keys in memcached +//----------------------------------------------------------------------------- +bool CGCBase::BMemcachedDelete( const CUtlVector<CUtlString> &vecKeys ) +{ + CProtoBufMsg<CGCMsgMemCachedDelete> msgRequest( k_EGCMsgMemCachedDelete ); + for ( int i = 0; i < vecKeys.Count(); ++i ) + { + msgRequest.Body().add_keys( vecKeys[i].String() ); + } + + if( !BSendSystemMessage( msgRequest ) ) + return false; + + // There is no reply to deleting in memcached + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Overload for a single key/value +//----------------------------------------------------------------------------- +bool CGCBase::BMemcachedDelete( const CUtlString &strKey ) +{ + CUtlVector<CUtlString> vecKeys( 0, 1 ); + vecKeys.AddToTail( strKey ); + return BMemcachedDelete( vecKeys ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the key's values from memcached +//----------------------------------------------------------------------------- +bool CGCBase::BYieldingMemcachedGet( const CUtlVector<CUtlString> &vecKeys, CUtlVector<GCMemcachedGetResult_t> &vecResults ) +{ + CProtoBufMsg<CGCMsgMemCachedGet> msgRequest( k_EGCMsgMemCachedGet ); + for ( int i = 0; i < vecKeys.Count(); ++i ) + { + msgRequest.Body().add_keys( vecKeys[i].String() ); + } + msgRequest.ExpectingReply( GJobCur().GetJobID() ); + if( !BSendSystemMessage( msgRequest ) ) + return false; + + CProtoBufMsg<CGCMsgMemCachedGetResponse> msgResponse; + if( !GJobCur().BYieldingWaitForMsg( &msgResponse, k_EGCMsgMemCachedGetResponse ) ) + { + EmitWarning( SPEW_GC, LOG_ALWAYS, "Didn't get reply from IS for BYieldingMemcachedGet\n" ); + return false; + } + + Assert( msgRequest.Body().keys_size() == msgResponse.Body().values_size() ); + if ( msgRequest.Body().keys_size() != msgResponse.Body().values_size() ) + { + EmitWarning( SPEW_GC, LOG_ALWAYS, "Mismatched reply from IS for BYieldingMemcachedGet, asked for %d keys, got %d back\n", (int)msgRequest.Body().keys_size(), (int)msgResponse.Body().values_size() ); + return false; // Doesn't match what we asked for! + } + + vecResults.Purge(); + vecResults.EnsureCapacity( msgResponse.Body().values_size() ); + for ( int i = 0; i < msgResponse.Body().values_size(); ++i ) + { + GCMemcachedGetResult_t &result = vecResults[ vecResults.AddToTail() ]; + result.m_bKeyFound = msgResponse.Body().values(i).found(); + if ( result.m_bKeyFound ) + { + result.m_bufValue.Copy( &(*msgResponse.Body().values(i).value().begin()), msgResponse.Body().values(i).value().size() ); + } + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Overload for a single key/value +//----------------------------------------------------------------------------- +bool CGCBase::BYieldingMemcachedGet( const CUtlString &strKeys, GCMemcachedGetResult_t &result ) +{ + CUtlVector<CUtlString> memcachedMemberKeys( 0, 1 ); + CUtlVector<GCMemcachedGetResult_t> memcachedResults; + + memcachedMemberKeys.AddToTail( strKeys ); + bool bRet = BYieldingMemcachedGet( memcachedMemberKeys, memcachedResults ); + if ( !bRet ) + return false; + + Assert( 1 == memcachedResults.Count() ); + if ( 1 != memcachedResults.Count() ) + return false; + + result.m_bKeyFound = memcachedResults[0].m_bKeyFound; + result.m_bufValue.Swap( memcachedResults[0].m_bufValue ); + return true; +} + + +//----------------------------------------------------------------------------- +bool CGCBase::BYieldingGetIPLocations( CUtlVector<uint32> &vecIPs, CUtlVector<CIPLocationInfo> &infos ) +{ + CProtoBufMsg<CGCMsgGetIPLocation> msgRequest( k_EGCMsgGetIPLocation ); + FOR_EACH_VEC( vecIPs, i ) + { + msgRequest.Body().add_ips( vecIPs[i] ); + } + + msgRequest.ExpectingReply( GJobCur().GetJobID() ); + if( !BSendSystemMessage( msgRequest ) ) + return false; + + // We don't need to worry about a reply mismatch in this case. The message + // has sufficient data so that we can match up the reply properly. + GJobCur().ClearFailedToReceivedMsgType( k_EGCMsgGetIPLocationResponse ); + + CProtoBufMsg<CGCMsgGetIPLocationResponse> msgResponse; + if( !GJobCur().BYieldingWaitForMsg( &msgResponse, k_EGCMsgGetIPLocationResponse ) ) + { + EmitWarning( SPEW_GC, LOG_ALWAYS, "Didn't get reply from IS for BYieldingGetIPLocation\n" ); + return false; + } + + for ( int i = 0; i < msgResponse.Body().infos_size(); i++ ) + { + infos.AddToTail( msgResponse.Body().infos( i ) ); + } + + return true; +} + +//----------------------------------------------------------------------------- +bool CGCBase::BYieldingUpdateGeoLocation( CUtlVector<CSteamID> const &requestedVecSteamIds ) +{ + CUtlVector<uint32> vecIPs; + CUtlVector<CSteamID> vecSteamIds; + + FOR_EACH_VEC( requestedVecSteamIds, i ) + { + const CSteamID memberSteamID = requestedVecSteamIds[i]; + CGCSession *pSession = FindUserOrGSSession( memberSteamID ); + if( pSession ) + { + if ( !pSession->GetIPPublic() ) + { + EmitInfo( SPEW_GC, 4, LOG_ALWAYS, "BYieldingUpdateGeoLocation Session %s IP == 0, unable to retrieve\n", memberSteamID.Render() ) ; + continue; + } + if ( !pSession->HasGeoLocation() ) + { + vecIPs.AddToTail( pSession->GetIPPublic() ); + vecSteamIds.AddToTail( memberSteamID ); + } + } + } + + if (!vecIPs.Count()) + return true; + +#define iptod(x) ((x)>>24&0xff), ((x)>>16&0xff), ((x)>>8&0xff), ((x)&0xff) + + FOR_EACH_VEC( vecIPs, i ) + { + EmitInfo( SPEW_GC, geolocation_spewlevel.GetInt(), geolocation_loglevel.GetInt(), "BYieldingUpdateGeoLocation GetIPLocation[%d] = (%s,%u.%u.%u.%u)\n", i, vecSteamIds[i].Render(), iptod( vecIPs[i] ) ) ; + } + + CUtlVector<CIPLocationInfo> infos; + if ( BYieldingGetIPLocations( vecIPs, infos ) ) + { + // The current IS has a bug where the IP will be blank/zero in the replies. If infos.Count() == vecIPs.Count() assume the order is correct + if ( vecSteamIds.Count() == vecIPs.Count() && vecIPs.Count() == infos.Count() ) + { + FOR_EACH_VEC( vecSteamIds, i ) + { + CGCSession *pSession = FindUserOrGSSession( vecSteamIds[i] ); + if ( pSession ) + { + EmitInfo( SPEW_GC, geolocation_spewlevel.GetInt(), geolocation_loglevel.GetInt(), "BYieldingUpdateGeoLocation[MATCHED] SetIPLocation[%s(%u.%u.%u.%u)] = (%6.3f,%6.3f)\n", pSession->GetSteamID().Render(), iptod( vecIPs[i] ), infos[i].latitude(), infos[i].longitude() ); + pSession->SetGeoLocation( infos[i].latitude(), infos[i].longitude() ); + } + } + } + else + { + FOR_EACH_VEC( vecSteamIds, i ) + { + FOR_EACH_VEC( infos, j ) + { + if ( infos[j].ip() == vecIPs[i] ) + { + CGCSession *pSession = FindUserOrGSSession( vecSteamIds[i] ); + if ( pSession ) + { + EmitInfo( SPEW_GC, 4, LOG_ALWAYS, "BYieldingUpdateGeoLocation[SEARCHED] SetIPLocation[%s(%u.%u.%u.%u)] = (%6.3f,%6.3f)\n", pSession->GetSteamID().Render(), iptod( vecIPs[i] ), infos[j].latitude(), infos[j].longitude() ); + pSession->SetGeoLocation( infos[j].latitude(), infos[j].longitude() ); + } + } + } + } + } + return true; + } + + return false; +} + +//----------------------------------------------------------------------------- +// Purpose: Populate the KeyValues with the stats +//----------------------------------------------------------------------------- +void CGCBase::SystemStats_Update( CGCMsgGetSystemStatsResponse &msgStats ) +{ + msgStats.set_active_jobs( m_JobMgr.CountJobs() ); + msgStats.set_yielding_jobs( m_JobMgr.CountYieldingJobs() ); + msgStats.set_user_sessions( m_hashUserSessions.Count() ); + msgStats.set_game_server_sessions( m_hashGSSessions.Count() ); + msgStats.set_socaches( m_mapSOCache.Count() ); + msgStats.set_socaches_to_unload( m_listCachesToUnload.Count() ); + msgStats.set_socaches_loading( m_rbtreeSOCachesBeingLoaded.Count() ); + msgStats.set_writeback_queue( m_vecCacheWritebacks.Count() ); + msgStats.set_steamid_locks( m_hashSteamIDLocks.Count() ); + msgStats.set_logon_queue( m_llStartPlaying.Count() ); + msgStats.set_logon_jobs( m_nStartPlayingJobCount ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns the singleton GC object +//----------------------------------------------------------------------------- +CGCBase *GGCBase() +{ + return g_pGCBase; +} + + +//----------------------------------------------------------------------------- +// Purpose: Spews information about the active locks on the GC +//----------------------------------------------------------------------------- +int LockSortFunc( CLock * const *lhs, CLock * const *rhs ) +{ + return (*rhs)->GetWaitingCount() - (*lhs)->GetWaitingCount(); +} + +void CGCBase::DumpSteamIDLocks( bool bFull, int nMax ) +{ + CUtlVector<CLock *> vecLocks; + for( CLock *pLock = m_hashSteamIDLocks.PvRecordFirst(); NULL != pLock; pLock = m_hashSteamIDLocks.PvRecordNext( pLock ) ) + { + if( pLock->BIsLocked() ) + { + vecLocks.AddToTail( pLock ); + } + } + + vecLocks.Sort( LockSortFunc ); + + if( nMax > vecLocks.Count() || bFull ) + { + nMax = vecLocks.Count(); + } + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%d locks total. %d locked, %d displayed\n", m_hashSteamIDLocks.Count(), vecLocks.Count(), nMax ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "Lock Holding Job First Waiting Job Wait Count Lock Time\n" ); + + for( int nLock = 0; nLock < nMax; nLock++ ) + { + CLock *pLock = vecLocks[nLock]; + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%-24s %-22s %-22s %-11d %d\n", + pLock->GetName(), + pLock->GetJobLocking() ? pLock->GetJobLocking()->GetName() : "--", + pLock->GetJobWaitingQueueHead() ? pLock->GetJobWaitingQueueHead()->GetName() : "--", + pLock->GetWaitingCount(), + (int) ( pLock->GetMicroSecondsSinceLock() / k_nMillion ) ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Dumps informations about currently running jobs +//----------------------------------------------------------------------------- +void CGCBase::DumpJobs( const char *pszJobName, int nMax, int nPrintLocksMax ) const +{ + m_JobMgr.DumpJobs( pszJobName, nMax, nPrintLocksMax ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Dumps information about a specific job +//----------------------------------------------------------------------------- +void CGCBase::DumpJob( JobID_t jobID, int nPrintLocksMax ) const +{ + m_JobMgr.DumpJob( jobID, nPrintLocksMax ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns counts of core objects +//----------------------------------------------------------------------------- +int CGCBase::GetSOCacheCount() const +{ + return m_mapSOCache.Count(); +} + +bool CGCBase::IsSOCached( const CSharedObject *pObj, uint32 nTypeID ) const +{ + // OPT: If there are many caches, this is very slow - it would be faster have a ref count on the shared object to track this. + // However this is debug only code. +#if defined( DEBUG ) + FOR_EACH_MAP_FAST( m_mapSOCache, i ) + { + CGCSharedObjectCache *pCache = m_mapSOCache[ i ]; + if ( pCache->IsObjectCached( pObj, nTypeID ) ) + { + return true; + } + if ( pCache->IsObjectDirty( pObj ) ) + { + Assert( false ); + return true; + } + } +#else + AssertMsg( false, "Calling IsSOCached() in release mode. This is a debug only function" ); +#endif + return false; +} + +int CGCBase::GetUserSessionCount() const +{ + return m_hashUserSessions.Count(); +} + +int CGCBase::GetGSSessionCount() const +{ + return m_hashGSSessions.Count(); +} + +//----------------------------------------------------------------------------- +// Purpose: Mark that we are shutting down +//----------------------------------------------------------------------------- +void CGCBase::SetIsShuttingDown() +{ + m_bIsShuttingDown = true; + GetJobMgr().SetIsShuttingDown(); +} + +//----------------------------------------------------------------------------- +// Purpose: Sets whether we are profiling or not +//----------------------------------------------------------------------------- +void CGCBase::SetProfilingEnabled( bool bEnabled ) +{ + if ( bEnabled ) + { + m_bStartProfiling = true; + } + else + { + m_bStopProfiling = true; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Sets whether to spew about vprof imbalances +//----------------------------------------------------------------------------- +void CGCBase::SetDumpVprofImbalances( bool bEnabled ) +{ + m_bDumpVprofImbalances = bEnabled; +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns whether we are spewing vprof imbalances +//----------------------------------------------------------------------------- +bool CGCBase::GetVprofImbalances() +{ + return m_bDumpVprofImbalances; +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns a steam ID for a user-provided input. Works with accountID, +// steam account name, or steam ID. +//----------------------------------------------------------------------------- +CSteamID CGCBase::YieldingGuessSteamIDFromInput( const char *pchInput ) +{ + AssertRunningJob(); + + if( !pchInput ) + { + EmitError( SPEW_CONSOLE, "Invalid NULL string passed to YieldingGuessSteamIDFromInput\n" ); + return CSteamID(); + } + + EUniverse localUniverse = m_pHost->GetUniverse(); + + // Is it a 64 bit Steam ID? + if ( pchInput[0] >= '0' && pchInput[0] <= '9' ) + { + CSteamID steamID( V_atoui64( pchInput ) ); + if ( steamID.IsValid() ) + return steamID; + } + + // quoted + + // See if it's a profile link. If it is, clip the SteamID from it. + const char *pszProfilePrepend = "steamcommunity.com/profiles/"; + int iInputLen = Q_strlen(pchInput); + int iProfilePrependLen = Q_strlen(pszProfilePrepend); + const char *pszFound = NULL; + if ( (pszFound = Q_stristr( pchInput, pszProfilePrepend )) != NULL ) + { + if ( iInputLen > ((pszFound + iProfilePrependLen) - pchInput) ) + { + CSteamID steamID; + steamID.SetFromString( (pszFound + iProfilePrependLen), localUniverse ); + if ( steamID.IsValid() ) + return steamID; + } + } + + // See if it's an id link. + const char *pszIDPrepend = "steamcommunity.com/id/"; + int iIDPrependLen = Q_strlen(pszIDPrepend); + if ( (pszFound = Q_stristr( pchInput, pszIDPrepend )) != NULL ) + { + if ( iInputLen > ((pszFound + iIDPrependLen) - pchInput) ) + { + char szMaxURL[512]; + Q_strncpy( szMaxURL, (pszFound + iIDPrependLen), sizeof(szMaxURL) ); + + // Trim off a trailing slash + int iURLLen = Q_strlen(szMaxURL); + if ( szMaxURL[iURLLen-1] == '/' || pchInput[iURLLen-1] == '\\' ) + { + szMaxURL[iURLLen-1] = '\0'; + } + + CUtlVector< CSteamID > vecIDs; + if ( BYieldingLookupAccount( k_EFindAccountTypeURL, szMaxURL, &vecIDs ) ) + { + // Should only ever find a single account for a URL + if ( vecIDs.Count() == 1 ) + return vecIDs[0]; + } + } + } + + CGCMsg< MsgGCEmpty_t > msg( k_EGCMsgLookupAccountFromInput ); + msg.AddStrData( pchInput ); + msg.ExpectingReply( GJobCur().GetJobID() ); + if( !BSendSystemMessage( msg ) ) + { + EmitError( SPEW_CONSOLE, "Unable to query GCHost in YieldingGuessSteamIDFromInput\n" ); + return CSteamID(); + } + + CGCMsg< MsgGCLookupAccountResponse > msgReply; + if( !GJobCur().BYieldingWaitForMsg( &msgReply, k_EGCMsgGenericReply ) ) + { + EmitError( SPEW_CONSOLE, "No response from GCHost in YieldingGuessSteamIDFromInput\n" ); + return CSteamID(); + } + + return CSteamID( msgReply.Body().m_ulSteamID ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns all matching Steam IDs for the specified query. +// Returns: true if a response was received from Steam. The list may still be +// empty in that case. +//----------------------------------------------------------------------------- +bool CGCBase::BYieldingLookupAccount( EAccountFindType eFindType, const char *pchInput, CUtlVector< CSteamID > *prSteamIDs ) +{ + if ( eFindType == k_EFindAccountTypeURL ) + { + CSteamAPIRequest apiRequest( k_EHTTPMethodGET, "ISteamUser", "ResolveVanityURL", 1 ); + apiRequest.SetGETParamString( "vanityurl", pchInput ); + + KeyValuesAD kvAPIResponse( "response" ); + CUtlString sWebApiErrMsg; + EResult eResult = YieldingSendWebAPIRequest( apiRequest, kvAPIResponse, sWebApiErrMsg, false ); + if ( k_EResultOK != eResult ) + { + // Emit an error on the less-common errors + if ( k_EResultNoMatch != eResult ) + { + EmitError( SPEW_GC, "WebAPI error looking up vanity URL by GC. %s\n", sWebApiErrMsg.String() ); + } + + return false; + } + + prSteamIDs->AddToTail( CSteamID( kvAPIResponse->GetUint64( "steamid" ) ) ); + + return true; + } + else + { + CProtoBufMsg< CMsgAMFindAccounts > msg( k_EGCMsgFindAccounts ); + msg.Body().set_search_type( eFindType ); + msg.Body().set_search_string( pchInput ); + msg.ExpectingReply( GJobCur().GetJobID() ); + + if( !BSendSystemMessage( msg ) ) + { + EmitError( SPEW_GC, "Unable to send GCMsgFindAccounts\n" ); + return false; + } + + CProtoBufMsg< CMsgAMFindAccountsResponse > response; + if( !GJobCur().BYieldingWaitForMsg( &response, k_EGCMsgGenericReply ) ) + { + EmitError( SPEW_GC, "No response to GCMsgFindAccounts\n" ); + return false; + } + + for( int i=0; i<response.Body().steam_id_size(); i++ ) + { + prSteamIDs->AddToTail( CSteamID( response.Body().steam_id( i ) ) ); + } + + return true; + } +} + +GC_CON_COMMAND( gc_search_vanityurl, "Tests searching for an account by vanity URL" ) +{ + CUtlVector< CSteamID > vecIDs; + if ( GGCBase()->BYieldingLookupAccount( k_EFindAccountTypeURL, args[1], &vecIDs ) ) + { + Msg( "Search success.\n" ); + FOR_EACH_VEC( vecIDs, i ) + { + CSteamID result = vecIDs[i]; + Msg( "Result: %llu\n", result.ConvertToUint64() ); + } + } + else + { + Msg( "Search failure.\n" ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: Dumps a summary of the GC's status +//----------------------------------------------------------------------------- +bool CGCBase::BYieldingRecordSupportAction( const CSteamID & actorID, const CSteamID & targetID, const char *pchData, const char *pchNote ) +{ + CGCMsg< MsgGCRecordSupportAction_t > msgRecordSupportAction( k_EGCMsgRecordSupportAction ); + msgRecordSupportAction.Body().m_unAccountID = targetID.GetAccountID(); + msgRecordSupportAction.Body().m_unActorID = actorID.GetAccountID(); + msgRecordSupportAction.AddStrData( pchData ); + msgRecordSupportAction.AddStrData( pchNote ); + msgRecordSupportAction.ExpectingReply( GJobCur().GetJobID() ); + GGCBase()->BSendSystemMessage( msgRecordSupportAction ); + + CGCMsg< MsgGCEmpty_t > msgReply; + if( !GJobCur().BYieldingWaitForMsg( &msgReply, k_EGCMsgGenericReply ) ) + { + EmitError( SPEW_GC, "No reply received to support action message\n" ); + return false; + } + else + { + return true; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Posts a steam alert to the alert alias for this GC's app. +//----------------------------------------------------------------------------- +void CGCBase::PostAlert( EAlertType eAlertType, bool bIsCritical, const char *pchAlertText, const CUtlVector< CUtlString > *pvecExtendedInfo, bool bAlsoSpew ) +{ + CProtoBufMsg< CMsgNotifyWatchdog > msg( k_EGCMsgPostAlert ); + msg.Body().set_alert_type( eAlertType ); + msg.Body().set_critical( bIsCritical ); + + if( !pvecExtendedInfo ) + { + msg.Body().set_text( pchAlertText ); + } + else + { + // put all the messages in one giant string and set that as the text + + // figure out how big "giant" is + uint32 unSize = Q_strlen( pchAlertText ) + 2; // header + \n + null + FOR_EACH_VEC( *pvecExtendedInfo, nLine ) + { + unSize += pvecExtendedInfo->Element( nLine ).Length(); + } + + // walk the strings again to assemble the buffer + CUtlBuffer bufMessage( 0, unSize, CUtlBuffer::TEXT_BUFFER ); + bufMessage.PutString( pchAlertText ); + bufMessage.PutString( "\n" ); + FOR_EACH_VEC( *pvecExtendedInfo, nLine ) + { + bufMessage.PutString( pvecExtendedInfo->Element( nLine ).Get() ); + } + + msg.Body().set_text( (const char *)bufMessage.Base() ); + } + + if( bAlsoSpew ) + { + EmitError( SPEW_GC, "%s", msg.Body().text().c_str() ); + } + + BSendSystemMessage( msg ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Fills the vector with all package IDs this account has a license to +//----------------------------------------------------------------------------- +bool CGCBase::BYieldingGetAccountLicenses( const CSteamID & steamID, CUtlVector< PackageLicense_t > & vecPackages ) +{ + CProtoBufMsg< CMsgAMGetLicenses > msg( k_EGCMsgGetLicenses ); + msg.Body().set_steamid( steamID.ConvertToUint64() ); + msg.ExpectingReply( GJobCur().GetJobID() ); + if( !BSendSystemMessage( msg ) ) + { + EmitWarning( SPEW_GC, SPEW_ALWAYS, "Unable to send GetAccountLicenses system message\n" ); + return false; + } + + CProtoBufMsg< CMsgAMGetLicensesResponse > msgReply; + if( !GJobCur().BYieldingWaitForMsg( &msgReply, k_EGCMsgGenericReply ) ) + { + EmitWarning( SPEW_GC, SPEW_ALWAYS, "Timeout waiting for GetAccountLicenses reply\n" ); + return false; + } + + if( msgReply.Body().result() != k_EResultOK ) + { + EmitWarning( SPEW_GC, SPEW_ALWAYS, "GetAccountLicenses for %s failed with %d\n", steamID.Render(), msgReply.Body().result() ); + return false; + } + + vecPackages.RemoveAll(); + vecPackages.EnsureCapacity( msgReply.Body().license_size() ); + + for( int i=0; i < msgReply.Body().license_size(); i++ ) + { + const CMsgPackageLicense &msgPackage = msgReply.Body().license( i ); + + //skip packages that they directly don't own (they may be lent to them via library sharing, and we don't want to grant based on that). + //we count account ID of zero as matching so we can deal with old Steam versions that didn't provide this field + if( ( msgPackage.owner_id() != steamID.GetAccountID() ) && ( msgPackage.owner_id() != 0 ) ) + continue; + + PackageLicense_t package; + package.m_unPackageID = msgPackage.package_id(); + package.m_rtimeCreated = msgPackage.time_created(); + vecPackages.AddToTail( package ); + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Fills the vector with all package IDs this account has a license to +//----------------------------------------------------------------------------- +bool CGCBase::BYieldingAddFreeLicense( const CSteamID & steamID, uint32 unPackageID, uint32 unIPPublic, const char *pchStoreCountryCode ) +{ + CProtoBufMsg< CMsgAMAddFreeLicense > msg( k_EGCMsgAddFreeLicense ); + msg.Body().set_steamid( steamID.ConvertToUint64() ); + msg.Body().set_packageid( unPackageID ); + if( unIPPublic ) + msg.Body().set_ip_public( unIPPublic ); + if( pchStoreCountryCode ) + msg.Body().set_store_country_code( pchStoreCountryCode ); + msg.ExpectingReply( GJobCur().GetJobID() ); + if( !BSendSystemMessage( msg ) ) + { + EmitWarning( SPEW_GC, SPEW_ALWAYS, "Unable to send GetAccountLicenses system message\n" ); + return false; + } + + CProtoBufMsg< CMsgAMAddFreeLicenseResponse > msgReply; + if( !GJobCur().BYieldingWaitForMsg( &msgReply, k_EGCMsgAddFreeLicenseResponse ) ) + { + EmitWarning( SPEW_GC, SPEW_ALWAYS, "Timeout waiting for GetAccountLicenses reply\n" ); + return false; + } + + if( msgReply.Body().eresult() != k_EResultOK ) + { + EmitWarning( SPEW_GC, SPEW_ALWAYS, "BYieldingAddFreeLicense for %s failed with %d\n", steamID.Render(), msgReply.Body().eresult() ); + return false; + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Fills the vector with all package IDs this account has a license to +//----------------------------------------------------------------------------- +int CGCBase::YieldingGrantGuestPass( const CSteamID & steamID, uint32 unPackageID, uint32 unPassesToGrant, int32 nDaysToExpiration ) +{ + CProtoBufMsg<CMsgAMGrantGuestPasses2> msg( k_EGCMsgGrantGuestPass ); + msg.Body().set_steam_id( steamID.ConvertToUint64() ); + msg.Body().set_package_id( unPackageID ); + msg.Body().set_passes_to_grant( unPassesToGrant ); + msg.Body().set_days_to_expiration( nDaysToExpiration ); + msg.ExpectingReply( GJobCur().GetJobID() ); + if( !BSendSystemMessage( msg ) ) + { + EmitWarning( SPEW_GC, SPEW_ALWAYS, "Unable to send GrantGuestPass system message\n" ); + return 0; + } + + CProtoBufMsg<CMsgAMGrantGuestPasses2Response> msgReply; + if( !GJobCur().BYieldingWaitForMsg( &msgReply, k_EGCMsgGrantGuestPassResponse ) ) + { + EmitWarning( SPEW_GC, SPEW_ALWAYS, "Timeout waiting for GrantGuestPass reply\n" ); + return 0; + } + + if( msgReply.Body().eresult() != k_EResultOK ) + { + EmitWarning( SPEW_GC, SPEW_ALWAYS, "YieldingGrantGuestPass for %s failed with %d\n", steamID.Render(), msgReply.Body().eresult() ); + return 0; + } + + return msgReply.Body().passes_granted(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets data for an account +//----------------------------------------------------------------------------- +const CAccountDetails *CGCBase::YieldingGetAccountDetails( const CSteamID & steamID, bool bForceReload ) +{ + return m_AccountDetailsManager.YieldingGetAccountDetails( steamID, bForceReload ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets the current persona name for an account +//----------------------------------------------------------------------------- +const char *CGCBase::YieldingGetPersonaName( const CSteamID & steamID, const char *szUnknownName ) +{ + const char *szPersonaName = m_AccountDetailsManager.YieldingGetPersonaName( steamID ); + return szPersonaName ? szPersonaName : szUnknownName; +} + +//----------------------------------------------------------------------------- +// Purpose: Clears a persona name from the cache +//----------------------------------------------------------------------------- +void CGCBase::ClearCachedPersonaName( const CSteamID & steamID ) +{ + m_AccountDetailsManager.ClearCachedPersonaName( steamID ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Tells us to load the persona name for a user, but not wait on it +//----------------------------------------------------------------------------- +void CGCBase::PreloadPersonaName( const CSteamID & steamID ) +{ + m_AccountDetailsManager.PreloadPersonaName( steamID ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Sends a message to the web API servers letting them know what the +// methods and interfaces are for this GC. +//----------------------------------------------------------------------------- +bool CGCBase::BSendWebApiRegistration() +{ + // if we aren't initialized enough to have a GCHost, just skip this + // registration request. We'll register later in our init process. + if( !m_pHost ) + return false; + + if( CGCWebAPIInterfaceMapRegistrar::VecInstance().Count() > 0 ) + { + CGCMsg< MsgGCWebAPIRegisterInterfaces_t > msgWebRegistration( k_EGCMsgWebAPIRegisterInterfaces ); + msgWebRegistration.Body().m_cInterfaces = CGCWebAPIInterfaceMapRegistrar::VecInstance().Count(); + CUtlBuffer bufRegistrations; + FOR_EACH_VEC( CGCWebAPIInterfaceMapRegistrar::VecInstance(), nInterface ) + { + KeyValues *pkvInterface = CGCWebAPIInterfaceMapRegistrar::VecInstance()[ nInterface ](); + Assert( pkvInterface ); + if( !pkvInterface ) + return false; + + KVPacker packer; + packer.WriteAsBinary( pkvInterface, bufRegistrations ); + pkvInterface->deleteThis(); + } + msgWebRegistration.AddVariableLenData( bufRegistrations.Base(), bufRegistrations.TellPut() ); + if( !BSendSystemMessage( msgWebRegistration ) ) + return false; + } + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Dumps a summary of the GC's status +//----------------------------------------------------------------------------- +void CGCBase::Dump() const +{ + char rtimeBuf[k_RTimeRenderBufferSize]; + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "GC Status for %d: path=%s\n", m_unAppID, m_sPath.Get() ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\tLogon Surge: %s\n", BIsInLogonSurge() ? "Yes" : "No" ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\tStartPlaying: waiting=%d, jobs running=%d of %d\n", m_llStartPlaying.Count(), m_nStartPlayingJobCount, cv_concurrent_start_playing_limit.GetInt() ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\tJobs: active=%d, yielding=%d\n", m_JobMgr.CountJobs(), m_JobMgr.CountYieldingJobs() ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\tSessions: user=%d, gameserver=%d\n", m_hashUserSessions.Count(), m_hashGSSessions.Count() ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\tCaches: %d (%d waiting to unload, %d currently loading, %s %d /+ %d)\n", m_mapSOCache.Count(), m_listCachesToUnload.Count(), m_rbtreeSOCachesBeingLoaded.Count(), + ( ( ( m_jobidFlushInventoryCacheAccounts == k_GIDNil ) || !m_JobMgr.BJobExists( m_jobidFlushInventoryCacheAccounts ) ) ? "last flushed" : "currently flushing" ), + m_numFlushInventoryCacheAccountsLastScheduled, m_rbFlushInventoryCacheAccounts.Count() ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\tWriteback Queue: %d (oldest: %s)\n", m_vecCacheWritebacks.Count(), m_vecCacheWritebacks.Count() > 0 ? CRTime::Render( m_vecCacheWritebacks[0]->GetWritebackTime(), rtimeBuf ) : "none" ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\tYieldingRequestSession: %d active\n", m_nRequestSessionJobsActive ); + m_AccountDetailsManager.Dump(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Dumps a summary of the GC's status +//----------------------------------------------------------------------------- +const char *CGCBase::GetCDNURL() const +{ + if( m_sCDNURL.IsEmpty() ) + { + switch( m_pHost->GetUniverse() ) + { + case k_EUniverseDev: + case k_EUniverseBeta: + m_sCDNURL.Format( "http://cdn.beta.steampowered.com/apps/%d/", GetAppID() ); + break; + case k_EUniversePublic: + default: + m_sCDNURL.Format( "http://media.steampowered.com/apps/%d/", GetAppID() ); + break; + } + } + + return m_sCDNURL.Get(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Prints an assert to the console +//----------------------------------------------------------------------------- + + +void CGCBase::AssertCallbackFunc( const char *pchFile, int nLine, const char *pchMessage ) +{ + if ( !ThreadInMainThread() ) // !KLUDGE! + { + EmitWarning( SPEW_GC, 4, "Thread assert %s(%d): %s\n", pchFile, nLine, pchMessage ); + return; + } + + // Our spew handler should have already spewed this once, no need to spew it again + //EmitError( SPEW_CONSOLE, "%s (%d): %s\n", V_GetFileName( pchFile ), nLine, pchMessage ); + if ( !Plat_IsInDebugSession() ) + { + + char rchCleanedJobName[48] = ""; + if ( ThreadInMainThread() && g_pJobCur != NULL ) + { + const char *pszJobName = g_pJobCur->GetName(); + int l = 0; + while ( l < sizeof(rchCleanedJobName)-1 ) + { + char c = pszJobName[l]; + if ( c == '\0' ) + break; + if ( !V_isalnum( c ) ) + { + c = '_'; + } + rchCleanedJobName[l] = c; + ++l; + } + rchCleanedJobName[l] = 0; + } + + // Throttle writing of minidumps on a file / line / job basis + CFmtStr sFileAndLine( "assert_%s(%d)%s%s", + V_GetFileName( pchFile ), + nLine, + rchCleanedJobName[0] ? "_" : "", + rchCleanedJobName + ); + + static CUtlDict< CCopyableUtlVector< RTime32 > > s_dictAsserts; + + int iDict = s_dictAsserts.Find( sFileAndLine.Access() ); + if ( !s_dictAsserts.IsValidIndex( iDict ) ) + { + iDict = s_dictAsserts.Insert( sFileAndLine.Access() ); + } + + CCopyableUtlVector< RTime32 > &vecTimes = s_dictAsserts[iDict]; + + int nStale = 0; + while ( nStale < vecTimes.Count() && ( CRTime::RTime32TimeCur() - vecTimes[nStale] ) > (uint32)cv_assert_minidump_window.GetInt() ) + { + nStale++; + } + vecTimes.RemoveMultipleFromHead( nStale ); + + bool bWriteDump = ( vecTimes.Count() < cv_assert_max_minidumps_in_window.GetInt() ); + if ( bWriteDump ) + { + vecTimes.AddToTail( CRTime::RTime32TimeCur() ); + + CUtlString sCurJob; + if ( ThreadInMainThread() && g_pJobCur != NULL ) + { + sCurJob.Format( "[From job %s]\n", g_pJobCur->GetName() ); + } + + // Write the dump + CUtlString sDumpComment; + sDumpComment.Format( "%s%s%s(%d): %s", + GGCBase()->GetIsShuttingDown() ? "[During shutdown]\n" : "", // Asserts during shutdown are much more often spurious. Let's make it clear if a shutdown happens during shutdown + sCurJob.String(), // The name of the current job name is often an incredibly useful piece of info. If the dumps are not valid, this can narrow the search space immensely + pchFile, + nLine, + pchMessage + ); + SetMinidumpComment( sDumpComment.String() ); + WriteMiniDump( sFileAndLine.Access() ); + SetMinidumpComment( "" ); // just for grins + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Claims all the memory for the GC +//----------------------------------------------------------------------------- +void CGCBase::Validate( CValidator &validator, const char *pchName ) +{ + VPROF_BUDGET( "CGCBase::Validate", VPROF_BUDGETGROUP_STEAM ); + + // these are INSIDE the function instead of outside so the interface + // doesn't change +#ifdef DBGFLAG_VALIDATE + VALIDATE_SCOPE(); + + // Validate the global message list + g_theMessageList.Validate( validator, "g_theMessageList" ); + + // Validate the network global memory pool + g_MemPoolMsg.Validate( validator, "g_MemPoolMsg" ); + + CNetPacketPool::ValidateGlobals( validator ); + + CJobMgr::ValidateStatics( validator, "CJobMgr" ); + CJob::ValidateStatics( validator, "CJob" ); + ValidateTempTextBuffers( validator ); + + ValidateObj( m_JobMgr ); + ValidateObj( m_sPath ); + + ValidateObj( m_hashUserSessions ); + for( CGCUserSession **ppSession = m_hashUserSessions.PvRecordFirst(); ppSession != NULL; ppSession = m_hashUserSessions.PvRecordNext( ppSession ) ) + { + + ValidatePtr( *ppSession ); + } + ValidateObj( m_hashGSSessions ); + for( CGCGSSession **ppSession = m_hashGSSessions.PvRecordFirst(); ppSession != NULL; ppSession = m_hashGSSessions.PvRecordNext( ppSession ) ) + { + ValidatePtr( *ppSession ); + } + + // validate the SQL access layer + CRecordBase::ValidateStatics( validator, "CRecordBase" ); + GSchemaFull().Validate( validator, "GSchemaFull" ); + CRecordInfo::ValidateStatics( validator, "CRecordInfo" ); + CSharedObject::ValidateStatics( validator ); + + OnValidate( validator, pchName ); +#endif // DBGFLAG_VALIDATE +} + +EResult YieldingSendWebAPIRequest( CSteamAPIRequest &request, KeyValues *pKVResponse, CUtlString &errMsg, bool b200MeansSuccess ) +{ + CHTTPResponse apiResponse; + if ( !GGCBase()->BYieldingSendHTTPRequest( &request, &apiResponse ) ) + { + errMsg.Format( "Did not get a response" ); + return k_EResultTimeout; + } + + if ( k_EHTTPStatusCode200OK != apiResponse.GetStatusCode() ) + { + errMsg.Format( "HTTP status code %d", apiResponse.GetStatusCode() ); + + // if ( k_EResultOK != pKVResponse->GetInt( "result", k_EResultFail ) ) + // { + // EmitError( SPEW_GC, "Web call to %s failed with error %d: %s\n", + // request.GetURL(), + // pKVResponse->GetInt( "error/errorcode", k_EResultFail ), + // pKVResponse->GetString( "error/errordesc" ) ); + // return pKVResponse->GetInt( "error/errorcode", k_EResultFail ); + // } + + return k_EResultFail; + } + + + if ( apiResponse.GetBodyBuffer() ) + { + pKVResponse->UsesEscapeSequences( true ); + if ( !pKVResponse->LoadFromBuffer( "webResponse", *apiResponse.GetBodyBuffer() ) ) + { + errMsg.Format( "Failed to parse keyvalues result" ); + return k_EResultFail; + } + } + + if ( b200MeansSuccess ) + { + return k_EResultOK; + } + + int result = pKVResponse->GetInt( "success", -1 ); + if ( result < 0 ) + { + errMsg = "Reply missing result code"; + return k_EResultFail; + } + errMsg = pKVResponse->GetString( "message", "" ); + if ( result != k_EResultOK && errMsg.IsEmpty() ) + { + errMsg = "(Unknown error)"; + } + return (EResult)result; +} + +GC_CON_COMMAND( ip_geolocation, "<a.b.c.d> Perform geolocation lookup" ) +{ + if ( args.ArgC() < 2 ) + { + EmitError( SPEW_GC, "Pass at least one IP to lookup\n" ); + return; + } + + // Get List of IP's to query + CUtlVector<uint32> vecIPs; + for ( int i = 1 ; i < args.ArgC() ; ++i ) + { + netadr_t adr; + adr.SetFromString( args[i] ); + if ( adr.GetIPHostByteOrder() == 0 ) + { + EmitInfo( SPEW_GC, 1, 1, "%s is not a valid IP\n", args[i] ); + } + else + { + vecIPs.AddToTail( adr.GetIPHostByteOrder() ); + } + } + if ( vecIPs.Count() <= 0 ) + return; + + // Do the query + CUtlVector<CIPLocationInfo> vecInfos; + vecInfos.SetCount( vecIPs.Count() ); + GGCBase()->BYieldingGetIPLocations( vecIPs, vecInfos ); + for ( int i = 0 ; i < vecInfos.Count() ; ++i ) + { + netadr_t adr( vecInfos[i].ip(), 0 ); + EmitInfo( SPEW_GC, 1, 1, "%s: %.1f, %.1f\n", adr.ToString( true ), vecInfos[i].latitude(), vecInfos[i].longitude() ); + } +} + +} // namespace GCSDK + + + diff --git a/gcsdk/gcclient.cpp b/gcsdk/gcclient.cpp new file mode 100644 index 0000000..01eba37 --- /dev/null +++ b/gcsdk/gcclient.cpp @@ -0,0 +1,601 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Holds the CGCClient class +// +//============================================================================= + +#include "stdafx.h" +#include "gcclient.h" +#include "steam/isteamgamecoordinator.h" +#include "gcsdk_gcmessages.pb.h" + +namespace GCSDK +{ + +//#define SOCDebug(...) Msg( __VA_ARGS__ ) +#define SOCDebug(...) ((void)0) + +//------------------------------------------------------------------------------ +// Purpose: Constructor +//------------------------------------------------------------------------------ +CGCClient::CGCClient( ISteamGameCoordinator *pSteamGameCoordinator, bool bGameserver ) +: m_pSteamGameCoordinator( NULL ), + m_memMsg( 0, 1024 ), +#ifndef STEAM + m_callbackGCMessageAvailable( NULL, NULL ), +#endif + m_mapSOCache( DefLessFunc(CSteamID) ) +{ +#ifndef STEAM + if( bGameserver ) + { + m_callbackGCMessageAvailable.SetGameserverFlag(); + } +#endif + if( pSteamGameCoordinator ) + { + DbgVerify( BInit( pSteamGameCoordinator ) ); + } +} + + +//------------------------------------------------------------------------------ +// Purpose: Constructor +//------------------------------------------------------------------------------ +CGCClient::~CGCClient( ) +{ + Uninit(); + + FOR_EACH_MAP_FAST( m_mapSOCache, i ) + { + delete m_mapSOCache[i]; + } + m_mapSOCache.RemoveAll(); +} + + +//------------------------------------------------------------------------------ +// Purpose: Performs the every-frame work required by the GC Client. Mostly that +// means running yielding jobs. +// Inputs: ulLimitMicroseconds - The target number of microseconds worth of +// work to do this time through the loop. +// Outputs: Returns true if there is still work to do that was skipped because +// time ran out. +//------------------------------------------------------------------------------ +bool CGCClient::BMainLoop( uint64 ulLimitMicroseconds, uint64 ulFrameTimeMicroseconds ) +{ + // Don't do any work if not initialized + if ( !m_pSteamGameCoordinator ) + return false; + + CLimitTimer limitTimer; + limitTimer.SetLimit( ulLimitMicroseconds ); + CJobTime::UpdateJobTime( ulFrameTimeMicroseconds ? ulFrameTimeMicroseconds : k_cMicroSecPerShellFrame ); + + bool bWorkRemaining = m_JobMgr.BFrameFuncRunSleepingJobs( limitTimer ); + bWorkRemaining |= m_JobMgr.BFrameFuncRunYieldingJobs( limitTimer ); + return bWorkRemaining; +} + + +//------------------------------------------------------------------------------ +// Purpose: Sends a message to the GC +// Inputs: unMsgType - the type ID of the message to send +// pubData - The data for the message we're sending +// cubData - The number of bytes of data in this message including any +// variable-lengthed data. +// Outputs: Returns false if the send failed. A return value of true doesn't +// mean that the message was necessarily received by the GC just that +// it didn't fail in obvious ways on the client. +//------------------------------------------------------------------------------ +bool CGCClient::BSendMessage( uint32 unMsgType, const uint8 *pubData, uint32 cubData ) +{ + if( m_pSteamGameCoordinator ) + return m_pSteamGameCoordinator->SendMessage( unMsgType, pubData, cubData ) == k_EGCResultOK; + else + return false; +} + + +//------------------------------------------------------------------------------ +// Purpose: Sends a message to the GC +// Inputs: msg - The message to send +// Outputs: Returns false if the send failed. A return value of true doesn't +// mean that the message was necessarily received by the GC just that +// it didn't fail in obvious ways on the client. +//------------------------------------------------------------------------------ +bool CGCClient::BSendMessage( const CGCMsgBase& msg ) +{ + return BSendMessage( msg.Hdr().m_eMsg, msg.PubPkt() + sizeof(GCMsgHdr_t), msg.CubPkt() - sizeof(GCMsgHdr_t) ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Used to send protobuf messages to the GC +//----------------------------------------------------------------------------- +class CProtoBufGCClientSendHandler : public CProtoBufMsgBase::IProtoBufSendHandler +{ +public: + CProtoBufGCClientSendHandler( CGCClient *pGCClient ) + : m_pClient( pGCClient ) {} + virtual bool BAsyncSend( MsgType_t eMsg, const uint8 *pubMsgBytes, uint32 cubSize ) + { + g_theMessageList.TallySendMessage( eMsg & ~k_EMsgProtoBufFlag, cubSize ); + VPROF_BUDGET( "CGCClient", VPROF_BUDGETGROUP_STEAM ); + { + VPROF_BUDGET( "CGCClient - BSendGCMsgToClient (ProtoBuf)", VPROF_BUDGETGROUP_STEAM ); + return m_pClient->BSendMessage( eMsg | k_EMsgProtoBufFlag, pubMsgBytes, cubSize ); + } + } + +private: + CGCClient *m_pClient; +}; + + +//----------------------------------------------------------------------------- +// Purpose: Sends a message to the given SteamID +//----------------------------------------------------------------------------- +bool CGCClient::BSendMessage( const CProtoBufMsgBase& msg ) +{ + CProtoBufGCClientSendHandler sender( this ); + return msg.BAsyncSend( sender ); +} + + +//------------------------------------------------------------------------------ +// Purpose: Callback handler for the GCMessageAvailable_t callback. Handles +// incoming messages. +// Inputs: pCallback - the callback from Steam +//------------------------------------------------------------------------------ +void CGCClient::OnGCMessageAvailable( GCMessageAvailable_t *pCallback ) +{ + uint32 cubData; + uint32 unMsgType; + while( m_pSteamGameCoordinator && m_pSteamGameCoordinator->IsMessageAvailable( &cubData ) ) + { + // Get the size of the full message. sizeof( GCMsgHdr_t ) was not sent in the binary data + uint32 unFullSize = cubData + sizeof( GCMsgHdr_t ); + m_memMsg.EnsureCapacity( unFullSize ); + uint8 *pFullPacket = m_memMsg.Base(); + uint8 *pPacketFromGC = pFullPacket+sizeof(GCMsgHdr_t); + + EGCResults eResult = m_pSteamGameCoordinator->RetrieveMessage( &unMsgType, pPacketFromGC, m_memMsg.Count() - sizeof( GCMsgHdr_t ), &cubData ); + Assert( eResult == k_EGCResultOK ); + if( eResult == k_EGCResultOK ) + { + if( unMsgType & k_EMsgProtoBufFlag ) + { + CNetPacket *pGCPacket = CNetPacketPool::AllocNetPacket(); + pGCPacket->Init( cubData, pPacketFromGC ); + CIMsgNetPacketAutoRelease pMsgNetPacket( pGCPacket ); + + // Safety check against malformed packet + if ( pMsgNetPacket.Get() != NULL ) + { + + // dispatch the packet + GetJobMgr().BRouteMsgToJob( this, pMsgNetPacket.Get(), JobMsgInfo_t( pMsgNetPacket->GetEMsg(), pMsgNetPacket->GetSourceJobID(), pMsgNetPacket->GetTargetJobID(), k_EServerTypeGCClient ) ); + + // keep track of how much we've sent/received this message + g_theMessageList.TallySendMessage( pMsgNetPacket->GetEMsg(), cubData ); + } + + // release the packet + pGCPacket->Release(); + } + else + { + Assert( 0 == (unMsgType & k_EMsgProtoBufFlag ) ); + + // get the header so we can fix it up + GCMsgHdrEx_t *pHdr = (GCMsgHdrEx_t *)pFullPacket; + pHdr->m_eMsg = unMsgType; + pHdr->m_ulSteamID = CSteamID().ConvertToUint64(); + + // make a new packet for the message so we can dispatch it + // The CNetPacket takes ownership of the buffer allocated above + CNetPacket *pGCPacket = CNetPacketPool::AllocNetPacket(); + pGCPacket->Init( unFullSize, pFullPacket ); + CIMsgNetPacketAutoRelease pMsgNetPacket( pGCPacket ); + + // Safety check against malformed packet + if ( pMsgNetPacket.Get() != NULL ) + { + + // dispatch the packet + GetJobMgr().BRouteMsgToJob( this, pMsgNetPacket.Get(), JobMsgInfo_t( pMsgNetPacket->GetEMsg(), pMsgNetPacket->GetSourceJobID(), pMsgNetPacket->GetTargetJobID(), k_EServerTypeGCClient ) ); + + // keep track of how much we've sent/received this message + g_theMessageList.TallySendMessage( pMsgNetPacket->GetEMsg(), cubData ); + } + + // release the packet + pGCPacket->Release(); + } + } + } +} + + +//------------------------------------------------------------------------------ +// Purpose: Performs all the initialization for the GC Client instance +// Outputs: Returns false if the initialization failed +//------------------------------------------------------------------------------ +bool CGCClient::BInit( ISteamGameCoordinator *pSteamGameCoordinator ) +{ + // Set the job pool size. Threads get lazily created so if no code + // is using the thread pool, no threads will be created. + m_JobMgr.SetThreadPoolSize( GetCPUInformation()->m_nLogicalProcessors - 1 ); + + MsgRegistrationFromEnumDescriptor( EGCSystemMsg_descriptor(), GCSDK::MT_GC ); + + m_pSteamGameCoordinator = pSteamGameCoordinator; +#ifndef STEAM + m_callbackGCMessageAvailable.Register( this, &CGCClient::OnGCMessageAvailable ); +#endif + + // process any messages that are already waiting + if( m_pSteamGameCoordinator ) + { + OnGCMessageAvailable( NULL ); + } + + return true; +} + + +//------------------------------------------------------------------------------ +// Purpose: Performs all the uninitialization for the GC Client instance +//------------------------------------------------------------------------------ +void CGCClient::Uninit( ) +{ +#ifndef STEAM + m_callbackGCMessageAvailable.Unregister(); +#endif + m_pSteamGameCoordinator = NULL; + + // Clear and remove the SO caches + unsigned short nMapIndex = m_mapSOCache.FirstInorder(); + while ( m_mapSOCache.IsValidIndex( nMapIndex ) ) + { + unsigned short nNextMapIndex = m_mapSOCache.NextInorder( nMapIndex ); + + CGCClientSharedObjectCache *pSOCache = m_mapSOCache[nMapIndex]; + Assert( pSOCache ); + if ( pSOCache ) + { + // Send notifications, but only if we were actually subscribed + if ( pSOCache->BIsSubscribed() ) + { + pSOCache->NotifyUnsubscribe(); + } + + // Delete the entry + delete pSOCache; + m_mapSOCache.RemoveAt( nMapIndex ); + } + + nMapIndex = nNextMapIndex; + } +} + + +//------------------------------------------------------------------------------ +// Purpose: Finds the SO cache for this steam ID. If bCreateIfMissing is false, +// NULL will be returned if the cache can't be found +//------------------------------------------------------------------------------ +CGCClientSharedObjectCache *CGCClient::FindSOCache( const CSteamID & steamID, bool bCreateIfMissing ) +{ + CUtlMap< CSteamID, CGCClientSharedObjectCache * >::IndexType_t nCache = m_mapSOCache.Find( steamID ); + if( m_mapSOCache.IsValidIndex( nCache ) ) + return m_mapSOCache[nCache]; + else + { + if( bCreateIfMissing ) + { + Assert( steamID.IsValid() ); + if ( !steamID.IsValid() ) + return NULL; + + CGCClientSharedObjectCache *pCache = new CGCClientSharedObjectCache( steamID ); + m_mapSOCache.Insert( steamID, pCache ); + return pCache; + } + else + { + return NULL; + } + } +} + +//------------------------------------------------------------------------------ +// Purpose: Add a listener to the SO cache, creating it if necessary +//------------------------------------------------------------------------------ +void CGCClient::AddSOCacheListener( const CSteamID &ownerID, ISharedObjectListener *pListener ) +{ + Assert( ownerID.IsValid() ); + CGCClientSharedObjectCache *pCache = FindSOCache( ownerID, true ); + Assert( pCache ); + pCache->AddListener( pListener ); +} + +//------------------------------------------------------------------------------ +// Purpose: Remove listener from the SO cache, if he is listening +//------------------------------------------------------------------------------ +bool CGCClient::RemoveSOCacheListener( const CSteamID &ownerID, ISharedObjectListener *pListener ) +{ + Assert ( this != NULL ); // Damn people - check your pointers before calling! + Assert( ownerID.IsValid() ); + CGCClientSharedObjectCache *pCache = FindSOCache( ownerID, false ); + if ( pCache == NULL ) + return false; // cache doesn't exist, so we could't have ben listening + return pCache->RemoveListener( pListener ); +} + +//------------------------------------------------------------------------------ +// Purpose: Notify that the given SO cache has been unsubscribed +//------------------------------------------------------------------------------ +void CGCClient::NotifySOCacheUnsubscribed( const CSteamID & ownerID ) +{ + + CUtlMap< CSteamID, CGCClientSharedObjectCache * >::IndexType_t nCache = m_mapSOCache.Find( ownerID ); + if( m_mapSOCache.IsValidIndex( nCache ) ) + { + + CGCClientSharedObjectCache *pSOCache = m_mapSOCache[nCache]; + + // Ignore requests to remove caches that were never subscribed + if ( pSOCache->BIsSubscribed() ) + { + SOCDebug( "NotifySOCacheUnsubscribed(%s) [in cache, subscribed]\n", ownerID.Render() ); + pSOCache->NotifyUnsubscribe(); + } + else + { + SOCDebug( "NotifySOCacheUnsubscribed(%s) [in cache, not subscribed]\n", ownerID.Render() ); + } + } + else + { + SOCDebug( "NotifySOCacheUnsubscribed(%s) [not in cache]\n", ownerID.Render() ); + } +} + +//------------------------------------------------------------------------------ +// Purpose: Dump everything about everyone +//------------------------------------------------------------------------------ +void CGCClient::Dump() +{ + FOR_EACH_MAP( m_mapSOCache, idx ) + { + m_mapSOCache[ idx ]->Dump(); + } +} + +//------------------------------------------------------------------------------ +// Purpose: Finds the shared object for this steam ID and key object +//------------------------------------------------------------------------------ +CSharedObject *CGCClient::FindSharedObject( const CSteamID & ownerID, const CSharedObject & soIndex ) +{ + CGCClientSharedObjectCache *pCache = FindSOCache( ownerID, false ); + if( pCache ) + return pCache->FindSharedObject( soIndex ); + else + return NULL; +} + + +//------------------------------------------------------------------------------ +// Purpose: Validates all the statics in the GCSDKLib that need to be validated +// when linked directly into the steam servers. +//------------------------------------------------------------------------------ +#ifdef DBGFLAG_VALIDATE +void CGCClient::ValidateStatics( CValidator &validator ) +{ + // Validate the global message list + g_theMessageList.Validate( validator, "g_theMessageList" ); + + // Validate the network global memory pool + g_MemPoolMsg.Validate( validator, "g_MemPoolMsg" ); + + CNetPacketPool::ValidateGlobals( validator ); + + CJobMgr::ValidateStatics( validator, "CJobMgr" ); + CJob::ValidateStatics( validator, "CJob" ); + ValidateTempTextBuffers( validator ); + CSharedObject::ValidateStatics( validator ); + + // validate the SQL access layer + CRecordBase::ValidateStatics( validator, "CRecordBase" ); + GSchemaFull().Validate( validator, "GSchemaFull" ); + CRecordInfo::ValidateStatics( validator, "CRecordInfo" ); +} +#endif // DBGFLAG_VALIDATE + + +class CGCSOCreateJob : public CGCClientJob +{ +public: + CGCSOCreateJob( CGCClient *pClient ) : CGCClientJob( pClient ) {} + + virtual bool BYieldingRunGCJob( GCSDK::IMsgNetPacket *pNetPacket ) + { + CProtoBufMsg<CMsgSOSingleObject> msg( pNetPacket ); + SOCDebug( "CGCSOCreateJob(owner=%s, type=%d)\n", CSteamID( msg.Body().owner() ).Render(), msg.Body().type_id() ); + CGCClientSharedObjectCache *pSOCache = m_pGCClient->FindSOCache( msg.Body().owner() ); + if ( pSOCache ) + { + pSOCache->BCreateFromMsg( msg.Body().type_id(), msg.Body().object_data().data(), msg.Body().object_data().size() ); + Assert( msg.Body().has_version() ); + pSOCache->SetVersion( msg.Body().version() ); + } + return true; + } + +}; + +GC_REG_JOB( CGCClient, CGCSOCreateJob, "CGCSOCreateJob", k_ESOMsg_Create, GCSDK::k_EServerTypeGCClient ); + +class CGCSODestroyJob : public CGCClientJob +{ +public: + CGCSODestroyJob( CGCClient *pClient ) : CGCClientJob( pClient ) {} + + virtual bool BYieldingRunGCJob( GCSDK::IMsgNetPacket *pNetPacket ) + { + CProtoBufMsg<CMsgSOSingleObject> msg( pNetPacket ); + SOCDebug( "CGCSODestroyJob(owner=%s, type=%d)\n", CSteamID( msg.Body().owner() ).Render(), msg.Body().type_id() ); + CGCClientSharedObjectCache *pCache = m_pGCClient->FindSOCache( msg.Body().owner(), false ); + if( pCache ) + { + pCache->BDestroyFromMsg( msg.Body().type_id(), msg.Body().object_data().data(), msg.Body().object_data().size() ); + Assert( msg.Body().has_version() ); + pCache->SetVersion( msg.Body().version() ); + } + return true; + } + +}; + +GC_REG_JOB( CGCClient, CGCSODestroyJob, "CGCSODestroyJob", k_ESOMsg_Destroy, GCSDK::k_EServerTypeGCClient ); + +class CGCSOUpdateJob : public CGCClientJob +{ +public: + CGCSOUpdateJob( CGCClient *pClient ) : CGCClientJob( pClient ) {} + + virtual bool BYieldingRunGCJob( GCSDK::IMsgNetPacket *pNetPacket ) + { + CProtoBufMsg<CMsgSOSingleObject> msg( pNetPacket ); + SOCDebug( "CGCSOUpdateJob(owner=%s, type=%d)\n", CSteamID( msg.Body().owner() ).Render(), msg.Body().type_id() ); + CGCClientSharedObjectCache *pSOCache = m_pGCClient->FindSOCache( msg.Body().owner() ); + if ( pSOCache ) + { + pSOCache->BUpdateFromMsg( msg.Body().type_id(), msg.Body().object_data().data(), msg.Body().object_data().size() ); + Assert( msg.Body().has_version() ); + pSOCache->SetVersion( msg.Body().version() ); + } + return true; + } + +}; + +GC_REG_JOB( CGCClient, CGCSOUpdateJob, "CGCSOUpdateJob", k_ESOMsg_Update, GCSDK::k_EServerTypeGCClient ); + +class CGCSOUpdateMultipleJob : public CGCClientJob +{ +public: + CGCSOUpdateMultipleJob( CGCClient *pClient ) : CGCClientJob( pClient ) {} + + virtual bool BYieldingRunGCJob( GCSDK::IMsgNetPacket *pNetPacket ) + { + CProtoBufMsg<CMsgSOMultipleObjects> msg( pNetPacket ); + SOCDebug( "CGCSOUpdateJob(owner=%s)\n", CSteamID( msg.Body().owner() ).Render() ); + CGCClientSharedObjectCache *pSOCache = m_pGCClient->FindSOCache( msg.Body().owner() ); + if ( pSOCache ) + { + pSOCache->m_context.PreSOUpdate( eSOCacheEvent_Incremental ); + + for ( int i = 0; i < msg.Body().objects_size(); ++i ) + { + const CMsgSOMultipleObjects_SingleObject &objMessage = msg.Body().objects( i ); + SOCDebug( " type %d\n", objMessage.type_id() ); + pSOCache->BUpdateFromMsg( objMessage.type_id(), objMessage.object_data().data(), objMessage.object_data().size() ); + } + + pSOCache->m_context.PostSOUpdate( eSOCacheEvent_Incremental ); + pSOCache->SetVersion( msg.Body().version() ); + } + return true; + } + +}; + +GC_REG_JOB( CGCClient, CGCSOUpdateMultipleJob, "CGCSOUpdateMultipleJob", k_ESOMsg_UpdateMultiple, GCSDK::k_EServerTypeGCClient ); + +class CGCSOCacheSubscribedJob : public CGCClientJob +{ +public: + CGCSOCacheSubscribedJob( CGCClient *pClient ) : CGCClientJob( pClient ) {} + + virtual bool BYieldingRunGCJob( GCSDK::IMsgNetPacket *pNetPacket ) + { + CProtoBufMsg< CMsgSOCacheSubscribed > msg ( pNetPacket ); + CGCClientSharedObjectCache *pSOCache = m_pGCClient->FindSOCache( msg.Body().owner(), true ); + + Assert( pSOCache ); + if( pSOCache ) + { + SOCDebug( "CGCSOCacheSubscribedJob(owner=%s) [in cache]\n", CSteamID( msg.Body().owner() ).Render() ); + DbgVerify( pSOCache->BParseCacheSubscribedMsg( msg.Body() ) ); + } + else + { + SOCDebug( "CGCSOCacheSubscribedJob(owner=%s) [not in cache]\n", CSteamID( msg.Body().owner() ).Render() ); + } + + m_pGCClient->Test_CacheSubscribed( pSOCache->GetOwner() ); + + return true; + } + +}; + +GC_REG_JOB( CGCClient, CGCSOCacheSubscribedJob, "CGCSOCacheSubscribedJob", k_ESOMsg_CacheSubscribed, GCSDK::k_EServerTypeGCClient ); + +class CGCSOCacheUnsubscribedJob : public CGCClientJob +{ +public: + CGCSOCacheUnsubscribedJob( CGCClient *pClient ) : CGCClientJob( pClient ) {} + + virtual bool BYieldingRunGCJob( GCSDK::IMsgNetPacket *pNetPacket ) + { + CProtoBufMsg< CMsgSOCacheUnsubscribed > msg( pNetPacket ); + SOCDebug( "CGCSOCacheUnsubscribedJob(owner=%s)\n", CSteamID( msg.Body().owner() ).Render() ); + m_pGCClient->NotifySOCacheUnsubscribed( msg.Body().owner() ); + + return true; + } + +}; + +GC_REG_JOB( CGCClient, CGCSOCacheUnsubscribedJob, "CGCSOCacheUnsubscribedJob", k_ESOMsg_CacheUnsubscribed, GCSDK::k_EServerTypeGCClient ); + +class CGCSOCacheSubscriptionCheck : public CGCClientJob +{ +public: + CGCSOCacheSubscriptionCheck( CGCClient *pClient ) : CGCClientJob( pClient ) {} + + virtual bool BYieldingRunGCJob( GCSDK::IMsgNetPacket *pNetPacket ) + { + CProtoBufMsg< CMsgSOCacheSubscriptionCheck > msg ( pNetPacket ); + CGCClientSharedObjectCache *pSOCache = m_pGCClient->FindSOCache( msg.Body().owner(), false ); + + // if we do not have the cache or it is out-of-date, request a refresh + if ( pSOCache == NULL || !pSOCache->BIsInitialized() || pSOCache->GetVersion() != msg.Body().version() ) + { + SOCDebug( "CGCSOCacheSubscriptionCheck(owner=%s) -- need refresh\n", CSteamID( msg.Body().owner() ).Render() ); + CProtoBufMsg< CMsgSOCacheSubscriptionRefresh > msg_response( k_ESOMsg_CacheSubscriptionRefresh ); + msg_response.Body().set_owner( msg.Body().owner() ); + m_pGCClient->BSendMessage( msg_response ); + } + else + { + SOCDebug( "CGCSOCacheSubscriptionCheck(owner=%s) -- up-to-date, no refresh needed\n", CSteamID( msg.Body().owner() ).Render() ); + + // This is one method by which the GC notifies us that we are subscribed. + if ( !pSOCache->BIsSubscribed() ) + { + pSOCache->NotifyResubscribedUpToDate(); + Assert( pSOCache->BIsSubscribed() ); + } + } + return true; + } + +}; + +GC_REG_JOB( CGCClient, CGCSOCacheSubscriptionCheck, "CGCSOCacheSubscriptionCheck", k_ESOMsg_CacheSubscriptionCheck, GCSDK::k_EServerTypeGCClient ); + +} // namespace GCSDK diff --git a/gcsdk/gcclient_sharedobjectcache.cpp b/gcsdk/gcclient_sharedobjectcache.cpp new file mode 100644 index 0000000..506ca44 --- /dev/null +++ b/gcsdk/gcclient_sharedobjectcache.cpp @@ -0,0 +1,586 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Extra functionality on top of CGCClientSharedObjectCache for GCClients +// +//============================================================================= + +#include "stdafx.h" +#include <time.h> +#include "gcsdk/gcclient_sharedobjectcache.h" +#include "gcsdk_gcmessages.pb.h" +#include <typeinfo> + +namespace GCSDK +{ + +//#define SOCDebug(...) Msg( __VA_ARGS__ ) +#define SOCDebug(...) ((void)0) + +//---------------------------------------------------------------------------- +// Purpose: constructor +//---------------------------------------------------------------------------- +CGCClientSharedObjectContext::CGCClientSharedObjectContext( const CSteamID & steamIDOwner ) + : m_steamIDOwner( steamIDOwner ) +{ + +} + + +//---------------------------------------------------------------------------- +// Purpose: Adds a new Listener to the cache. All objects in the cache will +// be sent as create messages to the new Listener +//---------------------------------------------------------------------------- +bool CGCClientSharedObjectContext::BAddListener( ISharedObjectListener *pListener ) +{ + if( m_vecListeners.HasElement( pListener ) ) + return false; + + m_vecListeners.AddToTail( pListener ); + return true; +} + + +//---------------------------------------------------------------------------- +// Purpose: Removes a Listener from the cache. All objects in the cache +// will have destroy messages sent for them to the new Listener. +//---------------------------------------------------------------------------- +bool CGCClientSharedObjectContext::BRemoveListener( ISharedObjectListener *pListener ) +{ + return m_vecListeners.FindAndRemove( pListener ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Send created/updated/destroyed calls on to all the listeners in the +// context +//---------------------------------------------------------------------------- +void CGCClientSharedObjectContext::SOCreated( const CSharedObject *pObject, ESOCacheEvent eEvent ) const +{ + tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s", __FUNCTION__ ); + FOR_EACH_VEC( m_vecListeners, nListener ) + { + m_vecListeners[nListener]->SOCreated( m_steamIDOwner, pObject, eEvent ); + } +} + +void CGCClientSharedObjectContext::PreSOUpdate( ESOCacheEvent eEvent ) const +{ + tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s", __FUNCTION__ ); + FOR_EACH_VEC( m_vecListeners, nListener ) + { + m_vecListeners[nListener]->PreSOUpdate( m_steamIDOwner, eEvent ); + } +} + +void CGCClientSharedObjectContext::SOUpdated( const CSharedObject *pObject, ESOCacheEvent eEvent ) const +{ + tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s", __FUNCTION__ ); + FOR_EACH_VEC( m_vecListeners, nListener ) + { + m_vecListeners[nListener]->SOUpdated( m_steamIDOwner, pObject, eEvent ); + } +} + +void CGCClientSharedObjectContext::PostSOUpdate( ESOCacheEvent eEvent ) const +{ + tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s", __FUNCTION__ ); + FOR_EACH_VEC( m_vecListeners, nListener ) + { + m_vecListeners[nListener]->PostSOUpdate( m_steamIDOwner, eEvent ); + } +} + +void CGCClientSharedObjectContext::SODestroyed( const CSharedObject *pObject, ESOCacheEvent eEvent ) const +{ + tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s", __FUNCTION__ ); + FOR_EACH_VEC( m_vecListeners, nListener ) + { + m_vecListeners[nListener]->SODestroyed( m_steamIDOwner, pObject, eEvent ); + } +} + +void CGCClientSharedObjectContext::SOCacheSubscribed( const CSteamID & steamIDOwner, ESOCacheEvent eEvent ) const +{ + tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s", __FUNCTION__ ); + FOR_EACH_VEC( m_vecListeners, nListener ) + { + m_vecListeners[nListener]->SOCacheSubscribed( steamIDOwner, eEvent ); + } +} + +void CGCClientSharedObjectContext::SOCacheUnsubscribed( const CSteamID & steamIDOwner, ESOCacheEvent eEvent ) const +{ + tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s", __FUNCTION__ ); + FOR_EACH_VEC( m_vecListeners, nListener ) + { + m_vecListeners[nListener]->SOCacheUnsubscribed( steamIDOwner, eEvent ); + } +} + + +//---------------------------------------------------------------------------- +// Purpose: Constructor +//---------------------------------------------------------------------------- +CGCClientSharedObjectTypeCache::CGCClientSharedObjectTypeCache( int nTypeID, const CGCClientSharedObjectContext & context ) + : m_context( context ), CSharedObjectTypeCache( nTypeID ) +{ + +} + + +//---------------------------------------------------------------------------- +// Purpose: Destructor +//---------------------------------------------------------------------------- +CGCClientSharedObjectTypeCache::~CGCClientSharedObjectTypeCache() +{ +} + + +//---------------------------------------------------------------------------- +// Purpose: Parses a cache subscribed message. +//---------------------------------------------------------------------------- +bool CGCClientSharedObjectTypeCache::BParseCacheSubscribedMsg( const CMsgSOCacheSubscribed_SubscribedType & msg, CUtlVector<CSharedObject*> &vecCreatedObjects, CUtlVector<CSharedObject*> &vecUpdatedObjects, CUtlVector<CSharedObject*> &vecObjectsToDestroy ) +{ + CSharedObjectVec vecUntouchedObjects; + for ( uint32 i = 0; i < GetCount(); i++ ) + { + vecUntouchedObjects.AddToTail( GetObject( i ) ); + } + + for( uint16 usObject = 0; usObject < msg.object_data_size(); usObject++ ) + { + bool bUpdatedExisting = false; + CSharedObject *pObject = BCreateFromMsg( msg.object_data( usObject ).data(), msg.object_data( usObject ).size(), &bUpdatedExisting ); + if ( pObject == NULL) + { + Assert( pObject ); + return false; + } + + // if an object was updated, remove it from the untouched list + if ( bUpdatedExisting ) + { + int index = vecUntouchedObjects.Find( pObject ); + if ( index != vecUntouchedObjects.InvalidIndex() ) + { + vecUntouchedObjects[index] = NULL; + } + vecUpdatedObjects.AddToTail( pObject ); + } + else + { + vecCreatedObjects.AddToTail( pObject ); + } + } + + // all objects that weren't in the SubscribedMsg should be destroyed + for ( int i = 0; i < vecUntouchedObjects.Count(); i++ ) + { + if ( vecUntouchedObjects[i] == NULL ) + continue; + + CSharedObject *pObject = RemoveObject( *vecUntouchedObjects[i] ); + Assert( pObject ); + if( pObject ) + vecObjectsToDestroy.AddToTail( pObject ); + } + + return true; +} + +void CGCClientSharedObjectTypeCache::RemoveAllObjects( CUtlVector<CSharedObject*> &vecObjects ) +{ + + // Go in reverse order to avoid O(n^2) shifting the items in the array + for ( int i = GetCount() - 1; i >= 0; i-- ) + { + CSharedObject *pObject = RemoveObjectByIndex( i ); + Assert( pObject ); + if ( pObject ) + vecObjects.AddToTail( pObject ); + } +} + + +//---------------------------------------------------------------------------- +// Purpose: Processes a received create message for an object of this type on +// the client/gameserver +//---------------------------------------------------------------------------- +CSharedObject *CGCClientSharedObjectTypeCache::BCreateFromMsg( const void *pvData, uint32 unSize, bool *bUpdatedExisting ) +{ + CUtlBuffer bufCreate( pvData, unSize, CUtlBuffer::READ_ONLY ); + CSharedObject *pNewObj = CSharedObject::Create( GetTypeID() ); + Assert( pNewObj ); + if( !pNewObj ) + { + EmitError( SPEW_SHAREDOBJ, "Unable to create object of type %d\n", GetTypeID() ); + return NULL; + } + + if( !pNewObj->BParseFromMessage( bufCreate ) ) + { + delete pNewObj; + return NULL; + } + + // Existing object? + CSharedObject *pObj = FindSharedObject( *pNewObj ); + if( pObj ) + { + pObj->Copy( *pNewObj ); + delete pNewObj; + if ( bUpdatedExisting ) + { + *bUpdatedExisting = true; + } + return pObj; + } + + // New object + AddObject( pNewObj ); + if ( bUpdatedExisting ) + { + *bUpdatedExisting = false; + } + return pNewObj; +} + + +//---------------------------------------------------------------------------- +// Purpose: Processes a received destroy message for an object of this type on +// the client/gameserver +//---------------------------------------------------------------------------- +bool CGCClientSharedObjectTypeCache::BDestroyFromMsg( const void *pvData, uint32 unSize ) +{ + CUtlBuffer bufDestroy( pvData, unSize, CUtlBuffer::READ_ONLY ); + CSharedObject *pIndexObj = CSharedObject::Create( GetTypeID() ); + if( !pIndexObj->BParseFromMessage( bufDestroy ) ) + { + delete pIndexObj; + return false; + } + + CSharedObject *pObject = RemoveObject( *pIndexObj ); + if( pObject ) + { + m_context.SODestroyed( pObject, eSOCacheEvent_Incremental ); + delete pObject; + } + + delete pIndexObj; + return true; +} + + +//---------------------------------------------------------------------------- +// Purpose: Processes a received destroy message for an object of this type on +// the client/gameserver +//---------------------------------------------------------------------------- +bool CGCClientSharedObjectTypeCache::BUpdateFromMsg( const void *pvData, uint32 unSize ) +{ + CUtlBuffer bufUpdate( pvData, unSize, CUtlBuffer::READ_ONLY ); + CSharedObject *pIndexObj = CSharedObject::Create( GetTypeID() ); + AssertMsg1( pIndexObj, "Unable to create index object of type %d", GetTypeID() ); + if( !pIndexObj ) + return false; + if( !pIndexObj->BParseFromMessage( bufUpdate ) ) + { + delete pIndexObj; + return false; + } + + CSharedObject *pObj = FindSharedObject( *pIndexObj ); + bool bRet = false; + if( pObj ) + { + bufUpdate.SeekGet( CUtlBuffer::SEEK_HEAD, 0 ); + + bRet = pObj->BUpdateFromNetwork( *pIndexObj ); + m_context.SOUpdated( pObj, eSOCacheEvent_Incremental ); + } + + delete pIndexObj; + return bRet; +} + + +//---------------------------------------------------------------------------- +// Purpose: Constructor +//---------------------------------------------------------------------------- +CGCClientSharedObjectCache::CGCClientSharedObjectCache( const CSteamID & steamIDOwner ) + : m_context( steamIDOwner ), + m_bInitialized( false ), + m_bSubscribed( false ) +{ + +} + + +//---------------------------------------------------------------------------- +// Purpose: Destructor +//---------------------------------------------------------------------------- +CGCClientSharedObjectCache::~CGCClientSharedObjectCache() +{ +} + + +//---------------------------------------------------------------------------- +// Purpose: Process an incoming create message on a client/gameserver. +//---------------------------------------------------------------------------- +bool CGCClientSharedObjectCache::BParseCacheSubscribedMsg( const CMsgSOCacheSubscribed & msg ) +{ + + // Assume all type caches will be untouched + CUtlVector<int> vecUntouchedTypes; + for ( int i = FirstTypeCacheIndex(); i != InvalidTypeCacheIndex(); i = NextTypeCacheIndex( i ) ) + { + CSharedObjectTypeCache *pTypeCache = GetTypeCacheByIndex( i ); + if ( pTypeCache ) + { + vecUntouchedTypes.AddToTail( pTypeCache->GetTypeID() ); + } + } + + // List of objects created, updated, and removed + CUtlVector<CSharedObject*> vecCreatedObjects; + CUtlVector<CSharedObject*> vecUpdatedObjects; + CUtlVector<CSharedObject*> vecObjectsToDestroy; + + bool bResult = true; + + // Scan types in message + for( uint16 usObject = 0; usObject < msg.objects_size(); usObject++ ) + { + const CMsgSOCacheSubscribed_SubscribedType & msgType = msg.objects( usObject ); + + // Find or create the type + CGCClientSharedObjectTypeCache *pTypeCache = CreateTypeCache( msgType.type_id() ); + if ( pTypeCache ) + { + int index = vecUntouchedTypes.Find( pTypeCache->GetTypeID() ); + if ( index != vecUntouchedTypes.InvalidIndex() ) + { + vecUntouchedTypes[index] = -1; + } + } + Assert( pTypeCache ); + if( !pTypeCache || !pTypeCache->BParseCacheSubscribedMsg( msgType, vecCreatedObjects, vecUpdatedObjects, vecObjectsToDestroy ) ) + bResult = false; + } + + // any type caches that weren't in the SubscribedMsg should be cleared + for ( int i = FirstTypeCacheIndex(); i != InvalidTypeCacheIndex(); i = NextTypeCacheIndex( i ) ) + { + CGCClientSharedObjectTypeCache *pTypeCache = GetTypeCacheByIndex( i ); + if ( vecUntouchedTypes.Find( pTypeCache->GetTypeID() ) != vecUntouchedTypes.InvalidIndex() ) + { + pTypeCache->RemoveAllObjects( vecObjectsToDestroy ); + } + } + + // Which event is happening? + ESOCacheEvent eNotificationEvent = eSOCacheEvent_Subscribed; + if ( m_bSubscribed ) + eNotificationEvent = eSOCacheEvent_Resubscribed; + + // Set version, assuming we didn't have any problems. If we hit any problems, + // we want to force a refresh + if ( bResult ) + SetVersion( msg.version() ); + + // Mark that the cache has been initialized by the server + m_bInitialized = true; + m_bSubscribed = true; + + // + // Send notifications + // + + // Initial cache subscribed + m_context.SOCacheSubscribed( GetOwner(), eNotificationEvent ); + + // Deletions + for ( int i = 0 ; i < vecObjectsToDestroy.Count() ; ++i ) + { + m_context.SODestroyed( vecObjectsToDestroy[i], eNotificationEvent ); + delete vecObjectsToDestroy[i]; + } + + // Updates + for ( int i = 0 ; i < vecUpdatedObjects.Count() ; ++i ) + { + m_context.SOUpdated( vecUpdatedObjects[i], eNotificationEvent ); + } + + // Created + for ( int i = 0 ; i < vecCreatedObjects.Count() ; ++i ) + { + m_context.SOUpdated( vecCreatedObjects[i], eNotificationEvent ); + } + + // Return true if everything parsed OK, or false + // if we had at least one failure + return bResult; +} + + +//---------------------------------------------------------------------------- +// Purpose: Process an incoming create message on a client/gameserver. +//---------------------------------------------------------------------------- +void CGCClientSharedObjectCache::NotifyUnsubscribe() +{ + if ( m_bSubscribed ) + { + m_bSubscribed = false; + m_context.SOCacheUnsubscribed( GetOwner(), eSOCacheEvent_Unsubscribed ); + } + else + { + AssertMsg( m_bSubscribed, "GC Sending us Unsubscribed message when we weren't subscribed" ); // Might not be a bug, but something worth checking + } +} + +//---------------------------------------------------------------------------- +// Purpose: GC is telling us that the version we have is up-to-date,a nd we are subscribed +//---------------------------------------------------------------------------- +void CGCClientSharedObjectCache::NotifyResubscribedUpToDate() +{ + if ( !m_bSubscribed ) + { + Assert( m_bInitialized ); + m_bSubscribed = true; + m_context.SOCacheSubscribed( GetOwner(), eSOCacheEvent_Subscribed ); + } + else + { + AssertMsg( m_bSubscribed, "Got NotifyResubscribedUpToDate when we were already subscribed?" ); // Might not be a bug, but something worth checking + } +} + + +//---------------------------------------------------------------------------- +// Purpose: Process an incoming create message on a client/gameserver. +//---------------------------------------------------------------------------- +bool CGCClientSharedObjectCache::BCreateFromMsg( int nTypeID, const void *pvData, uint32 unSize ) +{ + // We should be subscribed + if ( !m_bInitialized || !m_bSubscribed ) + { + // Note: We can go down and come back up without the GC knowing this. + // So this can happen + //Assert( m_bInitialized ); + //Assert( m_bSubscribed ); + //EmitWarning( SPEW_SHAREDOBJ, 1, "Received SOCache incremental update for cache we were not subscribed to (object type %d)\n", nTypeID ); + } + + // Locate / create the type cache + CGCClientSharedObjectTypeCache *pTypeCache = CreateTypeCache( nTypeID ); + + // Create the message or update existing + bool bUpdatedExisting = false; + CSharedObject *pObject = pTypeCache->BCreateFromMsg( pvData, unSize, &bUpdatedExisting ); + if ( pObject == NULL ) + return false; + + // Send notifications to listeners + if ( bUpdatedExisting ) + { + // This can happen --- see comment at the top of this function + //Assert( !bUpdatedExisting ); // shouldn't the GC know what it's already sent us? This is weird + m_context.SOUpdated( pObject, eSOCacheEvent_Incremental ); + } + else + { + m_context.SOCreated( pObject, eSOCacheEvent_Incremental ); + } + + return true; +} + + +//---------------------------------------------------------------------------- +// Purpose: Processes an incoming destroy message on a client/gameserver. +//---------------------------------------------------------------------------- +bool CGCClientSharedObjectCache::BDestroyFromMsg( int nTypeID, const void *pvData, uint32 unSize ) +{ + CGCClientSharedObjectTypeCache *pTypeCache = FindTypeCache( nTypeID ); + if( pTypeCache ) + { + return pTypeCache->BDestroyFromMsg( pvData, unSize ); + } + else + { + return false; + } +} + +//---------------------------------------------------------------------------- +// Purpose: Processes an incoming update message on a client/gameserver. +//---------------------------------------------------------------------------- +bool CGCClientSharedObjectCache::BUpdateFromMsg( int nTypeID, const void *pvData, uint32 unSize ) +{ + CGCClientSharedObjectTypeCache *pTypeCache = FindTypeCache( nTypeID ); + if( pTypeCache ) + { + return pTypeCache->BUpdateFromMsg( pvData, unSize ); + } + else + { + return false; + } +} + +//---------------------------------------------------------------------------- +// Purpose: Adds a listener object to be notified of object changes in this +// cache. The shared object cache does not own this object and will +// not free it. +//---------------------------------------------------------------------------- +void CGCClientSharedObjectCache::AddListener( ISharedObjectListener *pListener ) +{ + Assert( pListener ); + if ( !m_context.BAddListener( pListener ) ) + return; // was already listening, no action needed + + SOCDebug( "[%s] Adding listener %s\n", GetOwner().Render(), typeid( *pListener ).name() ); + + // If we're already subscribed, then immediately send notifications + if( BIsSubscribed() ) + { + pListener->SOCacheSubscribed( GetOwner(), eSOCacheEvent_ListenerAdded ); + for ( int i = FirstTypeCacheIndex(); i != InvalidTypeCacheIndex(); i = NextTypeCacheIndex( i ) ) + { + CGCClientSharedObjectTypeCache *pTypeCache = GetTypeCacheByIndex( i ); + for ( uint32 j = 0 ; j < pTypeCache->GetCount() ; ++j ) + { + CSharedObject *pObject = pTypeCache->GetObject( j ); + pListener->SOCreated( GetOwner(), pObject, eSOCacheEvent_ListenerAdded ); + } + } + } +} + + +//---------------------------------------------------------------------------- +// Purpose: Removes a listener object from the list to be notified of changes +// to this object cache. +//---------------------------------------------------------------------------- +bool CGCClientSharedObjectCache::RemoveListener( ISharedObjectListener *pListener ) +{ + Assert( pListener ); + if ( !m_context.BRemoveListener( pListener ) ) + return false; // wasn't already listening, nothing to do + + SOCDebug( "[%s] Removing listener %s\n", GetOwner().Render(), typeid( *pListener ).name() ); + + // If we were subscribed, then the listener's last subscribe notification + // was a "you are subscribed." Send him an unsubscribed notification + // so he doesn't think he's still subscribed. + if( BIsSubscribed() ) + { + pListener->SOCacheUnsubscribed( GetOwner(), eSOCacheEvent_ListenerRemoved ); + } + + return true; +} + +} // namespace GCSDK diff --git a/gcsdk/gcconstants.cpp b/gcsdk/gcconstants.cpp new file mode 100644 index 0000000..5e4e83d --- /dev/null +++ b/gcsdk/gcconstants.cpp @@ -0,0 +1,45 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Enum name to string and vice versa code +// +//============================================================================= + + +#include "stdafx.h" +#include "enumutils.h" +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + +#ifdef GC + ENUMSTRINGS_START( EForeignKeyAction ) + {k_EForeignKeyActionNoAction, "NO ACTION" }, + {k_EForeignKeyActionCascade, "CASCADE" }, + {k_EForeignKeyActionSetNULL, "SET NULL" }, + ENUMSTRINGS_REVERSE( EForeignKeyAction, k_EForeignKeyActionNoAction ) + + ENUMSTRINGS_START( EGCSQLType ) + {k_EGCSQLTypeInvalid, "k_EGCSQLTypeInvalid" }, + {k_EGCSQLType_Blob, "k_EGCSQLType_Blob" }, + {k_EGCSQLType_String, "k_EGCSQLType_String" }, + {k_EGCSQLType_int8, "k_EGCSQLType_int8" }, + {k_EGCSQLType_int16, "k_EGCSQLType_int16" }, + {k_EGCSQLType_int32, "k_EGCSQLType_int32" }, + {k_EGCSQLType_int64, "k_EGCSQLType_int64" }, + {k_EGCSQLType_float, "k_EGCSQLType_float" }, + {k_EGCSQLType_double, "k_EGCSQLType_double" }, + {k_EGCSQLType_Image, "k_EGCSQLType_Image" }, + ENUMSTRINGS_REVERSE( EGCSQLType, k_EGCSQLTypeInvalid ) +#endif + + ENUMSTRINGS_START( EUniverse ) + { k_EUniverseInvalid, "Invalid" }, + { k_EUniversePublic, "Public" }, + { k_EUniverseBeta, "Beta" }, + { k_EUniverseInternal, "Internal" }, + { k_EUniverseDev, "Dev" }, + ENUMSTRINGS_REVERSE( EUniverse, k_EUniverseInvalid ) + +} // namespace GCSDK diff --git a/gcsdk/gcdirtyfield.cpp b/gcsdk/gcdirtyfield.cpp new file mode 100644 index 0000000..10e0ed7 --- /dev/null +++ b/gcsdk/gcdirtyfield.cpp @@ -0,0 +1,190 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Network dirty field marker for shared objects +// +//============================================================================= +#include "stdafx.h" +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + +CSharedObjectDirtyFieldList::CSharedObjectDirtyFieldList( CSharedObject *obj ) +: m_obj( obj ) +, m_firstFieldBits( 0 ) +, m_pExtendedFields( NULL ) +{ +} + +CSharedObjectDirtyFieldList::~CSharedObjectDirtyFieldList() +{ + if ( m_pExtendedFields ) + { + delete m_pExtendedFields; + } +} + +CSharedObject *CSharedObjectDirtyFieldList::Obj() const +{ + return m_obj; +} + +void CSharedObjectDirtyFieldList::DirtyField( int index ) +{ + // if the field index fits within our dirty fields struct, set a bit to track it + if ( index < 32 ) + { + m_firstFieldBits |= (1 << index); + } + // if the field index is too big, store it in our backup structure; this is less efficient but more + // flexible, especially when dealing with fields that themselves contain flags so would be interpreted + // as huge numbers + else + { + if ( !m_pExtendedFields ) + m_pExtendedFields = new CUtlVector<int>; + + if ( !m_pExtendedFields->HasElement( index ) ) + m_pExtendedFields->AddToTail( index ); + } +} + +void CSharedObjectDirtyFieldList::GetDirtyFieldSet( CUtlVector<int> &fieldSet ) const +{ + fieldSet.Purge(); + if( !m_firstFieldBits && !m_pExtendedFields ) + { + return; + } + + if( m_firstFieldBits ) + { + for( int i = 0; i < 32; i++ ) + { + // handle dirty bits + if( m_firstFieldBits & ( 1 << i ) ) + { + fieldSet.AddToTail( i ); + } + } + } + + // handle higher order dirty-fields list if present + if ( m_pExtendedFields ) + { + fieldSet.AddVectorToTail( *m_pExtendedFields ); + } +} + +//----------------------------------------------------------------------------- +CSharedObjectDirtyList::CSharedObjectDirtyList() +{ +} + +CSharedObjectDirtyList::~CSharedObjectDirtyList() +{ +} + +void CSharedObjectDirtyList::DirtyObjectField( CSharedObject *obj, int field ) +{ + MEM_ALLOC_CREDIT_CLASS(); + Assert( obj != NULL ); + if (!obj) + { + return; + } + + int index = FindIndexByObj( obj ); + if ( index == InvalidIndex() ) + { + index = m_sharedObjectDirtyFieldList.AddToTail( CSharedObjectDirtyFieldList( obj ) ); + } + + m_sharedObjectDirtyFieldList[index].DirtyField( field ); +} + +int CSharedObjectDirtyList::FindIndexByObj( const CSharedObject *pObj ) const +{ + // Slow method for now, we can speed this up with a map later + for ( int i = 0; i < m_sharedObjectDirtyFieldList.Count(); ++i ) + { + if( m_sharedObjectDirtyFieldList[i].Obj() == pObj ) + { + return i; + } + } + + return InvalidIndex(); +} + +bool CSharedObjectDirtyList::HasElement( const CSharedObject *pObj ) const +{ + return FindIndexByObj( pObj ) != InvalidIndex(); +} + +bool CSharedObjectDirtyList::GetDirtyFieldSetByIndex( int index, CSharedObject **ppObj, CUtlVector<int> &fieldSet ) const +{ + VPROF_BUDGET( "CSharedObjectDirtyList::GetDirtyFieldSetByIndex", VPROF_BUDGETGROUP_STEAM ); + if( !m_sharedObjectDirtyFieldList.IsValidIndex( index ) ) + { + fieldSet.Purge(); + if( ppObj ) + { + *ppObj = NULL; + } + return false; + } + const CSharedObjectDirtyFieldList &fieldList = m_sharedObjectDirtyFieldList[index]; + if( ppObj ) + { + *ppObj = fieldList.Obj(); + } + fieldList.GetDirtyFieldSet( fieldSet ); + return true; +} + +bool CSharedObjectDirtyList::GetDirtyFieldSetByObj( CSharedObject *pObj, CUtlVector<int> &fieldSet ) +{ + int index = FindIndexByObj( pObj ); + if ( index == InvalidIndex() ) + { + fieldSet.Purge(); + return false; + } + + CSharedObjectDirtyFieldList &fieldList = m_sharedObjectDirtyFieldList[index]; + fieldList.GetDirtyFieldSet( fieldSet ); + return true; +} + +bool CSharedObjectDirtyList::FindAndRemove( CSharedObject *pObj ) +{ + int index = FindIndexByObj( pObj ); + if ( index == InvalidIndex() ) + { + return false; + } + + m_sharedObjectDirtyFieldList.Remove( index ); + return true; +} + +void CSharedObjectDirtyList::RemoveAll() +{ + m_sharedObjectDirtyFieldList.RemoveAll(); +} + +#ifdef DBGFLAG_VALIDATE +void CSharedObjectDirtyList::Validate( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE(); + + ValidateObj( m_sharedObjectDirtyFieldList ); +} +#endif + +} // namespace GCSDK + + + diff --git a/gcsdk/gchost.cpp b/gcsdk/gchost.cpp new file mode 100644 index 0000000..6ff28f7 --- /dev/null +++ b/gcsdk/gchost.cpp @@ -0,0 +1,26 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Holds a pointer to the GC's host's interface +// +//============================================================================= +#include "stdafx.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ +IGameCoordinatorHost *g_pGCHost = NULL; + +IGameCoordinatorHost *GGCHost() +{ + return g_pGCHost; +} + + +void SetGCHost( IGameCoordinatorHost *pHost ) +{ + g_pGCHost = pHost; +} + +} // namespace GCSDK diff --git a/gcsdk/gcinterface.cpp b/gcsdk/gcinterface.cpp new file mode 100644 index 0000000..f3d408b --- /dev/null +++ b/gcsdk/gcinterface.cpp @@ -0,0 +1,1085 @@ +//====== Copyright �, Valve Corporation, All rights reserved. ================= +// +// Purpose: Defines the GC interface exposed to the host +// +//============================================================================= + + +#include "stdafx.h" +#include "winlite.h" + +#include "tier0/minidump.h" +#include "tier1/interface.h" +#include "appframework/iappsystemgroup.h" +#include "filesystem.h" +#include "vstdlib/cvar.h" +#include "signal.h" + +#include "gcsdk/steamextra/rtime.h" +#include "gcsdk/directory.h" +#include "gcsdk/gcinterface.h" + + +#define WINDOWS_LEAN_AND_MEAN +#if !defined( _WIN32_WINNT ) +#define _WIN32_WINNT 0x0403 +#endif +#include <windows.h> + + +namespace GCSDK +{ +static GCConVar cv_assert_minidump_window( "assert_minidump_window", "28800", "Size of the minidump window in seconds. Each unique assert will dump at most assert_max_minidumps_in_window times in this many seconds" ); +static GCConVar cv_assert_max_minidumps_in_window( "assert_max_minidumps_in_window", "5", "The amount of times each unique assert will write a dump in assert_minidump_window seconds" ); +static GCConVar enable_assert_minidumps( "enable_assert_minidumps", "1", "An emergency shutoff to prevent the recording or tracking of asserts" ); +static GCConVar filter_blank_lines( "filter_blank_lines", "1", "Prevents blank lines from being written or logged" ); + +//----------------------------------------------------------------------------- +// Purpose: Creates a global pointer to the interface and exposes it to the host +//----------------------------------------------------------------------------- +CGCInterface g_GCInterface; +EXPOSE_SINGLE_INTERFACE_GLOBALVAR( CGCInterface, IGameCoordinator, GAMECOORDINATOR_INTERFACE_VERSION, g_GCInterface ); + +int32 CGCInterface::CDisableAssertRateLimit::s_nDisabledCount = 0; + +// Force the linker to include this even though we're in a static lib +void ForceIncludeGCInterface() +{ + #pragma comment( linker, "/INCLUDE:" __FUNCDNAME__ ) + void *pUnused = &__g_CreateCGCInterfaceIGameCoordinator_reg; + pUnused = NULL; + +#ifdef DEBUG + // Adds a note for the deploy tool to not let it prop with a debug GCSDK + printf( "is a debug binary" ); +#endif +} + + +//----------------------------------------------------------------------------- +// Purpose: Overrides the spew func used by Msg and DMsg to print to the console +//----------------------------------------------------------------------------- +class CConsoleLoggingListener : public ILoggingListener +{ +public: + virtual void Log( const LoggingContext_t *pContext, const tchar *pMessage ) + { + const char *pszFmt = ( sizeof( tchar ) == sizeof( char ) ) ? "%hs" : "%ls"; + switch ( pContext->m_Severity ) + { + default: + case LS_MESSAGE: + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, pszFmt, pMessage ); + break; + + case LS_WARNING: + EmitWarning( SPEW_CONSOLE, SPEW_ALWAYS, pszFmt, pMessage ); + break; + + case LS_ERROR: + case LS_HIGHEST_SEVERITY: + EmitError( SPEW_CONSOLE, pszFmt, pMessage ); + break; + + case LS_ASSERT: + //if this assert is in a job, display the name of the job as well + if ( ThreadInMainThread() && ( g_pJobCur != NULL ) ) + { + pszFmt = ( sizeof( tchar ) == sizeof( char ) ) ? "[Job %s] %hs" : "[Job %s] %ls"; + EmitAssertError( SPEW_CONSOLE, pszFmt, g_pJobCur->GetName(), pMessage ); + } + else + { + EmitAssertError( SPEW_CONSOLE, pszFmt, pMessage ); + } + break; + } + } +}; +static CNonFatalLoggingResponsePolicy s_NonFatalLoggingResponsePolicy; +static CConsoleLoggingListener s_ConsoleLoggingListener; + + +//----------------------------------------------------------------------------- +// Purpose: Prints an assert to the console +//----------------------------------------------------------------------------- +class CGCAssertionFailureListener : public IAssertionFailureListener +{ +public: + CGCAssertionFailureListener( void ) + : IAssertionFailureListener( false ) + { + } + + virtual void *AssertionFailure( const char *pFormattedMsg, const tchar *pchFile, int nLine, const tchar *pchFunction, const tchar *pchRawExpression, int nInstanceReportCount, AssertionType_t nType, bool bFatal ) OVERRIDE + { + if ( Plat_IsInDebugSession() ) + return NULL; + + bool bShouldWriteMinidump = false; + GGCInterface()->RecordAssert( pchFile, nLine, pFormattedMsg, &bShouldWriteMinidump ); + + return bShouldWriteMinidump ? this : NULL; + } + + virtual void MiniDumpHandler( const MiniDumpHandlerData_t &HandlerData, const char *pFormattedMsg, const tchar *pchFile, int nLine, const tchar *pchFunction, const tchar *pchRawExpression, int nInstanceReportCount, AssertionType_t nType, bool bFatal ) OVERRIDE + { + //re-route to default minidump handler (treat it the same as a crash) + CFmtStr minidumpNameToken( "assert_%s_%d", V_GetFileName( pchFile ), nLine ); + MiniDumpOptionalData_t optionalData( minidumpNameToken.Access() ); + + MiniDumpHandlerData_t modifiableHandlerData( HandlerData ); + modifiableHandlerData.SetOptionalData( optionalData ); + + //write to disk + Tier0GenericMiniDumpHandlerEx( modifiableHandlerData, NULL, MINIDUMP_ADDITIONAL_FLAG_PRINT_MESSAGE ); + } +}; +static CGCAssertionFailureListener sg_GCAssertionFailureHandler; + + +static void ProtobufLogHandler( ::google::protobuf::LogLevel level, const char* filename, int line, const std::string& message ) +{ + EG_MSG( g_EGMessages, "Protobuf %s(%d): %s\n", filename, line, message.c_str() ); + AssertFatalMsg( level != google::protobuf::LOGLEVEL_FATAL, "Fatal protobuf assert %s(%d): %s", filename, line, message.c_str() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Initializes the underlying libraries +//----------------------------------------------------------------------------- +static class CGCAppSystemGroup : public CAppSystemGroup +{ +public: + CGCAppSystemGroup() {} + void SetPath ( const char *pchBinaryPath ) { m_sBinaryPath = pchBinaryPath; } + + // Implementation of IAppSystemGroup + virtual bool Create() OVERRIDE + { + AppModule_t cvarModule = LoadModule( VStdLib_GetICVarFactory() ); + AddSystem( cvarModule, CVAR_INTERFACE_VERSION ); + + AppSystemInfo_t appSystems[] = + { + { "filesystem_stdio.dll", FILESYSTEM_INTERFACE_VERSION }, + { "", "" } // Required to terminate the list + }; + + CUtlVector<CUtlString> vecFullPaths; + AppSystemInfo_t *pSystem = appSystems; + while( pSystem->m_pModuleName[0] != '\0' ) + { + CUtlString &strNewPath = vecFullPaths[ vecFullPaths.AddToTail() ]; + strNewPath.Format( "%s%s%s", m_sBinaryPath.Get(), CORRECT_PATH_SEPARATOR_S, pSystem->m_pModuleName ); + pSystem->m_pModuleName = strNewPath.Get(); + + pSystem++; + } + + return AddSystems( appSystems ); + } + + virtual bool PreInit() OVERRIDE + { + CreateInterfaceFn factory = GetFactory(); + ConnectTier1Libraries( &factory, 1 ); + ConnectTier2Libraries( &factory, 1 ); + + if( !g_pFullFileSystem ) + return false; + + if ( !g_pCVar ) + return false; + + ConVar_Register(); + + return true; + } + + virtual void PostShutdown() OVERRIDE + { + ConVar_Unregister(); + DisconnectTier2Libraries(); + DisconnectTier1Libraries(); + } + + virtual void Destroy() OVERRIDE {} + + // this should never be called + virtual int Main( ) OVERRIDE { return -1; } + +private: + CUtlString m_sBinaryPath; +} g_gcAppSystemGroup; + + +//----------------------------------------------------------------------------- +// Purpose: Gets the global instance +//----------------------------------------------------------------------------- +CGCInterface *GGCInterface() +{ + return &g_GCInterface; +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CGCInterface::CGCInterface() + : m_pGCHost( NULL ) + , m_pGC( NULL ) + , m_pGCDirProcess( NULL ) + , m_nAppID( k_uAppIdInvalid ) + , m_eUniverse( k_EUniverseInvalid ) + , m_bDevMode( false ) + , m_ullGID( 0 ) + , m_bLogCaptureEnabled( false ) + , m_nVersion( 0 ) + , m_hParentProcess( NULL ) +{ +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CGCInterface::~CGCInterface() +{ + m_BlockEmitStrings.PurgeAndDeleteElements(); + ClearAssertInfo(); + delete m_pGC; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets the actual GC referred to by the interface +//----------------------------------------------------------------------------- +IGameCoordinator *CGCInterface::GetGC() +{ + return m_pGC; +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns true if the GC is running in a dev environment +//----------------------------------------------------------------------------- +bool CGCInterface::BIsDevMode() const +{ + return m_bDevMode; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets the GC's appID +//----------------------------------------------------------------------------- +AppId_t CGCInterface::GetAppID() const +{ + return m_nAppID; +} + +//----------------------------------------------------------------------------- +// Purpose: Gets the directory gc.dll is running in +//----------------------------------------------------------------------------- +const char *CGCInterface::GetGCDLLPath() const +{ + return m_sGCDLLPath; +} + +//----------------------------------------------------------------------------- +// Purpose: Reads the config KV from the disk +//----------------------------------------------------------------------------- +bool CGCInterface::BReadConfigDirectory( KeyValuesAD& configValues ) +{ + // Read the config file + const char *pchBaseConfigName = NULL; + + switch( GetUniverse() ) + { + case k_EUniversePublic: pchBaseConfigName = "gcconfig_public.vdf"; break; + case k_EUniverseBeta: pchBaseConfigName = "gcconfig_beta.vdf"; break; + case k_EUniverseInternal: pchBaseConfigName = "gcconfig_internal.vdf"; break; + case k_EUniverseDev: pchBaseConfigName = "gcconfig_dev.vdf"; break; + } + + if( !pchBaseConfigName || !configValues->LoadFromFile( g_pFullFileSystem, pchBaseConfigName, "CONFIG" ) ) + { + GCSDK::EmitError( SPEW_GC, "Unable to read config file: %s. Aborting.\n", pchBaseConfigName ? pchBaseConfigName : "unknown universe specified" ); + return false; + } + + //load up our directory + if ( !GDirectory()->BInit( configValues->FindKey( "directory" ) ) ) + { + GCSDK::EmitError( SPEW_GC, "Unable to find 'directory' key within config file %s.\n", pchBaseConfigName ); + return false; + } + + return true; +} + +bool CGCInterface::BReadConvars( KeyValuesAD& configValues ) +{ + //load the standard global convars + InitConVars( configValues->FindKey( "convars" ) ); + + //we can't load more if we don't have a directory as we don't know our GC type + if( !m_pGCDirProcess ) + { + AssertMsg( false, "Attempted to read console variables without any GC type specified" ); + return false; + } + + //get convars for this specific configuration name + InitConVars( configValues->FindKey( CFmtStr( "%s-process-convars", m_pGCDirProcess->GetName() ) ) ); + + //now load the GC type specific convars. Note that these can stomp, so we must do all of them in sequence + for( uint32 nInstance = 0; nInstance < m_pGCDirProcess->GetTypeInstanceCount(); nInstance++ ) + { + const char* pszTypeName = GDirectory()->GetNameForGCType( m_pGCDirProcess->GetTypeInstance( nInstance )->GetType() ); + InitConVars( configValues->FindKey( CFmtStr( "%s-type-convars", pszTypeName ) ) ); + } + + //see if they have a special config associated with this GC + if( m_pGCDirProcess->GetConfig( ) ) + { + const char* pszAdditionalConvars = m_pGCDirProcess->GetConfig()->GetString( "convars", NULL ); + if( pszAdditionalConvars ) + { + //now load the convars that are specific to this instance + InitConVars( configValues->FindKey( CFmtStr( "%s-convars", pszAdditionalConvars ) ) ); + } + } + + + if ( k_EUniverseDev != GetUniverse() ) + { + // See if there's a convar override file + KeyValuesAD pkvSavedConvars( "convars" ); + + if( pkvSavedConvars->LoadFromFile( g_pFullFileSystem, CFmtStr( "%u_%s_%s_savedconvars.vdf", GetAppID(), m_pGCDirProcess->GetName(), PchNameFromEUniverse( GetUniverse() ) ), "CONFIG" ) ) + { + InitConVars( pkvSavedConvars ); + } + else + { + EmitInfo( SPEW_GC, SPEW_ALWAYS, LOG_ALWAYS, "Unable to read saved convars file. Continuing with defaults.\n" ); + } + } + + return true; +} + + + +//----------------------------------------------------------------------------- +// Purpose: Sets the values of convars from the given KV +//----------------------------------------------------------------------------- +void CGCInterface::InitConVars( KeyValues *pkvConvars ) +{ + // init all the convars + if( !pkvConvars ) + return; + + FOR_EACH_VALUE( pkvConvars, pkvVar ) + { + if ( !pkvVar->GetString() ) + { + EmitWarning( SPEW_CONSOLE, SPEW_ALWAYS, "variable %s missing value, skipping\n", pkvVar->GetName() ); + } + + ConVar *pVar = NULL; + const char *pchSuffix = V_strrchr( pkvVar->GetName(), '_' ); + if ( NULL != pchSuffix && 0 == V_strcmp( pchSuffix, CFmtStr( "_%u", GetAppID() ) ) ) + { + pVar = g_pCVar->FindVar( pkvVar->GetName() ); + } + else + { + pVar = g_pCVar->FindVar( CFmtStr( "%s_%u", pkvVar->GetName(), GetAppID() ) ); + } + + if ( !pVar ) + { + EmitWarning( SPEW_CONSOLE, SPEW_ALWAYS, "config file references unknown convar %s\n", pkvVar->GetName() ); + } + else + { + pVar->SetValue( pkvVar->GetString() ); + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Writes the current non-default convars to disk +//----------------------------------------------------------------------------- +bool CGCInterface::BSaveConvars() +{ + //do nothing if we haven't loaded the directory + if( !m_pGCDirProcess ) + return false; + + // copy all the non-default convars to the config + KeyValuesAD pkvConvars( "convars" ); + + ICvar::Iterator iter( g_pCVar ); + for ( iter.SetFirst(); iter.IsValid(); iter.Next() ) + { + const ConCommandBase *pCommand = iter.Get(); + const GCConVar *pVar = dynamic_cast<const GCConVar *>( pCommand ); + + if( pVar && 0 != Q_strcmp( pVar->GetString(), pVar->GetDefault() ) ) + { + KeyValues *pkvVar = pkvConvars->FindKey( pVar->GetBaseName(), true ); + pkvVar->SetStringValue( pVar->GetString() ); + } + } + + return pkvConvars->SaveToFile( g_pFullFileSystem, CFmtStr( "%u_%s_%s_savedconvars.vdf", GetAppID(), m_pGCDirProcess->GetName(), PchNameFromEUniverse( GetUniverse() ) ), "CONFIG" ); +} + +//----------------------------------------------------------------------------- +// Purpose: Construct a Steam ID for a client, given an account ID +//----------------------------------------------------------------------------- +CSteamID CGCInterface::ConstructSteamIDForClient( AccountID_t unAccountID ) const +{ + return CSteamID( unAccountID, m_eUniverse, k_EAccountTypeIndividual ); +} + +//----------------------------------------------------------------------------- +void CGCInterface::ClearAssertWindowCounts() +{ + FOR_EACH_DICT_FAST( m_dictAsserts, nCurrFile ) + { + FOR_EACH_VEC( *m_dictAsserts[ nCurrFile ], nCurrAssert ) + { + ( *m_dictAsserts[ nCurrFile ] )[ nCurrAssert ]->m_nWindowFired = 0; + } + } +} + +//----------------------------------------------------------------------------- +void CGCInterface::ClearAssertInfo() +{ + FOR_EACH_DICT_FAST( m_dictAsserts, nCurrAssert ) + { + m_dictAsserts[ nCurrAssert ]->PurgeAndDeleteElements(); + } + m_dictAsserts.PurgeAndDeleteElements(); +} + +//----------------------------------------------------------------------------- +// Purpose: Records an assert and optionally passes back if we should write +// a minidump based on it +//----------------------------------------------------------------------------- +void CGCInterface::RecordAssert( const char *pchFile, int nLine, const char *pchMessage, bool *pbShouldWriteMinidump ) +{ + //assume we are not writing a dump by default + if( pbShouldWriteMinidump ) + *pbShouldWriteMinidump = false; + + //handle an emergency disable of asserts + if( !enable_assert_minidumps.GetBool() ) + return; + + //get our entry in our map + int iDict = m_dictAsserts.Find( pchFile ); + if ( !m_dictAsserts.IsValidIndex( iDict ) ) + { + iDict = m_dictAsserts.Insert( pchFile, new CUtlVector< AssertInfo_t* > ); + } + + CUtlVector< AssertInfo_t* > &vecAsserts = *m_dictAsserts[iDict]; + //see if we have an entry for this line already + AssertInfo_t* pAssert = NULL; + FOR_EACH_VEC( vecAsserts, nCurrAssert ) + { + if( ( uint32 )nLine == vecAsserts[ nCurrAssert ]->m_nLine ) + { + pAssert = vecAsserts[ nCurrAssert ]; + break; + } + } + + //one wasn't already in the list, so we need to create and insert it + if( !pAssert ) + { + pAssert = new AssertInfo_t; + pAssert->m_nLine = nLine; + pAssert->m_sMsg = pchMessage; + pAssert->m_nWindowFired = 0; + pAssert->m_nTotalFired = 0; + pAssert->m_nTotalRecorded = 0; + vecAsserts.AddToTail( pAssert ); + + //also, remove any newlines from the asserts. The default assert inserts them and this creates problems for a lot of the exporting of the data from SQL into Excel + pAssert->m_sMsg = pAssert->m_sMsg.Replace( '\n', ' ' ); + } + + //update our stats + pAssert->m_nTotalFired++; + pAssert->m_nWindowFired++; + + //remove any recorded asserts that are older than our window, so that we can record new asserts + int nStale = 0; + CUtlVector< RTime32 >& vecTimes = pAssert->m_vRecordTimes; + const RTime32 nStaleTime = CRTime::RTime32TimeCur() - (uint32)cv_assert_minidump_window.GetInt(); + while ( ( nStale < vecTimes.Count() ) && ( vecTimes[nStale] < nStaleTime ) ) + { + nStale++; + } + vecTimes.RemoveMultipleFromHead( nStale ); + + //see if we have room in how many asserts we want to track, if so, we want to record this assert + if ( ( vecTimes.Count() < cv_assert_max_minidumps_in_window.GetInt() ) || ( CDisableAssertRateLimit::s_nDisabledCount > 0 ) ) + { + vecTimes.AddToTail( CRTime::RTime32TimeCur() ); + pAssert->m_nTotalRecorded++; + if( pbShouldWriteMinidump ) + *pbShouldWriteMinidump = true; + } +} + +//flag indicating whether or not we should force a crash if we encounter an exit +static bool g_bCrashIfExitDetected = false; + +//callback handler registered to force a crash on exit conditions so we can track when/why the GC ever exits +static void GCForceCrash( bool bForceCrash ) +{ + if( bForceCrash ) + { + //we just want to initiate a crash, so that we can get a call stack + int* pForceCrash = NULL; + *pForceCrash = 100; + } +} + +static void ExitHandler() { GCForceCrash( g_bCrashIfExitDetected ); } +static void AbortHandler( int ) { GCForceCrash( true ); } +static void PureCallHandler() { GCForceCrash( true ); } + +static void InvalidCRTParamHandler(const wchar_t* expression, + const wchar_t* function, + const wchar_t* file, + unsigned int line, + uintptr_t pReserved) { GCForceCrash( true ); } + +static void InstallExceptionHandlers( bool bCrashOnNormalExit ) +{ + //don't crash on exit while in dev universe + g_bCrashIfExitDetected = bCrashOnNormalExit; + Plat_CollectMiniDumpsForFatalErrors(); + //and register one with the at exit handler + atexit( ExitHandler ); + //and register an abort handler + signal( SIGABRT, AbortHandler ); + //CRT invalid parameter handler + _set_invalid_parameter_handler( InvalidCRTParamHandler ); + //Pure virtual function call handler + _set_purecall_handler( PureCallHandler ); + + MiniDumpRegisterForUnhandledExceptions(); +} + +//----------------------------------------------------------------------------- +// Purpose: Loads the config, figures out what GC we should be running, and +// creates it +//----------------------------------------------------------------------------- +bool CGCInterface::BAsyncInit( uint32 unAppID, const char *pchDebugName, int iGCIndex, IGameCoordinatorHost *pHost ) +{ + //called to handle registration of exception handlers so that we will always crash rather than an unexpected termination + InstallExceptionHandlers( pHost->GetUniverse() != k_EUniverseDev ); + + // Make sure we can't deploy debug GCs outside the dev environment +#ifdef _DEBUG + if ( pHost->GetUniverse() != k_EUniverseDev ) + { + pHost->EmitSpew( SPEW_GC.GetName(), SPEW_ERROR, SPEW_ALWAYS, LOG_ALWAYS, + CFmtStr( "The GC for App %u is a debug binary. Shutting down.\n", unAppID ) ); + return false; + } +#endif + + //report if we are 64 or 32 bit for easier tracking during transition + COMPILE_TIME_ASSERT( sizeof( tchar ) == sizeof( char ) ); + pHost->EmitSpew( SPEW_GC.GetName(), SPEW_ERROR, SPEW_ALWAYS, LOG_ALWAYS, CFmtStr( "Initializing %d bit GC, Dir Index %d, PID:%u\n", ( uint32 )( sizeof( void* ) * 8 ), iGCIndex, GetCurrentProcessId() ).Access() ); + pHost->EmitSpew( SPEW_GC.GetName(), SPEW_ERROR, SPEW_ALWAYS, LOG_ALWAYS, CFmtStr1024( "Command Line: %s\n", Plat_GetCommandLine() ).Access() ); + + CommandLine()->CreateCmdLine( Plat_GetCommandLine() ); + + //get our machine name + { + char szMachineName[ MAX_COMPUTERNAME_LENGTH + 1 ]; + DWORD nBufferSize = ARRAYSIZE( szMachineName ); + GetComputerName( szMachineName, &nBufferSize ); + m_sMachineName = szMachineName; + } + + //open our a handle to our parent + { + uint32 nParentPID = MAX( 0, CommandLine()->ParmValue( "-parentpid", 0 ) ); + if( nParentPID == 0 ) + { + pHost->EmitSpew( SPEW_GC.GetName(), SPEW_ERROR, SPEW_ALWAYS, LOG_ALWAYS, "Parent process ID was not specified via -parentpid, unable to get information about the launching process\n" ); + } + else + { + m_hParentProcess = OpenProcess( PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, nParentPID ); + if( !m_hParentProcess ) + { + pHost->EmitSpew( SPEW_GC.GetName(), SPEW_ERROR, SPEW_ALWAYS, LOG_ALWAYS, "Unable to open the parent process with read access. Unable to get information about the launching process\n" ); + } + } + } + + static bool s_bInitCalled = false; + if ( s_bInitCalled ) + { + pHost->EmitSpew( SPEW_GC.GetName(), SPEW_ERROR, SPEW_ALWAYS, LOG_ALWAYS, "BInit called twice on the game IGameCoordinator" ); + return false; + } + s_bInitCalled = true; + + // Set basic variables + m_pGCHost = pHost; + m_nAppID = unAppID; + m_sDebugName = pchDebugName; + m_eUniverse = (EUniverse)m_pGCHost->GetUniverse(); + + // Initialize core systems + CRTime::UpdateRealTime(); + RandomSeed( CRTime::RTime32TimeCur() ); + + // Gets the path our dll is loaded from + HMODULE hModuleGC; + char rgchGCModuleFile[MAX_PATH+1] = ".\\"; + char rgchGCModulePath[MAX_PATH+1] = ".\\"; + if ( ::GetModuleHandleExA( GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (LPCSTR)&g_GCInterface, &hModuleGC ) ) + { + ::GetModuleFileNameA( hModuleGC, rgchGCModuleFile, MAX_PATH ); + V_strcpy_safe( rgchGCModulePath, rgchGCModuleFile ); + if ( char *pSlash = strrchr( rgchGCModulePath, '\\' ) ) + pSlash[1] = 0; + } + // Full path to GC.DLL (with final slash) is now in rgchGCModulePath + m_sGCDLLPath = rgchGCModulePath; + + CFmtStr sContentPath = rgchGCModulePath; + CFmtStr sBinaryPath = rgchGCModulePath; + if( Q_stristr( rgchGCModulePath, "bin\\gc\\x64" ) != NULL ) + { + Q_MakeAbsolutePath( sContentPath.Access(), FMTSTR_STD_LEN, "..\\..\\..", rgchGCModulePath ); + Q_MakeAbsolutePath( sBinaryPath.Access(), FMTSTR_STD_LEN, "..\\..\\..\\..\\bin\\x64", rgchGCModulePath ); + m_bDevMode = true; + m_sDevBinaryName = rgchGCModuleFile; + } + else if( Q_stristr( rgchGCModulePath, "bin\\gc" ) != NULL ) + { + Q_MakeAbsolutePath( sContentPath.Access(), FMTSTR_STD_LEN, "..\\..", rgchGCModulePath ); + Q_MakeAbsolutePath( sBinaryPath.Access(), FMTSTR_STD_LEN, "..\\..\\..\\bin", rgchGCModulePath ); + m_bDevMode = true; + m_sDevBinaryName = rgchGCModuleFile; + } + else + { + //launch through standard GC, so try and extract the version from our path (not a great solution, should extend interface so that + //the GCH provides us with the version it expects). The format is ....\vNNN\ so try and extract that + CUtlString sGCPath = rgchGCModulePath; + sGCPath.StripTrailingSlash(); + if ( const char *pSlash = strrchr( sGCPath.Get(), '\\' ) ) + { + //skip over the slash, and verify that we have a 'v' before the version + if( tolower( pSlash[ 1 ] ) == 'v' ) + { + //grab the version number + m_nVersion = ( uint32 )max( 0, atoi( pSlash + 2 ) ); + } + } + } + + // Starts logging + LoggingSystem_PushLoggingState(); + LoggingSystem_SetLoggingResponsePolicy( &s_NonFatalLoggingResponsePolicy ); + LoggingSystem_RegisterLoggingListener( &s_ConsoleLoggingListener ); + + // Select folder and prefix for dumps (using just the dir index for now, but update this later once we have the name + Tier0GenericMiniDumpHandler_SetFilenamePrefix( CFmtStr( "dumps\\gc%d_idx%d", unAppID, iGCIndex ) ); + + // Make sure dialogs don't come up and hang the process in production + if ( !m_bDevMode ) + { + Plat_EnableHeadlessMode(); + } + RegisterAssertionFailureListener( &sg_GCAssertionFailureHandler ); + + // Initialize GIDs + m_ullGID = 0; + m_ullGID = (uint64)iGCIndex << 56; // 8 bits of process id + m_ullGID |= (uint64)CRTime::RTime32TimeCur() << 24; // 32 bits of UTC time in seconds + // 24 bits/second of incremental counter space + + // This system assumes there are less than 256 GCs. Make sure of that + AssertMsg( iGCIndex >= 0 && iGCIndex < 256, "iGCIndex out of range. There can only be 256 GC processes for an app" ); + if ( iGCIndex < 0 || iGCIndex >= 256 ) + return false; + + // Make sure the protobuf library won't exitprocess without dumping + ::google::protobuf::SetLogHandler( ProtobufLogHandler ); + + g_gcAppSystemGroup.SetPath( sBinaryPath ); + if( g_gcAppSystemGroup.Startup() < 0 ) + return false; + + g_pFullFileSystem->AddSearchPath( sContentPath, "GAME" ); + g_pFullFileSystem->AddSearchPath( rgchGCModulePath, "CONFIG" ); // config files go with gc.dll + + // load the config file first thing so that we can use it for all the other startup code + KeyValuesAD configKeys( "config" ); + if( !BReadConfigDirectory( configKeys ) ) + { + return false; + } + + // Find this GC in the config and create it + m_pGCDirProcess = GDirectory()->GetProcess( iGCIndex ); + if ( NULL == m_pGCDirProcess ) + { + GCSDK::EmitError( SPEW_CONSOLE, "Failed to start GC for index %d.\n", iGCIndex ); + return false; + } + + CDirectory::GCFactory_t pfnFactory = GDirectory()->GetFactoryForProcessType( m_pGCDirProcess->GetProcessType() ); + if ( NULL == pfnFactory ) + { + GCSDK::EmitError( SPEW_CONSOLE, "Failed to start GC for index %d (type %s). Got a NULL factory function, likely missing registration for this type\n", iGCIndex, m_pGCDirProcess->GetProcessType() ); + return false; + } + + //now that we have more information about which GC we are, update our minidump name to reflect this + Tier0GenericMiniDumpHandler_SetFilenamePrefix( CFmtStr( "dumps\\gc%d_%s", unAppID, m_pGCDirProcess->GetName() ) ); + + //now that we know our GC type, we can actually load up our convars (which are dependent on this info) + if( !BReadConvars( configKeys ) ) + return false; + + // Init the GC. Not passing along the host because the interface layer owns it + // and chooses what to expose + m_pGC = pfnFactory( m_pGCDirProcess ); + return m_pGC->BAsyncInit( unAppID, pchDebugName, iGCIndex, NULL ); +} + +//----------------------------------------------------------------------------- +// Purpose: Generates a number that's guaranteed unique across all GC processes +// for this app. It is also guaranteed to have never been used by previous +// processes. +//----------------------------------------------------------------------------- +GID_t CGCInterface::GenerateGID() +{ + return ++m_ullGID; +} + +//we have the GC index encoded in the high bits when we init the gid, so just extract that +uint32 CGCInterface::GetGCDirIndexFromGID( GID_t gid ) +{ + return ( uint32 )( gid >> 56 ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets the universe the GC is currently running in +//----------------------------------------------------------------------------- +EUniverse CGCInterface::GetUniverse() const +{ + // Gets the current universe + return m_eUniverse; +} + + +//----------------------------------------------------------------------------- +// Purpose: Wrappers for GCHost functions that allow GCInterface to hook the calls +//----------------------------------------------------------------------------- +bool CGCInterface::BProcessSystemMessage( uint32 unGCSysMsgType, const void *pubData, uint32 cubData ) +{ + AssertMsg( unGCSysMsgType != 0, "Message sent without an ID. Did you use the wrong constructor?" ); + + //track this message that we are sending (always just strip off the protobuff flag so it works with all message types) + g_theMessageList.TallySendMessage( unGCSysMsgType & ~k_EMsgProtoBufFlag, cubData, eMsgSource_System ); + + VPROF_BUDGET( "GCHost", VPROF_BUDGETGROUP_STEAM ); + { + VPROF_BUDGET( "GCHost - ProcessSystemMessage", VPROF_BUDGETGROUP_STEAM ); + return m_pGCHost->BProcessSystemMessage( unGCSysMsgType, pubData, cubData ); + } +} + + +//----------------------------------------------------------------------------- +bool CGCInterface::BSendMessageToClient( uint64 ullSteamID, uint32 unMsgType, const void *pubData, uint32 cubData ) +{ + AssertMsg( unMsgType != 0, "Message sent without an ID. Did you use the wrong constructor?" ); + + //sanity check on our side that we are sending with a valid steam ID. Useful to catch message failures on the GC side since otherwise it must be caught in the GCH side + if( ullSteamID == k_steamIDNil.ConvertToUint64() ) + { + AssertMsg( false, "Message %d sent to invalid steam ID. This message will not be processed.", unMsgType & ~k_EMsgProtoBufFlag ); + return false; + } + + //track this message that we are sending (always just strip off the protobuff flag so it works with all message types) + g_theMessageList.TallySendMessage( unMsgType & ~k_EMsgProtoBufFlag, cubData, eMsgSource_Client ); + + VPROF_BUDGET( "GCHost", VPROF_BUDGETGROUP_STEAM ); + { + VPROF_BUDGET( "GCHost - SendMessageToClient", VPROF_BUDGETGROUP_STEAM ); + return m_pGCHost->BSendMessageToClient( ullSteamID, unMsgType, pubData, cubData ); + } +} + + +//----------------------------------------------------------------------------- +bool CGCInterface::BSendMessageToGC( int iGCServerIDTarget, uint32 unMsgType, const void *pubData, uint32 cubData ) +{ + AssertMsg( unMsgType != 0, "Message sent without an ID. Did you use the wrong constructor?" ); + + //track this message that we are sending (always just strip off the protobuff flag so it works with all message types) + g_theMessageList.TallySendMessage( unMsgType & ~k_EMsgProtoBufFlag, cubData, eMsgSource_GC ); + + VPROF_BUDGET( "GCHost", VPROF_BUDGETGROUP_STEAM ); + { + VPROF_BUDGET( "GCHost - SendMessageToGC", VPROF_BUDGETGROUP_STEAM ); + return m_pGCHost->BSendMessageToGC( iGCServerIDTarget, unMsgType, pubData, cubData ); + } +} + +//----------------------------------------------------------------------------- +void CGCInterface::AddBlockEmitString( const char* pszStr, bool bBlockConsole, bool bBlockLog ) +{ + BlockString_t* pStr = new BlockString_t; + pStr->m_sStr = pszStr; + pStr->m_bBlockConsole = bBlockConsole; + pStr->m_bBlockLog = bBlockLog; + m_BlockEmitStrings.AddToTail( pStr ); +} + +//----------------------------------------------------------------------------- +void CGCInterface::ClearBlockEmitStrings() +{ + m_BlockEmitStrings.PurgeAndDeleteElements(); +} + +//----------------------------------------------------------------------------- +void CGCInterface::EmitSpew( const char *pchGroupName, SpewType_t spewType, int iSpewLevel, int iLevelLog, const char *pchMsg ) +{ + //see if this output is being squelched by going through our blocked string + FOR_EACH_VEC( m_BlockEmitStrings, nStr ) + { + //if our output contains the blocked string, turn off the severity for that level + const BlockString_t* pStr = m_BlockEmitStrings[ nStr ]; + if( V_stristr( pchMsg, pStr->m_sStr ) != NULL ) + { + if( pStr->m_bBlockConsole ) + iSpewLevel = SPEW_NEVER; + if( pStr->m_bBlockLog ) + iLevelLog = LOG_NEVER; + } + } + + //see if this is just a blank line + if( filter_blank_lines.GetBool() && pchMsg ) + { + bool bIsBlankLine = true; + for( const char* pszCurrChar = pchMsg; *pszCurrChar; pszCurrChar++ ) + { + if( !V_isspace( *pszCurrChar ) ) + { + bIsBlankLine = false; + break; + } + } + if( bIsBlankLine ) + { + return; + } + } + + if ( m_bLogCaptureEnabled ) + { + m_vecLogCapture[m_vecLogCapture.AddToTail()].Set( pchMsg ); + } + + VPROF_BUDGET( "GCHost", VPROF_BUDGETGROUP_STEAM ); + { + VPROF_BUDGET( "GCHost - EmitMessage", VPROF_BUDGETGROUP_STEAM ); + m_pGCHost->EmitSpew( pchGroupName, spewType, iSpewLevel, iLevelLog, pchMsg ); + } +} + + +//----------------------------------------------------------------------------- +void CGCInterface::AsyncSQLQuery( IGCSQLQuery *pQuery, int eSchemaCatalog ) +{ + VPROF_BUDGET( "GCHost", VPROF_BUDGETGROUP_STEAM ); + { + VPROF_BUDGET( "GCHost - SQLQuery", VPROF_BUDGETGROUP_STEAM ); + m_pGCHost->AsyncSQLQuery( pQuery, eSchemaCatalog ); + } +} + + +//----------------------------------------------------------------------------- +void CGCInterface::SetStartupComplete( bool bSuccess ) +{ + m_pGCHost->StartupComplete( bSuccess ); +} + + +//----------------------------------------------------------------------------- +void CGCInterface::SetShutdownComplete() +{ + m_pGCHost->ShutdownComplete(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Passthrough implementations of the rest of the interface +//----------------------------------------------------------------------------- +void CGCInterface::Unload() +{ + if ( m_pGC ) + m_pGC->Unload(); + + g_gcAppSystemGroup.Shutdown(); +} + + +//----------------------------------------------------------------------------- +bool CGCInterface::BAsyncShutdown() +{ + bool bResult = false; + if ( m_pGC ) + bResult = m_pGC->BAsyncShutdown(); + + //if they have requested a shutdown, go ahead and allow exit + g_bCrashIfExitDetected = false; + + return bResult; +} + + +//----------------------------------------------------------------------------- +bool CGCInterface::BMainLoopOncePerFrame( uint64 ulLimitMicroseconds ) +{ + if ( !m_pGC ) + return false; + else + return m_pGC->BMainLoopOncePerFrame( ulLimitMicroseconds ); +} + + +//----------------------------------------------------------------------------- +bool CGCInterface::BMainLoopUntilFrameCompletion( uint64 ulLimitMicroseconds ) +{ + if ( !m_pGC ) + return false; + else + return m_pGC->BMainLoopUntilFrameCompletion( ulLimitMicroseconds ); +} + + +//----------------------------------------------------------------------------- +void CGCInterface::HandleMessageFromClient( uint64 ullSenderID, uint32 unMsgType, void *pubData, uint32 cubData ) +{ + if ( NULL == pubData || 0 == cubData ) + { + EG_ERROR( g_EGMessages, "Received invalid message from user %s. MessageID: %u pubData: %p cubData: %u\n", CSteamID( ullSenderID ).Render(), unMsgType, pubData, cubData ); + } + else if ( m_pGC ) + { + g_theMessageList.TallyReceiveMessage( unMsgType & ~k_EMsgProtoBufFlag, cubData, eMsgSource_Client ); + m_pGC->HandleMessageFromClient( ullSenderID, unMsgType, pubData, cubData ); + } +} + + +//----------------------------------------------------------------------------- +void CGCInterface::HandleMessageFromSystem( uint32 unGCSysMsgType, void *pubData, uint32 cubData ) +{ + if ( NULL == pubData || 0 == cubData ) + { + EG_ERROR( g_EGMessages, "Received invalid message from system. MessageID: %u pubData: %p cubData: %u\n", unGCSysMsgType, pubData, cubData ); + } + else if ( m_pGC ) + { + g_theMessageList.TallyReceiveMessage( unGCSysMsgType & ~k_EMsgProtoBufFlag, cubData, eMsgSource_System ); + m_pGC->HandleMessageFromSystem( unGCSysMsgType, pubData, cubData ); + } +} + + +//----------------------------------------------------------------------------- +void CGCInterface::HandleMessageFromGC( int iGCServerIDSender, uint32 unMsgType, void *pubData, uint32 cubData ) +{ + if ( NULL == pubData || 0 == cubData ) + { + EG_ERROR( g_EGMessages, "Received invalid message from GC. MessageID: %u pubData: %p cubData: %u\n", iGCServerIDSender, pubData, cubData ); + } + else if ( m_pGC ) + { + g_theMessageList.TallyReceiveMessage( unMsgType & ~k_EMsgProtoBufFlag, cubData, eMsgSource_GC ); + m_pGC->HandleMessageFromGC( iGCServerIDSender, unMsgType, pubData, cubData ); + } +} + + +//----------------------------------------------------------------------------- +void CGCInterface::StartLogCapture() +{ + m_bLogCaptureEnabled = true; +} + + +//----------------------------------------------------------------------------- +void CGCInterface::EndLogCapture() +{ + m_bLogCaptureEnabled = false; +} + + +//----------------------------------------------------------------------------- +const CUtlVector<CUtlString> *CGCInterface::GetLogCapture() +{ + return &m_vecLogCapture; +} + + +//----------------------------------------------------------------------------- +void CGCInterface::ClearLogCapture() +{ + m_vecLogCapture.RemoveAll(); +} + + +} + +//For the GC, we want to force a crash when we get stack corruption errors so +// that we can analyze the dumps and fix the problem. To disable this behavior, +// comment out the function below. + +// TEMP: Disabled on VS2013+ because I didn't know how to fix the linking problems +#if defined( _MSC_VER ) && ( _MSC_VER < 1800 ) +extern "C" +{ + #if defined (_X86_) + __declspec(noreturn) void __cdecl __report_gsfailure(void) + #else /* defined (_X86_) */ + __declspec(noreturn) void __cdecl __report_gsfailure(ULONGLONG StackCookie) + #endif /* defined (_X86_) */ + { + MiniDumpOptionalData_t optionalData( _T("stack_corruption") ); + InvokeMiniDumpHandler( NULL, &optionalData ); + static uint32* pNull = NULL; + *pNull = 0; + } +} +#endif
\ No newline at end of file diff --git a/gcsdk/gcjob.cpp b/gcsdk/gcjob.cpp new file mode 100644 index 0000000..f70c249 --- /dev/null +++ b/gcsdk/gcjob.cpp @@ -0,0 +1,1590 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +#include "stdafx.h" + +#include "msgprotobuf.h" +#include "smartptr.h" +#include "rtime.h" +#include "gcsdk/gcreportprinter.h" +#include <winsock.h> + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + +GCConVar cv_webapi_result_size_limit( "webapi_result_size_limit", "50000000" ); +GCConVar cv_webapi_serialize_threads( "webapi_serialize_threads", "1", 0, "Switch for serializing webAPI responses on threads" ); +static GCConVar webapi_account_tracking( "webapi_account_tracking", "1", "Controls whether or not account tracking stats are collected for web api usage" ); +static GCConVar webapi_kill_switch( "webapi_kill_switch", "0", "When set to zero this will block no web api calls, when set to 1 this will block all web api except those sent by steam. To block steam, use the webapi_enable_steam_<priority> controls" ); +static GCConVar webapi_kill_switch_error_response( "webapi_kill_switch_error_response", "1", "Determines if a response should be sent when the kill switch kills a web api request" ); +static GCConVar webapi_rate_limit_calls_per_min( "webapi_rate_limit_calls_per_min", "600", "Determines how many messages can be sent from an account via the web api per minute. <0 disables this limiting" ); +static GCConVar webapi_rate_limit_mb_per_min( "webapi_rate_limit_mb_per_min", "20", "Determines how many megabytes of data can be sent to an account via the web api per minute. <0 disables this limiting" ); +static GCConVar webapi_elevated_rate_limit_calls_per_min( "webapi_elevated_rate_limit_calls_per_min", "-1", "Determines how many messages can be sent from an account via the web api per minute for elevated accounts. <0 disables this limiting" ); +static GCConVar webapi_elevated_rate_limit_mb_per_min( "webapi_elevated_rate_limit_mb_per_min", "-1", "Determines how many megabytes of data can be sent to an account via the web api per minute for elevated accounts. <0 disables this limiting" ); +static GCConVar webapi_ip_rate_limit( "webapi_ip_rate_limit", "1", "Controls whether or not we rate limit based upon IPs or just accounts" ); + +//----------------------------------------------------------------------------- +// CWebAPIAccountTracker +// +// Utility that tracks web api calls based upon accesses made by various users +//----------------------------------------------------------------------------- +class CWebAPIAccountTracker +{ +public: + //called when a web api request is made to track the call. A return value of true indicates that it should be allowed, + //false indicates it should be blocked + bool TrackUser( AccountID_t nID, uint32 nIP ); + //called once the size of the response is known and will track bandwidth and caller attributed to the provided function + void TrackFunction( AccountID_t nID, uint32 nIP, const char* pszFunction, uint32 nResponseSize ); + + //called to reset all permissions to default + void ResetAccountPermissions(); + //called to associate a permission level with an account + void SetAccountPermission( AccountID_t nID, EWebAPIAccountLevel eLevel ); + + //completely resets accumulated stats + void ResetStats(); + //resets just the profile stats + void ResetProfileStats(); + + //different caller report filters that can be used + enum EDumpCaller + { + eDumpCaller_All, + eDumpCaller_Blocked, + eDumpCaller_Status, + eDumpCaller_Calls, + }; + + //different reports that can be provided + void DumpTotalCallers( EDumpCaller eFilter, const char* pszFunctionFilter = NULL ) const; + void DumpTotalIPs( EDumpCaller eFilter, const char* pszFunctionFilter = NULL ) const; + void DumpCaller( AccountID_t nID ) const; + void DumpIP( uint32 nIP ) const; + void DumpFunctions() const; + void DumpProfile( bool bAllTime ) const; + void DumpSteamServers() const; + + //given a steam server name, this will return the identifier of that server + uint32 GetSteamServerID( const char* pszServer ); + + //for steam level requests, we have a priority provided, and to track stats separately we break them up to different account IDs instead of just zero + static const uint32 k_nSteamIP_High = 2; + static const uint32 k_nSteamIP_Normal = 1; + static const uint32 k_nSteamIP_Low = 0; + +private: + + //called to get the starting time of this rate interval + RTime32 GetRateIntervalStart( AccountID_t nCaller ) const; + + struct SCallerStats + { + SCallerStats() : + m_nBlockedCalls( 0 ), + m_eLevel( eWebAPIAccountLevel_RateLimited ) + { + ResetRateInterval( 0 ); + } + + void ResetRateInterval( RTime32 nStart ) + { + m_nRateIntervalStartTime = nStart; + m_nRateIntervalCalls = 0; + m_nRateIntervalBytes = 0; + } + + //the most recent rate interval that we received a message from the user (used to expire old counts) + RTime32 m_nRateIntervalStartTime; + //how many messages have been sent within this rate interval (used for rate limiting) + uint32 m_nRateIntervalCalls; + //how many bytes have been sent for this account during this interval + uint32 m_nRateIntervalBytes; + //total number of blocked calls + uint32 m_nBlockedCalls; + //flags associated with this caller, used to block/whitelist, etc + EWebAPIAccountLevel m_eLevel; + }; + + struct SFunctionStats + { + //track the number of calls and bandwidth. The profile versions are separate and used for displaying profiles over a window + uint32 m_nTotalCalls; + uint32 m_nProfileCalls; + uint32 m_nMaxBytes; + uint32 m_nProfileMaxBytes; + uint64 m_nTotalBytes; + uint64 m_nProfileBytes; + + //a map of who has called us, which consists of the caller account and IP as the key + struct SCaller + { + bool operator==( const SCaller& rhs ) const { return m_nAccountID == rhs.m_nAccountID && m_nIP == rhs.m_nIP; } + AccountID_t m_nAccountID; + uint32 m_nIP; + }; + struct SCalls + { + uint32 m_nCalls; + uint64 m_nBytes; + }; + CUtlHashMapLarge< SCaller, SCalls > m_Callers; + }; + + //a structure used to simplify reporting so a vector can just be built of these, and then provided to the report function which will handle sorting it and displaying + struct SReportRow + { + SReportRow( const char* pszFunction, uint32 nCalls, uint64 nSize ) : + m_pszFunction( pszFunction ), + m_nCalls( nCalls ), + m_nSize( nSize ) + {} + + const char* m_pszFunction; + uint32 m_nCalls; + uint64 m_nSize; + }; + + struct SSteamServer + { + CUtlString m_sName; + uint32 m_nID; + }; + + //called to find an existing user, or create one if not in the list already + SCallerStats* CreateAccountUser( AccountID_t nID, RTime32 nRateIntervalStart ); + SCallerStats* CreateIPUser( uint32 nIP, RTime32 nRateIntervalStart ); + + //called to print a report of the provided report rows as either an ID list or a function list. This will re-sort the provided vector + static void PrintReport( const CUtlVector< SReportRow >& vec ); + + //how many seconds are in a rate interval + static const uint32 knRateIntervalTimeS = 60; + + CUtlHashMapLarge< AccountID_t, SCallerStats > m_AccountCallers; + CUtlHashMapLarge< uint32, SCallerStats > m_IPCallers; + CUtlHashMapLarge< uintp, SFunctionStats* > m_Functions; + CUtlHashMapLarge< const char*, SSteamServer*, CaseSensitiveStrEquals, MurmurHash3ConstCharPtr > m_SteamServers; + CJobTime m_ProfileTime; +}; + +//our global profiler +static CWebAPIAccountTracker g_WebAPIAccountTracker; + +void WebAPIAccount_ResetAllPermissions() +{ + g_WebAPIAccountTracker.ResetAccountPermissions(); +} + +void WebAPIAccount_SetPermission( AccountID_t nID, EWebAPIAccountLevel eLevel ) +{ + g_WebAPIAccountTracker.SetAccountPermission( nID, eLevel ); +} + +bool WebAPIAccount_BTrackUserAndValidate( AccountID_t nID, uint32 unIP ) +{ + return g_WebAPIAccountTracker.TrackUser( nID, unIP ); +} + +RTime32 CWebAPIAccountTracker::GetRateIntervalStart( AccountID_t nCaller ) const +{ + //we shift the time by the account ID so that all users don't wrap at the same time which can cause a temporary surge in web API requests + RTime32 curTime = CRTime::RTime32TimeCur() + ( nCaller % knRateIntervalTimeS ); + return curTime - ( curTime % knRateIntervalTimeS ); +} + +CWebAPIAccountTracker::SCallerStats* CWebAPIAccountTracker::CreateAccountUser( AccountID_t nID, RTime32 nRateIntervalStart ) +{ + int nIndex = m_AccountCallers.Find( nID ); + if( nIndex == m_AccountCallers.InvalidIndex() ) + { + nIndex = m_AccountCallers.Insert( nID ); + SCallerStats& caller = m_AccountCallers[ nIndex ]; + caller.m_nRateIntervalStartTime = nRateIntervalStart; + + //account ID is always unrestricted! + if( nID == 0 ) + caller.m_eLevel = eWebAPIAccountLevel_Unlimited; + } + + return &m_AccountCallers[ nIndex ]; +} + +CWebAPIAccountTracker::SCallerStats* CWebAPIAccountTracker::CreateIPUser( uint32 nIP, RTime32 nRateIntervalStart ) +{ + int nIndex = m_IPCallers.Find( nIP ); + if( nIndex == m_IPCallers.InvalidIndex() ) + { + nIndex = m_IPCallers.Insert( nIP ); + SCallerStats& caller = m_IPCallers[ nIndex ]; + caller.m_nRateIntervalStartTime = nRateIntervalStart; + } + + return &m_IPCallers[ nIndex ]; +} + +uint32 CWebAPIAccountTracker::GetSteamServerID( const char* pszServer ) +{ + int nIndex = m_SteamServers.Find( pszServer ); + if( nIndex == m_SteamServers.InvalidIndex() ) + { + SSteamServer* pServer = new SSteamServer; + pServer->m_sName = pszServer; + pServer->m_nID = m_SteamServers.Count(); + m_SteamServers.Insert( pServer->m_sName, pServer ); + return pServer->m_nID; + } + + return m_SteamServers[ nIndex ]->m_nID; +} + +void CWebAPIAccountTracker::DumpSteamServers() const +{ + CGCReportPrinter rp; + rp.AddStringColumn( "ID" ); + rp.AddStringColumn( "Server" ); + + FOR_EACH_MAP_FAST( m_SteamServers, nServer ) + { + const uint32 nID = m_SteamServers[ nServer ]->m_nID; + rp.StrValue( CFmtStr( "%u.%u.%u.%u", iptod( nID << 8 ) ) ); + rp.StrValue( m_SteamServers[ nServer ]->m_sName ); + rp.CommitRow(); + } + + rp.SortReport( "ID", false ); + rp.PrintReport( SPEW_CONSOLE ); +} + +//determines what the resulting account level access should be based upon the access rights of the IP address and the account +static EWebAPIAccountLevel DetermineAccessLevel( EWebAPIAccountLevel eAccount, EWebAPIAccountLevel eIP ) +{ + //unrestricted users should always be allowed, regardless of the IP range that they are making requests from, same with unlimited IP addresses + if( ( eAccount == eWebAPIAccountLevel_Unlimited ) || ( eIP == eWebAPIAccountLevel_Unlimited ) ) + return eWebAPIAccountLevel_Unlimited; + + //otherwise, if either is blocked, then block + if( ( eAccount == eWebAPIAccountLevel_Blocked ) || ( eIP == eWebAPIAccountLevel_Blocked ) ) + return eWebAPIAccountLevel_Blocked; + + //now we are dealing with default case versus elevated. Elevated wins over default + if( ( eAccount == eWebAPIAccountLevel_Elevated ) || ( eIP == eWebAPIAccountLevel_Elevated ) ) + return eWebAPIAccountLevel_Elevated; + + //default + return eWebAPIAccountLevel_RateLimited; +} + +bool CWebAPIAccountTracker::TrackUser( AccountID_t nID, uint32 nIP ) +{ + if( !webapi_account_tracking.GetBool() ) + return true; + + //first off update their aggregate caller stats + { + //what is our current time, and at what time did this rate interval start + const RTime32 rateIntervalStart = GetRateIntervalStart( nID ); + + //see if this account is completely blocked + SCallerStats* pAccountCaller = CreateAccountUser( nID, rateIntervalStart ); + SCallerStats* pIPCaller = CreateIPUser( nIP, rateIntervalStart ); + + //determine what our policy should be based upon the access level of the IP and the user + EWebAPIAccountLevel eAccessLevel = DetermineAccessLevel( pAccountCaller->m_eLevel, pIPCaller->m_eLevel ); + + //if we are blocked, just bail now + if( eAccessLevel == eWebAPIAccountLevel_Blocked ) + { + pAccountCaller->m_nBlockedCalls++; + pIPCaller->m_nBlockedCalls++; + return false; + } + + //reset the rate interval tracking + if( pAccountCaller->m_nRateIntervalStartTime < rateIntervalStart ) + pAccountCaller->ResetRateInterval( rateIntervalStart ); + if( pIPCaller->m_nRateIntervalStartTime < rateIntervalStart ) + pIPCaller->ResetRateInterval( rateIntervalStart ); + + //now handle rate limiting + if( ( eAccessLevel == eWebAPIAccountLevel_RateLimited ) || ( eAccessLevel == eWebAPIAccountLevel_Elevated ) ) + { + //determine the rate we want to limit + int32 nCallsPerMin = ( eAccessLevel == eWebAPIAccountLevel_RateLimited ) ? webapi_rate_limit_calls_per_min.GetInt() : webapi_elevated_rate_limit_calls_per_min.GetInt(); + int32 nBytesPerMin = ( ( eAccessLevel == eWebAPIAccountLevel_RateLimited ) ? webapi_rate_limit_mb_per_min.GetInt() : webapi_elevated_rate_limit_mb_per_min.GetInt() ) * 1024 * 1024; + + //see if this account is rate limited + if( ( eAccessLevel == eWebAPIAccountLevel_RateLimited ) ) + { + bool bAllow = true; + + //see if we are being limited based upon call rate limiting (tracking based upon ip and account) Note that + //we don't return until we've dones stat tracking for both so the reports are accurate and capture it at both levels + if( ( nCallsPerMin >= 0 && pAccountCaller->m_nRateIntervalCalls >= ( uint32 )nCallsPerMin ) || + ( nBytesPerMin >= 0 && pAccountCaller->m_nRateIntervalBytes >= ( uint32 )nBytesPerMin ) ) + { + pAccountCaller->m_nBlockedCalls++; + bAllow = false; + } + + if( webapi_ip_rate_limit.GetBool() ) + { + if( ( nCallsPerMin >= 0 && pIPCaller->m_nRateIntervalCalls >= ( uint32 )nCallsPerMin ) || + ( nBytesPerMin >= 0 && pIPCaller->m_nRateIntervalBytes >= ( uint32 )nBytesPerMin ) ) + { + pIPCaller->m_nBlockedCalls++; + bAllow = false; + } + } + + if( !bAllow ) + return false; + } + } + } + + return true; +} + +void CWebAPIAccountTracker::TrackFunction( AccountID_t nID, uint32 nIP, const char* pszFunction, uint32 nResponseSize ) +{ + if( !webapi_account_tracking.GetBool() ) + return; + + //update the bytes for that user + { + int nCallerIndex = m_AccountCallers.Find( nID ); + if( nCallerIndex != m_AccountCallers.InvalidIndex() ) + { + SCallerStats& caller = m_AccountCallers[ nCallerIndex ]; + caller.m_nRateIntervalBytes += nResponseSize; + caller.m_nRateIntervalCalls++; + } + } + + //update the bytes for that user and for their IP + { + int nCallerIndex = m_IPCallers.Find( nIP ); + if( nCallerIndex != m_IPCallers.InvalidIndex() ) + { + SCallerStats& caller = m_IPCallers[ nCallerIndex ]; + caller.m_nRateIntervalBytes += nResponseSize; + caller.m_nRateIntervalCalls++; + } + } + + //now update the function specific stats + { + int nFunctionIndex = m_Functions.Find( ( uintp )pszFunction ); + if( nFunctionIndex == m_Functions.InvalidIndex() ) + { + SFunctionStats* pNewStats = new SFunctionStats; + pNewStats->m_nTotalCalls = 0; + pNewStats->m_nProfileCalls = 0; + pNewStats->m_nTotalBytes = 0; + pNewStats->m_nProfileBytes = 0; + pNewStats->m_nMaxBytes = 0; + pNewStats->m_nProfileMaxBytes = 0; + nFunctionIndex = m_Functions.Insert( ( uintp )pszFunction, pNewStats ); + } + + //update our stats + SFunctionStats& function = *m_Functions[ nFunctionIndex ]; + function.m_nTotalCalls++; + function.m_nProfileCalls++; + function.m_nTotalBytes += nResponseSize; + function.m_nProfileBytes += nResponseSize; + function.m_nMaxBytes = MAX( function.m_nMaxBytes, nResponseSize ); + function.m_nProfileMaxBytes = MAX( function.m_nProfileMaxBytes, nResponseSize ); + + //update caller stats + { + struct SFunctionStats::SCaller caller; + caller.m_nAccountID = nID; + caller.m_nIP = nIP; + + int nCallerIndex = function.m_Callers.Find( caller ); + if( nCallerIndex == function.m_Callers.InvalidIndex() ) + { + nCallerIndex = function.m_Callers.Insert( caller ); + function.m_Callers[ nCallerIndex ].m_nCalls = 1; + function.m_Callers[ nCallerIndex ].m_nBytes = nResponseSize; + } + else + { + function.m_Callers[ nCallerIndex ].m_nCalls++; + function.m_Callers[ nCallerIndex ].m_nBytes += nResponseSize; + } + } + } +} + +void CWebAPIAccountTracker::SetAccountPermission( AccountID_t nID, EWebAPIAccountLevel eLevel ) +{ + SCallerStats* pCaller = CreateAccountUser( nID, GetRateIntervalStart( nID ) ); + pCaller->m_eLevel = eLevel; +} + +void CWebAPIAccountTracker::ResetAccountPermissions() +{ + FOR_EACH_MAP_FAST( m_AccountCallers, nCaller ) + { + m_AccountCallers[ nCaller ].m_eLevel = eWebAPIAccountLevel_RateLimited; + } + FOR_EACH_MAP_FAST( m_IPCallers, nCaller ) + { + m_IPCallers[ nCaller ].m_eLevel = eWebAPIAccountLevel_RateLimited; + } +} + +void CWebAPIAccountTracker::ResetStats() +{ + FOR_EACH_MAP_FAST( m_AccountCallers, nCaller ) + { + m_AccountCallers[ nCaller ].ResetRateInterval( GetRateIntervalStart( m_AccountCallers.Key( nCaller ) ) ); + m_AccountCallers[ nCaller ].m_nBlockedCalls = 0; + } + FOR_EACH_MAP_FAST( m_IPCallers, nCaller ) + { + m_IPCallers[ nCaller ].ResetRateInterval( GetRateIntervalStart( m_IPCallers.Key( nCaller ) ) ); + m_IPCallers[ nCaller ].m_nBlockedCalls = 0; + } + m_Functions.PurgeAndDeleteElements(); +} + +void CWebAPIAccountTracker::ResetProfileStats() +{ + FOR_EACH_MAP_FAST( m_Functions, nFunction ) + { + m_Functions[ nFunction ]->m_nProfileCalls = 0; + m_Functions[ nFunction ]->m_nProfileBytes = 0; + m_Functions[ nFunction ]->m_nProfileMaxBytes = 0; + } + m_ProfileTime.SetToJobTime(); +} + +static const int k_cSteamIDRenderedMaxLen = 36; + +//----------------------------------------------------------------------------- +// Purpose: Renders the steam ID to a buffer with an admin console link. NOTE: for convenience of +// calling code, this code returns a pointer to a static buffer and is NOT thread-safe. +// Output: buffer with rendered Steam ID +//----------------------------------------------------------------------------- +static const char * CSteamID_RenderLink( const CSteamID & steamID ) +{ + // longest length of returned string is k_cBufLen + // <link cmd="steamid64 %llu"></link> => 30 + 20 == 50 + // 50 + k_cSteamIDRenderedMaxLen + 1 + const int k_cBufLen = 50 + k_cSteamIDRenderedMaxLen + 1; + + const int k_cBufs = 4; // # of static bufs to use (so people can compose output with multiple calls to RenderLink() ) + static char rgchBuf[k_cBufs][k_cBufLen]; + static int nBuf = 0; + char * pchBuf = rgchBuf[nBuf]; // get pointer to current static buf + nBuf++; // use next buffer for next call to this method + nBuf %= k_cBufs; + + Q_snprintf( pchBuf, k_cBufLen, "<link cmd=\"steamid64 %llu\">%s</link>", steamID.ConvertToUint64(), steamID.Render() ); + return pchBuf; +} + + +//----------------------------------------------------------------------------- +// Purpose: Renders the passed-in steam ID to a buffer with admin console link. NOTE: for convenience +// of calling code, this code returns a pointer to a static buffer and is NOT thread-safe. +// Input: 64-bit representation of Steam ID to render +// Output: buffer with rendered Steam ID link +//----------------------------------------------------------------------------- +static const char * CSteamID_RenderLink( uint64 ulSteamID ) +{ + CSteamID steamID( ulSteamID ); + return CSteamID_RenderLink( steamID ); +} + + + +void CWebAPIAccountTracker::DumpCaller( AccountID_t nID ) const +{ + const CSteamID steamID = GGCInterface()->ConstructSteamIDForClient( nID ); + //cache the account name here so we don't yield while we have indices + CUtlString sPersona = GGCBase()->YieldingGetPersonaName( steamID, "[Unknown]" ); + + //dump high level user stats + int nCallerIndex = m_AccountCallers.Find( nID ); + if( nCallerIndex == m_AccountCallers.InvalidIndex() ) + { + EG_MSG( SPEW_CONSOLE, "User %u not found in any web api calls\n", nID ); + return; + } + + //a map of IP addresses that have been used by this account + CUtlHashMapLarge< uint32, SFunctionStats::SCalls > ipCalls; + + //now each function they called + uint64 nTotalBytes = 0; + uint32 nTotalCalls = 0; + CUtlVector< SReportRow > vFuncs; + FOR_EACH_MAP_FAST( m_Functions, nFunc ) + { + //add up how many calls they made to this function across all IPs + uint64 nFnBytes = 0; + uint32 nFnCalls = 0; + FOR_EACH_MAP_FAST( m_Functions[ nFunc ]->m_Callers, nCaller ) + { + const CWebAPIAccountTracker::SFunctionStats::SCaller& caller = m_Functions[ nFunc ]->m_Callers.Key( nCaller ); + if( caller.m_nAccountID == nID ) + { + const CWebAPIAccountTracker::SFunctionStats::SCalls& calls = m_Functions[ nFunc ]->m_Callers[ nCaller ]; + nFnBytes += calls.m_nBytes; + nFnCalls += calls.m_nCalls; + + int nIPIndex = ipCalls.Find( caller.m_nIP ); + if( nIPIndex == ipCalls.InvalidIndex() ) + { + SFunctionStats::SCalls toAdd; + toAdd.m_nBytes = calls.m_nBytes; + toAdd.m_nCalls = calls.m_nCalls; + ipCalls.Insert( caller.m_nIP, toAdd ); + } + else + { + ipCalls[ nIPIndex ].m_nBytes += calls.m_nBytes; + ipCalls[ nIPIndex ].m_nBytes += calls.m_nCalls; + } + } + } + + if( nFnCalls > 0 ) + { + vFuncs.AddToTail( SReportRow( ( const char* )m_Functions.Key( nFunc ), nFnCalls, nFnBytes ) ); + } + nTotalBytes += nFnBytes; + nTotalCalls += nFnCalls; + } + + const SCallerStats& caller = m_AccountCallers[ nCallerIndex ]; + EG_MSG( SPEW_CONSOLE, "---------------------------------------------------\n" ); + EG_MSG( SPEW_CONSOLE, "User %s: \"%s\"\n", CSteamID_RenderLink( steamID ), sPersona.String() ); + double fTotalMB = nTotalBytes / ( 1024.0 * 1024.0 ); + double fMBPerHour = fTotalMB / ( GGCBase()->GetGCUpTime() / 3600.0 ); + double fCallsPerHour = nTotalCalls / ( GGCBase()->GetGCUpTime() / 3600.0 ); + EG_MSG( SPEW_CONSOLE, "\tAccess: %u, Total Calls: %u, Blocked calls: %u, Total: %.2fMB, MB/h: %.2f, Calls/h: %.0f\n", caller.m_eLevel, nTotalCalls, caller.m_nBlockedCalls, fTotalMB, fMBPerHour, fCallsPerHour ); + //don't let someone accidentally change Steam's access! + if( nID != 0 ) + { + if( caller.m_eLevel == eWebAPIAccountLevel_RateLimited ) + { + EG_MSG( SPEW_CONSOLE, "\t<link cmd=\"webapi_account_set_access %u %d\">[Block Account]</link>", nID, eWebAPIAccountLevel_Blocked ); + EG_MSG( SPEW_CONSOLE, "\t<link cmd=\"webapi_account_set_access %u %d\">[Elevate Account]</link>\n", nID, eWebAPIAccountLevel_Elevated ); + } + else if( caller.m_eLevel == eWebAPIAccountLevel_Blocked ) + { + EG_MSG( SPEW_CONSOLE, "\t<link cmd=\"webapi_account_set_access %u %d\">[Unblock Account]</link>\n", nID, eWebAPIAccountLevel_RateLimited ); + } + else if( caller.m_eLevel == eWebAPIAccountLevel_Elevated ) + { + EG_MSG( SPEW_CONSOLE, "\t<link cmd=\"webapi_account_set_access %u %d\">[Demote Account]</link>\n", nID, eWebAPIAccountLevel_RateLimited ); + } + } + + //print a report of the IP addresses that they are calling from + { + CGCReportPrinter rp; + rp.AddStringColumn( "IP" ); + rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB ); + + FOR_EACH_MAP_FAST( ipCalls, nIP ) + { + rp.StrValue( CFmtStr( "%u.%u.%u.%u", iptod( ipCalls.Key( nIP ) ) ), CFmtStr( "webapi_account_dump_ip %u", ipCalls.Key( nIP ) ) ); + rp.IntValue( ipCalls[ nIP ].m_nCalls ); + rp.IntValue( ipCalls[ nIP ].m_nBytes ); + rp.CommitRow(); + } + + rp.SortReport( "MB" ); + rp.PrintReport( SPEW_CONSOLE ); + } + + //and print a report of all the functions that they've called + PrintReport( vFuncs ); +} + +void CWebAPIAccountTracker::DumpIP( uint32 nIP ) const +{ + //dump high level user stats + int nCallerIndex = m_IPCallers.Find( nIP ); + if( nCallerIndex == m_IPCallers.InvalidIndex() ) + { + EG_MSG( SPEW_CONSOLE, "IP %u not found in any web api calls\n", nIP ); + return; + } + + //a map of IP addresses that have been used by this account + CUtlHashMapLarge< AccountID_t, SFunctionStats::SCalls > accountCalls; + + //now each function they called + uint64 nTotalBytes = 0; + uint32 nTotalCalls = 0; + CUtlVector< SReportRow > vFuncs; + FOR_EACH_MAP_FAST( m_Functions, nFunc ) + { + //add up how many calls they made to this function across all IPs + uint64 nFnBytes = 0; + uint32 nFnCalls = 0; + FOR_EACH_MAP_FAST( m_Functions[ nFunc ]->m_Callers, nCaller ) + { + const CWebAPIAccountTracker::SFunctionStats::SCaller& caller = m_Functions[ nFunc ]->m_Callers.Key( nCaller ); + if( caller.m_nIP == nIP ) + { + const CWebAPIAccountTracker::SFunctionStats::SCalls& calls = m_Functions[ nFunc ]->m_Callers[ nCaller ]; + nFnBytes += calls.m_nBytes; + nFnCalls += calls.m_nCalls; + + int nAccountIndex = accountCalls.Find( caller.m_nAccountID ); + if( nAccountIndex == accountCalls.InvalidIndex() ) + { + SFunctionStats::SCalls toAdd; + toAdd.m_nBytes = calls.m_nBytes; + toAdd.m_nCalls = calls.m_nCalls; + accountCalls.Insert( caller.m_nAccountID, toAdd ); + GGCBase()->PreloadPersonaName( GGCInterface()->ConstructSteamIDForClient( caller.m_nAccountID ) ); + } + else + { + accountCalls[ nAccountIndex ].m_nBytes += calls.m_nBytes; + accountCalls[ nAccountIndex ].m_nBytes += calls.m_nCalls; + } + } + } + + if( nFnCalls > 0 ) + { + vFuncs.AddToTail( SReportRow( ( const char* )m_Functions.Key( nFunc ), nFnCalls, nFnBytes ) ); + } + nTotalBytes += nFnBytes; + nTotalCalls += nFnCalls; + } + + const SCallerStats& caller = m_IPCallers[ nCallerIndex ]; + EG_MSG( SPEW_CONSOLE, "---------------------------------------------------\n" ); + EG_MSG( SPEW_CONSOLE, "IP %u.%u.%u.%u\n", iptod( nIP ) ); + double fTotalMB = nTotalBytes / ( 1024.0 * 1024.0 ); + double fMBPerHour = fTotalMB / ( GGCBase()->GetGCUpTime() / 3600.0 ); + double fCallsPerHour = nTotalCalls / ( GGCBase()->GetGCUpTime() / 3600.0 ); + EG_MSG( SPEW_CONSOLE, "\tAccess: %u, Total Calls: %u, Blocked calls: %u, Total: %.2fMB, MB/h: %.2f, Calls/h: %.0f\n", caller.m_eLevel, nTotalCalls, caller.m_nBlockedCalls, fTotalMB, fMBPerHour, fCallsPerHour ); + + //print a report of the accounts that they are calling from + { + CGCReportPrinter rp; + rp.AddSteamIDColumn( "Account" ); + rp.AddStringColumn( "Persona" ); + rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB ); + + FOR_EACH_MAP_FAST( accountCalls, nAccount ) + { + CSteamID steamID = GGCInterface()->ConstructSteamIDForClient( accountCalls.Key( nAccount ) ); + rp.SteamIDValue( steamID, CFmtStr( "webapi_account_dump_caller %u", steamID.GetAccountID() ) ); + rp.StrValue( GGCBase()->YieldingGetPersonaName( steamID, "[unknown]" ), CFmtStr( "webapi_account_dump_caller %u", steamID.GetAccountID() ) ); + rp.IntValue( accountCalls[ nAccount ].m_nCalls ); + rp.IntValue( accountCalls[ nAccount ].m_nBytes ); + rp.CommitRow(); + } + + rp.SortReport( "MB" ); + rp.PrintReport( SPEW_CONSOLE ); + } + + //and print a report of all the functions that they've called + PrintReport( vFuncs ); +} + +struct SCallerReportStats +{ + uint32 m_nFunctions; + uint32 m_nCalls; + uint64 m_nBytes; +}; + +void CWebAPIAccountTracker::DumpTotalCallers( EDumpCaller eFilter, const char* pszFunctionFilter ) const +{ + //accumulate stats for each unique caller + CUtlHashMapLarge< AccountID_t, SCallerReportStats > mapCallers; + FOR_EACH_MAP_FAST( m_Functions, nCurrFunction ) + { + //handle filtering out functions we don't care about + const char* pszFunctionName = ( const char* )m_Functions.Key( nCurrFunction ); + if( pszFunctionFilter && ( V_stristr( pszFunctionName, pszFunctionFilter ) == NULL ) ) + continue; + + const SFunctionStats& function = *m_Functions[ nCurrFunction ]; + FOR_EACH_MAP_FAST( function.m_Callers, nCurrCaller ) + { + const AccountID_t key = function.m_Callers.Key( nCurrCaller ).m_nAccountID; + + //add this account + int nStatIndex = mapCallers.Find( key ); + if( nStatIndex == mapCallers.InvalidIndex() ) + { + SCallerReportStats stats; + stats.m_nFunctions = 0; + stats.m_nCalls = 0; + stats.m_nBytes = 0; + nStatIndex = mapCallers.Insert( key, stats ); + GGCBase()->PreloadPersonaName( GGCInterface()->ConstructSteamIDForClient( key ) ); + } + + mapCallers[ nStatIndex ].m_nFunctions += 1; + mapCallers[ nStatIndex ].m_nCalls += function.m_Callers[ nCurrCaller ].m_nCalls; + mapCallers[ nStatIndex ].m_nBytes += function.m_Callers[ nCurrCaller ].m_nBytes; + } + } + + CGCReportPrinter rp; + rp.AddStringColumn( "Account" ); + rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB ); + rp.AddIntColumn( "Blocked", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "Access", CGCReportPrinter::eSummary_None ); + rp.AddIntColumn( "APIs", CGCReportPrinter::eSummary_None ); + rp.AddIntColumn( "MB/h", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB ); + rp.AddIntColumn( "Calls/h", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "KB/c", CGCReportPrinter::eSummary_None ); + rp.AddStringColumn( "UserName" ); + + const double fUpTimeHrs = MAX( GGCBase()->GetGCUpTime() / 3600.0, 0.01 ); + + //now show the report + FOR_EACH_MAP_FAST( m_AccountCallers, nCurrCaller ) + { + //apply filters to our results + if( ( eFilter == eDumpCaller_Blocked ) && ( m_AccountCallers[ nCurrCaller ].m_nBlockedCalls == 0 ) ) + continue; + if( ( eFilter == eDumpCaller_Status ) && ( m_AccountCallers[ nCurrCaller ].m_eLevel == eWebAPIAccountLevel_RateLimited ) ) + continue; + + const AccountID_t accountID = m_AccountCallers.Key( nCurrCaller ); + const SCallerReportStats* pStats = NULL; + int nCollectedStats = mapCallers.Find( accountID ); + if( nCollectedStats != mapCallers.InvalidIndex() ) + { + pStats = &mapCallers[ nCollectedStats ]; + } + + //filter out users that didn't make any calls if appropriate + if( ( eFilter == eDumpCaller_Calls ) && ( !pStats || ( pStats->m_nCalls == 0 ) ) ) + continue; + + const CSteamID steamID( GGCInterface()->ConstructSteamIDForClient( accountID ) ); + rp.StrValue( steamID.Render() , CFmtStr( "webapi_account_dump_caller %u", accountID ) ); + rp.IntValue( ( pStats ) ? pStats->m_nCalls : 0 ); + rp.IntValue( ( pStats ) ? pStats->m_nBytes : 0 ); + rp.IntValue( m_AccountCallers[ nCurrCaller ].m_nBlockedCalls ); + rp.IntValue( m_AccountCallers[ nCurrCaller ].m_eLevel ); + rp.IntValue( ( pStats ) ? pStats->m_nFunctions : 0 ); + rp.IntValue( ( int64 )( ( ( pStats ) ? pStats->m_nBytes : 0 ) / fUpTimeHrs ) ); + rp.IntValue( ( int64 )( ( ( pStats ) ? pStats->m_nCalls : 0 ) / fUpTimeHrs ) ); + rp.IntValue( ( pStats && pStats->m_nCalls > 0 ) ? ( pStats->m_nBytes / pStats->m_nCalls ) / 1024 : 0 ); + rp.StrValue( GGCBase()->YieldingGetPersonaName( steamID, "[unknown]" ) ); + rp.CommitRow(); + } + + const char* pszSort = "MB/h"; + if( eFilter == eDumpCaller_Blocked ) + pszSort = "Blocked"; + else if( eFilter == eDumpCaller_Status ) + pszSort = "Access"; + + rp.SortReport( pszSort ); + rp.PrintReport( SPEW_CONSOLE ); +} + +void CWebAPIAccountTracker::DumpTotalIPs( EDumpCaller eFilter, const char* pszFunctionFilter ) const +{ + //accumulate stats for each unique caller + CUtlHashMapLarge< uint32, SCallerReportStats > mapIPs; + FOR_EACH_MAP_FAST( m_Functions, nCurrFunction ) + { + //handle filtering out functions we don't care about + const char* pszFunctionName = ( const char* )m_Functions.Key( nCurrFunction ); + if( pszFunctionFilter && ( V_stristr( pszFunctionName, pszFunctionFilter ) == NULL ) ) + continue; + + const SFunctionStats& function = *m_Functions[ nCurrFunction ]; + FOR_EACH_MAP_FAST( function.m_Callers, nCurrCaller ) + { + const uint32 key = function.m_Callers.Key( nCurrCaller ).m_nIP; + + //add this account + int nStatIndex = mapIPs.Find( key ); + if( nStatIndex == mapIPs.InvalidIndex() ) + { + SCallerReportStats stats; + stats.m_nFunctions = 0; + stats.m_nCalls = 0; + stats.m_nBytes = 0; + nStatIndex = mapIPs.Insert( key, stats ); + } + + mapIPs[ nStatIndex ].m_nFunctions += 1; + mapIPs[ nStatIndex ].m_nCalls += function.m_Callers[ nCurrCaller ].m_nCalls; + mapIPs[ nStatIndex ].m_nBytes += function.m_Callers[ nCurrCaller ].m_nBytes; + } + } + + CGCReportPrinter rp; + rp.AddStringColumn( "IP" ); + rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB ); + rp.AddIntColumn( "Blocked", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "Access", CGCReportPrinter::eSummary_None ); + rp.AddIntColumn( "APIs", CGCReportPrinter::eSummary_None ); + rp.AddIntColumn( "MB/h", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB ); + rp.AddIntColumn( "Calls/h", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "KB/c", CGCReportPrinter::eSummary_None ); + + const double fUpTimeHrs = MAX( GGCBase()->GetGCUpTime() / 3600.0, 0.01 ); + + //now show the report + FOR_EACH_MAP_FAST( m_IPCallers, nCurrCaller ) + { + //apply filters to our results + if( ( eFilter == eDumpCaller_Blocked ) && ( m_IPCallers[ nCurrCaller ].m_nBlockedCalls == 0 ) ) + continue; + if( ( eFilter == eDumpCaller_Status ) && ( m_IPCallers[ nCurrCaller ].m_eLevel == eWebAPIAccountLevel_RateLimited ) ) + continue; + + const uint32 nIP = m_IPCallers.Key( nCurrCaller ); + const SCallerReportStats* pStats = NULL; + int nCollectedStats = mapIPs.Find( nIP ); + if( nCollectedStats != mapIPs.InvalidIndex() ) + { + pStats = &mapIPs[ nCollectedStats ]; + } + + //filter out users that didn't make any calls if appropriate + if( ( eFilter == eDumpCaller_Calls ) && ( !pStats || ( pStats->m_nCalls == 0 ) ) ) + continue; + + rp.StrValue( CFmtStr( "%u.%u.%u.%u", iptod( nIP ) ), CFmtStr( "webapi_account_dump_ip %u", nIP ) ); + rp.IntValue( ( pStats ) ? pStats->m_nCalls : 0 ); + rp.IntValue( ( pStats ) ? pStats->m_nBytes : 0 ); + rp.IntValue( m_IPCallers[ nCurrCaller ].m_nBlockedCalls ); + rp.IntValue( m_IPCallers[ nCurrCaller ].m_eLevel ); + rp.IntValue( ( pStats ) ? pStats->m_nFunctions : 0 ); + rp.IntValue( ( int64 )( ( ( pStats ) ? pStats->m_nBytes : 0 ) / fUpTimeHrs ) ); + rp.IntValue( ( int64 )( ( ( pStats ) ? pStats->m_nCalls : 0 ) / fUpTimeHrs ) ); + rp.IntValue( ( pStats && pStats->m_nCalls > 0 ) ? ( pStats->m_nBytes / pStats->m_nCalls ) / 1024 : 0 ); + rp.CommitRow(); + } + + const char* pszSort = "MB/h"; + if( eFilter == eDumpCaller_Blocked ) + pszSort = "Blocked"; + else if( eFilter == eDumpCaller_Status ) + pszSort = "Access"; + + rp.SortReport( pszSort ); + rp.PrintReport( SPEW_CONSOLE ); +} + +void CWebAPIAccountTracker::DumpFunctions() const +{ + CUtlVector< SReportRow > vFuncs; + vFuncs.EnsureCapacity( m_Functions.Count() ); + + FOR_EACH_MAP_FAST( m_Functions, i ) + { + vFuncs.AddToTail( SReportRow( ( const char* )m_Functions.Key( i ), m_Functions[ i ]->m_nTotalCalls, m_Functions[ i ]->m_nTotalBytes ) ); + } + PrintReport( vFuncs ); +} + +void CWebAPIAccountTracker::DumpProfile( bool bAllTime ) const +{ + //accumulate totals so we can do percentage + uint32 nTotalCalls = 0; + uint64 nTotalBytes = 0; + FOR_EACH_MAP_FAST( m_Functions, nFunction ) + { + if( bAllTime ) + { + nTotalCalls += m_Functions[ nFunction ]->m_nTotalCalls; + nTotalBytes += m_Functions[ nFunction ]->m_nTotalBytes; + } + else + { + nTotalCalls += m_Functions[ nFunction ]->m_nProfileCalls; + nTotalBytes += m_Functions[ nFunction ]->m_nProfileBytes; + } + } + + //determine how much time we are covering, and come up with a scale to normalize the values + uint64 nSampleMicroS = ( bAllTime ) ? ( uint64 )GGCBase()->GetGCUpTime() * 1000000 : m_ProfileTime.CServerMicroSecsPassed(); + double fToPS = ( nSampleMicroS > 0 ) ? 1000000.0 / ( double )nSampleMicroS : 1.0; + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "Web API Profile: Sampled %.2f seconds\n", nSampleMicroS / ( 1000.0 * 1000.0 ) ); + + CGCReportPrinter rp; + rp.AddStringColumn( "Web API Name" ); + rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB ); + rp.AddFloatColumn( "%", CGCReportPrinter::eSummary_Total, 1 ); + rp.AddFloatColumn( "KBPS", CGCReportPrinter::eSummary_Total, 1 ); + rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total ); + rp.AddFloatColumn( "%", CGCReportPrinter::eSummary_Total, 1 ); + rp.AddFloatColumn( "CallsPS", CGCReportPrinter::eSummary_Total, 1 ); + rp.AddIntColumn( "MaxKB", CGCReportPrinter::eSummary_Max ); + + FOR_EACH_MAP_FAST( m_Functions, nFunction ) + { + const SFunctionStats* pFunc = m_Functions[ nFunction ]; + uint32 nCalls = ( bAllTime ) ? pFunc->m_nTotalCalls : pFunc->m_nProfileCalls; + uint32 nMax = ( bAllTime ) ? pFunc->m_nMaxBytes : pFunc->m_nProfileMaxBytes; + uint64 nBytes = ( bAllTime ) ? pFunc->m_nTotalBytes : pFunc->m_nProfileBytes; + + rp.StrValue( ( const char* )m_Functions.Key( nFunction ) ); + rp.IntValue( nBytes ); + rp.FloatValue( ( 100.0 * nBytes ) / nTotalBytes ); + rp.FloatValue( ( nBytes / 1024.0 ) * fToPS ); + rp.IntValue( nCalls ); + rp.FloatValue( ( 100.0 * nCalls ) / nTotalCalls ); + rp.FloatValue( nCalls * fToPS ); + rp.IntValue( nMax / 1024 ); + rp.CommitRow(); + } + + rp.SortReport( "KBPS" ); + rp.PrintReport( SPEW_CONSOLE ); +} + +void CWebAPIAccountTracker::PrintReport( const CUtlVector< SReportRow >& vec ) +{ + CGCReportPrinter rp; + + //now print it out based upon the type + rp.AddStringColumn( "Function" ); + rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB ); + rp.AddFloatColumn( "Calls/h", CGCReportPrinter::eSummary_Total, 0 ); + rp.AddFloatColumn( "MB/h", CGCReportPrinter::eSummary_Total ); + + const double fUpTimeHrs = MAX( GGCBase()->GetGCUpTime() / 3600.0, 0.01 ); + + for( int i = 0; i < vec.Count(); i++ ) + { + rp.StrValue( vec[ i ].m_pszFunction, CFmtStr( "webapi_account_dump_function_callers %s", vec[ i ].m_pszFunction ) ); + rp.IntValue( vec[ i ].m_nCalls ); + rp.IntValue( vec[ i ].m_nSize ); + rp.FloatValue( vec[ i ].m_nCalls / fUpTimeHrs ); + rp.FloatValue( ( vec[ i ].m_nSize / ( 1024.0 * 1024.0 ) ) / fUpTimeHrs ); + rp.CommitRow(); + } + + rp.SortReport( "MB/h" ); + rp.PrintReport( SPEW_CONSOLE ); +} + + +GC_CON_COMMAND( webapi_account_dump_steam_servers, "Dumps the ID listings of the various steam servers encoded in the IP address of Steam requests" ) +{ + g_WebAPIAccountTracker.DumpSteamServers(); +} + +GC_CON_COMMAND( webapi_account_dump_callers, "Dumps the most frequent callers of web api's for the current run of the GC" ) +{ + g_WebAPIAccountTracker.DumpTotalCallers( CWebAPIAccountTracker::eDumpCaller_Calls ); +} + +GC_CON_COMMAND( webapi_account_dump_ips, "Dumps the most frequent ip callers of web api's for the current run of the GC" ) +{ + g_WebAPIAccountTracker.DumpTotalIPs( CWebAPIAccountTracker::eDumpCaller_Calls ); +} + +GC_CON_COMMAND( webapi_account_dump_blocked_callers, "Dumps the callers that have been blocked and how many calls have been blocked" ) +{ + g_WebAPIAccountTracker.DumpTotalCallers( CWebAPIAccountTracker::eDumpCaller_Blocked ); +} + +GC_CON_COMMAND( webapi_account_dump_caller_access, "Dumps the access rights of any caller that is not the default rate limiting" ) +{ + g_WebAPIAccountTracker.DumpTotalCallers( CWebAPIAccountTracker::eDumpCaller_Status ); +} + +GC_CON_COMMAND( webapi_account_dump_functions, "Dumps the most frequently called web api functions" ) +{ + g_WebAPIAccountTracker.DumpFunctions(); +} + +GC_CON_COMMAND_PARAMS( webapi_account_dump_function_callers, 1, "<function name> - Dumps the most frequent callers of functions that match the provided substring" ) +{ + g_WebAPIAccountTracker.DumpTotalCallers( CWebAPIAccountTracker::eDumpCaller_Calls, args[ 1 ] ); +} + +GC_CON_COMMAND_PARAMS( webapi_account_dump_caller, 1, "<caller account> - Dumps the functions that the provided account ID has been calling the most" ) +{ + g_WebAPIAccountTracker.DumpCaller( ( AccountID_t )V_atoui64( args[ 1 ] ) ); +} + +GC_CON_COMMAND_PARAMS( webapi_account_dump_ip, 1, "<ip> - Dumps the functions that the provided ip has been calling the most" ) +{ + g_WebAPIAccountTracker.DumpIP( ( AccountID_t )V_atoui64( args[ 1 ] ) ); +} + +GC_CON_COMMAND( webapi_account_reset_stats, "Forces a reset of all stats collected for web api account stats" ) +{ + g_WebAPIAccountTracker.ResetStats(); +} + +//utility class for dumping out the profile results after time has expired +static void DumpWebAPIProfile() +{ + g_WebAPIAccountTracker.DumpProfile( false ); +} + +GC_CON_COMMAND_PARAMS( webapi_profile, 1, "<seconds to profile> Turns on web api profiling for N seconds and dumps the results" ) +{ + float fSeconds = MAX( 1.0f, atof( args[ 1 ] ) ); + g_WebAPIAccountTracker.ResetProfileStats(); + static CGlobalScheduledFunction s_DumpProfile; + s_DumpProfile.ScheduleMS( DumpWebAPIProfile, fSeconds * 1000.0f ); +} + +//console commands to control web API profiling +GC_CON_COMMAND( webapi_profile_reset, "Turns on web api profiling" ) +{ + g_WebAPIAccountTracker.ResetProfileStats(); +} + +GC_CON_COMMAND( webapi_profile_dump, "Displays stats collected while web api profiling was enabled" ) +{ + g_WebAPIAccountTracker.DumpProfile( false ); +} + +GC_CON_COMMAND( webapi_profile_dump_total, "Displays stats collected while web api profiling was enabled" ) +{ + g_WebAPIAccountTracker.DumpProfile( true ); +} + +//----------------------------------------------------------------------------- +// Purpose: Sends a message and waits for a response +// Input: steamIDTarget - The entity this message is going to +// msgOut - The message to send +// nTimeoutSec - Number of seconds to wait for a response +// pMsgIn - Pointer to the message that will contain the response +// eMsg - The type of message the response should be +// Returns: True is the response was received, false otherwise. The contents +// of pMsgIn will be valid only if the function returns true. +//----------------------------------------------------------------------------- +bool CGCJob::BYldSendMessageAndGetReply( CSteamID &steamIDTarget, CGCMsgBase &msgOut, uint nTimeoutSec, CGCMsgBase *pMsgIn, MsgType_t eMsg ) +{ + IMsgNetPacket *pNetPacket = NULL; + + if ( !BYldSendMessageAndGetReply( steamIDTarget, msgOut, nTimeoutSec, &pNetPacket ) ) + return false; + + pMsgIn->SetPacket( pNetPacket ); + + if ( pMsgIn->Hdr().m_eMsg != eMsg ) + return false; + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Sends a message and waits for a response +// Input: steamIDTarget - The entity this message is going to +// msgOut - The message to send +// nTimeoutSec - Number of seconds to wait for a response +// ppNetPackets - Pointer to a IMsgNetPacket pointer which will contain +// the response +// Returns: True is the response was received, false otherwise. *ppNetPacket +// will point to a valid packet only if the function returns true. +//----------------------------------------------------------------------------- +bool CGCJob::BYldSendMessageAndGetReply( CSteamID &steamIDTarget, CGCMsgBase &msgOut, uint nTimeoutSec, IMsgNetPacket **ppNetPacket ) +{ + msgOut.ExpectingReply( GetJobID() ); + + if ( !m_pGC->BSendGCMsgToClient( steamIDTarget, msgOut ) ) + return false; + + SetJobTimeout( nTimeoutSec ); + return BYieldingWaitForMsg( ppNetPacket ); +} + +//----------------------------------------------------------------------------- +// Purpose: BYldSendMessageAndGetReply, ProtoBuf edition +//----------------------------------------------------------------------------- +bool CGCJob::BYldSendMessageAndGetReply( CSteamID &steamIDTarget, CProtoBufMsgBase &msgOut, uint nTimeoutSec, CProtoBufMsgBase *pMsgIn, MsgType_t eMsg ) +{ + IMsgNetPacket *pNetPacket = NULL; + + if ( !BYldSendMessageAndGetReply( steamIDTarget, msgOut, nTimeoutSec, &pNetPacket ) ) + return false; + + pMsgIn->InitFromPacket( pNetPacket ); + + if ( pMsgIn->GetEMsg() != eMsg ) + return false; + + return true; +} + +bool CGCJob::BYldSendMessageAndGetReply( CSteamID &steamIDTarget, CProtoBufMsgBase &msgOut, uint nTimeoutSec, IMsgNetPacket **ppNetPacket ) +{ + msgOut.ExpectingReply( GetJobID() ); + + if ( !m_pGC->BSendGCMsgToClient( steamIDTarget, msgOut ) ) + return false; + + SetJobTimeout( nTimeoutSec ); + return BYieldingWaitForMsg( ppNetPacket ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CGCWGJob::CGCWGJob( CGCBase *pGCBase ) +: CGCJob( pGCBase ), + m_steamID( k_steamIDNil ), + m_pWebApiFunc( NULL ) +{ +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CGCWGJob::~CGCWGJob() +{ + +} + + +//----------------------------------------------------------------------------- +// Purpose: Receive a k_EMsgWGRequest and manage keyvalues serialization/deserialization +//----------------------------------------------------------------------------- +bool CGCWGJob::BYieldingRunGCJob( IMsgNetPacket * pNetPacket ) +{ + CGCMsg<MsgGCWGRequest_t> msg( pNetPacket ); + + CUtlString strRequestName; + KeyValuesAD pkvRequest( "request" ); + KeyValuesAD pkvResponse( "response" ); + + m_steamID = CSteamID( msg.Body().m_ulSteamID ); + + msg.BReadStr( &strRequestName ); + + //deserialize KV + m_bufRequest.Clear(); + m_bufRequest.Put( msg.PubReadCur(), msg.Body().m_cubKeyValues ); + KVPacker packer; + if ( !packer.ReadAsBinary( pkvRequest, m_bufRequest ) ) + { + AssertMsg( false, "Failed to deserialize key values from WG request" ); + CGCWGJobMgr::SendErrorMessage( msg, "Failed to deserialize key values from WG request", k_EResultInvalidParam ); + return false; + } + + if( !BVerifyParams( msg, pkvRequest, m_pWebApiFunc ) ) + return false; + + bool bRet = BYieldingRunJobFromRequest( pkvRequest, pkvResponse ); + + + // request failed, set success for wg + if ( pkvResponse->IsEmpty( "success" ) ) + { + pkvResponse->SetInt( "success", bRet ? k_EResultOK : k_EResultFail ); + } + + // send response msg + CGCWGJobMgr::SendResponse( msg, pkvResponse, bRet ); + + return true; +} + +bool CGCWGJob::BVerifyParams( const CGCMsg<MsgGCWGRequest_t> & msg, KeyValues *pkvRequest, const WebApiFunc_t * pWebApiFunc ) +{ + // we've found the function; now validate the call + for ( int i = 0; i < Q_ARRAYSIZE( pWebApiFunc->m_rgParams ); i++ ) + { + if ( !pWebApiFunc->m_rgParams[i].m_pchParam ) + break; + + // just simple validation for now; make sure the key exists + if ( !pWebApiFunc->m_rgParams[i].m_bOptional && !pkvRequest->FindKey( pWebApiFunc->m_rgParams[i].m_pchParam ) ) + { + CGCWGJobMgr::SendErrorMessage( msg, + CFmtStr( "Error: Missing parameter '%s' from web request '%s'\n", pWebApiFunc->m_rgParams[i].m_pchParam, pWebApiFunc->m_pchRequestName ), + k_EResultInvalidParam ); + EmitWarning( SPEW_GC, 2, "Error: Missing parameter '%s' from web request '%s'\n", pWebApiFunc->m_rgParams[i].m_pchParam, pWebApiFunc->m_pchRequestName ); + return false; + } + } + + return true; +} + + +void CGCWGJob::SetErrorMessage( KeyValues *pkvErr, const char *pchErrorMsg, int32 nResult ) +{ + CGCWGJobMgr::SetErrorMessage( pkvErr, pchErrorMsg, nResult ); +} + + +//----------------------------------------------------------------------------- +// CGCJobVerifySession - A job that asks steam if a given user is still connected +// and cleans up the session if the user is gone +//----------------------------------------------------------------------------- +bool CGCJobVerifySession::BYieldingRunGCJob() +{ + if ( !m_pGC->BYieldingLockSteamID( m_steamID, __FILE__, __LINE__ ) ) + return false; + + m_pGC->YieldingRequestSession( m_steamID ); + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CWebAPIJob::CWebAPIJob( CGCBase *pGC, EWebAPIOutputFormat eDefaultOutputFormat ) +: CGCJob( pGC ), m_eDefaultOutputFormat( eDefaultOutputFormat ) +{ +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CWebAPIJob::~CWebAPIJob() +{ +} + +//----------------------------------------------------------------------------- +// Called to handle converting a web api response to a serialized result and free the object as well on a background thread + +class CEmitWebAPIData +{ +public: + CEmitWebAPIData( CMsgHttpRequest* pRequest) : m_bResult( false ), m_Request( pRequest ) {} + + //inputs + CHTTPRequest m_Request; + + //outputs + CHTTPResponse m_Response; + std::string m_sSerializedResponse; + bool m_bResult; +}; + +//the worker thread function that is responsible for serializing the response from web api values to a text buffer, freeing the web api value tree, and serializing the message to a protobuf for +//direct sending. If this function fails (a false response value) the message will not be serialized +static void ThreadedEmitFormattedOutputWrapperAndFreeResponse( CWebAPIResponse *pResponse, CEmitWebAPIData* pEmitData, EWebAPIOutputFormat eDefaultFormat, size_t unMaxResultSize ) +{ + // parse the output type that we want the result to be in + EWebAPIOutputFormat eOutputFormat = eDefaultFormat; + const char *pszParamOutput = pEmitData->m_Request.GetGETParamString( "format", NULL ); + if ( !pszParamOutput ) + pszParamOutput = pEmitData->m_Request.GetPOSTParamString( "format", NULL ); + + if ( pszParamOutput ) + { + if ( Q_stricmp( pszParamOutput, "xml" ) == 0 ) + eOutputFormat = k_EWebAPIOutputFormat_XML; + else if ( Q_stricmp( pszParamOutput, "vdf" ) == 0 ) + eOutputFormat = k_EWebAPIOutputFormat_VDF; + else if ( Q_stricmp( pszParamOutput, "json" ) == 0 ) + eOutputFormat = k_EWebAPIOutputFormat_JSON; + } + + pEmitData->m_bResult = pResponse->BEmitFormattedOutput( eOutputFormat, *( pEmitData->m_Response.GetBodyBuffer() ), unMaxResultSize ); + delete pResponse; + + //update the response code on the output (can probably be done elsewhere), but must be done before we pack the message below + switch( eOutputFormat ) + { + case k_EWebAPIOutputFormat_JSON: + pEmitData->m_Response.SetResponseHeaderValue( "content-type", "application/json; charset=UTF-8" ); + break; + case k_EWebAPIOutputFormat_XML: + pEmitData->m_Response.SetResponseHeaderValue( "content-type", "text/xml; charset=UTF-8" ); + break; + case k_EWebAPIOutputFormat_VDF: + pEmitData->m_Response.SetResponseHeaderValue( "content-type", "text/vdf; charset=UTF-8" ); + break; + default: + break; + } + + //if successful, we can go ahead and convert this all the way into a completely serialized form for sending over the wire + if( pEmitData->m_bResult ) + { + CMsgHttpResponse msgResponse; + pEmitData->m_Response.SerializeIntoProtoBuf( msgResponse ); + msgResponse.SerializeToString( &pEmitData->m_sSerializedResponse ); + } +} + +//called to respond to a web api request with the specified response value +static void WebAPIRespondToRequest( const char* pszName, uint32 nSenderIP, const CHTTPResponse& response, const CProtoBufMsg< CMsgWebAPIRequest >& msg ) +{ + VPROF_BUDGET( "WebAPI - sending result", VPROF_BUDGETGROUP_STEAM ); + CProtoBufMsg<CMsgHttpResponse> msgResponse( k_EGCMsgWebAPIJobRequestHttpResponse, msg ); + response.SerializeIntoProtoBuf( msgResponse.Body() ); + GGCBase()->BReplyToMessage( msgResponse, msg ); + + //track this message in the web API response + g_WebAPIAccountTracker.TrackFunction( msg.Body().api_key().account_id(), nSenderIP, pszName, msgResponse.Body().body().size() ); +} + +//responses to the web api message with an error code +static void WebAPIRespondWithError( const char* pszName, uint32 nSenderIP, const CProtoBufMsg< CMsgWebAPIRequest >& msg, EHTTPStatusCode statusCode ) +{ + CHTTPResponse response; + response.SetStatusCode( statusCode ); + WebAPIRespondToRequest( pszName, nSenderIP, response, msg ); +} + +//parses an IP address out of the provided string. This will be the last IP address in the list +static uint32 ParseIPAddrFromForwardHeader( const char* pszHeader ) +{ + //find the last comma in the string, our IP address follows that + const char* pszStart = V_strrchr( pszHeader, ',' ); + //if no match, then we just have the ip address, otherwise advance past the comma + if( !pszStart ) + pszStart = pszHeader; + else + pszStart++; + + //skip leading spaces + while( V_isspace( *pszStart ) ) + pszStart++; + + return ntohl( inet_addr( pszStart ) ); +} + +//----------------------------------------------------------------------------- +// Purpose: Receive a k_EMsgWebAPIJobRequest and manage serialization/deserialization to +// web request/response objects +//----------------------------------------------------------------------------- +bool CWebAPIJob::BYieldingRunJobFromMsg( IMsgNetPacket * pNetPacket ) +{ + VPROF_BUDGET( "WebAPI", VPROF_BUDGETGROUP_STEAM ); + + CProtoBufMsg<CMsgWebAPIRequest> msg( pNetPacket ); + + //make sure all the required parameters were present + bool bMsgParsedOK = msg.Body().has_api_key() + && msg.Body().has_interface_name() + && msg.Body().has_method_name() + && msg.Body().has_request() + && msg.Body().has_version(); + + if( !bMsgParsedOK ) + { + WebAPIRespondWithError( GetName(), 0, msg, k_EHTTPStatusCode400BadRequest ); + return true; + } + + //determine the account that sent this request + const AccountID_t nSenderAccountID = msg.Body().api_key().account_id(); + uint32 nSenderIP = 0; + + //if this isn't a system request, try and identify the IP address of the sender so we can rate limit accordingly + if( nSenderAccountID != 0 ) + { + const int nNumHeaders = msg.Body().request().headers_size(); + for( int nHeader = 0; nHeader < nNumHeaders; nHeader++ ) + { + if( strcmp( msg.Body().request().headers( nHeader ).name().c_str(), "X-Forwarded-For" ) != 0 ) + continue; + + //this is our IP address + nSenderIP = ParseIPAddrFromForwardHeader( msg.Body().request().headers( nHeader ).value().c_str() ); + } + + //see if we have the kill switch turned on + if( webapi_kill_switch.GetBool() ) + { + if( webapi_kill_switch_error_response.GetBool() ) + WebAPIRespondWithError( GetName(), nSenderIP, msg, k_EHTTPStatusCode503ServiceUnavailable ); + return true; + } + } + else + { +// !FIXME! DOTAMERGE +// //determine the priority of this steam request +// uint32 nPriority; +// nSenderIP = GetSteamRequestIPAddress( msg.Body().request(), nPriority ); +// //and allow for a kill switch based upon this priority +// if( ( ( nPriority == CWebAPIAccountTracker::k_nSteamIP_Low ) && !webapi_enable_steam_low.GetBool() ) || +// ( ( nPriority == CWebAPIAccountTracker::k_nSteamIP_Normal ) && !webapi_enable_steam_normal.GetBool() ) || +// ( ( nPriority == CWebAPIAccountTracker::k_nSteamIP_High ) && !webapi_enable_steam_high.GetBool() ) ) +// { +// if( webapi_kill_switch_error_response.GetBool() ) +// WebAPIRespondWithError( GetName(), 0, msg ); +// return true; +// } + } + + //track stats for this account, and handle rate limiting + if( !g_WebAPIAccountTracker.TrackUser( nSenderAccountID, nSenderIP ) ) + { + if( webapi_kill_switch_error_response.GetBool() ) + WebAPIRespondWithError( GetName(), nSenderIP, msg, k_EHTTPStatusCode429TooManyRequests ); + return true; + } + + //allocate the data that we'll use to fill out the request and send it to the background thread for work + CPlainAutoPtr< CEmitWebAPIData > pEmitData( new CEmitWebAPIData( const_cast< CMsgHttpRequest* >( &msg.Body().request() ) ) ); + CHTTPRequest& request = pEmitData->m_Request; + CHTTPResponse& response = pEmitData->m_Response; + + { + VPROF_BUDGET( "WebAPI - Prepare msg", VPROF_BUDGETGROUP_STEAM ); + + m_webAPIKey.DeserializeFromProtoBuf( msg.Body().api_key() ); + } + + CPlainAutoPtr< CWebAPIResponse > pwebAPIResponse( new CWebAPIResponse() ); + + { + VPROF_BUDGET( "WebAPI - Process msg", VPROF_BUDGETGROUP_STEAM ); + if( !BYieldingRunJobFromAPIRequest( msg.Body().interface_name().c_str(), msg.Body().method_name().c_str(), msg.Body().version(), &request, &response, pwebAPIResponse.Get() ) ) + { + //error executing our job + WebAPIRespondWithError( GetName(), nSenderIP, msg, k_EHTTPStatusCode500InternalServerError ); + return false; + } + } + +// !FIXME! DOTAMERGE +// //see if they want to re-route this request +// if( m_nRerouteRequest >= 0 ) +// { +// if( ( uint32 )m_nRerouteRequest == m_pGC->GetGCDirIndex() ) +// { +// AssertMsg( false, "Error: WebAPI %s attempting to re-route a web api message to itself (%d)", GetName(), m_nRerouteRequest ); +// } +// else +// { +// //route to the other GC and discard this message +// CProtoBufMsg< CMsgGCMsgWebAPIJobRequestForwardResponse > msgRoute( k_EGCMsgWebAPIJobRequestForwardResponse ); +// msgRoute.Body().set_dir_index( m_nRerouteRequest ); +// m_pGC->BReplyToMessage( msgRoute, msg ); +// } +// return true; +// } + + VPROF_BUDGET( "WebAPI - Emitting result", VPROF_BUDGETGROUP_STEAM ); + + response.SetStatusCode( pwebAPIResponse->GetStatusCode() ); + response.SetExpirationHeaderDeltaFromNow( pwebAPIResponse->GetExpirationSeconds() ); + if( pwebAPIResponse->GetLastModified() ) + response.SetHeaderTimeValue( "last-modified", pwebAPIResponse->GetLastModified() ); + + // if we aren't allowed to have a message body on this, simply send the result back now + if( !CHTTPUtil::BStatusCodeAllowsBody( pwebAPIResponse->GetStatusCode() ) ) + { + AssertMsg( pwebAPIResponse->GetRootValue() == NULL, "Response HTTP status code %d doesn't allow a body, but one was present", pwebAPIResponse->GetStatusCode() ); + //since we didn't have a body to serialize, we need to handle sending just the response + WebAPIRespondToRequest( GetName(), nSenderIP, response, msg ); + return true; + } + + //let the job convert the formatting and free our response for us (quite costly). Note that we detach the web API response since this job will free it + bool bThreadFuncSucceeded = BYieldingWaitForThreadFunc( CreateFunctor( ThreadedEmitFormattedOutputWrapperAndFreeResponse, pwebAPIResponse.Detach(), pEmitData.Get(), m_eDefaultOutputFormat, (size_t)cv_webapi_result_size_limit.GetInt() ) ); + + //if we called the function successfully and had a valid result, just send back the preserialized body + if( bThreadFuncSucceeded && pEmitData->m_bResult ) + { + m_pGC->BReplyToMessageWithPreSerializedBody( k_EGCMsgWebAPIJobRequestHttpResponse, msg, ( const byte* )pEmitData->m_sSerializedResponse.c_str(), pEmitData->m_sSerializedResponse.size() ); + g_WebAPIAccountTracker.TrackFunction( nSenderAccountID, nSenderIP, GetName(), pEmitData->m_sSerializedResponse.size() ); + } + else + { + //we failed to generate a response, see if we ran out of space + if( response.GetBodyBuffer()->TellMaxPut() > cv_webapi_result_size_limit.GetInt() ) + { + // !FIXME! DOTAMERGE + //CGCAlertInfo alert( "WebAPIResponseSize", "WebAPI request %s failed to emit because it exceeded %d characters", request.GetURL(), cv_webapi_result_size_limit.GetInt() ); + // + //switch( request.GetEHTTPMethod() ) + //{ + //case k_EHTTPMethodGET: + // { + // const uint32 nNumParams = request.GetGETParamCount(); + // for( uint32 nParam = 0; nParam < nNumParams; nParam++ ) + // { + // alert.AddExtendedInfoLine( "%s=%s", request.GetGETParamName( nParam ), request.GetGETParamValue( nParam ) ); + // } + // } + // break; + //case k_EHTTPMethodPOST: + // { + // const uint32 nNumParams = request.GetPOSTParamCount(); + // for( uint32 nParam = 0; nParam < nNumParams; nParam++ ) + // { + // alert.AddExtendedInfoLine( "%s=%s", request.GetPOSTParamName( nParam ), request.GetPOSTParamValue( nParam ) ); + // } + // } + // break; + //} + // + //GGCBase()->PostAlert( alert ); + GGCBase()->PostAlert( k_EAlertTypeInfo, false, CFmtStr( "WebAPI request %s failed to emit because it exceeded %d characters", request.GetURL(), cv_webapi_result_size_limit.GetInt() ) ); + } + else + { + EG_WARNING( SPEW_GC, "WebAPI %s - request %s failed to emit for unknown reason\n", GetName(), request.GetURL() ); + } + //make sure that they get an error code back + WebAPIRespondWithError( GetName(), nSenderIP, msg, k_EHTTPStatusCode500InternalServerError ); + } + + return true; +} + +void CWebAPIJob::AddLocalizedString( CWebAPIValues *pOutDefn, const char *pchFieldName, const char *pchKeyName, ELanguage eLang, bool bReturnTokenIfNotFound ) +{ + // NULL keys we just skip + if( !pchKeyName ) + return; + + const char *pchValue; + if( eLang == k_Lang_None ) + { + pchValue = pchKeyName; + } + else + { + pchValue = GGCBase()->LocalizeToken( pchKeyName, eLang, bReturnTokenIfNotFound ); + } + + if( pchValue ) + { + pOutDefn->CreateChildObject( pchFieldName )->SetStringValue( pchValue ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: A wrapper to call BEmitFormattedOutput and pass out a return value +// since functors don't make return values available. +//----------------------------------------------------------------------------- +/*static*/ void CWebAPIJob::ThreadedEmitFormattedOutputWrapper( CWebAPIResponse *pResponse, EWebAPIOutputFormat eFormat, CUtlBuffer *poutputBuffer, size_t unMaxResultSize, bool *pbResult ) +{ + *pbResult = pResponse->BEmitFormattedOutput( eFormat, *poutputBuffer, unMaxResultSize ); +} + + +} // namespace GCSDK diff --git a/gcsdk/gcleaderboardapi.cpp b/gcsdk/gcleaderboardapi.cpp new file mode 100644 index 0000000..2883acc --- /dev/null +++ b/gcsdk/gcleaderboardapi.cpp @@ -0,0 +1,93 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//============================================================================= + +#include "stdafx.h" + +#include "gcsdk/enumutils.h" +#include "gcsdk/gcbase.h" +#include "gcsdk/http.h" +#include "gcsdk/job.h" +#include "steam/isteamuserstats.h" +#include "gcleaderboardapi.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +using namespace GCSDK; + +// @note Tom Bui: copied from steam +ENUMSTRINGS_START( ELeaderboardSortMethod ) +{ k_ELeaderboardSortMethodNone, "" }, +{ k_ELeaderboardSortMethodAscending, "Ascending" }, +{ k_ELeaderboardSortMethodDescending, "Descending" }, +ENUMSTRINGS_REVERSE( ELeaderboardSortMethod, k_ELeaderboardSortMethodNone ) + +ENUMSTRINGS_START( ELeaderboardDisplayType ) +{ k_ELeaderboardDisplayTypeNone, "" }, +{ k_ELeaderboardDisplayTypeNumeric, "Numeric" }, +{ k_ELeaderboardDisplayTypeTimeSeconds, "Seconds" }, +{ k_ELeaderboardDisplayTypeTimeMilliSeconds, "MilliSeconds" }, +ENUMSTRINGS_REVERSE( ELeaderboardDisplayType, k_ELeaderboardDisplayTypeNone ) + +ENUMSTRINGS_START( ELeaderboardUploadScoreMethod ) +{ k_ELeaderboardUploadScoreMethodNone, "" }, +{ k_ELeaderboardUploadScoreMethodKeepBest, "KeepBest" }, +{ k_ELeaderboardUploadScoreMethodForceUpdate, "ForceUpdate" }, +ENUMSTRINGS_REVERSE( ELeaderboardUploadScoreMethod, k_ELeaderboardUploadScoreMethodNone ) + +namespace GCSDK +{ + uint32 Leaderboard_YieldingFind( const char *pName, ELeaderboardSortMethod eLeaderboardSortMethod, ELeaderboardDisplayType eLeaderboardDisplayType, bool bCreateIfNotFound ) + { + CSteamAPIRequest apiRequest( k_EHTTPMethodPOST, "ISteamLeaderboards", "FindOrCreateLeaderboard", 1 ); + apiRequest.SetPOSTParamUInt32( "appid", GGCBase()->GetAppID() ); + apiRequest.SetPOSTParamString( "name", pName ); + + apiRequest.SetPOSTParamString( "sortmethod", PchNameFromELeaderboardSortMethod( eLeaderboardSortMethod ) ); + apiRequest.SetPOSTParamString( "displaytype", PchNameFromELeaderboardDisplayType( eLeaderboardDisplayType ) ); + apiRequest.SetPOSTParamBool( "createifnotfound", bCreateIfNotFound ); + + KeyValuesAD kvAPIResponse( "response" ); + const EResult eCallResult = GGCBase()->YieldingSendHTTPRequestKV( &apiRequest, kvAPIResponse ); + const EResult eAPIResult = eCallResult == k_EResultOK ? static_cast<EResult>( kvAPIResponse->GetInt( "result", k_EResultFail ) ) : k_EResultFail; + if ( eAPIResult == k_EResultOK ) + { + KeyValues *pKVEntry = kvAPIResponse->FindKey( pName ); + if ( pKVEntry ) + { + return pKVEntry->GetInt( "leaderBoardID", kInvalidLeaderboardID ); + } + } + return kInvalidLeaderboardID; + } + + bool Leaderboard_YieldingSetScore( uint32 unLeaderboardID, const CSteamID &steamID, ELeaderboardUploadScoreMethod eLeaderboardUploadScoreMethod, int score ) + { + Assert( unLeaderboardID != kInvalidLeaderboardID ); + if ( unLeaderboardID == kInvalidLeaderboardID ) + return false; + + CSteamAPIRequest apiRequest( k_EHTTPMethodPOST, "ISteamLeaderboards", "SetLeaderboardScore", 1 ); + apiRequest.SetPOSTParamUInt32( "appid", GGCBase()->GetAppID() ); + apiRequest.SetPOSTParamUInt32( "leaderboardid", unLeaderboardID ); + apiRequest.SetPOSTParamUInt64( "steamid", steamID.ConvertToUint64() ); + apiRequest.SetPOSTParamInt32( "score", score ); + + apiRequest.SetPOSTParamString( "scoremethod", PchNameFromELeaderboardUploadScoreMethod( eLeaderboardUploadScoreMethod ) ); + + KeyValuesAD kvAPIResponse( "response" ); + const EResult eCallResult = GGCBase()->YieldingSendHTTPRequestKV( &apiRequest, kvAPIResponse ); + const EResult eAPIResult = eCallResult == k_EResultOK ? static_cast<EResult>( kvAPIResponse->GetInt( "result", k_EResultFail ) ) : k_EResultFail; + if ( eAPIResult == k_EResultOK ) + return true; + + EmitError( SPEW_GC, __FUNCTION__ ": error code %u/%u setting leaderboard %u to %i (%s) for user '%s'.\n", + eCallResult, eAPIResult, unLeaderboardID, score, PchNameFromELeaderboardUploadScoreMethod( eLeaderboardUploadScoreMethod ), steamID.Render() ); + return false; + } + +}; // namespace GCSDK diff --git a/gcsdk/gclogger.cpp b/gcsdk/gclogger.cpp new file mode 100644 index 0000000..09a3b2e --- /dev/null +++ b/gcsdk/gclogger.cpp @@ -0,0 +1,263 @@ +//========= Copyright (c), Valve LLC, All rights reserved. ============ +// +// Purpose: +// +// $NoKeywords: $ +//============================================================================= + + +#include "stdafx.h" +#include "steamextra/gamecoordinator/igamecoordinatorhost.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +//----------------------------------------------------------------------------- +// Maximum length of a sprintf'ed logging message. +//----------------------------------------------------------------------------- +const int MAX_LOGGING_MESSAGE_LENGTH = 2048; + +namespace GCSDK +{ + +int g_nMaxSpewLevel = 4; +int g_nMaxLogLevel = 4; + +//----------------------------------------------------------------------------- +// Purpose: Sends an event back to the GC host for logging +// Input: pchGroupName - group to log to +// spewType - the type of the message +// iLevel - level of spew ( 0..4 ) +// iLevelLog - level for logging +// pchMsg - printf format +//----------------------------------------------------------------------------- +void EmitBaseMessageV( const char *pchGroupName, SpewType_t spewType, int iSpewLevel, int iLevelLog, const char *pchMsg, va_list vaArgs ) +{ + VPROF_BUDGET( "GCHost", VPROF_BUDGETGROUP_STEAM ); + + char pchBuf[ MAX_LOGGING_MESSAGE_LENGTH ]; + Q_vsnprintf( pchBuf, MAX_LOGGING_MESSAGE_LENGTH, pchMsg, vaArgs ); + +#ifdef GC +// !FIXME! DOTAMERGE +// // If this is coming from the context of a job, then allow that job to +// // override our emit via a specified handler. +// if ( g_pJobCur ) +// { +// IJobEmitSpewHandler *pHandler = g_pJobCur->GetEmitSpewHandler(); +// if ( pHandler ) +// { +// bool bOutputSpew = pHandler->OnEmitSpew( pchGroupName, spewType, iSpewLevel, iLevelLog, pchBuf ); +// if ( !bOutputSpew ) +// { +// return; +// } +// } +// } + + { + VPROF_BUDGET( "GCHost - EmitMessage", VPROF_BUDGETGROUP_STEAM ); + // !FIXME! DOTAMERGE + //GGCInterface()->EmitSpew( pchGroupName, spewType, iSpewLevel, iLevelLog, pchBuf ); + GGCHost()->EmitMessage( pchGroupName, spewType, iSpewLevel, iLevelLog, pchBuf ); + } +#else + // TODO: Actually log it + DevMsg( "%s", pchBuf ); +#endif +} + +//similar to the above, but takes in a group, and handles filtering messages out that are disabled, and extracting other info from the group +void EmitBaseMessageV( const CGCEmitGroup& Group, SpewType_t spewType, int iConsoleLevel, int iLogLevel, const char *pchMsg, va_list vaArgs ) +{ + int iClampConsoleLevel = ( iConsoleLevel <= MIN( Group.GetConsoleLevel(), g_nMaxSpewLevel ) ) ? SPEW_ALWAYS : SPEW_NEVER; + int iClampLogLevel = ( iLogLevel <= MIN( Group.GetLogLevel(), g_nMaxLogLevel ) ) ? LOG_ALWAYS : LOG_NEVER; + if( ( iClampConsoleLevel != SPEW_NEVER ) || ( iClampLogLevel != LOG_NEVER ) ) + EmitBaseMessageV( Group.GetName(), spewType, iClampConsoleLevel, iClampLogLevel, pchMsg, vaArgs ); +} + +//------------------------------ +// AssertError + +void CGCEmitGroup::Internal_AssertError( const char *pchMsg, ... ) const +{ + va_list args; + va_start( args, pchMsg ); + AssertErrorV( pchMsg, args ); + va_end( args ); +} + +void CGCEmitGroup::AssertErrorV( const char *pchMsg, va_list vaArgs ) const +{ + EmitBaseMessageV( *this, SPEW_ASSERT, 1, 1, pchMsg, vaArgs ); +} + +//------------------------------ +// Error + +void CGCEmitGroup::Internal_Error( const char *pchMsg, ... ) const +{ + va_list args; + va_start( args, pchMsg ); + ErrorV( pchMsg, args ); + va_end( args ); +} + +void CGCEmitGroup::ErrorV( const char *pchMsg, va_list vaArgs ) const +{ + EmitBaseMessageV( *this, SPEW_ERROR, 1, 1, pchMsg, vaArgs ); +} + +//------------------------------ +// Warning + +void CGCEmitGroup::Internal_Warning( const char *pchMsg, ... ) const +{ + va_list args; + va_start( args, pchMsg ); + WarningV( pchMsg, args ); + va_end( args ); +} + +void CGCEmitGroup::WarningV( const char *pchMsg, va_list vaArgs ) const +{ + EmitBaseMessageV( *this, SPEW_WARNING, 2, 2, pchMsg, vaArgs ); +} + +//------------------------------ +// Msg + +void CGCEmitGroup::Internal_Msg( const char *pchMsg, ... ) const +{ + va_list args; + va_start( args, pchMsg ); + MsgV( pchMsg, args ); + va_end( args ); +} + +void CGCEmitGroup::MsgV( const char *pchMsg, va_list vaArgs ) const +{ + EmitBaseMessageV( *this, SPEW_MESSAGE, 3, 3, pchMsg, vaArgs ); +} + +//------------------------------ +// Verbose + +void CGCEmitGroup::Internal_Verbose( const char *pchMsg, ... ) const +{ + va_list args; + va_start( args, pchMsg ); + VerboseV( pchMsg, args ); + va_end( args ); +} + +void CGCEmitGroup::VerboseV( const char *pchMsg, va_list vaArgs ) const +{ + EmitBaseMessageV( *this, SPEW_MESSAGE, 4, 4, pchMsg, vaArgs ); +} + + +//------------------------------ +// Verbose + +void CGCEmitGroup::Internal_BoldMsg( const char *pchMsg, ... ) const +{ + va_list args; + va_start( args, pchMsg ); + BoldMsgV( pchMsg, args ); + va_end( args ); +} + +void CGCEmitGroup::BoldMsgV( const char *pchMsg, va_list vaArgs ) const +{ + // !FIXME! DOTAMERGE + //EmitBaseMessageV( *this, SPEW_BOLD_MESSAGE, 1, 1, pchMsg, vaArgs ); + EmitBaseMessageV( *this, SPEW_MESSAGE, 1, 1, pchMsg, vaArgs ); +} + + +//------------------------------ +// General Emit + + +void CGCEmitGroup::Internal_Emit( EMsgLevel eLvl, PRINTF_FORMAT_STRING const char *pchMsg, ... ) const +{ + va_list args; + va_start( args, pchMsg ); + EmitV( eLvl, pchMsg, args ); + va_end( args ); +} + +void CGCEmitGroup::EmitV( EMsgLevel eLvl, PRINTF_FORMAT_STRING const char *pchMsg, va_list vaArgs ) const +{ + switch( eLvl ) + { + case kMsg_Error: ErrorV( pchMsg, vaArgs ); break; + case kMsg_Warning: WarningV( pchMsg, vaArgs ); break; + case kMsg_Msg: MsgV( pchMsg, vaArgs ); break; + case kMsg_Verbose: VerboseV( pchMsg, vaArgs ); break; + default: + AssertMsg1( false, "Unexpected error level of %d provided to GCEmitGroup::Emit", eLvl ); + break; + } +} + + + +//--------------------------------------------------------------------- +// Legacy Interface +//--------------------------------------------------------------------- + +void EGInternal_EmitInfo( const CGCEmitGroup& Group, int iLevel, int iLevelLog, const char *pchMsg, ... ) +{ + va_list args; + va_start( args, pchMsg ); + EmitBaseMessageV( Group, SPEW_MESSAGE, iLevel, iLevelLog, pchMsg, args ); + va_end( args ); +} + +void EmitInfoV( const CGCEmitGroup& Group, int iLevel, int iLevelLog, const char *pchMsg, va_list vaArgs ) +{ + EmitBaseMessageV( Group, SPEW_MESSAGE, iLevel, iLevelLog, pchMsg, vaArgs ); +} + +void EmitWarning( const CGCEmitGroup& Group, int iLevel, const char *pchMsg, ... ) +{ + va_list args; + va_start( args, pchMsg ); + EmitBaseMessageV( Group, SPEW_WARNING, iLevel, iLevel, pchMsg, args ); + va_end( args ); +} + +void EmitError( const CGCEmitGroup& Group, const char *pchMsg, ... ) +{ + va_list args; + va_start( args, pchMsg ); + EmitBaseMessageV( Group, SPEW_ERROR, 1, 1, pchMsg, args ); + va_end( args ); +} + +// Emit an assert-like error, generating a minidump +void EmitAssertError( const CGCEmitGroup& Group, const char *pchMsg, ... ) +{ + va_list args; + va_start( args, pchMsg ); + EmitBaseMessageV( Group, SPEW_ASSERT, 1, 1, pchMsg, args ); + va_end( args ); +} + +//legacy group types +DECLARE_GC_EMIT_GROUP( SPEW_SYSTEM_MISC, system ); +DECLARE_GC_EMIT_GROUP( SPEW_JOB, job ); +DECLARE_GC_EMIT_GROUP( SPEW_CONSOLE, console ); +DECLARE_GC_EMIT_GROUP( SPEW_GC, gc ); +DECLARE_GC_EMIT_GROUP( SPEW_SQL, sql ); +DECLARE_GC_EMIT_GROUP( SPEW_NETWORK, network ); +DECLARE_GC_EMIT_GROUP( SPEW_SHAREDOBJ, sharedobj ); +DECLARE_GC_EMIT_GROUP( SPEW_MICROTXN, microtxn ); +DECLARE_GC_EMIT_GROUP( SPEW_PROMO, promo ); +DECLARE_GC_EMIT_GROUP( SPEW_PKGITEM, pkgitem ); +DECLARE_GC_EMIT_GROUP( SPEW_ECONOMY, econ ); +DECLARE_GC_EMIT_GROUP( SPEW_THREADS, threads ); + +} // namespace GCSDK diff --git a/gcsdk/gcmsg.cpp b/gcsdk/gcmsg.cpp new file mode 100644 index 0000000..63e0955 --- /dev/null +++ b/gcsdk/gcmsg.cpp @@ -0,0 +1,94 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Holds function bodies for GCSDK +// +//============================================================================= + +#include "stdafx.h" +#include "gcmsg.h" +#include "msgprotobuf.h" + +#include "tier0/memdbgoff.h" + +#ifdef GC +namespace GCSDK +{ + IMPLEMENT_CLASS_MEMPOOL( CStructNetPacket, 1000, UTLMEMORYPOOL_GROW_SLOW ); +} +#endif + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + +CUtlString GCMsgHdr_t::GetHeaderDescription( ) +{ + CUtlString desc; + desc.Format( "gc msg %s, SteamID %llu", PchMsgNameFromEMsg( m_eMsg ), m_ulSteamID ); + return desc; +} + +CUtlString GCMsgHdrEx_t::GetHeaderDescription( ) +{ + CUtlString desc; + desc.Format( "gc msg %s, SteamID %llu, version %hd, job source %llu, job target %llu", PchMsgNameFromEMsg( m_eMsg ), m_ulSteamID, m_nHdrVersion, m_JobIDSource, m_JobIDTarget ); + return desc; +} + +CUtlString GCMsgInterHdr_t::GetHeaderDescription( ) +{ + CUtlString desc; + desc.Format( "gc inter msg %s, SourceAppID %d, version %hd, job source %llu, job target %llu", PchMsgNameFromEMsg( m_eMsg ), m_unSourceAppId, m_nHdrVersion, m_JobIDSource, m_JobIDTarget ); + return desc; +} + +//----------------------------------------------------------------------------- +// Purpose: Takes a pNetPacket, then converts it to the appropriate actual message +// type and return the IMsgNetPacket interface. +//----------------------------------------------------------------------------- +IMsgNetPacket *IMsgNetPacketFromCNetPacket( CNetPacket *pNetPacket ) +{ + uint32 cubPkt = pNetPacket->CubData(); + uint8* pubPkt = pNetPacket->PubData(); + + if ( cubPkt >= sizeof( ProtoBufMsgHeader_t ) && ( ( ( ProtoBufMsgHeader_t *) pubPkt )->m_EMsgFlagged & k_EMsgProtoBufFlag ) ) + { + CProtoBufNetPacket *pMsgNetPacket = new CProtoBufNetPacket( pNetPacket, GCProtoBufMsgSrc_Unspecified, CSteamID(), 0, 0x7FFFFFFF & ( ( ProtoBufMsgHeader_t *) pubPkt )->m_EMsgFlagged ); + if ( !pMsgNetPacket->IsValid() ) + { + pMsgNetPacket->Release(); + return NULL; + } + + return pMsgNetPacket; + } + else + { + if ( cubPkt >= sizeof( GCMsgHdrEx_t ) ) + { + CStructNetPacket *pMsgNetPacket = (CStructNetPacket*)malloc( sizeof( CStructNetPacket ) ); + Construct( pMsgNetPacket, pNetPacket ); + return pMsgNetPacket; + } + } + + EmitError( SPEW_GC, "IMsgNetPacketFromCNetPacket: malformed packet, size %d bytes\n", cubPkt ); + for ( uint32 i = 0 ; i < cubPkt ; i += 16 ) + { + char buf[512]; + char *d = buf; + d += Q_snprintf(d, 12, "%08X:", i ); + for ( uint32 j = i ; j < i+16 && j < cubPkt ; ++j ) + { + d += Q_snprintf(d, 8, " %02X", pubPkt[j] ); + } + *d = '\0'; + EmitError( SPEW_GC, "%s\n", buf ); + } + + AssertMsg( false, "IMsgNetPacketFromCNetPacket couldn't detect any appropriate msg format, returning NULL." ); + return NULL; +} +} // namespace GCSDK diff --git a/gcsdk/gcparalleljobfarm.cpp b/gcsdk/gcparalleljobfarm.cpp new file mode 100644 index 0000000..5d15f9b --- /dev/null +++ b/gcsdk/gcparalleljobfarm.cpp @@ -0,0 +1,119 @@ +//====== Copyright (c), Valve Corporation, All rights reserved. ======= +// +// Purpose: Implements parallel job farming process +// +//============================================================================= + +#include "stdafx.h" +#include "rtime.h" +#include "gcparalleljobfarm.h" + + +namespace GCSDK +{ + +bool IYieldingParallelFarmJobHandler::BYieldingExecuteParallel( int numJobsParallel, char const *pchJobName, uint nTimeoutSec ) +{ + AssertRunningJob(); + + if ( !pchJobName ) + pchJobName = GJobCur().GetName(); + + struct CParallelFarmHeapData_t + { + explicit CParallelFarmHeapData_t( IYieldingParallelFarmJobHandler *pHandler, int numJobsFarmLimit ) + { + m_pHandler = pHandler; + m_jobIdParent = GJobCur().GetJobID(); + m_numJobsFarmed = 0; + m_numJobsFarmLimit = MAX( 1, numJobsFarmLimit ); + m_iJobSequenceCounter = 0; + m_bErrorEncountered = false; + m_bWorkloadCompleted = false; + } + + IYieldingParallelFarmJobHandler *m_pHandler; + JobID_t m_jobIdParent; + int m_numJobsFarmLimit; + int m_numJobsFarmed; + int m_iJobSequenceCounter; + bool m_bErrorEncountered; + bool m_bWorkloadCompleted; + }; + CParallelFarmHeapData_t *pHeapData = new CParallelFarmHeapData_t( this, numJobsParallel ); + + class CYieldingParallelFarmJob : public CGCJob + { + public: + CYieldingParallelFarmJob( CGCBase *pGC, CParallelFarmHeapData_t *pJobData, char const *pchJobName, uint nTimeoutSec ) : CGCJob( pGC, pchJobName ) + , m_pJobData( pJobData ), m_iJobSequenceCounter( pJobData->m_iJobSequenceCounter ), m_nTimeoutSec( nTimeoutSec ) + { + } + virtual bool BYieldingRunJob( void *pvStartParam ) + { + if ( m_nTimeoutSec ) + SetJobTimeout( m_nTimeoutSec ); + + bool bWorkloadCompleted = false; + bool bResult = m_pJobData->m_pHandler + ? m_pJobData->m_pHandler->BYieldingRunWorkload( m_iJobSequenceCounter, &bWorkloadCompleted ) + : false; + + if ( !bResult ) + m_pJobData->m_bErrorEncountered = true; + else if ( bWorkloadCompleted ) + m_pJobData->m_bWorkloadCompleted = true; + + -- m_pJobData->m_numJobsFarmed; + + if ( !m_pJobData->m_bErrorEncountered && !m_pJobData->m_bWorkloadCompleted ) + { + CYieldingParallelFarmJob *pFarmedJob = new CYieldingParallelFarmJob( m_pGC, m_pJobData, GetName(), m_nTimeoutSec ); + ++ m_pJobData->m_numJobsFarmed; + ++ m_pJobData->m_iJobSequenceCounter; + pFarmedJob->StartJobDelayed( NULL ); + } + + if ( !m_pJobData->m_numJobsFarmed ) + { // No more farmed jobs to wait for + m_pGC->GetJobMgr().BRouteWorkItemCompletedDelayed( m_pJobData->m_jobIdParent, false ); + } + + return bResult; + } + + protected: + CParallelFarmHeapData_t *m_pJobData; + int m_iJobSequenceCounter; + uint m_nTimeoutSec; + }; + + for ( ; ; ++ pHeapData->m_iJobSequenceCounter ) + { + if ( pHeapData->m_numJobsFarmed < pHeapData->m_numJobsFarmLimit ) + { + CYieldingParallelFarmJob *pFarmedJob = new CYieldingParallelFarmJob( GGCBase(), pHeapData, pchJobName, nTimeoutSec ); + ++ pHeapData->m_numJobsFarmed; + pFarmedJob->StartJobDelayed( NULL ); + } + else + { + if ( !GJobCur().BYieldingWaitForWorkItem( pchJobName ) ) + { + EmitError( SPEW_GC, "YieldingExecuteParallel: failed to sync with %u farmed work items.\n", pHeapData->m_numJobsFarmed ); + pHeapData->m_bErrorEncountered = true; + pHeapData->m_pHandler = NULL; // handler itself may become invalid when the function returns + return false; // leak pHeapData because work items might still be running and this can avoid a crash (this condition is abnormal) + } + + break; + } + } + + bool bResult = pHeapData->m_bWorkloadCompleted && !pHeapData->m_bErrorEncountered; + delete pHeapData; + return bResult; +} + + +} diff --git a/gcsdk/gcreportprinter.cpp b/gcsdk/gcreportprinter.cpp new file mode 100644 index 0000000..9671437 --- /dev/null +++ b/gcsdk/gcreportprinter.cpp @@ -0,0 +1,489 @@ +//========= Copyright (c), Valve LLC, All rights reserved. ============ +// +// Purpose: +// +// $NoKeywords: $ +//============================================================================= + + +#include "stdafx.h" +#include "gcreportprinter.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + + +CGCReportPrinter::Variant_t::Variant_t() : + m_nInt( 0 ), + m_fFloat( 0 ) +{} + +CGCReportPrinter::CGCReportPrinter() +{ +} + +CGCReportPrinter::~CGCReportPrinter() +{ + Clear(); +} + +bool CGCReportPrinter::AddStringColumn( const char* pszColumn ) +{ + //don't allow adding columns if data is already present + if( m_Rows.Count() > 0 ) + return false; + + int nIndex = m_Columns.AddToTail(); + Column_t& col = m_Columns[ nIndex ]; + col.m_sName = pszColumn; + col.m_eType = eCol_String; + col.m_eSummary = eSummary_None; + col.m_nNumDecimals = 0; + col.m_eIntDisplay = eIntDisplay_Normal; + return true; +} + +bool CGCReportPrinter::AddIntColumn( const char* pszColumn, ESummaryType eSummary, EIntDisplayType eIntDisplay /* = eIntDisplay_Normal */ ) +{ + //don't allow adding columns if data is already present + if( m_Rows.Count() > 0 ) + return false; + + int nIndex = m_Columns.AddToTail(); + Column_t& col = m_Columns[ nIndex ]; + col.m_sName = pszColumn; + col.m_eType = eCol_Int; + col.m_eSummary = eSummary; + col.m_nNumDecimals = 0; + col.m_eIntDisplay = eIntDisplay; + return true; +} + +bool CGCReportPrinter::AddFloatColumn( const char* pszColumn, ESummaryType eSummary, uint32 unNumDecimal /* = 2 */ ) +{ + //don't allow adding columns if data is already present + if( m_Rows.Count() > 0 ) + return false; + + int nIndex = m_Columns.AddToTail(); + Column_t& col = m_Columns[ nIndex ]; + col.m_sName = pszColumn; + col.m_eType = eCol_Float; + col.m_eSummary = eSummary; + col.m_nNumDecimals = unNumDecimal; + col.m_eIntDisplay = eIntDisplay_Normal; + return true; +} + +bool CGCReportPrinter::AddSteamIDColumn( const char* pszColumn ) +{ + //don't allow adding columns if data is already present + if( m_Rows.Count() > 0 ) + return false; + + int nIndex = m_Columns.AddToTail(); + Column_t& col = m_Columns[ nIndex ]; + col.m_sName = pszColumn; + col.m_eType = eCol_SteamID; + col.m_eSummary = eSummary_None; + col.m_nNumDecimals = 0; + col.m_eIntDisplay = eIntDisplay_Normal; + return true; +} + +void CGCReportPrinter::ClearData() +{ + m_Rows.PurgeAndDeleteElements(); +} + +//called to reset the entire report +void CGCReportPrinter::Clear() +{ + ClearData(); + m_Columns.Purge(); +} + +//called to commit the values that have been added as a new row +bool CGCReportPrinter::CommitRow() +{ + //only let full rows be committed + if( m_RowBuilder.Count() != m_Columns.Count() ) + return false; + if( m_Columns.IsEmpty() ) + return false; + + m_Rows.AddToTail( new TRow( m_RowBuilder ) ); + m_RowBuilder.RemoveAll(); + return true; +} + +//called to add the various data to the report, the order of this must match the columns that were added originally +bool CGCReportPrinter::StrValue( const char* pszStr, const char* pszLink ) +{ + //make sure we have a following column and that the type matches + if( ( m_RowBuilder.Count() >= m_Columns.Count() ) || ( m_Columns[ m_RowBuilder.Count() ].m_eType != eCol_String ) ) + return false; + + Variant_t& val = m_RowBuilder[ m_RowBuilder.AddToTail() ]; + val.m_sStr = pszStr; + val.m_sLink = pszLink; + return true; +} + +bool CGCReportPrinter::IntValue( int64 nValue, const char* pszLink ) +{ + //make sure we have a following column and that the type matches + if( ( m_RowBuilder.Count() >= m_Columns.Count() ) || ( m_Columns[ m_RowBuilder.Count() ].m_eType != eCol_Int ) ) + return false; + + Variant_t& val = m_RowBuilder[ m_RowBuilder.AddToTail() ]; + val.m_nInt = nValue; + val.m_sLink = pszLink; + return true; +} + +bool CGCReportPrinter::FloatValue( double fValue, const char* pszLink ) +{ + //make sure we have a following column and that the type matches + if( ( m_RowBuilder.Count() >= m_Columns.Count() ) || ( m_Columns[ m_RowBuilder.Count() ].m_eType != eCol_Float ) ) + return false; + + Variant_t& val = m_RowBuilder[ m_RowBuilder.AddToTail() ]; + val.m_fFloat = fValue; + val.m_sLink = pszLink; + return true; +} + +bool CGCReportPrinter::SteamIDValue( CSteamID id, const char* pszLink ) +{ + //make sure we have a following column and that the type matches + if( ( m_RowBuilder.Count() >= m_Columns.Count() ) || ( m_Columns[ m_RowBuilder.Count() ].m_eType != eCol_SteamID ) ) + return false; + + Variant_t& val = m_RowBuilder[ m_RowBuilder.AddToTail() ]; + val.m_SteamID = id; + val.m_sLink = pszLink; + return true; +} + +//class that implements sorting our rows based upon a provided column with ascending or descending ordering +class CReportRowSorter +{ +public: + + CReportRowSorter( bool bDescending, uint32 nCol, CGCReportPrinter::EColumnType eType ) : + m_bDescending( bDescending ), m_nCol( nCol ), m_eType( eType ) + { + } + + bool operator()( const CGCReportPrinter::TRow* pR1, const CGCReportPrinter::TRow* pR2 ) + { + //to implement ascending vs descending, we can just flip our inputs + if( m_bDescending ) + std::swap( pR1, pR2 ); + + const CGCReportPrinter::Variant_t& v1 = ( *pR1 )[ m_nCol ]; + const CGCReportPrinter::Variant_t& v2 = ( *pR2 )[ m_nCol ]; + + switch( m_eType ) + { + case CGCReportPrinter::eCol_String: + return stricmp( v1.m_sStr, v2.m_sStr ) < 0; + case CGCReportPrinter::eCol_Int: + return v1.m_nInt < v2.m_nInt; + case CGCReportPrinter::eCol_Float: + return v1.m_fFloat < v2.m_fFloat; + case CGCReportPrinter::eCol_SteamID: + return v1.m_SteamID < v2.m_SteamID; + } + + return false; + } + + bool m_bDescending; + uint32 m_nCol; + CGCReportPrinter::EColumnType m_eType; +}; + + +//sorts the report based upon the specified column name +void CGCReportPrinter::SortReport( const char* pszColumn, bool bDescending ) +{ + //find our column + FOR_EACH_VEC( m_Columns, nCol ) + { + if( stricmp( m_Columns[ nCol ].m_sName, pszColumn ) == 0 ) + { + CReportRowSorter sorter( bDescending, nCol, m_Columns[ nCol ].m_eType ); + std::sort( m_Rows.begin(), m_Rows.end(), sorter ); + break; + } + } +} + +void CGCReportPrinter::SortReport( uint32 nColIndex, bool bDescending ) +{ + if( nColIndex < ( uint32 )m_Columns.Count() ) + { + CReportRowSorter sorter( bDescending, nColIndex, m_Columns[ nColIndex ].m_eType ); + std::sort( m_Rows.begin(), m_Rows.end(), sorter ); + } +} + +//utility to count the number of digits on the provided integer +static uint CountDigits( int64 nInt ) +{ + //the zero special case, since it would otherwise fall out of the loop too early + if( nInt == 0 ) + return 1; + + int nDigits = 0; + if( nInt < 0 ) + { + //for the minus sign + nDigits++; + } + + while( nInt != 0 ) + { + nInt /= 10; + nDigits++; + } + + return nDigits; +} + +static uint CountIntWidth( int64 nValue, CGCReportPrinter::EIntDisplayType eIntDisplay ) +{ + uint unDigits; + switch ( eIntDisplay ) + { + case CGCReportPrinter::eIntDisplay_Memory_MB: + // "1234.56 MB" + unDigits = CountDigits( nValue / k_nMegabyte ) + 1 + 2 + 3; + break; + + case CGCReportPrinter::eIntDisplay_Normal: + default: + // 12345678 + unDigits = CountDigits( nValue ); + break; + } + return unDigits; +} + +static const char * GetIntValueDisplay( int64 nValue, CGCReportPrinter::EIntDisplayType eIntDisplay ) +{ + static CFmtStr1024 s_fmtResult; + switch ( eIntDisplay ) + { + case CGCReportPrinter::eIntDisplay_Memory_MB: + // "1234.56 MB" + s_fmtResult.sprintf( "%lld.%02u MB", ( nValue / k_nMegabyte ), (uint32)( 100.0f * ( ( abs( nValue ) % k_nMegabyte ) / (float)k_nMegabyte ) ) ); + break; + + case CGCReportPrinter::eIntDisplay_Normal: + default: + // 12345678 + s_fmtResult.sprintf( "%lld", nValue ); + break; + } + return s_fmtResult; +} + +//called to print out the provided report +void CGCReportPrinter::PrintReport( CGCEmitGroup& eg, uint32 nTop ) +{ + //we need to determine our totals and maximum row widths for our columns first + CUtlVector< uint32 > vColWidths; + vColWidths.EnsureCapacity( m_Columns.Count() ); + CUtlVector< Variant_t > vSummary; + vSummary.EnsureCapacity( m_Columns.Count() ); + + CFmtStr1024 vMsg; + CFmtStr1024 vSeparator; + + FOR_EACH_VEC( m_Columns, nCol ) + { + const Column_t& col = m_Columns[ nCol ]; + uint32 nColWidth = V_strlen( col.m_sName ); + Variant_t summary; + + //run through all the values to find the row widths and the summary values + FOR_EACH_VEC( m_Rows, nRow ) + { + //bail after the first N elements + if( ( nTop > 0 ) && ( ( uint32 )nRow >= nTop ) ) + break; + + const Variant_t& v = ( *m_Rows[ nRow ] )[ nCol ]; + switch( col.m_eType ) + { + case eCol_String: + nColWidth = MAX( nColWidth, strlen( v.m_sStr ) ); + break; + case eCol_SteamID: + nColWidth = MAX( nColWidth, strlen( v.m_SteamID.Render() ) ); + break; + case eCol_Float: + nColWidth = MAX( nColWidth, CountDigits( ( int64 )v.m_fFloat ) + 1 + col.m_nNumDecimals ); + switch( col.m_eSummary ) + { + case eSummary_Max: + summary.m_fFloat = MAX( summary.m_fFloat, v.m_fFloat ); + break; + case eSummary_Total: + summary.m_fFloat += v.m_fFloat; + break; + } + break; + case eCol_Int: + nColWidth = MAX( nColWidth, CountIntWidth( v.m_nInt, col.m_eIntDisplay ) ); + switch( col.m_eSummary ) + { + case eSummary_Max: + summary.m_nInt = MAX( summary.m_nInt, v.m_nInt ); + break; + case eSummary_Total: + summary.m_nInt += v.m_nInt; + break; + } + break; + } + } + + //make sure the summary value contributes to the column width + switch( col.m_eType ) + { + case eCol_Float: + nColWidth = MAX( nColWidth, CountDigits( ( int64 )summary.m_fFloat ) + 1 + col.m_nNumDecimals ); + break; + case eCol_Int: + nColWidth = MAX( nColWidth, CountIntWidth( summary.m_nInt, col.m_eIntDisplay ) ); + break; + } + + //initialize our column sizes + vColWidths.AddToTail( nColWidth ); + vSummary.AddToTail( summary ); + + vMsg.AppendFormat( "%*s", nColWidth, col.m_sName.String() ); + vMsg.Append( ' ' ); + + for( uint32 nChar = 0; nChar < nColWidth; nChar++ ) + vSeparator.Append( '-' ); + vSeparator.Append( ' ' ); + } + + //now print our header + vMsg.Append( '\n' ); + vSeparator.Append( '\n' ); + + EG_MSG( eg, "%s", vMsg.String() ); + EG_MSG( eg, "%s", vSeparator.String() ); + + //buffer for compositing our value + CFmtStr1024 vValue; + + //now print each of our columns + FOR_EACH_VEC( m_Rows, nRow ) + { + //bail after the first N elements + if( ( nTop > 0 ) && ( ( uint32 )nRow >= nTop ) ) + break; + + vMsg.Clear(); + FOR_EACH_VEC( m_Columns, nCol ) + { + const Column_t& col = m_Columns[ nCol ]; + const Variant_t& v = ( *m_Rows[ nRow ] )[ nCol ]; + const uint32 nColWidth = vColWidths[ nCol ]; + + vValue.Clear(); + switch( col.m_eType ) + { + case eCol_String: + vValue.Append( v.m_sStr.String() ); + break; + case eCol_SteamID: + vValue.Append( v.m_SteamID.Render() ); + break; + case eCol_Float: + vValue.sprintf( "%.*f", col.m_nNumDecimals, v.m_fFloat ); + break; + case eCol_Int: + vValue.Append( GetIntValueDisplay( v.m_nInt, col.m_eIntDisplay ) ); + break; + } + + //print out spaces before we do the link (so we don't have the whole table underlined) + uint32 nValueLen = vValue.Length(); + uint32 nNumSpaces = nColWidth - MIN( nColWidth, nValueLen ); + for( uint32 nCurrSpace = 0; nCurrSpace < nNumSpaces; nCurrSpace++ ) + vMsg.Append( ' ' ); + + //print out the link if one is provided + if( !v.m_sLink.IsEmpty() ) + { + vMsg.AppendFormat( "<link cmd=\"%s\">", v.m_sLink.String() ); + vMsg.Append( vValue ); + vMsg.Append( "</link>" ); + } + else + { + //allow for steam ID special linking if no link is specified + if( col.m_eType == eCol_SteamID ) + { + // !FIXME! DOTAMERGE + //vMsg.Append( v.m_SteamID.RenderLink() ); + vMsg.Append( v.m_SteamID.Render() ); + } else + vMsg.Append( vValue ); + } + + vMsg.Append( ' ' ); + } + + vMsg.Append( '\n' ); + EG_MSG( eg, "%s", vMsg.String() ); + } + + //and finally our footer + EG_MSG( eg, "%s", vSeparator.String() ); + + //and our summary + { + vMsg.Clear(); + FOR_EACH_VEC( m_Columns, nCol ) + { + const Column_t& col = m_Columns[ nCol ]; + const Variant_t& v = vSummary[ nCol ]; + const uint32 nColWidth = vColWidths[ nCol ]; + + if( ( col.m_eType == eCol_String ) || ( col.m_eSummary == eSummary_None ) ) + { + vMsg.AppendFormat( "%*s ", nColWidth, "" ); + } + else + { + switch( col.m_eType ) + { + case eCol_Float: + vMsg.AppendFormat( "%*.*f ", nColWidth, col.m_nNumDecimals, v.m_fFloat ); + break; + case eCol_Int: + vMsg.AppendFormat( "%*s ", nColWidth, GetIntValueDisplay( v.m_nInt, col.m_eIntDisplay ) ); + break; + } + } + } + + vMsg.Append( '\n' ); + EG_MSG( eg, "%s", vMsg.String() ); + } +} + +} // namespace GCSDK diff --git a/gcsdk/gcsdk_game.vpc b/gcsdk/gcsdk_game.vpc new file mode 100644 index 0000000..279b07c --- /dev/null +++ b/gcsdk/gcsdk_game.vpc @@ -0,0 +1,160 @@ +//----------------------------------------------------------------------------- +// GCSDK.VPC +// +// Project Script for the Game Coordinator SDK +//----------------------------------------------------------------------------- + +$macro SRCDIR ".." +$Macro GENERATED_PROTO_DIR "$SRCDIR\gcsdk\generated_proto" + +$include "$SRCDIR\vpc_scripts\source_lib_base.vpc" +$include "$SRCDIR\vpc_scripts\protobuf_builder.vpc" +$include "$SRCDIR\gcsdk\steammessages_include.vpc" +$include "$SRCDIR\gcsdk\gcsdk_gcmessages_include.vpc" + +$Configuration +{ + $Compiler + { + //$PreprocessorDefinitions "$BASE;STEAM" + $AdditionalIncludeDirectories "$BASE;$SRCDIR\gcsdk\steamextra;..;..\public;..\public\gcsdk;$SRCDIR\thirdparty\JSON_parser;." + $PreprocessorDefinitions "$BASE;TF" // Some steamextra files have tweaks unfortunately + $Create/UsePrecompiledHeader "Use Precompiled Header (/Yu)" + $AdditionalOptions "$BASE /YlGCSDK_PCH_Symbol" + } +} + +$Project "gcsdk" +{ + $Folder "Source Files" + { + $File "stdafx.cpp" + { + $Configuration + { + $Compiler + { + $Create/UsePrecompiledHeader "Create Precompiled Header (/Yc)" + + // Generate a bogus symbol to fix the PCH "no debug symbols" warning + //$AdditionalOptions "$BASE /YlBogusGCSDKSymbol" + } + } + } + + // turn off the debug check for the debug build + $File "$SRCDIR\common\debug_lib_check.cpp" + { + $Configuration "Debug" + { + $ExcludedFromBuild "Yes" + } + } + + $File "gcclient.cpp" + $File "gcconstants.cpp" + $File "gclogger.cpp" + $File "gcmsg.cpp" + $File "job.cpp" + $File "jobmgr.cpp" + $File "jobtime.cpp" + $File "messagelist.cpp" + $File "msgprotobuf.cpp" + $File "netpacket.cpp" + $File "netpacketpool.cpp" + $File "protobufsharedobject.cpp" + $File "sharedobject.cpp" + $File "sharedobjectcache.cpp" + $File "gcclient_sharedobjectcache.cpp" + $File "steamextra\misc.cpp" + $File "steamextra\rtime.cpp" + $File "steamextra\steamid.cpp" + { + // Needs to #define something prior to including the PCH + $Configuration + { + $Compiler + { + $Create/UsePrecompiledHeader "Not Using Precompiled Headers" + } + } + } + $File "steamextra\tier1\hashglobals.cpp" + $File "steamextra\tier1\tsmempool.cpp" + $File "steamextra\tier1\tsmultimempool.cpp" + $File "workthreadpool.cpp" + $File "webapi_response.cpp" + $File "$SRCDIR\thirdparty\JSON_parser\JSON_parser.c" + { + $Configuration + { + $Compiler + { + $Create/UsePrecompiledHeader "Not Using Precompiled Headers" + } + } + } + } + $Folder "Header Files" + { + $File "stdafx.h" + + $File "$SRCDIR\public\gcsdk\enumutils.h" + $File "$SRCDIR\public\gcsdk\gcclient.h" + $File "$SRCDIR\public\gcsdk\gcclientsdk.h" + $File "$SRCDIR\public\gcsdk\gcconstants.h" + $File "$SRCDIR\public\gcsdk\gclogger.h" + $File "$SRCDIR\public\gcsdk\gcclientjob.h" + $File "$SRCDIR\public\gcsdk\gcmsg.h" + $File "$SRCDIR\public\gcsdk\gcsdk.h" + $File "$SRCDIR\public\gcsdk\gcschema.h" + $File "$SRCDIR\public\gcsdk\gcsession.h" + $File "$SRCDIR\public\gcsdk\gcsystemmsgs.h" + $File "$SRCDIR\public\gcsdk\job.h" + $File "$SRCDIR\public\gcsdk\jobmgr.h" + $File "$SRCDIR\public\gcsdk\jobtime.h" + $File "$SRCDIR\public\gcsdk\messagelist.h" + $File "$SRCDIR\public\gcsdk\msgbase.h" + $File "$SRCDIR\public\gcsdk\netpacket.h" + $File "$SRCDIR\public\gcsdk\msgprotobuf.h" + $File "$SRCDIR\public\gcsdk\netpacketpool.h" + $File "$SRCDIR\public\gcsdk\protobufsharedobject.h" + $File "$SRCDIR\public\gcsdk\refcount.h" + $File "$SRCDIR\public\gcsdk\sharedobject.h" + $File "$SRCDIR\public\gcsdk\sharedobjectcache.h" + $File "$SRCDIR\public\gcsdk\gcclient_sharedobjectcache.h" + $File "$SRCDIR\public\gcsdk\workthreadpool.h" + + $File "steamextra\clientenums.h" + $File "steamextra\misc.h" + $File "steamextra\rtime.h" + $File "steamextra\gamecoordinator\igamecoordinator.h" + $File "steamextra\gamecoordinator\igamecoordinatorhost.h" + $File "steamextra\gamecoordinator\igcsqlquery.h" + $File "steamextra\gamecoordinator\igcsqlresultsetlist.h" + $File "steamextra\steam\isteamgamecoordinator.h" + $File "steamextra\tier0\t0constants.h" + $File "steamextra\tier1\pearsonshash.h" + $File "steamextra\tier1\tsmempool.h" + $File "steamextra\tier1\tsmultimempool.h" + } + + $Folder "Protobuf Files" + { + $File "gcsystemmsgs.proto" + $Folder "Generated Files" + { + $DynamicFile "$GENERATED_PROTO_DIR/gcsystemmsgs.pb.h" + $DynamicFile "$GENERATED_PROTO_DIR/gcsystemmsgs.pb.cc" + { + $Configuration + { + $Compiler + { + $Create/UsePrecompiledHeader "Not Using Precompiled Headers" + } + } + } + } + } +} diff --git a/gcsdk/gcsdk_game_include.vpc b/gcsdk/gcsdk_game_include.vpc new file mode 100644 index 0000000..b726881 --- /dev/null +++ b/gcsdk/gcsdk_game_include.vpc @@ -0,0 +1,44 @@ +//----------------------------------------------------------------------------- +// gcsdk_include.vpc +// +// Project Script +//----------------------------------------------------------------------------- + +$MacroRequired "PLATFORM" +$MacroRequired GENERATED_PROTO_DIR + +$include "$SRCDIR\vpc_scripts\protobuf_builder.vpc" +$include "$SRCDIR\gcsdk\steammessages_include.vpc" + +$Configuration +{ + $General + { + $AdditionalProjectDependencies "gcsdk" + } +} + +$Project +{ + $File "$SRCDIR/gcsdk/gcsystemmsgs.proto" + $Folder "Generated Files" [!$OSXALL] + { + $DynamicFile "$GENERATED_PROTO_DIR/gcsystemmsgs.pb.h" + $DynamicFile "$GENERATED_PROTO_DIR/gcsystemmsgs.pb.cc" + { + $Configuration + { + $Compiler + { + $Create/UsePrecompiledHeader "Not Using Precompiled Headers" + } + } + } + } + + $Folder "Link Libraries" + { + $Lib gcsdk + } + +} diff --git a/gcsdk/gcsdk_gc.vpc b/gcsdk/gcsdk_gc.vpc new file mode 100644 index 0000000..3283614 --- /dev/null +++ b/gcsdk/gcsdk_gc.vpc @@ -0,0 +1,257 @@ +//----------------------------------------------------------------------------- +// GCSDK_gc.VPC +// +// Project Script for the Game Coordinator SDK (GC version) +//----------------------------------------------------------------------------- + +$MacroRequired "PLATSUBDIR" + +// The OUTLIBDIR for this library has long used $PLATFORM in the path so +// appending $PLATSUBDIR is redundant. +$macro SRCDIR ".." +$Macro OUTLIBDIR "$LIBPUBLIC" +$Macro GENERATED_PROTO_DIR "$SRCDIR\gcsdk\generated_proto_gc" + +$Linux +{ + $BuildForLinux 1 +} + +$include "$SRCDIR\vpc_scripts\source_lib_base.vpc" +$include "$SRCDIR\vpc_scripts\protobuf_builder.vpc" +$include "$SRCDIR\gcsdk\steammessages_include.vpc" +$include "$SRCDIR\gcsdk\gcsdk_gcmessages_include.vpc" + +$Configuration "Release" +{ + $General + { + $IntermediateDirectory ".\Release_gc$PLATSUBDIR" + } + + $Compiler + { + $PreprocessorDefinitions "$BASE;RELEASEASSERTS" + } +} + +$Configuration "Debug" +{ + $General + { + $IntermediateDirectory ".\Debug_gc$PLATSUBDIR" + } +} + +$Configuration +{ + $Compiler + { + $PreprocessorDefinitions "$BASE;GC" + $AdditionalIncludeDirectories "$BASE;$SRCDIR\gcsdk\steamextra;..;..\public;..\public\gcsdk;.;$SRCDIR\thirdparty\JSON_parser" + $Create/UsePrecompiledHeader "Use Precompiled Header (/Yu)" + $AdditionalOptions "$BASE /YlGCSDK_PCH_Symbol" + } +} + +$Project "gcsdk_gc" +{ + $Folder "Source Files" + { + // turn off the debug check for the debug build + $File "$SRCDIR\common\debug_lib_check.cpp" + { + $Configuration "Debug" + { + $ExcludedFromBuild "Yes" + } + } + + $File "stdafx.cpp" + { + $Configuration + { + $Compiler + { + $Create/UsePrecompiledHeader "Create Precompiled Header (/Yc)" + + // Generate a bogus symbol to fix the PCH "no debug symbols" warning + //$AdditionalOptions "$BASE /YlBogusGCSDKSymbol" + } + } + } + + $File "accountdetails.cpp" + $File "bufferpool.cpp" + $File "directory.cpp" + $File "framefunction.cpp" + $File "gcbase.cpp" + $File "gcconstants.cpp" + $File "gc_convar.cpp" + $File "gchost.cpp" + $File "gcjob.cpp" + $File "gcleaderboardapi.cpp" + $File "gclogger.cpp" + $File "gcmsg.cpp" + $File "gcdirtyfield.cpp" + $file "gcreportprinter.cpp" + $File "gcsession.cpp" + $File "gcsqlquery.cpp" + $File "gcparalleljobfarm.cpp" + $File "gcwebapi.cpp" + $File "gcwebapikey.cpp" + $File "gcwgjobmgr.cpp" + $File "gc_sharedobjectcache.cpp" + $File "http.cpp" + $File "job.cpp" + $File "jobmgr.cpp" + $File "jobtime.cpp" + $File "messagelist.cpp" + $File "msgprotobuf.cpp" + $File "netpacket.cpp" + $File "netpacketpool.cpp" + $File "protobufsharedobject.cpp" + $File "scheduledfunction.cpp" + $File "schemasharedobject.cpp" + $File "sdocache.cpp" + $File "sharedobject.cpp" + $File "sharedobjectcache.cpp" + $File "sharedobjecttransaction.cpp" + $File "webapi_response.cpp" + $File "workthreadpool.cpp" + $File "$SRCDIR\thirdparty\JSON_parser\JSON_parser.c" + { + $Configuration + { + $Compiler + { + $Create/UsePrecompiledHeader "Not Using Precompiled Headers" + } + } + } + + $File "steamextra\misc.cpp" + $File "steamextra\rtime.cpp" + $File "steamextra\steamid.cpp" + $File "steamextra\tier1\hashglobals.cpp" + $File "steamextra/tier1/murmurhash3.cpp" + $File "steamextra\tier1\tsmempool.cpp" + $File "steamextra\tier1\tsmultimempool.cpp" + + $Folder "SQLAccess" + { + $File "sqlaccess\columnset.cpp" + $File "sqlaccess\record.cpp" + $File "sqlaccess\recordinfo.cpp" + $File "sqlaccess\schema.cpp" + $File "sqlaccess\schemafull.cpp" + $File "sqlaccess\schemaupdate.cpp" + $File "sqlaccess\sqlaccess.cpp" + $File "sqlaccess\sqlrecord.cpp" + $File "sqlaccess\sqlutil.cpp" + } + } + + $Folder "Header Files" + { + $File "stdafx.h" + + $File "$SRCDIR\public\gcsdk\accountdetails.h" + $File "$SRCDIR\public\gcsdk\bufferpool.h" + $File "$SRCDIR/public/gcsdk/directory.h" + $File "$SRCDIR\public\gcsdk\enumutils.h" + $File "$SRCDIR\public\gcsdk\framefunction.h" + $File "$SRCDIR\public\gcsdk\gcbase.h" + $File "$SRCDIR\public\gcsdk\gcclient.h" + $File "$SRCDIR\public\gcsdk\gcclientsdk.h" + $File "$SRCDIR\public\gcsdk\gcconstants.h" + $File "$SRCDIR\public\gcsdk\gc_convar.h" + $File "$SRCDIR\public\gcsdk\gclogger.h" + $File "$SRCDIR\public\gcsdk\gcleaderboardapi.h" + $File "$SRCDIR\public\gcsdk\gchost.h" + $File "$SRCDIR\public\gcsdk\gcjob.h" + $File "$SRCDIR\public\gcsdk\gcclientjob.h" + $File "$SRCDIR\public\gcsdk\gcmsg.h" + $File "$SRCDIR\public\gcsdk\gcdirtyfield.h" + $File "$SRCDIR/public/gcsdk/gcreportprinter.h" + $File "$SRCDIR\public\gcsdk\gcsdk.h" + $File "$SRCDIR\public\gcsdk\gcschema.h" + $File "$SRCDIR\public\gcsdk\gcsession.h" + $File "$SRCDIR\public\gcsdk\gcsqlquery.h" + $File "$SRCDIR\public\gcsdk\gcsqlwritequeue.h" + $File "$SRCDIR/public/gcsdk/gcsystemaccess.h" + $File "$SRCDIR\public\gcsdk\gcsystemmsgs.h" + $File "$SRCDIR\public\gcsdk\gcparalleljobfarm.h" + $File "$SRCDIR\public\gcsdk\gcwebapi.h" + $File "$SRCDIR\public\gcsdk\gcwebapikey.h" + $File "$SRCDIR\public\gcsdk\gcwgjobmgr.h" + $File "$SRCDIR\public\gcsdk\gc_sharedobjectcache.h" + $File "$SRCDIR\public\gcsdk\http.h" + $File "$SRCDIR\public\gcsdk\job.h" + $File "$SRCDIR\public\gcsdk\jobmgr.h" + $File "$SRCDIR\public\gcsdk\jobtime.h" + $File "$SRCDIR\public\gcsdk\messagelist.h" + $File "$SRCDIR\public\gcsdk\msgbase.h" + $File "$SRCDIR\public\gcsdk\netpacket.h" + $File "$SRCDIR\public\gcsdk\msgprotobuf.h" + $File "$SRCDIR\public\gcsdk\netpacketpool.h" + $File "$SRCDIR\public\gcsdk\protobufsharedobject.h" + $File "$SRCDIR\public\gcsdk\refcount.h" + $File "$SRCDIR\public\gcsdk\scheduledfunction.h" + $File "$SRCDIR\public\gcsdk\schemasharedobject.h" + $File "$SRCDIR\public\gcsdk\sdocache.h" + $File "$SRCDIR\public\gcsdk\sharedobject.h" + $File "$SRCDIR\public\gcsdk\sharedobjectcache.h" + $File "$SRCDIR\public\gcsdk\sharedobjecttransaction.h" + $File "$SRCDIR\public\gcsdk\webapi_response.h" + $File "$SRCDIR\public\gcsdk\workthreadpool.h" + + $File "steamextra\clientenums.h" + $File "steamextra\misc.h" + $File "steamextra\rtime.h" + $File "steamextra\gamecoordinator\igamecoordinator.h" + $File "steamextra\gamecoordinator\igamecoordinatorhost.h" + $File "steamextra\gamecoordinator\igcsqlquery.h" + $File "steamextra\gamecoordinator\igcsqlresultsetlist.h" + $File "steamextra\steam\isteamgamecoordinator.h" + $File "steamextra\tier0\t0constants.h" + $File "steamextra/tier1/murmurhash3.h" + $File "steamextra\tier1\pearsonshash.h" + $File "steamextra\tier1\tsmempool.h" + $File "steamextra\tier1\tsmultimempool.h" + $File "steamextra\tier1\utlhashmaplarge.h" + $File "$SRCDIR\thirdparty\JSON_parser\JSON_parser.h" + + $Folder "SQLAccess" + { + $File "$SRCDIR\public\gcsdk\sqlaccess\columnset.h" + $File "$SRCDIR\public\gcsdk\sqlaccess\record.h" + $File "$SRCDIR\public\gcsdk\sqlaccess\recordinfo.h" + $File "$SRCDIR\public\gcsdk\sqlaccess\schema.h" + $File "$SRCDIR\public\gcsdk\sqlaccess\schemafull.h" + $File "$SRCDIR\public\gcsdk\sqlaccess\schemaupdate.h" + $File "$SRCDIR\public\gcsdk\sqlaccess\sqlaccess.h" + $File "$SRCDIR\public\gcsdk\sqlaccess\sqlrecord.h" + $File "$SRCDIR\public\gcsdk\sqlaccess\sqlutil.h" + } + } + + $Folder "Protobuf Files" + { + $File "gcsystemmsgs.proto" + $Folder "Generated Files" + { + $DynamicFile "$GENERATED_PROTO_DIR/gcsystemmsgs.pb.h" + $DynamicFile "$GENERATED_PROTO_DIR/gcsystemmsgs.pb.cc" + { + $Configuration + { + $Compiler + { + $Create/UsePrecompiledHeader "Not Using Precompiled Headers" + } + } + } + } + } +} diff --git a/gcsdk/gcsdk_gc_include.vpc b/gcsdk/gcsdk_gc_include.vpc new file mode 100644 index 0000000..4ac1db0 --- /dev/null +++ b/gcsdk/gcsdk_gc_include.vpc @@ -0,0 +1,46 @@ +//----------------------------------------------------------------------------- +// gcsdk_include.vpc +// +// Project Script +//----------------------------------------------------------------------------- + +$MacroRequired "PLATFORM" +$MacroRequired GENERATED_PROTO_DIR + +$include "$SRCDIR\vpc_scripts\protobuf_builder.vpc" +$include "$SRCDIR\gcsdk\steammessages_include.vpc" +$include "$SRCDIR\game\protobuf_include.vpc" + + +$Configuration +{ + $General + { + $AdditionalProjectDependencies "gcsdk_gc" + } +} + +$Project +{ + $File "$SRCDIR/gcsdk/gcsystemmsgs.proto" + $Folder "Generated Files" + { + $DynamicFile "$GENERATED_PROTO_DIR/gcsystemmsgs.pb.h" + $DynamicFile "$GENERATED_PROTO_DIR/gcsystemmsgs.pb.cc" + { + $Configuration + { + $Compiler + { + $Create/UsePrecompiledHeader "Not Using Precompiled Headers" + } + } + } + } + + $Folder "Link Libraries" + { + $Lib gcsdk_gc + } + +} diff --git a/gcsdk/gcsdk_gcmessages.proto b/gcsdk/gcsdk_gcmessages.proto new file mode 100644 index 0000000..bc197a3 --- /dev/null +++ b/gcsdk/gcsdk_gcmessages.proto @@ -0,0 +1,376 @@ +//====== Copyright 1996-2010, Valve Corporation, All rights reserved. ======= +// +// Purpose: The file defines our Google Protocol Buffers which are used in over +// the wire messages between servers as well as between the TF GC and TF gameservers +// and clients. +// +//============================================================================= + +// We care more about speed than code size +option optimize_for = SPEED; + +// We don't use the service generation functionality +option cc_generic_services = false; + + +// +// STYLE NOTES: +// +// Use CamelCase CMsgMyMessageName style names for messages. +// +// Use lowercase _ delimited names like my_steam_id for field names, this is non-standard for Steam, +// but plays nice with the Google formatted code generation. +// +// Try not to use required fields ever. Only do so if you are really really sure you'll never want them removed. +// Optional should be preffered as it will make versioning easier and cleaner in the future if someone refactors +// your message and wants to remove or rename fields. +// +// Use fixed64 for JobId_t, GID_t, or SteamID. This is appropriate for any field that is normally +// going to be larger than 2^56. Otherwise use int64 for 64 bit values that are frequently smaller +// than 2^56 as it will safe space on the wire in those cases. +// +// Similar to fixed64, use fixed32 for RTime32 or other 32 bit values that are frequently larger than +// 2^28. It will safe space in those cases, otherwise use int32 which will safe space for smaller values. +// An exception to this rule for RTime32 is if the value will frequently be zero rather than set to an actual +// time. +// + +import "steammessages.proto"; + +message CMsgSOIDOwner +{ + optional uint32 type = 1; + optional uint64 id = 2; +} + +// +// k_ESOMsg_Create +// k_ESOMsg_Update +// k_ESOMsg_Destroy +// +message CMsgSOSingleObject +{ + optional fixed64 owner = 1; // the steam ID of the owner of this object + optional int32 type_id = 2; // the shared object type ID of this object + optional bytes object_data = 3; // the actual data for the object + optional fixed64 version = 4; // version of the cache + + optional CMsgSOIDOwner owner_soid = 5; // The SOID that owns this object + optional uint32 service_id = 6; // The service that sent this update +}; + + +// +// k_ESOMsg_UpdateMultiple +// +message CMsgSOMultipleObjects +{ + message SingleObject + { + optional int32 type_id = 1; // the shared object type ID of this object + optional bytes object_data = 2; // the actual data for the object + }; + optional fixed64 owner = 1; // the steam ID of the owner of the objects + repeated SingleObject objects = 2; // a list of types + optional fixed64 version = 3; // version of the cache + optional CMsgSOIDOwner owner_soid = 6; // The SOID that owns this object + optional uint32 service_id = 7; // The service that sent this update +}; + + +// +// k_ESOMsg_CacheSubscribed +// +message CMsgSOCacheSubscribed +{ + message SubscribedType + { + optional int32 type_id = 1; // ID of the type for these objects + repeated bytes object_data = 2; // the data for all the objects of this type + }; + + optional fixed64 owner = 1; // the owner of this cache + repeated SubscribedType objects = 2; // a list of types + optional fixed64 version = 3; // version of the cache + optional CMsgSOIDOwner owner_soid = 4; // The SOID that owns this object + optional uint32 service_id = 5; // The service that is providing this SO cache information + repeated uint32 service_list = 6; // Other services that are providing parts of this cache and need to to be received for it to be complete + optional fixed64 sync_version = 7; // The unique ID of the sync to ensure all subscribes across services match. Ignored/not provided if no other GCs are involved +}; + +// +// k_ESOMsg_CacheSubscribedUpToDate +// +message CMsgSOCacheSubscribedUpToDate +{ + optional fixed64 version = 1; // version of the cache for this sub GC + optional CMsgSOIDOwner owner_soid = 2; // The SOID that owns this object + optional uint32 service_id = 3; // The service that is providing this SO cache information + repeated uint32 service_list = 4; // Other services that are providing parts of this cache and need to to be received for it to be complete + optional fixed64 sync_version = 5; // The unique ID of the sync to ensure all subscribes across services match. Ignored/not provided if no other GCs are involved +}; + +// +// k_ESOMsg_CacheUnsubscribed +// +message CMsgSOCacheUnsubscribed +{ + optional fixed64 owner = 1; // the owner of this cache +}; + + +// +// k_ESOMsg_CacheSubscriptionCheck +// +message CMsgSOCacheSubscriptionCheck +{ + optional fixed64 owner = 1; // the owner of the cache + optional fixed64 version = 2; // version of the cache + optional CMsgSOIDOwner owner_soid = 3; // The SOID that owns this object + optional uint32 service_id = 4; // The service associated with this version + repeated uint32 service_list = 5; // The other services that need to provide information on this cache + optional fixed64 sync_version = 6; // The unique ID of the sync to ensure all subscribes across services match. Ignored/not provided if no other GCs are involved +}; + + +// +// k_ESOMsg_CacheSubscriptionRefresh +// +message CMsgSOCacheSubscriptionRefresh +{ + optional fixed64 owner = 1; // the owner of the cache + optional CMsgSOIDOwner owner_soid = 2; // The SOID that owns this object +}; + +// +// Stored in memcached for each SO Cache +// +message CMsgSOCacheVersion +{ + optional fixed64 version = 1; // version of the cache +}; + +enum PartnerAccountType +{ + //the default for users, which is an account not linked to a provider + PARTNER_NONE = 0; + //linked to a Perfect World account + PARTNER_PERFECT_WORLD = 1; + //linked to a Nexon account + PARTNER_NEXON = 2; +}; + +message CMsgGCMultiplexMessage +{ + optional uint32 msgtype = 1; // ID of the message being sent + optional bytes payload = 2; // Serialized message to send + repeated fixed64 steamids = 3; // Clients to send the message to +}; + + +// k_EGCToGCMsgMasterAck +message CGCToGCMsgMasterAck +{ + optional uint32 dir_index = 1; // the index from the directory that this GC is responsible for + optional string machine_name = 3; // the machine name that this GC is running on so that we can quickly see the universe configuration + optional string process_name = 4; // the name of the process that this is running to ensure that it matches + repeated uint32 type_instances = 5; // the list of type instances contained within this process, must match the master directory +}; + +// k_EGCToGCMsgMasterAck_Response +message CGCToGCMsgMasterAck_Response +{ + optional int32 eresult = 1 [default = 2]; // Is this GC what the master thinks it should be? +}; + +// k_EGCToGCMsgMasterStartupComplete +message CGCToGCMsgMasterStartupComplete +{ + message GCInfo + { + optional uint32 dir_index = 1; + optional string machine_name = 2; + }; + + repeated GCInfo gc_info = 1; +}; + + +// k_EGCToGCMsgRouted +message CGCToGCMsgRouted +{ + optional uint32 msg_type = 1; // the type of the contained message + optional fixed64 sender_id = 2; // steam ID of the client that sent this. If nil, it came from the system instead. + optional bytes net_message = 3; // the binary blob contents of the contained message +}; + +// k_EGCToGCMsgRoutedReply +message CGCToGCMsgRoutedReply +{ + optional uint32 msg_type = 1; // the type of the contained message + optional bytes net_message = 2; // the binary blob contents of the contained message +}; + +// k_EMsgGCUpdateSubGCSessionInfo +message CMsgGCUpdateSubGCSessionInfo +{ + message CMsgUpdate + { + optional fixed64 steamid = 1; + optional fixed32 ip = 2; + optional bool trusted = 3; + }; + + repeated CMsgUpdate updates = 1; // the list of updates we should process (in order) +} + +// k_EMsgGCRequestSubGCSessionInfo +message CMsgGCRequestSubGCSessionInfo +{ + optional fixed64 steamid = 1; +} + +// k_EMsgGCRequestSubGCSessionInfoRespone +message CMsgGCRequestSubGCSessionInfoResponse +{ + optional fixed32 ip = 1; + optional bool trusted = 2; +} + +// k_EMsgGCToGCIncrementRecruitmentLevel +message CMsgGCToGCIncrementRecruitmentLevel +{ + optional fixed64 steamid = 1; +} + + +// +// CMsgSOCacheHaveVersion +// +// An array of these is sent to indicate what SO cache versions we currently have +// +message CMsgSOCacheHaveVersion +{ + optional CMsgSOIDOwner soid = 1; // ID of the cache we have + optional fixed64 version = 2; // version stamp we have of this cache + optional uint32 service_id = 3; // For partial caches, what service this version is associated with +}; + +// !FIXME! DOTAMERGE +//// +//// CMsgClientHello +//// +//message CMsgClientHello +//{ +// optional uint32 version = 1; +// repeated CMsgSOCacheHaveVersion socache_have_versions = 2; +// +// /// Game-specific value indicating what state the client is in, +// /// and what sort of session is wants. This is used to decide +// /// whether the client even needs a session, and if so, what +// /// priority we should assign +// optional uint32 client_session_need = 3; +// +// /// What special partner-specific launcher was used, if any? +// optional PartnerAccountType client_launcher = 4; +// +// /// Secret Key used to gain elevated status with the GC +// optional string secret_key = 5; +//}; +// +//// +//// CMsgClientWelcome +//// +//message CMsgClientWelcome +//{ +// optional uint32 version = 1; +// optional bytes game_data = 2; +// +// // List of caches to which the client is now subscribed, but +// // the GC thinks we don't have the latest data so it is included it here. +// repeated CMsgSOCacheSubscribed outofdate_subscribed_caches = 3; +// +// // List of caches to which the client is now subscribed, and the +// // GC thinks that we already have the latest version. (It is sending +// // the version number just to make sure.) +// repeated CMsgSOCacheSubscriptionCheck uptodate_subscribed_caches = 4; +// +// // location information +// message Location +// { +// optional float latitude = 1; +// optional float longitude = 2; +// optional string country = 3; +// } +// +// optional Location location = 5; +// +// optional bytes save_game_key = 6; +//}; + +/// Status of connection to GC. +// +// Don't change these values, as they need to be consistent across props, since they +// are central to handshaking with clients old and new! +enum GCConnectionStatus +{ + GCConnectionStatus_HAVE_SESSION = 0; // You have a session. (The status of one or more services may have changed.) + GCConnectionStatus_GC_GOING_DOWN = 1; // Notification that the entire GC system is going down. + GCConnectionStatus_NO_SESSION = 2; // You don't have a session. Please send a hello message to initiate the session creation process. + GCConnectionStatus_NO_SESSION_IN_LOGON_QUEUE = 3; // You don't have a session, but we have your info and will sign you on ASAP. No need to send it again. + GCConnectionStatus_NO_STEAM = 4; // Client-side only value. We aren't connected to Steam. + GCConnectionStatus_SUSPENDED = 5; // Client-side only value. We were told that we've been suspended from connecting to this GC. No need to try again later. +}; + +// +// CMsgConnectionStatus +// +message CMsgConnectionStatus +{ + optional GCConnectionStatus status = 1; + optional uint32 client_session_need = 2; // The last client session need state we got from you. + + // + // If they are in the queue, this tells them where they are. + // + optional int32 queue_position = 3; // your approximate position in the queue + optional int32 queue_size = 4; // how many people are in the queue + optional int32 wait_seconds = 5; // how long you've been waiting + optional int32 estimated_wait_seconds_remaining = 6; // estimated time until you are logged on. + + // TODO: Here we could include a list of services + // (perhaps just SO cache type ID's?) that + // are currently known to be offline. +}; + +//k_EMsgGCToGCSOCacheSubscribe +message CMsgGCToGCSOCacheSubscribe +{ + message CMsgHaveVersions + { + optional uint32 service_id = 1; + optional uint64 version = 2; + }; + + optional fixed64 subscriber = 1; // the ID of the user that is subscribing to a cache + optional fixed64 subscribe_to = 2; // the ID of the cache that is being subscribed to + optional fixed64 sync_version = 3; // the unique ID that is used to distinguish this synch across the various parts + repeated CMsgHaveVersions have_versions = 4; // the version that the client has already (used to skip unnecessary sends) +}; + +//k_EMsgGCToGCSOCacheUnsubscribe +message CMsgGCToGCSOCacheUnsubscribe +{ + optional fixed64 subscriber = 1; // if 0, then this means unsubscribe everyone from the cache + optional fixed64 unsubscribe_from = 2; // the cache we are unsubscribing from +}; + +// k_EMsgGCPingRequest +// k_EMsgGCPingReply +message CMsgGCClientPing +{ + // Remember, if you need to associate ping replies with requests, consider using the JobID field +}; + +// Do not remove this comment due to a bug on the Mac OS X protobuf compiler + diff --git a/gcsdk/gcsdk_gcmessages_include.vpc b/gcsdk/gcsdk_gcmessages_include.vpc new file mode 100644 index 0000000..cf12497 --- /dev/null +++ b/gcsdk/gcsdk_gcmessages_include.vpc @@ -0,0 +1,22 @@ +$Project +{ + $Folder "Protobuf Files" + { + $File "$SRCDIR\gcsdk\gcsdk_gcmessages.proto" + $Folder "Generated Files" + { + $DynamicFile "$GENERATED_PROTO_DIR\gcsdk_gcmessages.pb.h" + $DynamicFile "$GENERATED_PROTO_DIR\gcsdk_gcmessages.pb.cc" + { + $Configuration + { + $Compiler + { + $Create/UsePrecompiledHeader "Not Using Precompiled Headers" + } + } + } + } + } + +} diff --git a/gcsdk/gcsdk_include.vpc b/gcsdk/gcsdk_include.vpc new file mode 100644 index 0000000..eaebd0b --- /dev/null +++ b/gcsdk/gcsdk_include.vpc @@ -0,0 +1,13 @@ + + // Include the gcsdk static lib, excluded from builds as appropriate. + $Folder "Link Libraries" + { + $File "$SRCDIR\lib\$PLATFORM\release\gcsdk$_STATICLIB_EXT" + { + $Configuration "Debug" { $ExcludedFromBuild "Yes" } + } + $File "$SRCDIR\lib\$PLATFORM\debug\gcsdk$_STATICLIB_EXT" + { + $Configuration "Release" { $ExcludedFromBuild "Yes" } + } + } diff --git a/gcsdk/gcsession.cpp b/gcsdk/gcsession.cpp new file mode 100644 index 0000000..b739398 --- /dev/null +++ b/gcsdk/gcsession.cpp @@ -0,0 +1,607 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Holds the CGCSession class +// +//============================================================================= +#include "stdafx.h" +#include "gcsession.h" +#include "steamextra/rtime.h" +#include "gcsdk_gcmessages.pb.h" +#include "gcsdk/gcreportprinter.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +// Probably this makes more sense true by default, but we're spewing a ton and Fletcher says it +// isn't a big deal for TF so here we go. +GCConVar gs_session_assert_valid_addr_and_port( "gs_session_assert_valid_addr_and_port", "0" ); + +namespace GCSDK +{ + +DECLARE_GC_EMIT_GROUP( g_EGSessions, sessions ); +DECLARE_GC_EMIT_GROUP_DEFAULTS( g_EGRateLimit, ratelimit, 2, 3 ); + +GCConVar max_user_messages_per_second( "max_user_messages_per_second", "20", 0, "Maximum number of messages a user can send per second. 0 disables the rate limiting" ); +static GCConVar user_message_rate_limit_warning_period( "user_message_rate_limit_warning_period", "30", 0, "Number of seconds between warning about rate limiting for users" ); + +static GCConVar msg_rate_limit_report_user_bucket_1( "msg_rate_limit_report_user_bucket_1", "10", 0, "These values control where various users are bucketed in rate limiting reports to help identify how frequently users are running into rate limiting" ); +static GCConVar msg_rate_limit_report_user_bucket_2( "msg_rate_limit_report_user_bucket_2", "100", 0, "These values control where various users are bucketed in rate limiting reports to help identify how frequently users are running into rate limiting" ); + +static GCConVar msg_rate_limit_list_user( "msg_rate_limit_list_user", "0", 0, "When set to a user account ID, this will report all the messages that are rate limited for that user to the console" ); + +CMsgRateLimitTracker g_RateLimitTracker; + + +CMsgRateLimitTracker::CMsgRateLimitTracker() : + m_StartTime( CRTime::RTime32TimeCur() ) +{ +} + +void CMsgRateLimitTracker::TrackRateLimitedMsg( const CSteamID steamID, MsgType_t eMsgType ) +{ + //update message stat + { + uint32 nMsgIndex = m_MsgStats.Find( eMsgType ); + if( !m_MsgStats.IsValidIndex( nMsgIndex ) ) + { + nMsgIndex = m_MsgStats.Insert( eMsgType, 0 ); + } + m_MsgStats[ nMsgIndex ]++; + } + + //update user stats + { + uint32 nUserIndex = m_UserStats.Find( steamID ); + if( !m_UserStats.IsValidIndex( nUserIndex ) ) + { + nUserIndex = m_UserStats.Insert( steamID, 0 ); + } + m_UserStats[ nUserIndex ]++; + } + + //determine the severity to output the warning at. Assume verbose unless we are tracking a specific account ID (note that no account has 0 so 0 still effectively turns it off) + CGCEmitGroup::EMsgLevel eMsgLevel = CGCEmitGroup::kMsg_Verbose; + if( ( uint32 )msg_rate_limit_list_user.GetInt() == steamID.GetAccountID() ) + { + eMsgLevel = CGCEmitGroup::kMsg_Msg; + } + EG_EMIT( g_EGMessages, eMsgLevel, "Dropped message %s (%d) for user %s\n", PchMsgNameFromEMsg( eMsgType ), eMsgType, steamID.Render() ); +} + +void CMsgRateLimitTracker::ReportMsgStats() const +{ + CGCReportPrinter rp; + rp.AddStringColumn( "Msg" ); + rp.AddIntColumn( "Count", CGCReportPrinter::eSummary_Total ); + + FOR_EACH_MAP_FAST( m_MsgStats, nCurrMsg ) + { + rp.StrValue( PchMsgNameFromEMsg( m_MsgStats.Key( nCurrMsg ) ) ); + rp.IntValue( m_MsgStats[ nCurrMsg ] ); + rp.CommitRow(); + } + + rp.SortReport( "Count" ); + rp.PrintReport( SPEW_CONSOLE ); +} + +void CMsgRateLimitTracker::ReportTopUsers( uint32 nMinMsgs, uint32 nListTop ) const +{ + //collect a list of all messages, and sort them into order of frequency + CGCReportPrinter rp; + rp.AddSteamIDColumn( "User" ); + rp.AddIntColumn( "Count", CGCReportPrinter::eSummary_Total ); + + FOR_EACH_MAP_FAST( m_UserStats, nCurrMsg ) + { + rp.SteamIDValue( m_UserStats.Key( nCurrMsg ) ); + rp.IntValue( m_UserStats[ nCurrMsg ] ); + rp.CommitRow(); + } + + rp.SortReport( "Count" ); + rp.PrintReport( SPEW_CONSOLE, nListTop ); +} + +void CMsgRateLimitTracker::ReportUserStats() const +{ + //run through the users and aggregate stats + const uint32 nBucketLimit1 = ( uint32 )max( 0, min( msg_rate_limit_report_user_bucket_1.GetInt(), msg_rate_limit_report_user_bucket_2.GetInt() ) ); + const uint32 nBucketLimit2 = ( uint32 )max( 0, max( msg_rate_limit_report_user_bucket_1.GetInt(), msg_rate_limit_report_user_bucket_2.GetInt() ) ); + + uint32 nTotalMsg = 0; + uint32 nMaxUser = 0; + uint32 nBucketCount1 = 0; + uint32 nBucketCount2 = 0; + FOR_EACH_MAP_FAST( m_UserStats, nCurrMsg ) + { + //add user counts to the buckets + const uint32 nMsgs = m_UserStats[ nCurrMsg ]; + if( nMsgs <= nBucketLimit1 ) + nBucketCount1++; + else if( nMsgs <= nBucketLimit2 ) + nBucketCount2++; + + //add up our total number of offenses + nTotalMsg += nMsgs; + nMaxUser = max( nMaxUser, nMsgs ); + } + + EG_MSG( SPEW_CONSOLE, "Capture Duration: %ds\n", CRTime::RTime32TimeCur() - m_StartTime ); + EG_MSG( SPEW_CONSOLE, "Total Dropped Messages: %d\n", nTotalMsg ); + EG_MSG( SPEW_CONSOLE, "Message IDs: %d\n", m_MsgStats.Count() ); + EG_MSG( SPEW_CONSOLE, "Users: %d (peak: %d)\n", m_UserStats.Count(), nMaxUser ); + EG_MSG( SPEW_CONSOLE, " Below %d msgs: %d\n", nBucketLimit1, nBucketCount1 ); + EG_MSG( SPEW_CONSOLE, " Below %d msgs: %d\n", nBucketLimit2, nBucketCount2 ); +} + +void CMsgRateLimitTracker::ClearStats() +{ + m_StartTime = CRTime::RTime32TimeCur(); + m_UserStats.RemoveAll(); + m_MsgStats.RemoveAll(); +} + +//console command hooks +GC_CON_COMMAND( msg_rate_limit_dump, "Dumps stats about rate limiting of messages" ) +{ + g_RateLimitTracker.ReportUserStats(); + g_RateLimitTracker.ReportMsgStats(); + g_RateLimitTracker.ReportTopUsers( 0, 20 ); +} + +GC_CON_COMMAND( msg_rate_limit_dump_users, "Dumps a list of users that have been rate limited. Optional parameters can specify the number to dump or the minimum number of messages required." ) +{ + if( args.ArgC() < 3 ) + { + EG_MSG( SPEW_CONSOLE, "Proper usage is: %s <min messages> <top users> - Specify 0 for one or both to have it be ignored\n", args[ 0 ] ); + return; + } + g_RateLimitTracker.ReportTopUsers( ( uint32 )max( 0, atoi( args[ 1 ] ) ), ( uint32 )max( 0, atoi( args[ 2 ] ) ) ); +} + +GC_CON_COMMAND( msg_rate_limit_dump_msgs, "Dumps a list of messages that have been rate limited." ) +{ + g_RateLimitTracker.ReportMsgStats(); +} + +GC_CON_COMMAND( msg_rate_limit_clear, "Clears all the accumulated msg rate limit stats" ) +{ + g_RateLimitTracker.ClearStats(); +} + +//------------------------------------------------------------------------------------------ +// CSteamIDRateLimit +//------------------------------------------------------------------------------------------ + +CSteamIDRateLimit::CSteamIDRateLimit( const GCConVar& cvNumPerPeriod, const GCConVar* pcvPeriodS ) : + m_cvNumPerPeriod( cvNumPerPeriod ), + m_pcvPeriodS( pcvPeriodS ), + m_LastClear( CRTime::RTime32TimeCur() ), + m_FrameFunction( "SteamIDRateLimit", CBaseFrameFunction::k_EFrameType_RunOnce ) +{ + m_FrameFunction.Register( this, &CSteamIDRateLimit::OnFrameFn ); +} + +CSteamIDRateLimit::~CSteamIDRateLimit() +{ +} + +bool CSteamIDRateLimit::BIsRateLimited( CSteamID steamID, uint32 unMsgType ) +{ + int nIndex = m_Msgs.FindOrInsert( steamID, 0 ); + if( ++m_Msgs[ nIndex ] >= ( uint32 )m_cvNumPerPeriod.GetInt() ) + { + g_RateLimitTracker.TrackRateLimitedMsg( steamID, unMsgType ); + return true; + } + return false; +} + +bool CSteamIDRateLimit::OnFrameFn( const CLimitTimer& timer ) +{ + //if no period is specified, assume one second + int nIntervalS = ( m_pcvPeriodS ) ? MAX( 1, m_pcvPeriodS->GetInt() ) : 1; + if( CRTime::RTime32TimeCur() >= m_LastClear + nIntervalS ) + { + m_Msgs.RemoveAll(); + m_LastClear = CRTime::RTime32TimeCur(); + } + return false; +} + + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CGCSession::CGCSession( const CSteamID & steamID, CGCSharedObjectCache *pSOCache ) +: m_steamID( steamID ), + m_pSOCache( pSOCache ), + m_bIsShuttingDown( false ), + m_osType( k_eOSUnknown ), + m_bIsTestSession( false ), + m_bIsSecure( false ), + m_unIPPublic( 0 ), + m_flLatitude( 0.0f ), + m_flLongitude( 0.0f ) , + m_haveGeoLocation( false ), + m_bInitialized( false ), + m_rtLastMessageReceived( 0 ) +{ + m_jtLastMessageReceived.SetLTime( 0 ); + m_jtTimeSentPing.SetLTime( 0 ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CGCSession::~CGCSession() +{ +} + + +//----------------------------------------------------------------------------- +// Purpose: Checks the message against rate limiting. Returns true if we should +// drop the message. False otherwise. This default behavior is a very basic +// n messages / user / second rate limiting that's only meant to stop the +// worse abuses +//----------------------------------------------------------------------------- +bool CGCSession::BRateLimitMessage( MsgType_t unMsgType ) +{ + unMsgType &= ~k_EMsgProtoBufFlag; + if ( max_user_messages_per_second.GetInt() <= 0 ) + return false; + + RTime32 rtCur = CRTime::RTime32TimeCur(); + m_jtLastMessageReceived.SetToJobTime(); + if ( m_rtLastMessageReceived != rtCur ) + { + m_rtLastMessageReceived = rtCur; + m_unMessagesRecievedThisSecond = 0; + } + + m_unMessagesRecievedThisSecond++; + if ( m_unMessagesRecievedThisSecond > (uint32)max_user_messages_per_second.GetInt() ) + { + //log this message + g_RateLimitTracker.TrackRateLimitedMsg( GetSteamID(), unMsgType ); + return true; + } + + return false; +} + +//----------------------------------------------------------------------------- +// Purpose: The run function is called on each session (user and gameserver) +// approximately every k_nUserSessionRunInterval microseconds (or +// k_nGSSessionRunInterval for GS Sessions) +//----------------------------------------------------------------------------- +void CGCSession::Run() +{ + // These cached subscription messages are very expensive and only needed for a short period of time + // If we're hitting the run loop, it's been around long enough + GetSOCache()->ClearCachedSubscriptionMessage(); +} + + +//----------------------------------------------------------------------------- +bool CGCSession::GetGeoLocation( float &latitude, float &longittude ) const +{ + latitude = m_flLatitude; + longittude = m_flLongitude; + + return m_haveGeoLocation; +} + +//----------------------------------------------------------------------------- +void CGCSession::SetGeoLocation( float latitude, float longittude ) +{ + m_flLatitude = latitude; + m_flLongitude = longittude; + m_haveGeoLocation = true; +} + +//----------------------------------------------------------------------------- +// Purpose: Claims all the memory for the session object +//----------------------------------------------------------------------------- +#ifdef DBGFLAG_VALIDATE +void CGCSession::Validate( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE(); +} +#endif // DBGFLAG_VALIDATE + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CGCUserSession::~CGCUserSession() +{ + if ( m_steamIDGS.BGameServerAccount() ) + { + EmitError( SPEW_GC, "Destroying user %s while still connected to server %s\n", GetSteamID().Render(), GetSteamIDGS().Render() ); + } +} + +bool CGCUserSession::BInit() +{ + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Sets the session's game server to the given SteamID. This will +// cause the session to leave the current server it's on, it any +// Returns: True if the user's session was added to the GS's session. +// False if the session could not be found or if the user was already +// on the server. +//----------------------------------------------------------------------------- +bool CGCUserSession::BSetServer( const CSteamID &steamIDGS ) +{ + if ( steamIDGS == m_steamIDGS ) + return false; + + BLeaveServer(); + + if( steamIDGS.IsValid() ) + { + CGCGSSession *pGSSession = GGCBase()->FindGSSession( steamIDGS ); + if ( !pGSSession ) + { + EmitError( SPEW_GC, "User %s attempting to join server %s which has no session\n", GetSteamID().Render(), steamIDGS.Render() ); + return false; + } + + if ( !pGSSession->BAddUser( GetSteamID() ) ) + { + EmitWarning( SPEW_GC, SPEW_ALWAYS, "Server %s already had user %s in its user list\n", steamIDGS.Render(), GetSteamID().Render() ); + // Fall through + } + } + + m_steamIDGS = steamIDGS; + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Removes the session from the given game server +// Returns: True if the user's session was removed from the GS's session. +// False if the session could not be found or if the user was not found +// on the server. +//----------------------------------------------------------------------------- +bool CGCUserSession::BLeaveServer() +{ + if( m_steamIDGS.IsValid() ) + { + // Remember the last server we were connected to + m_steamIDGSPrev = m_steamIDGS; + + CGCGSSession *pGSSession = GGCBase()->FindGSSession( m_steamIDGS ); + if ( pGSSession ) + { + pGSSession->BRemoveUser( GetSteamID() ); + } + } + + m_steamIDGS = CSteamID(); + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Dumps useful information about this session +//----------------------------------------------------------------------------- +void CGCUserSession::Dump( bool bFull ) const +{ +// this is ifdef'd out in Steam because GCSDK can't depend on steamid.cpp +#ifndef STEAM + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "User Session %s (%s)\n", GetSteamID().Render(), BIsShuttingDown() ? "SHUTTING DOWN" : "Active" ); + CJob *pJob = GGCBase()->PJobHoldingLock( GetSteamID() ); + if( pJob ) + { + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\t LOCKED BY: %s\n", pJob->GetName() ); + } + if( bFull && GetSOCache() ) + GetSOCache()->Dump(); + if( GetSteamIDGS().BGameServerAccount() ) + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\tGameserver: %s\n", GetSteamIDGS().Render() ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\tOS: %d Secure: %d\n", GetOSType(), IsSecure() ? 1 : 0 ); +#endif +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CGCGSSession::CGCGSSession( const CSteamID & steamID, CGCSharedObjectCache *pCache, uint32 unServerAddr, uint16 usServerPort ) +: CGCSession( steamID, pCache ), m_unServerAddr( unServerAddr ), m_usServerPort( usServerPort ) +{ + if ( gs_session_assert_valid_addr_and_port.GetBool() ) + { + Assert( unServerAddr ); + Assert( usServerPort ); + } + + // Default our public IP to be the same as our IP address + m_unIPPublic = unServerAddr; +} + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CGCGSSession::~CGCGSSession() +{ + if ( m_vecUsers.Count() > 0 ) + { + EmitError( SPEW_GC, "Destroying game server %s while %d users are still connected\n", GetSteamID().Render(), m_vecUsers.Count() ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a user to the list of users active on the game server +// Returns: True if the user was added, false if the user was already on this +// server. +//----------------------------------------------------------------------------- +bool CGCGSSession::BAddUser( const CSteamID &steamIDUser ) +{ + if( m_vecUsers.HasElement( steamIDUser ) ) + return false; + + PreAddUser( steamIDUser ); + m_vecUsers.AddToTail( steamIDUser ); + PostAddUser( steamIDUser ); + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Called if our IP / port changes after session is started +//----------------------------------------------------------------------------- +void CGCGSSession::SetIPAndPort( uint32 unServerAddr, uint16 usServerPort ) +{ + // If we didn't have an override for the public IP, then also + // update the public IP. + // + // !KLUDGE! This is gross for two reasons: + // - First, do we really need two different fields? + // - Second, why can the IP change after the session is created? + // Shouldn't we force the session to be destroyed and recreated? + // It cannot *really* be the same "session", can it? + if ( m_unIPPublic == m_unServerAddr ) + m_unIPPublic = unServerAddr; + m_unServerAddr = unServerAddr; + m_usServerPort = usServerPort; +} + +//----------------------------------------------------------------------------- +// Purpose: Removes a user from the list of users active on the game server +// Returns: True if the user was added, false if the user was not already on +// this server. +//----------------------------------------------------------------------------- +bool CGCGSSession::BRemoveUser( const CSteamID &steamIDUser ) +{ + int nIndex = m_vecUsers.Find( steamIDUser ); + if ( !m_vecUsers.IsValidIndex( nIndex ) ) + return false; + + PreRemoveUser( steamIDUser ); + m_vecUsers.Remove( nIndex ); + PostRemoveUser( steamIDUser ); + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Removes all users from the list of game server users. +//----------------------------------------------------------------------------- +void CGCGSSession::RemoveAllUsers() +{ + if ( 0 == m_vecUsers.Count() ) + return; + + PreRemoveAllUsers(); + + // Iterate all the users and tell them to leave this server. + // Using back because the users will remove themselves from + // this list during this function + FOR_EACH_VEC_BACK( m_vecUsers, i ) + { + CGCUserSession *pUserSession = GGCBase()->FindUserSession( m_vecUsers[i] ); + if ( pUserSession ) + { + pUserSession->BLeaveServer(); + } + } + + // Catch anyone we don't have a session for anymore + m_vecUsers.RemoveAll(); + + PostRemoveAllUsers(); +} + +#define iptod(x) ((x)>>24&0xff), ((x)>>16&0xff), ((x)>>8&0xff), ((x)&0xff) + +//----------------------------------------------------------------------------- +// Purpose: Dumps useful information about this session +//----------------------------------------------------------------------------- +void CGCGSSession::Dump( bool bFull ) const +{ +// this is ifdef'd out in Steam because GCSDK can't depend on steamid.cpp +#ifndef STEAM + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "GS Session %s\n", GetSteamID().Render() ); + CJob *pJob = GGCBase()->PJobHoldingLock( GetSteamID() ); + if( pJob ) + { + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\t LOCKED BY: %s\n", pJob->GetName() ); + } + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\t%d users:\n", m_vecUsers.Count() ); + FOR_EACH_VEC( m_vecUsers, nUser ) + { + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\t\t%s\n", m_vecUsers[nUser].Render() ); + } + if( GetSOCache() ) + { + if ( bFull ) + { + GetSOCache()->Dump(); + } + else + { + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\t SO Cache Version: %llu\n", GetSOCache()->GetVersion() ); + } + } +#endif +} + + +//----------------------------------------------------------------------------- +// Purpose: Claims all the memory for the session object +//----------------------------------------------------------------------------- +#ifdef DBGFLAG_VALIDATE +void CGCGSSession::Validate( CValidator &validator, const char *pchName ) +{ + CGCSession::Validate( validator, pchName); + + VALIDATE_SCOPE(); + + ValidateObj( m_vecUsers ); +} +#endif // DBGFLAG_VALIDATE + + +//----------------------------------------------------------------------------- +// Purpose: Client says it needs the SO Cache +// Input : pNetPacket - received message +//----------------------------------------------------------------------------- +class CGCCacheSubscriptionRefresh: public CGCJob +{ +public: + CGCCacheSubscriptionRefresh( CGCBase *pGC ) : CGCJob( pGC ) { } + bool BYieldingRunJobFromMsg( IMsgNetPacket *pNetPacket ); +}; + +bool CGCCacheSubscriptionRefresh::BYieldingRunJobFromMsg( IMsgNetPacket *pNetPacket ) +{ + CProtoBufMsg<CMsgSOCacheSubscriptionRefresh> msg( pNetPacket ); + + CSteamID steamID( msg.Hdr().client_steam_id() ); + + CSteamID steamIDCacheOwner( msg.Body().owner() ); + CGCSharedObjectCache *pCache = m_pGC->FindSOCache( steamIDCacheOwner ); + if ( pCache == NULL || !pCache->BIsSubscribed( steamID ) ) + { + return false; + } + + pCache->SendSubscriberMessage( steamID ); + + return true; +} + +GC_REG_JOB( CGCBase, CGCCacheSubscriptionRefresh, "CGCCacheSubscriptionRefresh", k_ESOMsg_CacheSubscriptionRefresh, k_EServerTypeGC ); + +} // namespace GCSDK + diff --git a/gcsdk/gcsqlquery.cpp b/gcsdk/gcsqlquery.cpp new file mode 100644 index 0000000..955e912 --- /dev/null +++ b/gcsdk/gcsqlquery.cpp @@ -0,0 +1,151 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//=============================================================================// +#include "stdafx.h" + +#include "tier0/memdbgoff.h" + +namespace GCSDK +{ +IMPLEMENT_CLASS_MEMPOOL( CGCSQLQuery, 1000, UTLMEMORYPOOL_GROW_SLOW ); +IMPLEMENT_CLASS_MEMPOOL( CGCSQLQueryGroup, 1000, UTLMEMORYPOOL_GROW_SLOW ); +} + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ +//---------------------------------------------------------------------------- +// Purpose: Constructor. +//---------------------------------------------------------------------------- +// create queries on the heap with Alloc +CGCSQLQuery::CGCSQLQuery() +{ + m_pBufParams = GetBufferPool().GetBuffer(); +} + +//---------------------------------------------------------------------------- +// Purpose: Destructor. +//---------------------------------------------------------------------------- +CGCSQLQuery::~CGCSQLQuery() +{ + // free all the data + ClearParams(); + GetBufferPool().ReturnBuffer( m_pBufParams ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Gets a singleton buffer pool for bind params +//---------------------------------------------------------------------------- +static GCConVar sql_bind_param_max_pool_size_mb( "sql_bind_param_max_pool_size_mb", "10", "Maximum size in bytes of the SQL Bind Param buffer pool" ); +static GCConVar sql_bind_param_init_buffer_size( "sql_bind_param_init_buffer_size", "16384", "Initial buffer size for buffers in the SQL Bind Param buffer pool" ); +/*static*/ CBufferPool &CGCSQLQuery::GetBufferPool() +{ + static CBufferPool s_bufferPool( "SQL bind params", sql_bind_param_max_pool_size_mb, sql_bind_param_init_buffer_size ); + return s_bufferPool; +} + + +//---------------------------------------------------------------------------- +// Purpose: Allocates a bit of memory for the parameter and adds it to the list +//---------------------------------------------------------------------------- +void CGCSQLQuery::AddBindParamRaw( EGCSQLType eType, const byte *pubData, uint32 cubData ) +{ + Assert( m_vecParams.Count() < k_cMaxBindParams ); + if( m_vecParams.Count() >= k_cMaxBindParams ) + return; + + GCSQLBindParam_t ¶m = m_vecParams[ m_vecParams.AddToTail() ]; + param.m_eType = eType; + param.m_cubData = cubData; + + if ( cubData ) + { + param.m_nOffset = m_pBufParams->TellPut(); + m_pBufParams->Put( pubData, cubData ); + } + else + { + param.m_nOffset = 0; + } +} + + +//---------------------------------------------------------------------------- +// Purpose: frees all the bind params +//---------------------------------------------------------------------------- +void CGCSQLQuery::ClearParams() +{ + m_vecParams.RemoveAll(); + m_pBufParams->Clear(); +} + + +//---------------------------------------------------------------------------- +// Purpose: Constructor. +//---------------------------------------------------------------------------- +// create queries on the heap with Alloc +CGCSQLQueryGroup::CGCSQLQueryGroup() +: m_pResults( NULL ) +{ + +} + + +//---------------------------------------------------------------------------- +// Purpose: Sets the result pointer +//---------------------------------------------------------------------------- +CGCSQLQueryGroup::~CGCSQLQueryGroup() +{ + // free all the data + Clear(); + + if( m_pResults ) + m_pResults->Destroy(); +} + + +//---------------------------------------------------------------------------- +// Purpose: Adds a new query to the group +//---------------------------------------------------------------------------- +void CGCSQLQueryGroup::AddQuery( CGCSQLQuery *pQuery ) +{ + m_vecQueries.AddToTail( pQuery ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Sets the name (file and line) for the group +//---------------------------------------------------------------------------- +void CGCSQLQueryGroup::SetName( const char *sName ) +{ + m_sName = sName; +} + + +//---------------------------------------------------------------------------- +// Purpose: clears all the queries in the query group and resets its name +//---------------------------------------------------------------------------- +void CGCSQLQueryGroup::Clear() +{ + m_vecQueries.PurgeAndDeleteElements(); + m_sName = ""; +} + + +//---------------------------------------------------------------------------- +// Purpose: Sets the result pointer +//---------------------------------------------------------------------------- +void CGCSQLQueryGroup::SetResults( IGCSQLResultSetList *pResults ) +{ + if( m_pResults ) + m_pResults->Destroy(); + m_pResults = pResults; +} + +} // namespace GCSDK diff --git a/gcsdk/gcsystemaccess.cpp b/gcsdk/gcsystemaccess.cpp new file mode 100644 index 0000000..3a7d3d9 --- /dev/null +++ b/gcsdk/gcsystemaccess.cpp @@ -0,0 +1,690 @@ +//========= Copyright (c), Valve LLC, All rights reserved. ============ +// +// Purpose: +// +// $NoKeywords: $ +//============================================================================= + +#include "stdafx.h" +#include "gcsystemaccess.h" +#include "gcjob.h" +#include "gcsdk/gcreportprinter.h" + + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + + +static GCConVar gcaccess_enable( "gcaccess_enable", "1", "Kill switch that disables all gcaccess tracking and systems" ); + +DECLARE_GC_EMIT_GROUP( g_EGAccess, access ); + + +//----------------------------------------------------------------------------------------------------------------------------------------- +GCAccessSystem_t GenerateUniqueGCAccessID() +{ + static GCAccessSystem_t s_nID = 0; + return s_nID++; +} + +//----------------------------------------------------------------------------------------------------------------------------------------- +// CGCAccessSystem +//----------------------------------------------------------------------------------------------------------------------------------------- + +//a GCAccess system that has been registered +class CGCAccessSystem +{ +public: + //the name of the system for display + CUtlString m_sName; + //the unique ID for this system + GCAccessSystem_t m_nID; + //which actions should be enabled or disabled for this system in particular + uint32 m_nActions; + uint32 m_nSuppressActions; + + struct AccessStats_t + { + AccessStats_t(); + void Add( const AccessStats_t& rhs ); + //how many raw accesses have we had? + uint32 m_nNumValid; + //how many failed because the context didn't have rights? + uint32 m_nNumFail; + //how many failed because the associated ID wasn't valid? + uint32 m_nNumFailID; + }; + + struct TrackedJob_t + { + CUtlString m_sContext; + AccessStats_t m_Stats; + }; + + //which jobs have violated this system access rights + CUtlHashMapLarge< uintp, TrackedJob_t > m_Jobs; +}; + +//----------------------------------------------------------------------------------------------------------------------------------------- +// CGCAccessContext +//----------------------------------------------------------------------------------------------------------------------------------------- + +CGCAccessContext::CGCAccessContext() : + m_nActions( 0 ), + m_nSuppressActions( 0 ) +{ +} + +void CGCAccessContext::Init( const char* pszName, uint32 nActions, uint32 nSuppressActions ) +{ + m_sName = pszName; + m_nActions = nActions; + m_nSuppressActions = nSuppressActions; +} + +void CGCAccessContext::AddSystem( GCAccessSystem_t nSystem ) +{ + m_nSystems.InsertIfNotFound( nSystem ); +} + +bool CGCAccessContext::HasSystem( GCAccessSystem_t system ) const +{ + return ( m_nSystems.Find( system ) != m_nSystems.InvalidIndex() ); +} + +bool CGCAccessContext::IsSubsetOf( const CGCAccessContext& context ) const +{ + FOR_EACH_VEC( m_nSystems, nCurrSystem ) + { + if( !context.HasSystem( m_nSystems[ nCurrSystem ] ) ) + return false; + } + return true; +} + +void CGCAccessContext::AddSystemsFrom( const CGCAccessContext& context ) +{ + FOR_EACH_VEC( context.m_nSystems, nCurrSystem ) + { + AddSystem( context.m_nSystems[ nCurrSystem ] ); + } +} + +void CGCAccessContext::SetAction( EGCAccessAction eAction, bool bSet ) +{ + if( bSet ) + m_nActions |= eAction; + else + m_nActions &= ~( uint32 )eAction; +} + +void CGCAccessContext::SetSuppressAction( EGCAccessAction eAction, bool bSet ) +{ + if( bSet ) + m_nSuppressActions |= eAction; + else + m_nSuppressActions &= ~( uint32 )eAction; +} + +//----------------------------------------------------------------------------------------------------------------------------------------- +// CGCAccess +//----------------------------------------------------------------------------------------------------------------------------------------- + +static CGCAccess g_GCAccess; +CGCAccess& GGCAccess() +{ + return g_GCAccess; +} + +CGCAccessSystem::AccessStats_t::AccessStats_t() + : m_nNumFail( 0 ) + , m_nNumFailID( 0 ) + , m_nNumValid( 0 ) +{} + +void CGCAccessSystem::AccessStats_t::Add( const AccessStats_t& rhs ) +{ + m_nNumFail += rhs.m_nNumFail; + m_nNumFailID += rhs.m_nNumFailID; + m_nNumValid += rhs.m_nNumValid; +} + +CGCAccess::CGCAccess() : + m_nActions( GCAccessAction_TrackSuccess | GCAccessAction_TrackFail| GCAccessAction_ReturnFail ), + m_nSuppressActions( 0 ) +{ + m_GlobalContext.Init( "Global" ); +} + +void CGCAccess::RegisterSystem( const char* pszName, GCAccessSystem_t nID, uint32 nActions, uint32 nSupressActions ) +{ + //make sure that we don't have a conflict + int nIndex = m_Systems.Find( nID ); + if( nIndex != m_Systems.InvalidIndex() ) + { + // !FIXME! + // DOTAMERGE AssertMsg varargs + AssertMsg3( false, "Multiple systems conflciting in Register System: ID %d, original: %s, new %s", nID, m_Systems[ nIndex ]->m_sName.String(), pszName ); + return; + } + + CGCAccessSystem* pSystem = new CGCAccessSystem; + pSystem->m_sName = pszName; + pSystem->m_nID = nID; + pSystem->m_nActions = nActions; + pSystem->m_nSuppressActions = nSupressActions; + m_Systems.Insert( nID, pSystem ); +} + +bool CGCAccess::ValidateAccess( GCAccessSystem_t nSystem ) +{ + //global kill switch + if( !gcaccess_enable.GetBool() ) + return true; + + return InternalValidateAccess( nSystem, CSteamID(), CSteamID() ); +} + +bool CGCAccess::ValidateSteamIDAccess( GCAccessSystem_t nSystem, CSteamID steamID ) +{ + //global kill switch + if( !gcaccess_enable.GetBool() ) + return true; + + CSteamID expectedID = ( g_pJobCur ) ? g_pJobCur->SOVALIDATE_GetSteamID() : steamID; + return InternalValidateAccess( nSystem, steamID, expectedID ); +} + +bool CGCAccess::InternalValidateAccess( GCAccessSystem_t nSystem, CSteamID steamID, CSteamID expectedID ) +{ + //see if tracking for this system is disabled. This list is almost always empty, so just use a linear search for now. + FOR_EACH_VEC( m_SuppressAccess, nAction ) + { + if( m_SuppressAccess[ nAction ].m_nSystem == nSystem ) + return true; + } + + //assume the global context + const CGCAccessContext* pContext = &m_GlobalContext; + const char* pszJobName = "[global]"; + + //if we have a job, use it's context + if( g_pJobCur ) + { + const CGCJob* pJob = static_cast< const CGCJob* >( g_pJobCur ); + pszJobName = pJob->GetName(); + if( pJob->GetContext() ) + pContext = pJob->GetContext(); + } + + //is this a valid system to access + bool bValidSteamID = ( steamID == expectedID ); + bool bValidSystem = pContext->HasSystem( nSystem ); + bool bValidAccess = ( bValidSystem && bValidSteamID ); + + //determine the actions we want to do for tracking + uint32 nActions = pContext->GetActions() | m_nActions; + uint32 nSuppressActions = pContext->GetSuppressActions() | m_nSuppressActions; + + //look up the system that we are accessing + CGCAccessSystem* pSystem = NULL; + int nSystemIndex = m_Systems.Find( nSystem ); + if( nSystemIndex == m_Systems.InvalidIndex() ) + { + // !FIXME! DOTAMERGE + // AssertMsg varargs + AssertMsg1( false, "Error: Tracking a system that has not been registered. Make sure to register system %u", nSystem ); + } + else + { + //make sure that we have a stat entry for this job + pSystem = m_Systems[ nSystemIndex ]; + nActions |= pSystem->m_nActions; + nSuppressActions |= pSystem->m_nSuppressActions; + } + + //remove any suppressed actions + nActions &= ~nSuppressActions; + + //see if we need to track this access + bool bTrackSuccess = ( nActions & GCAccessAction_TrackSuccess ) && bValidAccess; + bool bTrackFail = ( nActions & GCAccessAction_TrackFail ) && !bValidAccess; + if( ( bTrackSuccess || bTrackFail ) && pSystem ) + { + //make sure that we have a stat entry for this job + int nJobIndex = pSystem->m_Jobs.Find( ( uintp )pszJobName ); + if( nJobIndex == pSystem->m_Jobs.InvalidIndex() ) + { + nJobIndex = pSystem->m_Jobs.Insert( ( uintp )pszJobName ); + pSystem->m_Jobs[ nJobIndex ].m_sContext = pContext->GetName(); + } + + //update our stats based upon what was valid and what wasn't + CGCAccessSystem::AccessStats_t& stats = pSystem->m_Jobs[ nJobIndex ].m_Stats; + if( bTrackSuccess ) + stats.m_nNumValid++; + + if( bTrackFail ) + { + if( !bValidSteamID ) + stats.m_nNumFailID++; + if( !bValidSystem ) + stats.m_nNumFail++; + } + } + + //see if this is a single access we want to try and track + FOR_EACH_VEC( m_SingleAsserts, nCurrAssert ) + { + SingleAssert_t* pAssert = m_SingleAsserts[ nCurrAssert ]; + if( nSystem == pAssert->m_System ) + { + if( V_stricmp( ( pAssert->m_bContext ) ? pContext->GetName() : pszJobName, pAssert->m_sContextOrJob ) == 0 ) + { + //log this assert + { + // !FIXME! DOTAMERGE + //CGCInterface::CDisableAssertRateLimit disableAssertLimit; + + // !FIXME! DOTAMERGE + // AssertMsg varargs + AssertMsg3( false, "GCSystemAccess Caught %s (context %s) calling into %s", pszJobName, pContext->GetName(), pSystem ? pSystem->m_sName.String() : "unknown" ); + } + + //and clear it + delete pAssert; + m_SingleAsserts.FastRemove( nCurrAssert ); + } + } + } + + //at this point, if it is a valid access, just bail + if( bValidAccess ) + return true; + + //otherwise, handle the failure case + if( nActions & ( GCAccessAction_Msg | GCAccessAction_Assert ) ) + { + int nSystemIndex = m_Systems.Find( nSystem ); + const char* pszSystem = ( nSystemIndex != m_Systems.InvalidIndex() ) ? m_Systems[ nSystemIndex ]->m_sName.String() : "<unregistered>"; + + //display a message based upon if it was a system or ID violation + CFmtStr1024 szMsg; + if( !bValidSystem ) + { + szMsg.sprintf( "Job %s Accessed invalid system %s (%u) while in context %s\n", pszJobName, pszSystem, nSystem, pContext->GetName() ); + } + else + { + szMsg.sprintf( "Job %s Accessed invalid steam ID %s but expected %s in system %s (%u) while in context %s\n", pszJobName, steamID.RenderLink(), expectedID.RenderLink(), pszSystem, nSystem, pContext->GetName() ); + } + + if( nActions & GCAccessAction_Msg ) + EG_MSG( g_EGAccess, "%s", szMsg.String() ); + if( nActions & GCAccessAction_Assert ) + { + // !FIXME! DOTAMERGE + // AssertMsg varargs + AssertMsg1( false, "%s", szMsg.String() ); + } + } + + return !( nActions & GCAccessAction_ReturnFail ); +} + +void CGCAccess::SuppressAccess( GCAccessSystem_t nSystem, bool bEnable ) +{ + //see if it is an existing item already suppressed that we need to modify the ref count of + FOR_EACH_VEC( m_SuppressAccess, nAccess ) + { + if( m_SuppressAccess[ nAccess ].m_nSystem == nSystem ) + { + if( bEnable ) + { + //to enable, we want to decrease the disable count, or remove if we are fully re-enabled + if( m_SuppressAccess[ nAccess ].m_nCount <= 1 ) + m_SuppressAccess.FastRemove( nAccess ); + else + m_SuppressAccess[ nAccess ].m_nCount--; + } + else + { + m_SuppressAccess[ nAccess ].m_nCount++; + } + return; + } + } + + //only add it to the list if we are disabling + if( !bEnable ) + { + SuppressAccess_t access; + access.m_nSystem = nSystem; + access.m_nCount = 1; + m_SuppressAccess.AddToTail( access ); + } +} + +void CGCAccess::ClearSystemStats() +{ + FOR_EACH_MAP_FAST( m_Systems, nSystem ) + { + m_Systems[ nSystem ]->m_Jobs.Purge(); + } +} + +bool CGCAccess::CatchSingleAssert( const char* pszSystem, bool bContext, const char* pszContextOrJob ) +{ + //find the system we are referencing so we don't add invalid breakpoints if we can avoid it + GCAccessSystem_t nSystemID = ( GCAccessSystem_t )-1; + FOR_EACH_MAP_FAST( m_Systems, nSystem ) + { + if( V_stricmp( m_Systems[ nSystem ]->m_sName, pszSystem ) == 0 ) + { + nSystemID = m_Systems.Key( nSystem ); + break; + } + } + + if( nSystemID == ( GCAccessSystem_t )-1 ) + return false; + + SingleAssert_t* pAssert = new SingleAssert_t; + pAssert->m_System = nSystemID; + pAssert->m_bContext = bContext; + pAssert->m_sContextOrJob = pszContextOrJob; + m_SingleAsserts.AddToTail( pAssert ); + return true; +} + +void CGCAccess::ClearSingleAsserts() +{ + m_SingleAsserts.PurgeAndDeleteElements(); +} + + +//given a display type and a stats object, determines if it meets the criteria +static bool ShouldDisplayStats( CGCAccess::EDisplay eDisplay, const CGCAccessSystem::AccessStats_t& stats ) +{ + if( eDisplay == CGCAccess::eDisplay_Referenced ) + return ( stats.m_nNumFail + stats.m_nNumFailID + stats.m_nNumValid ) > 0; + if( eDisplay == CGCAccess::eDisplay_Violations ) + return ( stats.m_nNumFail + stats.m_nNumFailID ) > 0; + if( eDisplay == CGCAccess::eDisplay_IDViolations ) + return stats.m_nNumFailID > 0; + + //unknown, so just assume all + return true; +} + + +void CGCAccess::ReportJobs( const char* pszContext, EDisplay eDisplay ) const +{ + //collect all of the job stats into a single summary table + CUtlHashMapLarge< uintp, CGCAccessSystem::TrackedJob_t > jobs; + FOR_EACH_MAP_FAST( m_Systems, nSystem ) + { + const CGCAccessSystem* pSystem = m_Systems[ nSystem ]; + FOR_EACH_MAP_FAST( pSystem->m_Jobs, nJob ) + { + const CGCAccessSystem::TrackedJob_t& job = pSystem->m_Jobs[ nJob ]; + //skip any contexts we don't care about + if( pszContext && ( V_stricmp( pszContext, job.m_sContext ) != 0 ) ) + continue; + if( !ShouldDisplayStats( eDisplay, job.m_Stats ) ) + continue; + + int nIndex = jobs.Find( pSystem->m_Jobs.Key( nJob ) ); + if( nIndex == jobs.InvalidIndex() ) + { + nIndex = jobs.Insert( pSystem->m_Jobs.Key( nJob ) ); + jobs[ nIndex ].m_sContext = job.m_sContext; + } + + jobs[ nIndex ].m_Stats.Add( job.m_Stats ); + } + } + + CGCReportPrinter rp; + rp.AddStringColumn( "Job" ); + rp.AddStringColumn( "Context" ); + rp.AddIntColumn( "Valid", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "Invalid", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "InvalidID", CGCReportPrinter::eSummary_Total ); + + FOR_EACH_MAP_FAST( jobs, nJob ) + { + rp.StrValue( ( const char* )jobs.Key( nJob ), CFmtStr( "gcaccess_dump_job \"%s\" %d", ( const char* )jobs.Key( nJob ), eDisplay ).String() ); + rp.StrValue( jobs[ nJob ].m_sContext, CFmtStr( "gcaccess_dump_context \"%s\" %d", jobs[ nJob ].m_sContext.String(), eDisplay ).String() ); + rp.IntValue( jobs[ nJob ].m_Stats.m_nNumValid ); + rp.IntValue( jobs[ nJob ].m_Stats.m_nNumFail ); + rp.IntValue( jobs[ nJob ].m_Stats.m_nNumFailID ); + rp.CommitRow(); + } + + rp.SortReport( "Job", false ); + rp.PrintReport( SPEW_CONSOLE ); +} + +void CGCAccess::ReportSystems( const char* pszContext, EDisplay eDisplay ) const +{ + CGCReportPrinter rp; + rp.AddStringColumn( "System" ); + rp.AddIntColumn( "ID", CGCReportPrinter::eSummary_None ); + rp.AddIntColumn( "Jobs", CGCReportPrinter::eSummary_None ); + rp.AddIntColumn( "Valid", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "Invalid", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "InvalidID", CGCReportPrinter::eSummary_Total ); + + FOR_EACH_MAP_FAST( m_Systems, nSystem ) + { + const CGCAccessSystem* pSystem = m_Systems[ nSystem ]; + CGCAccessSystem::AccessStats_t stats; + FOR_EACH_MAP_FAST( pSystem->m_Jobs, nJob ) + { + const CGCAccessSystem::TrackedJob_t& job = pSystem->m_Jobs[ nJob ]; + //skip any contexts we don't care about + if( pszContext && ( V_stricmp( pszContext, job.m_sContext ) != 0 ) ) + continue; + + stats.Add( job.m_Stats ); + } + + if( !ShouldDisplayStats( eDisplay, stats ) ) + continue; + + rp.StrValue( pSystem->m_sName, CFmtStr( "gcaccess_dump_system \"%s\" %d", pSystem->m_sName.String(), eDisplay ).String() ); + rp.IntValue( pSystem->m_nID ); + rp.IntValue( pSystem->m_Jobs.Count() ); + rp.IntValue( stats.m_nNumValid ); + rp.IntValue( stats.m_nNumFail ); + rp.IntValue( stats.m_nNumFailID ); + rp.CommitRow(); + } + + rp.SortReport( "System", false ); + rp.PrintReport( SPEW_CONSOLE ); +} + +void CGCAccess::FullReport( const char* pszSystemFilter, const char* pszContextFilter, const char* pszJobFilter, EDisplay eDisplay ) const +{ + CGCReportPrinter rp; + rp.AddStringColumn( "Job" ); + rp.AddStringColumn( "Context" ); + rp.AddStringColumn( "System" ); + rp.AddIntColumn( "Valid", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "Invalid", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "InvalidID", CGCReportPrinter::eSummary_Total ); + + FOR_EACH_MAP_FAST( m_Systems, nSystem ) + { + const CGCAccessSystem* pSystem = m_Systems[ nSystem ]; + if( pszSystemFilter && V_stricmp( pszSystemFilter, pSystem->m_sName ) ) + continue; + + FOR_EACH_MAP_FAST( pSystem->m_Jobs, nJob ) + { + const CGCAccessSystem::AccessStats_t& stats = pSystem->m_Jobs[ nJob ].m_Stats; + const char* pszJob = ( const char* )pSystem->m_Jobs.Key( nJob ); + const char* pszContext = ( const char* )pSystem->m_Jobs[ nJob ].m_sContext; + + if( !ShouldDisplayStats( eDisplay, stats ) ) + continue; + if( pszJobFilter && V_stricmp( pszJobFilter, pszJob ) ) + continue; + if( pszContextFilter && V_stricmp( pszContextFilter, pszContext ) ) + continue; + + rp.StrValue( pszJob, CFmtStr( "gcaccess_dump_job \"%s\" %d", pszJob, eDisplay ).String() ); + rp.StrValue( pszContext, CFmtStr( "gcaccess_dump_context \"%s\" %d", pszContext, eDisplay ).String() ); + rp.StrValue( pSystem->m_sName, CFmtStr( "gcaccess_dump_system \"%s\" %d", pSystem->m_sName.String(), eDisplay ).String() ); + rp.IntValue( stats.m_nNumValid ); + rp.IntValue( stats.m_nNumFail ); + rp.IntValue( stats.m_nNumFailID ); + rp.CommitRow(); + } + } + + rp.SortReport( "Context", false ); + rp.PrintReport( SPEW_CONSOLE ); +} + +void CGCAccess::DependencyReport( const char* pszSystem, EDisplay eDisplay ) const +{ + //first off, we need to find the system we are scanning for dependencies on + const CGCAccessSystem* pMatchSystem = NULL; + FOR_EACH_MAP_FAST( m_Systems, nSystem ) + { + const CGCAccessSystem* pSystem = m_Systems[ nSystem ]; + if( V_stricmp( pszSystem, pSystem->m_sName ) == 0 ) + { + pMatchSystem = pSystem; + break; + } + } + + if( !pMatchSystem ) + { + EG_MSG( SPEW_CONSOLE, "Unable to find system %s for dependency report\n", pszSystem ); + return; + } + + //now generate our report + CGCReportPrinter rp; + rp.AddStringColumn( "Job" ); + rp.AddStringColumn( "Context" ); + rp.AddStringColumn( "System" ); + rp.AddIntColumn( "Valid", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "Invalid", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "InvalidID", CGCReportPrinter::eSummary_Total ); + + FOR_EACH_MAP_FAST( m_Systems, nSystem ) + { + const CGCAccessSystem* pSystem = m_Systems[ nSystem ]; + //skip ourself + if( pSystem == pMatchSystem ) + continue; + + FOR_EACH_MAP_FAST( pSystem->m_Jobs, nJob ) + { + const CGCAccessSystem::AccessStats_t& stats = pSystem->m_Jobs[ nJob ].m_Stats; + const char* pszJob = ( const char* )pSystem->m_Jobs.Key( nJob ); + const char* pszContext = ( const char* )pSystem->m_Jobs[ nJob ].m_sContext; + + //skip any job that isn't using our match system + if( !pMatchSystem->m_Jobs.HasElement( ( uintp )pszJob ) ) + continue; + if( !ShouldDisplayStats( eDisplay, stats ) ) + continue; + + rp.StrValue( pszJob, CFmtStr( "gcaccess_dump_job \"%s\" %d", pszJob, eDisplay ).String() ); + rp.StrValue( pszContext, CFmtStr( "gcaccess_dump_context \"%s\" %d", pszContext, eDisplay ).String() ); + rp.StrValue( pSystem->m_sName, CFmtStr( "gcaccess_dump_system \"%s\" %d", pSystem->m_sName.String(), eDisplay ).String() ); + rp.IntValue( stats.m_nNumValid ); + rp.IntValue( stats.m_nNumFail ); + rp.IntValue( stats.m_nNumFailID ); + rp.CommitRow(); + } + } + + rp.SortReport( "System", false ); + rp.PrintReport( SPEW_CONSOLE ); + + +} + +GC_CON_COMMAND_PARAMS( gcaccess_dump_job, 1, "<Job Name> Dumps all of the accesses associated with the specified job" ) +{ + CGCAccess::EDisplay eDisplay = ( args.ArgC() >= 3 ) ? ( CGCAccess::EDisplay )V_atoi( args[ 2 ] ) : CGCAccess::eDisplay_Referenced; + GGCAccess().FullReport( NULL, NULL, args[ 1 ], eDisplay ); +} + +GC_CON_COMMAND_PARAMS( gcaccess_dump_context, 1, "<Job Name> Dumps all of the accesses associated with the specified context" ) +{ + CGCAccess::EDisplay eDisplay = ( args.ArgC() >= 3 ) ? ( CGCAccess::EDisplay )V_atoi( args[ 2 ] ) : CGCAccess::eDisplay_Referenced; + GGCAccess().FullReport( NULL, args[ 1 ], NULL, eDisplay ); +} + +GC_CON_COMMAND_PARAMS( gcaccess_dump_system, 1, "<Job Name> Dumps all of the accesses associated with the specified system" ) +{ + CGCAccess::EDisplay eDisplay = ( args.ArgC() >= 3 ) ? ( CGCAccess::EDisplay )V_atoi( args[ 2 ] ) : CGCAccess::eDisplay_Referenced; + GGCAccess().FullReport( args[ 1 ], NULL, NULL, eDisplay ); +} + +GC_CON_COMMAND_PARAMS( gcaccess_dump_context_jobs, 1, "<Context Name> Dumps all of the accesses associated with the specified context" ) +{ + CGCAccess::EDisplay eDisplay = ( args.ArgC() >= 3 ) ? ( CGCAccess::EDisplay )V_atoi( args[ 2 ] ) : CGCAccess::eDisplay_Referenced; + GGCAccess().ReportJobs( args[ 1 ], eDisplay ); +} + +GC_CON_COMMAND_PARAMS( gcaccess_dump_context_systems, 1, "<Context Name> Dumps all of the accesses associated with the specified context" ) +{ + CGCAccess::EDisplay eDisplay = ( args.ArgC() >= 3 ) ? ( CGCAccess::EDisplay )V_atoi( args[ 2 ] ) : CGCAccess::eDisplay_Referenced; + GGCAccess().ReportSystems( args[ 1 ], eDisplay ); +} + +GC_CON_COMMAND_PARAMS( gcaccess_dump_system_dependencies, 1, "<System Name> This will dump out for all jobs that reference the specified system, what other systems they reference" ) +{ + CGCAccess::EDisplay eDisplay = ( args.ArgC() >= 3 ) ? ( CGCAccess::EDisplay )V_atoi( args[ 2 ] ) : CGCAccess::eDisplay_Referenced; + GGCAccess().DependencyReport( args[ 1 ], eDisplay ); +} + +GC_CON_COMMAND( gcaccess_dump_jobs, "Dumps all the jobs that have been tracked with the gcaccess system as well as count of violations" ) +{ + CGCAccess::EDisplay eDisplay = ( args.ArgC() >= 2 ) ? ( CGCAccess::EDisplay )V_atoi( args[ 1 ] ) : CGCAccess::eDisplay_Referenced; + GGCAccess().ReportJobs( NULL, eDisplay); +} + +GC_CON_COMMAND( gcaccess_dump_systems, "Dumps all the systems that have been tracked with the gcaccess system as well as count of violations" ) +{ + CGCAccess::EDisplay eDisplay = ( args.ArgC() >= 2 ) ? ( CGCAccess::EDisplay )V_atoi( args[ 1 ] ) : CGCAccess::eDisplay_Referenced; + GGCAccess().ReportSystems( NULL, eDisplay ); +} + +GC_CON_COMMAND( gcaccess_dump_full, "Dumps all the information tracked by the gcaccess" ) +{ + CGCAccess::EDisplay eDisplay = ( args.ArgC() >= 2 ) ? ( CGCAccess::EDisplay )V_atoi( args[ 1 ] ) : CGCAccess::eDisplay_Referenced; + GGCAccess().FullReport( NULL, NULL, NULL, eDisplay ); +} + +GC_CON_COMMAND_PARAMS( gcaccess_catch_context, 2, "<system> <context> - Catches and generates an assert when the specified context calls into the provided system. This will only generate a single assert" ) +{ + if( !GGCAccess().CatchSingleAssert( args[ 1 ], true, args[ 2 ] ) ) + EG_MSG( SPEW_CONSOLE, "Unable to register catch, likely \'%s\' is not a valid system.", args[ 1 ] ); +} + +GC_CON_COMMAND_PARAMS( gcaccess_catch_job, 2, "<system> <job> - Catches and generates an assert when the specified job calls into the provided system. This will only generate a single assert" ) +{ + if( !GGCAccess().CatchSingleAssert( args[ 1 ], false, args[ 2 ] ) ) + EG_MSG( SPEW_CONSOLE, "Unable to register catch, likely \'%s\' is not a valid system.", args[ 1 ] ); +} + +GC_CON_COMMAND( gcaccess_catch_clear, "Clears all previously specified catches" ) +{ + GGCAccess().ClearSingleAsserts(); +} + +} //namespace GCSDK diff --git a/gcsdk/gcsystemmsgs.proto b/gcsdk/gcsystemmsgs.proto new file mode 100644 index 0000000..e5a1f53 --- /dev/null +++ b/gcsdk/gcsystemmsgs.proto @@ -0,0 +1,203 @@ +//====== Copyright 1996-2010, Valve Corporation, All rights reserved. ======= +// +// Purpose: The file defines our Google Protocol Buffers which are used in over +// the wire messages between servers as well as between clients and servers. +// +//============================================================================= + +// We care more about speed than code size +option optimize_for = SPEED; + +// We don't use the service generation functionality +option cc_generic_services = false; + + +// +// STYLE NOTES: +// +// Use CamelCase CMsgMyMessageName style names for messages. +// +// Use lowercase _ delimited names like my_steam_id for field names, this is non-standard for Steam, +// but plays nice with the Google formatted code generation. +// +// Try not to use required fields ever. Only do so if you are really really sure you'll never want them removed. +// Optional should be preffered as it will make versioning easier and cleaner in the future if someone refactors +// your message and wants to remove or rename fields. +// +// Use fixed64 for JobId_t, GID_t, or SteamID. This is appropriate for any field that is normally +// going to be larger than 2^56. Otherwise use int64 for 64 bit values that are frequently smaller +// than 2^56 as it will safe space on the wire in those cases. +// +// Similar to fixed64, use fixed32 for RTime32 or other 32 bit values that are frequently larger than +// 2^28. It will safe space in those cases, otherwise use int32 which will safe space for smaller values. +// An exception to this rule for RTime32 is if the value will frequently be zero rather than set to an actual +// time. +// + +enum EGCSystemMsg +{ + k_EGCMsgInvalid = 0; + k_EGCMsgMulti = 1; + + k_EGCMsgGenericReply = 10; + + k_EGCMsgSystemBase = 50; + k_EGCMsgAchievementAwarded = 51; + k_EGCMsgConCommand = 52; // A command from the GC's admin console + k_EGCMsgStartPlaying = 53; + k_EGCMsgStopPlaying = 54; + k_EGCMsgStartGameserver = 55; + k_EGCMsgStopGameserver = 56; + k_EGCMsgWGRequest = 57; + k_EGCMsgWGResponse = 58; + k_EGCMsgGetUserGameStatsSchema = 59; // Gets the user game stats schema for the app + k_EGCMsgGetUserGameStatsSchemaResponse = 60; + k_EGCMsgGetUserStatsDEPRECATED = 61; // Gets user game stats for a user + k_EGCMsgGetUserStatsResponse = 62; + k_EGCMsgAppInfoUpdated = 63; // Message sent to the GC when there has been an AppInfo update + k_EGCMsgValidateSession = 64; // Message sent by the GC when it wants to make sure a session exists + k_EGCMsgValidateSessionResponse = 65; // Message sent to the GC in response to ValidateSession + k_EGCMsgLookupAccountFromInput = 66; // Sent by the GC to lookup user. Reply is k_EGCMsgGenericReply + k_EGCMsgSendHTTPRequest = 67; // Message sent by the GC to do a generic HTTP request + k_EGCMsgSendHTTPRequestResponse = 68; // Response back to the GC with the results of the HTTP request + k_EGCMsgPreTestSetup = 69; // Reset the GC database (usually for testing purposes) + k_EGCMsgRecordSupportAction = 70; // Logs a support action + k_EGCMsgGetAccountDetails_DEPRECATED = 71; // DEPRECATED - Use protobuf-based k_EGCMsgGetAccountDetails //Requests the details for an account +// k_EGCMsgSendInterAppMessage = 72; // Sends a message to another app's GC + k_EGCMsgReceiveInterAppMessage = 73; // Receives a message from another app's GC + k_EGCMsgFindAccounts = 74; // queries the AMs for accounts by name + k_EGCMsgPostAlert = 75; // posts an alert to Steam + k_EGCMsgGetLicenses = 76; // asks Steam for the user's licenses + k_EGCMsgGetUserStats = 77; // Gets user game stats for a user + k_EGCMsgGetCommands = 78; // request for a list of commands from a gc console + k_EGCMsgGetCommandsResponse = 79; // response with a list of commands for a gc console + k_EGCMsgAddFreeLicense = 80; // request for for Steam to add a license to the specified free package + k_EGCMsgAddFreeLicenseResponse = 81; // response with the result of the attempt to add a free license + k_EGCMsgGetIPLocation = 82; // Get geolocation data for a specific IP + k_EGCMsgGetIPLocationResponse = 83; // Geolocation response + + k_EGCMsgSystemStatsSchema = 84; // Message sent by the GC specifying what its stats schema is + k_EGCMsgGetSystemStats = 85; // Message sent to the GC requesting its stats + k_EGCMsgGetSystemStatsResponse = 86; // Message sent by the GC with its stats + + k_EGCMsgSendEmail = 87; // Sent by the GC to send an email to a user + k_EGCMsgSendEmailResponse = 88; // Response with the result of the send request + k_EGCMsgGetEmailTemplate = 89; // Sent to the GC to request an email template + k_EGCMsgGetEmailTemplateResponse = 90; // Get email template response + k_EGCMsgGrantGuestPass = 91; // Sends a guest pass/gift to a user + k_EGCMsgGrantGuestPassResponse = 92; // Send guest pass response + k_EGCMsgGetAccountDetails = 93; // Requests the details for an account + k_EGCMsgGetAccountDetailsResponse = 94; // Requests the details for an account response + k_EGCMsgGetPersonaNames = 95; // Gets the persona names for a set of accounts + k_EGCMsgGetPersonaNamesResponse = 96; // Gets the persona names for a set of accounts response + k_EGCMsgMultiplexMsg = 97; // Sends a single message to multiple users + + // web API calls + k_EGCMsgWebAPIRegisterInterfaces = 101; // sent once at startup to register APIs + k_EGCMsgWebAPIJobRequest = 102; // sent when an actual request is made +// k_EGCMsgWebAPIRegistrationRequested = 103; // sent by the GC Host when it learns a web API server has started -- No longer sent by the GCH + k_EGCMsgWebAPIJobRequestHttpResponse = 104; // sent by the GC in response to k_EGCMsgWebAPIJobRequest - uses CMsgHttpResponse + k_EGCMsgWebAPIJobRequestForwardResponse = 105; // sent by the GC in response to k_EGCMsgWebAPIJobRequest - uses CMsgGCMsgWebAPIJobRequestForwardResponse + + + // Memcached + k_EGCMsgMemCachedGet = 200; // Get key(s) from memcached + k_EGCMsgMemCachedGetResponse = 201; // Retrieved keys + k_EGCMsgMemCachedSet = 202; // Set key(s) into memcached + k_EGCMsgMemCachedDelete = 203; // Delete key(s) from memcached + k_EGCMsgMemCachedStats = 204; // Request stats from memcached + k_EGCMsgMemCachedStatsResponse = 205; // Retrieved stats + + // SQL + k_EGCMsgSQLStats = 210; // Get the current SQL stats + k_EGCMsgSQLStatsResponse = 211; // Retrieved stats + + // Multi-GC setup + k_EGCMsgMasterSetDirectory = 220; // The master GC will send this back to the GCH indicating the sub GC's that need to be created + k_EGCMsgMasterSetDirectoryResponse = 221; // Response back from the GCH to the master GC regarding the successful receiving of the directory (note this doesn't mean all sub GCs are done starting up) + k_EGCMsgMasterSetWebAPIRouting = 222; // The master GC will send this back to the GCH indicating the routing details for WebAPI requests + k_EGCMsgMasterSetWebAPIRoutingResponse = 223; // Response back from the GCH to the master GC regarding the successful receipt and processing of k_EGCMsgMasterSetWebAPIRouting + k_EGCMsgMasterSetClientMsgRouting = 224; // The master GC will send this back to the GCH indicating the routing details for client messages + k_EGCMsgMasterSetClientMsgRoutingResponse = 225; // Response back from the GCH to the master GC regarding the successful receipt and processing of k_EGCMsgMasterSetClientMsgRouting + k_EGCMsgSetOptions = 226; // Each GC can send this back to the GCH indicating which client messages and data streams the GC is interested in + k_EGCMsgSetOptionsResponse = 227; // Response back from the GCH (optional - don't send a job ID with SetOptions to skip the response) + + // Newer GC SDK messages. All should be protobuf based. + k_EGCMsgSystemBase2 = 500; + k_EGCMsgGetPurchaseTrustStatus = 501; // Gets information about a user's purchase trustworthiness + k_EGCMsgGetPurchaseTrustStatusResponse = 502; // Gets information about a user's purchase trustworthiness response + k_EGCMsgUpdateSession = 503; // GCH update regarding the status of a session, only sent if the convar to enable these messages is set + k_EGCMsgGCAccountVacStatusChange = 504; // Accounts VAC status has changed + k_EGCMsgCheckFriendship = 505; // GC checking if two users are friends + k_EGCMsgCheckFriendshipResponse = 506; // Response to the GC regarding the users' friendship + k_EGCMsgGetPartnerAccountLink = 507; // Request the partner account link details for a user + k_EGCMsgGetPartnerAccountLinkResponse = 508; // Response to the GC with the partner account link details + k_EGCMsgVSReportedSuspiciousActivity = 509; // Suspicious activity detected on user's account + k_EGCMsgAccountTradeBanStatusChange = 510; // Account's trade ban status has changed + k_EGCMsgAccountLockStatusChange = 511; // Account's lock status has changed + k_EGCMsgDPPartnerMicroTxns = 512; // A set of microtransaction data relayed to the DP for insert or update. + k_EGCMsgDPPartnerMicroTxnsResponse = 513; // ... Response + k_EGCMsgGetIPASN = 514; // Get ASN data for a specific IP + k_EGCMsgGetIPASNResponse = 515; // IP ASN response + k_EGCMsgGetAppFriendsList = 516; // GC requesting a list of app-friends for a user (defined as friends who play the app requesting the friends list) + k_EGCMsgGetAppFriendsListResponse = 517; // ... Response + k_EGCMsgVacVerificationChange = 518; // From VS to GC + k_EGCMsgAccountPhoneNumberChange = 519; // From Phone Server to GC + k_EGCMsgAccountTwoFactorChange = 520; // From AM to GC + k_EGCMsgCheckClanMembership = 521; // Checks if an account is a member of a given clan + k_EGCMsgCheckClanMembershipResponse = 522; // Checks if an account is a member of a given clan response +}; + +enum ESOMsg +{ + k_ESOMsg_Create = 21; + k_ESOMsg_Update = 22; + k_ESOMsg_Destroy = 23; + k_ESOMsg_CacheSubscribed = 24; + k_ESOMsg_CacheUnsubscribed = 25; + k_ESOMsg_UpdateMultiple = 26; + k_ESOMsg_CacheSubscriptionCheck = 27; // need to get rid of this! + k_ESOMsg_CacheSubscriptionRefresh = 28; + k_ESOMsg_CacheSubscribedUpToDate = 29; +}; + +enum EGCBaseClientMsg +{ + k_EMsgGCPingRequest = 3001; // GC <=> client + k_EMsgGCPingResponse = 3002; // GC <=> client + + // WARNING: These messages were promoted into the gcsdk + // from base_gcmessages.proto (which is not part of the gcsdk put is + // "shared game code"). But we did not want to change the numbers + // for existing messages. Don't add new messages in the 4000 range! + k_EMsgGCClientWelcome = 4004; // GC => client + k_EMsgGCServerWelcome = 4005; // GC => server + k_EMsgGCClientHello = 4006; // client => GC + k_EMsgGCServerHello = 4007; // server => GC + + k_EMsgGCClientGoodbye = 4008; // GC => client + k_EMsgGCServerGoodbye = 4009; // GC => server + + // !FIXME! DOTAMERGE + //k_EMsgGCClientConnectionStatus = 4009; // GC => client + //k_EMsgGCServerConnectionStatus = 4010; // GC => server + +}; + +enum EGCToGCMsg +{ + k_EGCToGCMsgMasterAck = 150; // Sub GC => Master GC, confirm that we've started up successfully so the master can go when all sub GCs are ready + k_EGCToGCMsgMasterAckResponse = 151; // Master GC => Sub GC, the ack came through fine and everything is as expected + k_EGCToGCMsgRouted = 152; // Master GC => Sub GC, a routed message that originally came in from a client or the system and has been sent to another GC + k_EGCToGCMsgRoutedReply = 153; // Sub GC => Sub/Master GC, a reply message that needs to be sent back from the original GC that received the message + k_EMsgGCUpdateSubGCSessionInfo = 154; // Master GC => Other GCs + k_EMsgGCRequestSubGCSessionInfo = 155; // Sub GC => Master GC + k_EMsgGCRequestSubGCSessionInfoResponse = 156; // Master GC => Sub GC + k_EGCToGCMsgMasterStartupComplete = 157; // Master GC => Sub GC - Indicates that the master universe has completed startup, indicating that sub GC's are in a good spot to complete startup + k_EMsgGCToGCSOCacheSubscribe = 158; // Master GC => Sub GC - A steam ID has subscribed to another SO cache (or possibly their own) + k_EMsgGCToGCSOCacheUnsubscribe = 159; // Master GC => Sub GC - A steam ID has unsubscribed from another SO cache (or possibly their own) +} + + +// Do not remove this comment due to a bug on the Mac OS X protobuf compiler + diff --git a/gcsdk/gcwebapi.cpp b/gcsdk/gcwebapi.cpp new file mode 100644 index 0000000..af7783d --- /dev/null +++ b/gcsdk/gcwebapi.cpp @@ -0,0 +1,31 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: CWAPI code for GC access to the Web API server +// +//============================================================================= + +#include "stdafx.h" +#include "gcwebapi.h" +#include "enumutils.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +CUtlVector< GCWebAPIInterfaceMapCreationFunc_t > & CGCWebAPIInterfaceMapRegistrar::VecInstance() +{ + static CUtlVector< GCWebAPIInterfaceMapCreationFunc_t > sm_vecInterfaceFuncs; + return sm_vecInterfaceFuncs; +} + + +ENUMSTRINGS_START( EWebApiParamType ) +{ k_EWebApiParamTypeInvalid, "Invalid" }, +{ k_EWebApiParamTypeInt32, "int32" }, +{ k_EWebApiParamTypeUInt32, "uint32" }, +{ k_EWebApiParamTypeInt64, "int64" }, +{ k_EWebApiParamTypeUInt64, "uint64" }, +{ k_EWebApiParamTypeFloat, "float" }, +{ k_EWebApiParamTypeString, "string" }, +{ k_EWebApiParamTypeBool, "bool" }, +{ k_EWebApiParamTypeRawBinary, "rawbinary" } +ENUMSTRINGS_REVERSE( EWebApiParamType, k_EWebApiParamTypeInvalid ) diff --git a/gcsdk/gcwebapikey.cpp b/gcsdk/gcwebapikey.cpp new file mode 100644 index 0000000..7e9e722 --- /dev/null +++ b/gcsdk/gcwebapikey.cpp @@ -0,0 +1,50 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: implementation of CWebAPIKey +// +//============================================================================= + + +#include "stdafx.h" +#include "gcsdk/msgprotobuf.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + + +//----------------------------------------------------------------------------- +// Purpose: Clears key settings +//----------------------------------------------------------------------------- +void CWebAPIKey::Clear() +{ + m_unAccountID = 0; + m_unPublisherGroupID = 0; +} + + +//----------------------------------------------------------------------------- +// Purpose: Serializes the request into a message object (for proxying between +// back-end Steam servers). +//----------------------------------------------------------------------------- +void CWebAPIKey::SerializeIntoProtoBuf( CMsgWebAPIKey & apiKey ) const +{ + apiKey.set_status( m_eStatus ); + apiKey.set_account_id( m_unAccountID ); + apiKey.set_publisher_group_id( m_unPublisherGroupID ); + apiKey.set_key_id( m_unWebAPIKeyID ); + apiKey.set_domain( m_sDomain.Get() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Deserializes the response from a message object (for proxying between +// back-end Steam servers). +//----------------------------------------------------------------------------- +void CWebAPIKey::DeserializeFromProtoBuf( const CMsgWebAPIKey & apiKey ) +{ + m_eStatus = (EWebAPIKeyStatus)apiKey.status(); + m_unAccountID = apiKey.account_id(); + m_unPublisherGroupID = apiKey.publisher_group_id(); + m_unWebAPIKeyID = apiKey.key_id(); + m_sDomain = apiKey.domain().c_str(); +} diff --git a/gcsdk/gcwgjobmgr.cpp b/gcsdk/gcwgjobmgr.cpp new file mode 100644 index 0000000..c4b44ba --- /dev/null +++ b/gcsdk/gcwgjobmgr.cpp @@ -0,0 +1,210 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +#include "stdafx.h" +#include "enumutils.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + +ENUMSTRINGS_START( EGCWebApiPrivilege ) +{ k_EGCWebApiPriv_None, "None" }, +{ k_EGCWebApiPriv_Account, "Account" }, +{ k_EGCWebApiPriv_Approved, "Approved" }, +{ k_EGCWebApiPriv_Session, "Session" }, +{ k_EGCWebApiPriv_Support, "Support" }, +{ k_EGCWebApiPriv_Admin, "Admin" }, +{ k_EGCWebApiPriv_EditApp, "EditApp" }, +{ k_EGCWebApiPriv_MemberPublisher, "MemberPublisher" }, +{ k_EGCWebApiPriv_EditPublisher, "EditPublisher" }, +{ k_EGCWebApiPriv_AccountOptional, "AccountOptional" }, +ENUMSTRINGS_END( EGCWebApiPrivilege ); + + +//----------------------------------------------------------------------------- +// Purpose: Singleton accessor to registered WG jobs +//----------------------------------------------------------------------------- +CUtlDict< const JobCreationFunc_t* > &GMapGCWGJobCreationFuncs() +{ + static CUtlDict< const JobCreationFunc_t* > s_MapWGJobCreationFuncs; + return s_MapWGJobCreationFuncs; +} + + +//----------------------------------------------------------------------------- +// Purpose: Singleton accessor to registered WG jobs +//----------------------------------------------------------------------------- +CUtlDict< const WebApiFunc_t* > &GMapGCWGRequestInfo() +{ + static CUtlDict< const WebApiFunc_t* > s_MapWGRequestInfo; + return s_MapWGRequestInfo; +} + + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CGCWGJobMgr::CGCWGJobMgr( ) +{ + +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CGCWGJobMgr::~CGCWGJobMgr() +{ + +} + + +//----------------------------------------------------------------------------- +// Purpose: Hub has receieved a k_EMsgWGRequest message, find & dispatch WG Job +//----------------------------------------------------------------------------- +bool CGCWGJobMgr::BHandleMsg( IMsgNetPacket *pNetPacket ) +{ + CGCMsg<MsgGCWGRequest_t> msg( pNetPacket ); + AssertMsg( msg.GetEMsg() == k_EGCMsgWGRequest, "GCWGJobMgr asked to route a message, but it is not a MsgWGRequest" ); + + CUtlString strRequestName; + msg.BReadStr( &strRequestName ); + + int iJobType = GMapGCWGRequestInfo().Find( strRequestName.Get() ); + + if ( !GMapGCWGRequestInfo().IsValidIndex( iJobType ) ) + { + EmitError( SPEW_GC, "Unable to find GC WG handler %s", strRequestName.Get() ); + SendErrorMessage( msg, "Unknown method", k_EResultInvalidParam ); + return false; + } + + const WebApiFunc_t * pFunc = GMapGCWGRequestInfo()[iJobType]; + if( !BVerifyPrivileges( msg, pFunc ) ) + return false; + + CGCWGJob *job = (CGCWGJob *)(*((GMapGCWGJobCreationFuncs())[iJobType]))( GGCBase(), NULL ); + Assert( job ); + + job->SetWebApiFunc( pFunc ); + + job->StartJobFromNetworkMsg( pNetPacket, msg.Hdr().m_JobIDSource ); + return true; +} + + +bool CGCWGJobMgr::BVerifyPrivileges( const CGCMsg<MsgGCWGRequest_t> & msg, const WebApiFunc_t * pFunc ) +{ + if(msg.Body().m_unPrivilege != (uint32)pFunc->m_eRequiredPrivilege ) + { + SendErrorMessage( msg, + CFmtStr( "Privilege mismatch in %s gc call. Expected: %s, actual: %s", pFunc->m_pchRequestName, PchNameFromEGCWebApiPrivilege( pFunc->m_eRequiredPrivilege ), PchNameFromEGCWebApiPrivilege( (EGCWebApiPrivilege)msg.Body().m_unPrivilege ) ), + k_EResultAccessDenied ); + return false; + } + else + { + return true; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Sends an error message in response to a WG request +// Inputs: msg - The message we're responding to. This causes the error to +// be routed to the right place and eventually back to the right +// browser. +// pchErrorMsg - The message to display +// nResult - The error code to return +//----------------------------------------------------------------------------- +void CGCWGJobMgr::SendErrorMessage( const CGCMsg<MsgGCWGRequest_t> & msg, const char *pchErrorMsg, int32 nResult ) +{ + KeyValuesAD pkvErr( "error" ); + pkvErr->SetString( "error", pchErrorMsg ); + pkvErr->SetInt( "success", nResult ); + + SendResponse( msg, pkvErr, false ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Sets an error message in a response to a WG request +// Inputs: pkvResponse - The response KV block to set the error in +// pchErrorMsg - The message to display +// nResult - The error code to return +//----------------------------------------------------------------------------- +void CGCWGJobMgr::SetErrorMessage( KeyValues *pkvErr, const char *pchErrorMsg, int32 nResult ) +{ + pkvErr->SetString( "error", pchErrorMsg ); + pkvErr->SetInt( "success", nResult ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Sends a message in response to a WG request +// Inputs: msg - The message we're responding to. This causes the error to +// be routed to the right place and eventually back to the right +// browser. +// pkvResponse - The KeyValues containing the response data +//----------------------------------------------------------------------------- +void CGCWGJobMgr::SendResponse( const CGCMsg<MsgGCWGRequest_t> & msg, KeyValues *pkvResponse, bool bResult ) +{ + //prepare response msg + CGCMsg<MsgGCWGResponse_t> msgResponse( k_EGCMsgWGResponse, msg ); + CUtlBuffer bufResponse; + KVPacker packer; + if ( packer.WriteAsBinary( pkvResponse, bufResponse ) ) + { + msgResponse.AddVariableLenData( bufResponse.Base(), bufResponse.TellPut() ); + msgResponse.Body().m_cubKeyValues = bufResponse.TellPut(); + msgResponse.Body().m_bResult = bResult; + } + else + { + msgResponse.Body().m_cubKeyValues = 0; + AssertMsg( false, "Failed to serialize WG response" ); + } + msgResponse.Hdr().m_JobIDTarget = msg.Hdr().m_JobIDSource; + + GGCBase()->BSendSystemMessage( msgResponse ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a WG job handler to the global list +//----------------------------------------------------------------------------- +void CGCWGJobMgr::RegisterWGJob( const WebApiFunc_t *pWGJobType, const JobType_t *pJobCreationFunc ) +{ + const char *pchRequestName = pWGJobType->m_pchRequestName; + GMapGCWGJobCreationFuncs().Insert( pchRequestName, &(pJobCreationFunc->m_pJobFactory) ); + GMapGCWGRequestInfo().Insert( pchRequestName, pWGJobType ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the list of registered WG jobs +//----------------------------------------------------------------------------- +CUtlDict< const WebApiFunc_t* > &CGCWGJobMgr::GetWGRequestMap() +{ + return GMapGCWGRequestInfo(); +} + + +#ifdef DBGFLAG_VALIDATE +void CGCWGJobMgr::Validate( CValidator &validator, const char *pchName ) +{ +} + +void CGCWGJobMgr::ValidateStatics( CValidator &validator ) +{ + ValidateObj( GMapGCWGJobCreationFuncs() ); + ValidateObj( GMapGCWGRequestInfo() ); +} + +#endif // DBGFLAG_VALIDATE + + + +} // namespace GCSDK diff --git a/gcsdk/http.cpp b/gcsdk/http.cpp new file mode 100644 index 0000000..29f4e67 --- /dev/null +++ b/gcsdk/http.cpp @@ -0,0 +1,710 @@ +//====== Copyright � 1996-2010, Valve Corporation, All rights reserved. ======= +// +// Purpose: HTTP related enums and objects, stuff that both clients and server use should go here +// +//============================================================================= + +#include "stdafx.h" +#include <time.h> + +#include "msgprotobuf.h" +#include "rtime.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +using namespace GCSDK; + +//----------------------------------------------------------------------------- +// Purpose: Check if the given status code is one that by spec allows a body, or is +// one that "MUST NOT" include an entity body +//----------------------------------------------------------------------------- +bool CHTTPUtil::BStatusCodeAllowsBody( EHTTPStatusCode eHTTPStatus ) +{ + switch( eHTTPStatus ) + { + case k_EHTTPStatusCode100Continue: + case k_EHTTPStatusCode101SwitchingProtocols: + case k_EHTTPStatusCode204NoContent: + case k_EHTTPStatusCode205ResetContent: + case k_EHTTPStatusCode304NotModified: + return false; + + default: + break; + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Wrapper to ease use of Steam Web APIs +//----------------------------------------------------------------------------- +CSteamAPIRequest::CSteamAPIRequest( EHTTPMethod eMethod, const char *pchInterface, const char *pchMethod, int nVersion ) : + CHTTPRequest( eMethod, "api.steampowered.com", CFmtStr( "/%s/%s/v%04d/", pchInterface, pchMethod, nVersion ) ) +{ + if ( k_EHTTPMethodPOST == eMethod ) + { + SetPOSTParamString( "format", "vdf" ); + SetPOSTParamString( "key", GGCBase()->GetSteamAPIKey() ); + } + else + { + SetGETParamString( "format", "vdf" ); + SetGETParamString( "key", GGCBase()->GetSteamAPIKey() ); + } + + EUniverse universe = GGCHost()->GetUniverse(); + if ( universe == k_EUniverseBeta ) + { + SetHostname( "api-beta.steampowered.com" ); + } + else if ( universe == k_EUniverseDev ) + { + // Set this to your dev universe API endpoint if not local + SetHostname( "localhost:8282" ); + } +} + +CHTTPRequest::CHTTPRequest( CMsgHttpRequest* pProto ) +{ + Init( pProto ); +} + +//----------------------------------------------------------------------------- +// Purpose: Construct a request object, it's invalid until data is setup +//----------------------------------------------------------------------------- +CHTTPRequest::CHTTPRequest() +{ + Init( NULL ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Common init code +//----------------------------------------------------------------------------- +void CHTTPRequest::Init( CMsgHttpRequest* pProto ) +{ + //see if we are wrapping a proto that already exists + if( pProto ) + { + m_pProto = pProto; + m_bOwnProto = false; + } + else + { + m_pProto = new CMsgHttpRequest; + m_bOwnProto = true; + + SetEHTTPMethod( k_EHTTPMethodInvalid ); + // Default URL + SetURL( "/" ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CHTTPRequest::CHTTPRequest( EHTTPMethod eMethod, const char *pchHost, const char *pchRelativeURL ) +{ + Init( NULL ); + + SetEHTTPMethod( eMethod ); + SetHostname( pchHost ); + SetURL( pchRelativeURL ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CHTTPRequest::CHTTPRequest( EHTTPMethod eMethod, const char *pchAbsoluteURL ) +{ + Init( NULL ); + + SetEHTTPMethod( eMethod ); + + // We need to break the URL down into host + relativeURL + const char *pchHost = Q_strstr( pchAbsoluteURL, "://" ); + if ( pchHost ) + { + // Skip past protocol + pchHost += 3; + + // Find the next /, that is the beginning of the actual URL + const char *pchURL = Q_strstr( pchHost, "/"); + if ( !pchURL ) + { + // No URL specified after host? just default to / then. + SetURL( "/" ); + } + else + { + // Ok, now we must remove the query string + const char *pchQueryString = Q_strstr( pchURL, "?" ); + if ( !pchQueryString ) + { + // No query string, full thing is the URL + SetURL( pchURL ); + } + else + { + // URL is everything before query string + SetURLDirect( pchURL, pchQueryString - pchURL ); + + Assert( *pchQueryString == '?' ); + pchQueryString++; + if( *pchQueryString ) + { + m_pProto->mutable_get_params()->Clear(); + CUtlVector<CMsgHttpRequest_QueryParam> vecParams; + RetrieveURLEncodedData( pchQueryString, Q_strlen( pchQueryString ), vecParams ); + FOR_EACH_VEC( vecParams, i ) + { + *m_pProto->add_get_params() = vecParams[i]; + } + } + } + } + + // Is there a userinfo separator in the hostname portion? We don't support + // username/password authentication in the URL, but we must still skip it. + const char *pchUserinfoSep = strchr( pchHost, '@' ); + if ( pchUserinfoSep && ( !pchURL || pchUserinfoSep < pchURL ) ) + { + pchHost = pchUserinfoSep + 1; + } + + // If we found a URL only set upto that, otherwise set everything as host + if ( pchURL ) + SetHostnameDirect( pchHost, pchURL-pchHost ); + else + SetHostname( pchHost ); + } + else + { + AssertMsg( false, "Bad absolute URL to CHTTPRequest constructor, must start with protocol://" ); + EmitError( SPEW_GC, "Bad absolute URL to CHTTPRequest constructor, must start with protocol://" ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CHTTPRequest::~CHTTPRequest() +{ + if( m_bOwnProto ) + delete m_pProto; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the value of a GET parameter, this is case-insensitive by default. +//----------------------------------------------------------------------------- +const CMsgHttpRequest_QueryParam *CHTTPRequest::GetGETParam( const char *pchGetParamName, bool bMatchCase ) const +{ + const uint32 nNumParams = m_pProto->get_params_size(); + for( uint32 nParam = 0; nParam < nNumParams; nParam++ ) + { + const CMsgHttpRequest_QueryParam& param = m_pProto->get_params( nParam ); + if ( bMatchCase ) + { + if ( Q_strcmp( param.name().c_str(), pchGetParamName ) == 0 ) + return ¶m; + } + else + { + if ( Q_stricmp( param.name().c_str(), pchGetParamName ) == 0 ) + return ¶m; + } + } + + return NULL; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the value of a GET parameter, this is case-insensitive by default. +//----------------------------------------------------------------------------- +const char *CHTTPRequest::GetGETParamString( const char *pchGetParamName, const char *pchDefault, bool bMatchCase ) const +{ + const CMsgHttpRequest_QueryParam *pParam = GetGETParam( pchGetParamName, bMatchCase ); + if ( !pParam ) + return pchDefault; + + return pParam->value().c_str(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the value of a GET parameter, this is case-insensitive by default. +//----------------------------------------------------------------------------- +bool CHTTPRequest::GetGETParamBool( const char *pchGetParamName, bool bDefault, bool bMatchCase ) const +{ + const CMsgHttpRequest_QueryParam *pParam = GetGETParam( pchGetParamName, bMatchCase ); + if ( !pParam ) + return bDefault; + + return ( Q_atoi( pParam->value().c_str() ) != 0); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the value of a GET parameter, this is case-insensitive by default. +//----------------------------------------------------------------------------- +int32 CHTTPRequest::GetGETParamInt32( const char *pchGetParamName, int32 nDefault, bool bMatchCase ) const +{ + const CMsgHttpRequest_QueryParam *pParam = GetGETParam( pchGetParamName, bMatchCase ); + if ( !pParam ) + return nDefault; + + return Q_atoi( pParam->value().c_str() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the value of a GET parameter, this is case-insensitive by default. +//----------------------------------------------------------------------------- +uint32 CHTTPRequest::GetGETParamUInt32( const char *pchGetParamName, uint32 unDefault, bool bMatchCase ) const +{ + const CMsgHttpRequest_QueryParam *pParam = GetGETParam( pchGetParamName, bMatchCase ); + if ( !pParam ) + return unDefault; + + return (uint32)V_atoui64( pParam->value().c_str() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the value of a GET parameter, this is case-insensitive by default. +//----------------------------------------------------------------------------- +int64 CHTTPRequest::GetGETParamInt64( const char *pchGetParamName, int64 nDefault, bool bMatchCase ) const +{ + const CMsgHttpRequest_QueryParam *pParam = GetGETParam( pchGetParamName, bMatchCase ); + if ( !pParam ) + return nDefault; + + return V_atoi64( pParam->value().c_str() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the value of a GET parameter, this is case-insensitive by default. +//----------------------------------------------------------------------------- +uint64 CHTTPRequest::GetGETParamUInt64( const char *pchGetParamName, uint64 unDefault, bool bMatchCase ) const +{ + const CMsgHttpRequest_QueryParam *pParam = GetGETParam( pchGetParamName, bMatchCase ); + if ( !pParam ) + return unDefault; + + return V_atoui64( pParam->value().c_str() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the value of a GET parameter, this is case-insensitive by default. +//----------------------------------------------------------------------------- +float CHTTPRequest::GetGETParamFloat( const char *pchGetParamName, float fDefault, bool bMatchCase ) const +{ + const CMsgHttpRequest_QueryParam *pParam = GetGETParam( pchGetParamName, bMatchCase ); + if ( !pParam ) + return fDefault; + + return Q_atof( pParam->value().c_str() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the value of a POST parameter, this is case-insensitive by default. +//----------------------------------------------------------------------------- +const CMsgHttpRequest_QueryParam *CHTTPRequest::GetPOSTParam( const char *pchPostParamName, bool bMatchCase ) const +{ + const uint32 nNumParams = m_pProto->post_params_size(); + for( uint32 nParam = 0; nParam < nNumParams; nParam++ ) + { + const CMsgHttpRequest_QueryParam& param = m_pProto->post_params( nParam ); + if ( bMatchCase ) + { + if ( Q_strcmp( param.name().c_str(), pchPostParamName ) == 0 ) + return ¶m; + } + else + { + if ( Q_stricmp( param.name().c_str(), pchPostParamName ) == 0 ) + return ¶m; + } + } + + return NULL; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the value of a POST parameter, this is case-insensitive by default. +//----------------------------------------------------------------------------- +const char *CHTTPRequest::GetPOSTParamString( const char *pchGetParamName, const char *pchDefault, bool bMatchCase ) const +{ + const CMsgHttpRequest_QueryParam *pParam = GetPOSTParam( pchGetParamName, bMatchCase ); + if ( !pParam ) + return pchDefault; + + return pParam->value().c_str(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the value of a POST parameter, this is case-insensitive by default. +//----------------------------------------------------------------------------- +bool CHTTPRequest::GetPOSTParamBool( const char *pchGetParamName, bool bDefault, bool bMatchCase ) const +{ + const CMsgHttpRequest_QueryParam *pParam = GetPOSTParam( pchGetParamName, bMatchCase ); + if ( !pParam ) + return bDefault; + + return ( Q_atoi( pParam->value().c_str() ) != 0); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the value of a POST parameter, this is case-insensitive by default. +//----------------------------------------------------------------------------- +int32 CHTTPRequest::GetPOSTParamInt32( const char *pchGetParamName, int32 nDefault, bool bMatchCase ) const +{ + const CMsgHttpRequest_QueryParam *pParam = GetPOSTParam( pchGetParamName, bMatchCase ); + if ( !pParam ) + return nDefault; + + return Q_atoi( pParam->value().c_str() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the value of a POST parameter, this is case-insensitive by default. +//----------------------------------------------------------------------------- +uint32 CHTTPRequest::GetPOSTParamUInt32( const char *pchGetParamName, uint32 unDefault, bool bMatchCase ) const +{ + const CMsgHttpRequest_QueryParam *pParam = GetPOSTParam( pchGetParamName, bMatchCase ); + if ( !pParam ) + return unDefault; + + return (uint32)V_atoui64( pParam->value().c_str() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the value of a POST parameter, this is case-insensitive by default. +//----------------------------------------------------------------------------- +int64 CHTTPRequest::GetPOSTParamInt64( const char *pchGetParamName, int64 nDefault, bool bMatchCase ) const +{ + const CMsgHttpRequest_QueryParam *pParam = GetPOSTParam( pchGetParamName, bMatchCase ); + if ( !pParam ) + return nDefault; + + return V_atoi64( pParam->value().c_str() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the value of a POST parameter, this is case-insensitive by default. +//----------------------------------------------------------------------------- +uint64 CHTTPRequest::GetPOSTParamUInt64( const char *pchGetParamName, uint64 unDefault, bool bMatchCase ) const +{ + const CMsgHttpRequest_QueryParam *pParam = GetPOSTParam( pchGetParamName, bMatchCase ); + if ( !pParam ) + return unDefault; + + return V_atoui64( pParam->value().c_str() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the value of a POST parameter, this is case-insensitive by default. +//----------------------------------------------------------------------------- +float CHTTPRequest::GetPOSTParamFloat( const char *pchGetParamName, float fDefault, bool bMatchCase ) const +{ + const CMsgHttpRequest_QueryParam *pParam = GetPOSTParam( pchGetParamName, bMatchCase ); + if ( !pParam ) + return fDefault; + + return Q_atof( pParam->value().c_str() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Add a GET param to the request +//----------------------------------------------------------------------------- +void CHTTPRequest::SetGETParamRaw( const char *pchGetParamName, uint8 *pData, uint32 cubDataLen ) +{ + // See if it already exists, and overwrite then (case sensitive!) + CMsgHttpRequest_QueryParam *pParam = const_cast< CMsgHttpRequest_QueryParam* >( GetGETParam( pchGetParamName, true ) ); + if ( !pParam ) + { + // Add a new one then + pParam = m_pProto->add_get_params(); + pParam->set_name( pchGetParamName ); + } + + pParam->set_value( pData, cubDataLen ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Add a POST param to the request given a string for the name and value +//----------------------------------------------------------------------------- +void CHTTPRequest::SetPOSTParamRaw( const char *pchPostParamName, uint8 *pData, uint32 cubDataLen ) +{ + // See if it already exists, and overwrite then (case sensitive!) + CMsgHttpRequest_QueryParam *pParam = const_cast< CMsgHttpRequest_QueryParam* >( GetPOSTParam( pchPostParamName, true ) ); + if ( !pParam ) + { + // Add a new one then + pParam = m_pProto->add_post_params(); + pParam->set_name( pchPostParamName ); + } + + pParam->set_value( pData, cubDataLen ); +} + + +//----------------------------------------------------------------------------- +// Fetch a request headers value by header name +//----------------------------------------------------------------------------- +const char * CHTTPRequest::GetRequestHeaderValue( const char *pchRequestHeaderName, const char *pchDefault ) const +{ + const uint32 nNumHeaders = m_pProto->headers_size(); + for( uint32 nHeader = 0; nHeader < nNumHeaders; nHeader++ ) + { + const CMsgHttpRequest_RequestHeader& header = m_pProto->headers( nHeader ); + if( Q_stricmp( header.name().c_str(), pchRequestHeaderName ) == 0 ) + { + return header.value().c_str(); + } + } + + return pchDefault; +} + +//----------------------------------------------------------------------------- +// Set a header field for the request +//----------------------------------------------------------------------------- +void CHTTPRequest::SetRequestHeaderValue( const char *pchHeaderName, const char *pchHeaderString ) +{ + const uint32 nNumHeaders = m_pProto->headers_size(); + for( uint32 nHeader = 0; nHeader < nNumHeaders; nHeader++ ) + { + CMsgHttpRequest_RequestHeader* pHeader = m_pProto->mutable_headers( nHeader ); + if( Q_stricmp( pHeader->name().c_str(), pchHeaderName ) == 0 ) + { + pHeader->set_value( pchHeaderString ); + return; + } + } + + CMsgHttpRequest_RequestHeader* pHeader = m_pProto->add_headers(); + pHeader->set_name( pchHeaderName ); + pHeader->set_value( pchHeaderString ); +} + +//----------------------------------------------------------------------------- +// Purpose: Fetch a request header by header name and convert it to a time value +// This is just a helpful wrapper of GetRequestHeaderValue that deals with +// parsing the time value +//----------------------------------------------------------------------------- +RTime32 CHTTPRequest::GetRequestHeaderTimeValue( const char *pchRequestHeaderName, RTime32 rtDefault ) const +{ + const char *pchValue = GetRequestHeaderValue( pchRequestHeaderName ); + if( !pchValue ) + return rtDefault; + + return CRTime::RTime32FromHTTPDateString( pchValue ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Set the body data for the request +//----------------------------------------------------------------------------- +void CHTTPRequest::SetBodyData( const void *pubData, uint32 cubData ) +{ + m_pProto->set_body( pubData, cubData ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get data out of a query string +//----------------------------------------------------------------------------- +void CHTTPRequest::RetrieveURLEncodedData( const char *pchQueryString, int nQueryStrLen, CUtlVector<CMsgHttpRequest_QueryParam> &vecParams ) +{ + CUtlBuffer bufParamName( 0, 64 ); + CUtlBuffer bufParamValue( 0, 128 ); + + // We shouldn't get passed the ? from a query string, but if we do just skip it and parse ok anyway + if ( nQueryStrLen && pchQueryString[0] == '?' ) + { + ++pchQueryString; + --nQueryStrLen; + } + + bool bInParamValue = false; + int iTokenStart = 0; + for( int i=0; i < nQueryStrLen; ++i ) + { + if ( pchQueryString[i] == '=' && !bInParamValue ) + { + // = switches to value from name, starts a new value token + bufParamName.Put( &pchQueryString[iTokenStart], i - iTokenStart ); + bInParamValue = true; + iTokenStart = i + 1; + } + else if ( pchQueryString[i] == '&' ) + { + // & terminates a value and starts a new name token + if ( bInParamValue ) + { + bufParamValue.Put( &pchQueryString[iTokenStart], i - iTokenStart ); + + int iIndex = vecParams.AddToTail(); + CMsgHttpRequest_QueryParam *pParam = &vecParams[iIndex]; + + uint32 unNameLen = Q_URLDecode( (char*)bufParamName.Base(), bufParamName.TellPut(), (const char*)bufParamName.Base(), bufParamName.TellPut() ); + pParam->set_name( (const char*)bufParamName.Base(), unNameLen ); + + uint32 unDataLen = (uint32)Q_URLDecode( (char*)bufParamValue.Base(), bufParamValue.TellPut(), (const char*)bufParamValue.Base(), bufParamValue.TellPut() ); + pParam->set_value( (const uint8*)bufParamValue.Base(), unDataLen ); + } + bufParamName.Clear(); + bufParamValue.Clear(); + bInParamValue = false; + iTokenStart = i+1; + } + } + + // Use any left over value from the end of the query string + if ( bInParamValue ) + { + bufParamValue.Put( &pchQueryString[iTokenStart], nQueryStrLen - iTokenStart ); + + int iIndex = vecParams.AddToTail(); + CMsgHttpRequest_QueryParam *pParam = &vecParams[iIndex]; + + uint32 unNameLen = Q_URLDecode( (char*)bufParamName.Base(), bufParamName.TellPut(), (const char*)bufParamName.Base(), bufParamName.TellPut() ); + pParam->set_name( (const char*)bufParamName.Base(), unNameLen ); + + uint32 unDataLen = (uint32)Q_URLDecode( (char*)bufParamValue.Base(), bufParamValue.TellPut(), (const char*)bufParamValue.Base(), bufParamValue.TellPut() ); + pParam->set_value( (const uint8*)bufParamValue.Base(), unDataLen ); + } +} + + +//---------------------------------------------------------------------------- +// Purpose: Gets a singleton buffer pool for HTTP responses +//---------------------------------------------------------------------------- +static GCConVar http_response_max_pool_size_mb( "http_response_max_pool_size_mb", "10", "Maximum size in bytes of the HTTP Response buffer pool" ); +static GCConVar http_response_init_buffer_size( "http_response_init_buffer_size", "65536", "Initial buffer size for buffers in the HTTP Response buffer pool" ); +/*static*/ CBufferPool &CHTTPResponse::GetBufferPool() +{ + static CBufferPool s_bufferPool( "HTTP Response", http_response_max_pool_size_mb, http_response_init_buffer_size, CUtlBuffer::TEXT_BUFFER | CUtlBuffer::CONTAINS_CRLF ); + return s_bufferPool; +} + + +//----------------------------------------------------------------------------- +// Purpose: Construct a response object, it defaults to being a 500 internal server error response +//----------------------------------------------------------------------------- +CHTTPResponse::CHTTPResponse() : +m_pbufBody( GetBufferPool().GetBuffer() ), +m_eStatusCode( k_EHTTPStatusCode500InternalServerError ) +{ + m_pkvResponseHeaders = new KeyValues( "ResponseHeaders" ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CHTTPResponse::~CHTTPResponse() +{ + GetBufferPool().ReturnBuffer( m_pbufBody ); + m_pbufBody = NULL; + + if ( m_pkvResponseHeaders ) + m_pkvResponseHeaders->deleteThis(); + + m_pkvResponseHeaders = NULL; +} + +//----------------------------------------------------------------------------- +// Purpose: Sets the "expiration" response header to a given number of seconds +// ahead or behind the current time. You can set the Expires header directly as well, +// this is just a helper so you don't have to deal with time formatting. +//----------------------------------------------------------------------------- +void CHTTPResponse::SetExpirationHeaderDeltaFromNow( int32 nSecondsFromNow ) +{ + time_t rawtime; + time( &rawtime ); + rawtime += nSecondsFromNow; + SetHeaderTimeValue( "expires", rawtime ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Formats a time value and sets it as a header. This is a helper so +// you don't have to deal with time formatting +//----------------------------------------------------------------------------- +void CHTTPResponse::SetHeaderTimeValue( const char *pchHeaderName, RTime32 rtTimestamp ) +{ + char rgchDate[128]; + struct tm tmStruct; + time_t rawtime = rtTimestamp; + Plat_gmtime( &rawtime, &tmStruct ); + DbgVerify( strftime( rgchDate, 128, "%a, %d %b %Y %H:%M:%S GMT", &tmStruct ) ); + SetResponseHeaderValue( pchHeaderName, rgchDate ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Serializes the response into a message object (for proxying between +// back-end Steam servers). +//----------------------------------------------------------------------------- +void CHTTPResponse::SerializeIntoProtoBuf( CMsgHttpResponse & response ) const +{ + MEM_ALLOC_CREDIT(); + response.set_status_code( m_eStatusCode ); + + FOR_EACH_VALUE( m_pkvResponseHeaders, pkvRequestHeader ) + { + const char *pchName = pkvRequestHeader->GetName(); + const char *pchValue = pkvRequestHeader->GetString(); + + if ( pchName && pchValue ) + { + CMsgHttpResponse_ResponseHeader *pHeader = response.add_headers(); + pHeader->set_name( pchName ); + pHeader->set_value( pchValue ); + } + } + + if( m_pbufBody->TellPut() > 0 ) + response.set_body( m_pbufBody->Base(), m_pbufBody->TellPut() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Deserializes the response from a message object (for proxying between +// back-end Steam servers). +//----------------------------------------------------------------------------- +void CHTTPResponse::DeserializeFromProtoBuf( const CMsgHttpResponse & response ) +{ + m_eStatusCode = (EHTTPStatusCode)response.status_code(); + + for( int i=0; i<response.headers_size(); i++ ) + { + m_pkvResponseHeaders->SetString( response.headers(i).name().c_str(), response.headers(i).value().c_str() ); + } + + if( response.has_body() ) + m_pbufBody->Put( response.body().data(), response.body().size() ); +} + diff --git a/gcsdk/job.cpp b/gcsdk/job.cpp new file mode 100644 index 0000000..0eef357 --- /dev/null +++ b/gcsdk/job.cpp @@ -0,0 +1,1451 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//============================================================================= + + +#include "stdafx.h" + +#ifdef WIN32 +#include "typeinfo.h" +#else +#include <typeinfo> +#endif + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ +//----------------------------------------------------------------------------- +// Purpose: EJobPauseReason descriptions +//----------------------------------------------------------------------------- +static const char * const k_prgchJobPauseReason[] = +{ + "active", + "not started", + "netmsg", + "sleep for time", + "waiting for lock", + "yielding", + "SQL", + "work item", +}; + +COMPILE_TIME_ASSERT( ARRAYSIZE( k_prgchJobPauseReason ) == k_EJobPauseReasonCount ); + +CJob *g_pJobCur = NULL; + + +//----------------------------------------------------------------------------- +// Purpose: Delete a Job +// Input: pJob - The Job to delete +//----------------------------------------------------------------------------- +void CJob::DeleteJob( CJob *pJob ) +{ + // we can't delete the if we still have a pending work item + pJob->WaitForThreadFuncWorkItemBlocking(); + delete pJob; +} + +//----------------------------------------------------------------------------- +// Purpose: Constructor +// Input: pServerParent - The server we belong to +//----------------------------------------------------------------------------- +CJob::CJob( CJobMgr &jobMgr, const char *pchJobName ) : m_JobMgr( jobMgr ), m_pchJobName( pchJobName ) +{ + m_ePauseReason = k_EJobPauseReasonNotStarted; + + m_JobID = jobMgr.GetNewJobID(); + m_pJobType = NULL; + m_bWorkItemCanceled = false; + m_hCoroutine = Coroutine_Create( &BRunProxy, this ); + m_pvStartParam = NULL; + m_bRunFromMsg = false; + m_pJobPrev = NULL; + m_pWaitingOnLock = NULL; + m_pJobToNotifyOnLockRelease = NULL; + m_pWaitingOnWorkItem = NULL; + m_STimeStarted.SetToJobTime(); + m_STimeSwitched.SetToJobTime(); + m_STimeNextHeartbeat.SetFromJobTime( k_cMicroSecJobHeartbeat ); + m_bIsLongRunning = false; + m_cLocksAttempted = 0; + m_cLocksWaitedFor = 0; + m_flags.m_uFlags = 0; + m_cyclecountTotal = 0; + m_unWaitMsgType = 0; + + GetJobMgr().InsertJob( *this ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CJob::~CJob() +{ + // don't want SendMsgToConnection to call back into us, we *know* + // we are replying to these other jobs now + g_pJobCur = NULL; + + // reset the job pointer + g_pJobCur = m_pJobPrev; + + // remove from the job tracking list + GetJobMgr().RemoveJob( *this ); + + // Forcefully release any locks + ReleaseLocks(); + + // free any network messages we've allocated + FOR_EACH_VEC( m_vecNetPackets, i ) + { + m_vecNetPackets[i]->Release(); + } + m_vecNetPackets.RemoveAll(); + + AssertMsg2( 0 == GetDoNotYieldDepth(), "Job ending with %d open Do Not Yields. Are we missing a END_DO_NOT_YIELD()? Innermost delared at %s", + GetDoNotYieldDepth(), m_stackDoNotYieldGuards[m_stackDoNotYieldGuards.Head()] ); +} + + +//----------------------------------------------------------------------------- +// Purpose: if necessary wait for the pending work item to finish +//----------------------------------------------------------------------------- +void CJob::WaitForThreadFuncWorkItemBlocking() +{ + if ( m_pWaitingOnWorkItem ) + { + switch ( GetPauseReason() ) + { + case k_EJobPauseReasonWorkItem: + // force the workitem to be canceled in case it's still in the in-queue + m_pWaitingOnWorkItem->ForceTimeOut(); + + // we can't shutdown the job while it's work item is currently running + // alot of work items refernce back into the job object + while ( m_pWaitingOnWorkItem->BIsRunning() ) + ThreadSleep( 25 ); + + m_pWaitingOnWorkItem = NULL; + break; + +#if 0 // not used in gcsdk + case k_EJobPauseReasonGeneric: + AssertMsg1( ( !m_pWaitingForGeneric || ( m_pWaitingForGeneric == ( void * ) 1 ) ), "CJob::WaitForThreadFuncWorkItemBlocking job %s will leak generic heap object", GetName() ); + // Let another assert fire later, don't null-out: m_pWaitingForGeneric = NULL; + break; +#endif + + default: + AssertMsg1( false, "CJob::WaitForThreadFuncWorkItemBlocking job %s has unexpected work item state", GetName() ); + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: returns the name of the job +//----------------------------------------------------------------------------- +const char *CJob::GetName() const +{ + if ( m_pchJobName ) + return m_pchJobName; + else if ( m_pJobType ) + return m_pJobType->m_pchName; + else + return ""; +} + + +//----------------------------------------------------------------------------- +// Purpose: string description of why we're paused +//----------------------------------------------------------------------------- +const char *CJob::GetPauseReasonDescription() const +{ + static char srgchPauseReason[k_cSmallBuff]; + if ( GetPauseReason() < Q_ARRAYSIZE( k_prgchJobPauseReason ) ) + { + switch( GetPauseReason() ) + { + case k_EJobPauseReasonWaitingForLock: + { + Q_snprintf( srgchPauseReason, k_cSmallBuff, "WOL: 0x%x (%s)", (unsigned int)m_pWaitingOnLock, m_pWaitingOnLock ? m_pWaitingOnLock->GetName() : "null" ); + return srgchPauseReason; + } + + case k_EJobPauseReasonNetworkMsg: + { + const char *pchMsgType; + if( g_theMessageList.GetMessage( m_unWaitMsgType, &pchMsgType, 0xFF ) ) + { + Q_snprintf( srgchPauseReason, k_cSmallBuff, "NetMsg: %s", pchMsgType ); + return srgchPauseReason; + } + else + { + Q_snprintf( srgchPauseReason, k_cSmallBuff, "NetMsg: Unknown %d", m_unWaitMsgType ); + return srgchPauseReason; + } + } + + default: + return k_prgchJobPauseReason[ GetPauseReason() ]; + } + } + + return "undefined"; +} + + +//----------------------------------------------------------------------------- +// Purpose: accessor to get access to the JobMgr from the server we belong to +//----------------------------------------------------------------------------- +CJobMgr &CJob::GetJobMgr() +{ + return m_JobMgr; +} + + +//----------------------------------------------------------------------------- +// Purpose: Starts the job, based on the current network msg +// Input : hConnection - Connection that the message was received form +// pubPkt - The raw message packet +// cubPkt - The size of the message packet +//----------------------------------------------------------------------------- +void CJob::StartJobFromNetworkMsg( IMsgNetPacket *pNetPacket, const JobID_t &gidJobIDSrc ) +{ + // hang on to the packet with the message that started this job + AddPacketToList( pNetPacket, gidJobIDSrc ); + SetFromFromMsg( true ); + // start running this job + InitCoroutine(); + //and start executing the job + Continue(); +} + +//----------------------------------------------------------------------------- +// Purpose: Starts the job +//----------------------------------------------------------------------------- +void CJob::StartJob( void * pvStartParam ) +{ + // job must not be started + AssertMsg1( m_ePauseReason == k_EJobPauseReasonNotStarted, "CJob::StartJob() called twice on job %s\n", GetName() ); + // save the start params for this job + SetStartParam( pvStartParam ); + m_JobMgr.CheckThreadID(); + + // start running this job + InitCoroutine(); + Continue(); +} + +//----------------------------------------------------------------------------- +// Purpose: Starts the job in a suspended state +//----------------------------------------------------------------------------- + +void CJob::StartJobDelayed( void * pvStartParam ) +{ + // job must not be started + AssertMsg1( m_ePauseReason == k_EJobPauseReasonNotStarted, "CJob::StartJob() called twice on job %s\n", GetName() ); + // save the start params for this job + SetStartParam( pvStartParam ); + m_JobMgr.CheckThreadID(); + + //init the job, but don't start it + InitCoroutine(); + + //set our job as suspended (a yield) + m_ePauseReason = k_EJobPauseReasonYield; + m_JobMgr.AddDelayedJobToYieldList( *this ); +} + +//----------------------------------------------------------------------------- +// Purpose: setup the debug memory and job name before running a job the first time +//----------------------------------------------------------------------------- +void CJob::InitCoroutine() +{ + // make sure we have an appropriate chunk of memory to store + // our debug alloc info + if( MemAlloc_GetDebugInfoSize() ) + { + m_memAllocStack.EnsureCapacity( MemAlloc_GetDebugInfoSize() ); + + // Set the job name as the root + MemAlloc_InitDebugInfo( m_memAllocStack.Base(), GetName(), 0 ); + } + + // Set the job name + if ( !m_pJobType && !m_pchJobName ) + { +#ifdef _WIN32 + m_pchJobName = typeid( *this ).raw_name(); + if ( m_pchJobName[0] == '.' && m_pchJobName[1] == '?' && m_pchJobName[2] == 'A') + m_pchJobName += 4; + if ( m_pchJobName[0] == '?' && m_pchJobName[1] == '$' ) + m_pchJobName += 2; +#else + m_pchJobName = typeid( *this ).name(); +#endif + } +} + + +//----------------------------------------------------------------------------- +// Purpose: proxy function for starting the job in the coroutine +//----------------------------------------------------------------------------- +void CJob::BRunProxy( void *pvThis ) +{ + CJob *pJob = (CJob *)pvThis; + + // run the job + bool bJobReturn = false; + if ( pJob->m_bRunFromMsg ) + { + Assert( pJob->m_vecNetPackets.Count() > 0 ); + bJobReturn = pJob->BYieldingRunJobFromMsg( pJob->m_vecNetPackets.Head() ); + } + else + { + bJobReturn = pJob->BYieldingRunJob( pJob->m_pvStartParam ); + } + + pJob->m_flags.m_bits.m_bJobFailed = ( true != bJobReturn ); + + // kill it + DeleteJob( pJob ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds this packet to the linked list of packets for this job +//----------------------------------------------------------------------------- +void CJob::AddPacketToList( IMsgNetPacket *pNetPacket, const GID_t gidJobIDSrc ) +{ + Assert( pNetPacket ); + pNetPacket->AddRef(); + + m_vecNetPackets.AddToTail( pNetPacket ); +} + + +//----------------------------------------------------------------------------- +// Purpose: marks a net packet as being finished with, releases the packet and frees the memory +//----------------------------------------------------------------------------- +void CJob::ReleaseNetPacket( IMsgNetPacket *pNetPacket ) +{ + int iVec = m_vecNetPackets.Find( pNetPacket ); + if ( iVec != m_vecNetPackets.InvalidIndex() ) + { + pNetPacket->Release(); + m_vecNetPackets.Remove( iVec ); + } + else + { + AssertMsg( false, "Job failed trying to release a IMsgNetPacket it doesn't own" ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: continues the current job +//----------------------------------------------------------------------------- +void CJob::Continue() +{ + AssertNotRunningThisJob(); + + m_pJobPrev = g_pJobCur; + g_pJobCur = this; + + // Record frame we're starting in and start a timer to track how long we work for within the frame + // Also, add in how much time has passed since we were last paused to track heartbeat requirements + m_FastTimerDelta.Start(); + + m_STimeSwitched.SetToJobTime(); + + // Check if we need to heartbeat + if ( BJobNeedsToHeartbeat() ) + { + Heartbeat(); + } + + m_JobMgr.GetJobStats().m_cTimeslices++; + + m_ePauseReason = k_EJobPauseReasonNone; +#if defined(_WIN32) && defined(COROUTINE_TRACE) + const char *pchRawName = typeid( *this ).raw_name(); + if ( pchRawName[0] == '.' && pchRawName[1] == '?' && pchRawName[2] == 'A') + pchRawName += 4; + if ( pchRawName[0] == '?' && pchRawName[1] == '$' ) + pchRawName += 2; +#else + const char *pchRawName = ""; +#endif + + // Save debug credit "call stack" + void *pvSaveDebugInfo = GetJobMgr().GetMainMemoryDebugInfo(); + MemAlloc_SaveDebugInfo( pvSaveDebugInfo ); + MemAlloc_RestoreDebugInfo( m_memAllocStack.Base() ); + + // continue the coroutine, with the profiling if necessary + bool bJobStillActive; +#if defined( VPROF_ENABLED ) + if ( g_VProfCurrentProfile.IsEnabled() ) + { + VPROF_BUDGET( GetName(), VPROF_BUDGETGROUP_JOBS_COROUTINES ); + bJobStillActive = Coroutine_Continue( m_hCoroutine, pchRawName ); + } + else +#endif + { + bJobStillActive = Coroutine_Continue( m_hCoroutine, pchRawName ); + } + + // WARNING: MEMBER VARIABLES ARE NOW UNSAFE TO ACCESS - this CJob may be deleted + + // Restore debug credit call stack + if( bJobStillActive ) + { + // only save off debug info for jobs that are still running + MemAlloc_SaveDebugInfo( m_memAllocStack.Base() ); + } + MemAlloc_RestoreDebugInfo( pvSaveDebugInfo ); + +} + +void CJob::Debug() +{ + AssertNotRunningThisJob(); + + // This function will 'load' this coroutine then immediately + // break into the debugger. When execution is continued, it + // will pop back out to this context + + // So, we don't set m_pJobPrev or g_pJobCur because nobody + // would ever have the chance to see them anyway. + Coroutine_DebugBreak( m_hCoroutine ); +} + +//----------------------------------------------------------------------------- +// Purpose: pauses the current job +//----------------------------------------------------------------------------- +void CJob::Pause( EJobPauseReason eReason ) +{ + AssertRunningThisJob(); + AssertMsg1( 0 == m_stackDoNotYieldGuards.Count(), "Yielding while in a BEGIN_DO_NOT_YIELD() block declared at %s", m_stackDoNotYieldGuards[m_stackDoNotYieldGuards.Head()] ); + + g_pJobCur = m_pJobPrev; + + // End our timer so we know how much time we've spent + m_FastTimerDelta.End(); + m_cyclecountTotal += m_FastTimerDelta.GetDuration(); + + if ( m_FastTimerDelta.GetDuration().GetMicroseconds() > k_cMicroSecTaskGranularity * 10 ) + { + m_flags.m_bits.m_bLongInterYield = true; + } + // pause this job, remembering which frame and why + m_ePauseReason = eReason; + // We shouldn't have to set the frame -- it should be the same one + Assert( m_STimeSwitched.LTime() == CJobTime::LJobTimeCur() ); + Coroutine_YieldToMain(); +} + +void CJob::GenerateAssert( const char *pchMsg ) +{ + // Default message if they didn't provide a custom one + if ( !pchMsg ) + { + pchMsg = "Forced assert failure"; + } + + // Just for grins, allow this function to be called whether + // we are the current job or not + if ( this == g_pJobCur ) + { + AssertMsg1( !"Job assertion requested", "%s", pchMsg ); + } + else + { + Coroutine_DebugAssert( m_hCoroutine, pchMsg ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: pauses the job until a network msg for the job arrives +//----------------------------------------------------------------------------- +bool CJob::BYieldingWaitForMsg( IMsgNetPacket **ppNetPacket ) +{ + AssertRunningThisJob(); + *ppNetPacket = NULL; + + // await and retrieve the network message + if ( GetJobMgr().BYieldingWaitForMsg( *this ) ) + { + Assert( m_vecNetPackets.Count() > 0 ); + *ppNetPacket = m_vecNetPackets.Tail(); + return true; + } + + return false; +} + +//----------------------------------------------------------------------------- +// Purpose: pauses the job until a network msg for the job arrives +//----------------------------------------------------------------------------- +bool CJob::BYieldingWaitForMsg( CGCMsgBase *pMsg, MsgType_t eMsg ) +{ + IMsgNetPacket *pNetPacket = NULL; + + // Check if we already waited for a message of this type + // but timed out. If so, then we currently don't have a way + // to tell if the message we might receive is the reply + // to the old mesage, of the one we're about to send. + // So let's just disallow this entirely. + if ( BHasFailedToReceivedMsgType( eMsg ) ) + { + AssertMsg2( false, "Job %s cannot wait for msg %u, it has already failed to wait for that msg type.", GetName(), eMsg ); + return false; + } + + m_unWaitMsgType = eMsg; + if ( !BYieldingWaitForMsg( &pNetPacket) ) + { + // Remember this event, so we can at least detect if a reply comes late, we don't get confused. + MarkFailedToReceivedMsgType( eMsg ); + return false; + } + + pMsg->SetPacket( pNetPacket ); + + if ( pMsg->Hdr().m_eMsg != eMsg ) + { + // Remember this event, so we can at least detect if a reply comes late, we don't get confused. + MarkFailedToReceivedMsgType( eMsg ); + + AssertMsg2( false, "CJob::BYieldingWaitForMsg expected msg %u but received %u", eMsg, pMsg->Hdr().m_eMsg ); + return false; + } + + return true; +} + +//----------------------------------------------------------------------------- +bool CJob::BHasFailedToReceivedMsgType( MsgType_t m ) const +{ + FOR_EACH_VEC( m_vecMsgTypesFailedToReceive, i ) + { + if ( m_vecMsgTypesFailedToReceive[i] == m ) + return true; + } + return false; +} + +//----------------------------------------------------------------------------- +void CJob::MarkFailedToReceivedMsgType( MsgType_t m ) +{ + if ( !BHasFailedToReceivedMsgType( m ) ) + { + m_vecMsgTypesFailedToReceive.AddToTail( m ); + } +} + +//----------------------------------------------------------------------------- +void CJob::ClearFailedToReceivedMsgType( MsgType_t m ) +{ + m_vecMsgTypesFailedToReceive.FindAndFastRemove( m ); +} + +//----------------------------------------------------------------------------- +// Purpose: pauses the job until a network msg for the job arrives +//----------------------------------------------------------------------------- +bool CJob::BYieldingWaitForMsg( CProtoBufMsgBase *pMsg, MsgType_t eMsg ) +{ + IMsgNetPacket *pNetPacket = NULL; + + // Check if we already waited for a message of this type + // but timed out. If so, then we currently don't have a way + // to tell if the message we might receive is the reply + // to the old mesage, of the one we're about to send. + // So let's just disallow this entirely. + if ( BHasFailedToReceivedMsgType( eMsg ) ) + { + AssertMsg2( false, "Job %s cannot wait for msg %u, it has already failed to wait for that msg type.", GetName(), eMsg ); + return false; + } + + m_unWaitMsgType = eMsg; + if ( !BYieldingWaitForMsg( &pNetPacket) ) + { + // Remember this event, so we can at least detect if a reply comes late, we don't get confused. + MarkFailedToReceivedMsgType( eMsg ); + return false; + } + + pMsg->InitFromPacket( pNetPacket ); + + if ( pMsg->GetEMsg() != eMsg ) + { + // Remember this event, so we can at least detect if a reply comes late, we don't get confused. + MarkFailedToReceivedMsgType( eMsg ); + + EmitError( SPEW_GC, "CJob::BYieldingWaitForMsg expected msg %u but received %u", eMsg, pMsg->GetEMsg() ); + return false; + } + + return true; +} + +#ifdef GC + +bool CJob::BYieldingWaitForMsg( CGCMsgBase *pMsg, MsgType_t eMsg, const CSteamID &expectedID ) +{ + if( !BYieldingWaitForMsg( pMsg, eMsg ) ) + return false; + + if( pMsg->Hdr().m_ulSteamID != expectedID.ConvertToUint64() ) + { + EmitError( SPEW_GC, "CJob::BYieldingWaitForMsg expected reply from steam ID %s, but instead got a response from %s for message %d\n", expectedID.Render(), CSteamID( pMsg->Hdr().m_ulSteamID ).Render(), eMsg ); + return false; + } + + return true; +} + + +bool CJob::BYieldingWaitForMsg( CProtoBufMsgBase *pMsg, MsgType_t eMsg, const CSteamID &expectedID ) +{ + if( !BYieldingWaitForMsg( pMsg, eMsg ) ) + return false; + + if( pMsg->GetClientSteamID() != expectedID ) + { + EmitError( SPEW_GC, "CJob::BYieldingWaitForMsg expected reply from steam ID %s, but instead got a response from %s for message %d\n", expectedID.Render(), pMsg->GetClientSteamID().Render(), eMsg ); + return false; + } + + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: pauses the job until a network msg for the job arrives +//----------------------------------------------------------------------------- +bool CJob::BYieldingRunQuery( CGCSQLQueryGroup *pQueryGroup, ESchemaCatalog eSchemaCatalog ) +{ + AssertRunningThisJob(); + + // await and retrieve the network message + return GetJobMgr().BYieldingRunQuery( *this, pQueryGroup, eSchemaCatalog ); +} +#endif + + +//----------------------------------------------------------------------------- +// Purpose: pauses the job until a work item callback occurs +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CJob::BYieldingWaitForWorkItem( const char *pszWorkItemName ) +{ + AssertRunningThisJob(); + + // await the work item completion + return GetJobMgr().BYieldingWaitForWorkItem( *this, pszWorkItemName ); +} + + +//----------------------------------------------------------------------------- +// Purpose: CWorkItem for processing functions in CJob-derived classes on another thread +//----------------------------------------------------------------------------- +class CJobThreadFuncWorkItem : public CWorkItem +{ +public: + DECLARE_WORK_ITEM( CJobThreadFuncWorkItem ); + CJobThreadFuncWorkItem( CJob *pJob, JobThreadFunc_t jobThreadFunc, CFunctor *pFunctor ) : CWorkItem( pJob->GetJobID() ), + m_pJob( pJob ), + m_pJobThreadFunc( jobThreadFunc ), + m_pFunctor( pFunctor ) + { + } + + virtual bool ThreadProcess( CWorkThread *pThread ) + { + if ( m_pJobThreadFunc ) + (m_pJob->*m_pJobThreadFunc)(); + if ( m_pFunctor ) + (*m_pFunctor)(); + return true; + } + +private: + CJob *m_pJob; + JobThreadFunc_t m_pJobThreadFunc; + CFunctor *m_pFunctor; +}; + + +//----------------------------------------------------------------------------- +// Purpose: pauses the job until a work item callback occurs +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CJob::BYieldingWaitForThreadFuncWorkItem( CWorkItem *pItem ) +{ + AssertRunningThisJob(); + Assert( m_pWaitingOnWorkItem == NULL ); + Assert( pItem->GetJobID() == GetJobID() ); + + m_pWaitingOnWorkItem = pItem; + + // add it to a central thread pool + GetJobMgr().AddThreadedJobWorkItem( pItem ); + + // await the work item completion + bool bSuccess = GetJobMgr().BYieldingWaitForWorkItem( *this ); + + m_pWaitingOnWorkItem = NULL; + + return bSuccess; +} + + +//----------------------------------------------------------------------------- +// Purpose: pauses the job until a work item callback occurs +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CJob::BYieldingWaitForThreadFunc( CFunctor *jobFunctor ) +{ + // store off which function to launch when we're done + CJobThreadFuncWorkItem *pJobThreadFuncWorkItem = new CJobThreadFuncWorkItem( this, NULL, jobFunctor ); + + bool bSuccess = BYieldingWaitForThreadFuncWorkItem( pJobThreadFuncWorkItem ); + + // free the thread func + SafeRelease( jobFunctor ); + SAFE_RELEASE( pJobThreadFuncWorkItem ); + + return bSuccess; +} + + +//----------------------------------------------------------------------------- +// Purpose: Allows a job that was paused for a specific reason to resume +//----------------------------------------------------------------------------- +void CJob::EndPause( EJobPauseReason eExpectedState ) +{ + Assert( m_ePauseReason == eExpectedState ); + if( m_ePauseReason == eExpectedState ) + { + m_ePauseReason = k_EJobPauseReasonYield; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns the number of heartbeats to wait before timing out this job +//----------------------------------------------------------------------------- +uint32 CJob::CHeartbeatsBeforeTimeout() +{ + return k_cJobHeartbeatsBeforeTimeoutDefault; +} + + +//----------------------------------------------------------------------------- +// Purpose: Send heartbeat messages to our listeners during long operations +// to let them know we're still alive and avoid timeouts +// This should be called by the CJobMgr +//----------------------------------------------------------------------------- +void CJob::Heartbeat() +{ + // Reset our counter + m_STimeNextHeartbeat.SetFromJobTime( k_cMicroSecJobHeartbeat ); +} + + + + +//----------------------------------------------------------------------------- +// Purpose: waits for specified time and checks for timeout. Useful when you +// need to repeatedly sleep while waiting for something to happen. +// This function uses STime (server "pseudo" time) to determine +// timeout conditions. +// +// Input: cMicrosecondsToSleep - duration to sleep this call +// stimeStarted - the time to calculate timeout from. (Typically, +// the time you start calling this in a loop, passing the same +// start time each time you call this method.) +// nMicroSecLimit - duration from stimeStarted to consider timed out +// Output : Returns true if not timed out yet, false if timed out +//----------------------------------------------------------------------------- +bool CJob::BYieldingWaitTimeWithLimit( uint32 cMicrosecondsToSleep, CJobTime &stimeStarted, int64 nMicroSecLimit ) +{ + if ( stimeStarted.CServerMicroSecsPassed() > nMicroSecLimit ) + return false; + + return BYieldingWaitTime( cMicrosecondsToSleep ); +} + + +//----------------------------------------------------------------------------- +// Purpose: waits for specified time and checks for timeout. Useful when you +// need to repeatedly sleep while waiting for something to happen. +// This function uses RTime (wall-clock "real" time) to determine +// timeout conditions. +// +// Input: cMicrosecondsToSleep - duration to sleep this call +// nSecLimit - duration from stimeStarted to consider timed out +// Output : Returns true if not timed out yet, false if timed out +//----------------------------------------------------------------------------- +bool CJob::BYieldingWaitTimeWithLimitRealTime( uint32 cMicrosecondsToSleep, int nSecondsLimit ) +{ + return BYieldingWaitTime( cMicrosecondsToSleep ); +} + + +//----------------------------------------------------------------------------- +// Purpose: pauses the job for the specified amount of time +// Input : m_cMicrosecondsToSleep - microseconds to wait for +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CJob::BYieldingWaitTime( uint32 cMicrosecondsToSleep ) +{ + AssertRunningThisJob(); + return GetJobMgr().BYieldingWaitTime( *this, cMicrosecondsToSleep ); +} + +//----------------------------------------------------------------------------- +// Purpose: pauses the job until the next time the JobMgr Run() is called +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CJob::BYield() +{ + AssertRunningThisJob(); + return GetJobMgr().BYield( *this ); +} + +//----------------------------------------------------------------------------- +// Purpose: Pauses the job ONLY IF JobMgr decides it needs to based on time run and priority +// If pausing, pauses until the next time the JobMgr Run() is called +// Input: pbYielded - Set to true if we did yield +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CJob::BYieldIfNeeded( bool *pbYielded ) +{ + AssertRunningThisJob(); + + if ( pbYielded ) + *pbYielded = false; + + // Assume only low priority jobs need to yield + // Automatically bail out here if the job is not low priority + + return GetJobMgr().BYieldIfNeeded( *this, pbYielded ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Pauses the job for a single frame +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CJob::BYieldingWaitOneFrame() +{ + return BYieldingWaitTime( 1 ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Blocks until we acquire the lock on the specified object +// Input : *pLock - object to lock +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CJob::_BYieldingAcquireLock( CLock *pLock, const char *filename, int line ) +{ + AssertRunningThisJob(); + + // Skip the path info from the filename. It just maks the debug messages excessively long. + filename = V_GetFileName( filename ); + + // Is the lock locked by this job? If so, inc the ref count. + if ( pLock->GetJobLocking() == this ) + { + pLock->IncrementReference(); + return true; + } + + // jobs can have multiple locks as long as they are in priority order + FOR_EACH_VEC( m_vecLocks, i ) + { + if( m_vecLocks[i]->GetLockType() == pLock->GetLockType() ) + { + if( m_vecLocks[i]->GetLockSubType() <= pLock->GetLockSubType() ) + { + AssertMsg7( false, "Job %s Locking %s at %s:(%d) with yielding; holds lock %s from %s(%d)\n", + GetName(), + pLock->GetName(), + filename, line, + m_vecLocks[i]->GetName(), + m_vecLocks[i]->m_pFilename, m_vecLocks[i]->m_line ); + return false; + } + } + else if ( m_vecLocks[i]->GetLockType() < pLock->GetLockType() ) + { + AssertMsg7( false, "Job %s Locking %s at %s:(%d) with yielding; holds lock %s from %s(%d)\n", + GetName(), + pLock->GetName(), + filename, line, + m_vecLocks[i]->GetName(), + m_vecLocks[i]->m_pFilename, m_vecLocks[i]->m_line ); + return false; + } + } + + if( m_pWaitingOnLock != NULL ) + { + AssertMsg7( false, "Job (%s) locking %s at %s(%d); already waiting on %s at %s(%d).\n", + GetName(), + pLock->GetName(), + filename, line, + m_pWaitingOnLock->GetName(), + m_pWaitingOnLock->m_pFilename, m_pWaitingOnLock->m_line ); + return false; + } + + m_cLocksAttempted++; + if ( pLock->BIsLocked() ) + { + // tell the job we want the lock next + // But walking the entire linked list is slow so + // skip to the tail pointer + pLock->AddToWaitingQueue( this ); + + // We should be the tail of the list + Assert( NULL == m_pJobToNotifyOnLockRelease ); + + // yield until we get the lock + m_pWaitingOnLock = pLock; + m_pWaitingOnLockFilename = filename; + m_waitingOnLockLine = line; + m_cLocksWaitedFor++; + Pause( k_EJobPauseReasonWaitingForLock ); + m_pWaitingOnLock = NULL; + + // make sure we actually got it, instead of timing out + int index = m_vecLocks.Find( pLock ); + if ( index != m_vecLocks.InvalidIndex() && this == pLock->GetJobLocking() ) + { + pLock->IncrementReference(); + return true; + } + else + { + m_flags.m_bits.m_bLocksFailed = true; + EmitWarning( SPEW_JOB, LOG_ALWAYS, "Failed to get lock %s at %s(%d) after waiting in %s\n", pLock->GetName(), filename, line, GetName() ); + if ( m_vecLocks.Count() == 0 ) + { + EmitWarning( SPEW_JOB, LOG_ALWAYS, "m_vecLocks.Count(): %d, this: 0x%p, pLock->GetJobLocking(): %s (0x%p)\n", + m_vecLocks.Count(), this, pLock->GetJobLocking() ? pLock->GetJobLocking()->GetName() : "(null)", pLock->GetJobLocking() ); + } + else + { + EmitWarning( SPEW_JOB, LOG_ALWAYS, "m_vecLocks.Count(): %d this: 0x%p, pLock: 0x%p pLock->GetJobLocking(): %s (0x%p)\n", + m_vecLocks.Count(), this, pLock, pLock->GetJobLocking() ? pLock->GetJobLocking()->GetName() : "(null)", pLock->GetJobLocking() ); + FOR_EACH_VEC( m_vecLocks, i ) + { + EmitWarning( SPEW_JOB, LOG_ALWAYS, "m_vecLocks[%d]: %s (0x%p) %s(%d)\n", + i, m_vecLocks[i] ? m_vecLocks[i]->GetName() : "(null)", m_vecLocks[i], m_vecLocks[i]->m_pFilename, m_vecLocks[i]->m_line ); + } + } + return false; + } + } + else + { + // unused, take it for ourself + pLock->IncrementReference(); + _SetLock( pLock, filename, line ); + return true; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Either locks on the specified object immediately or returns failure +// Input : *pLock - object to lock +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CJob::_BAcquireLockImmediate( CLock *pLock, const char *filename, int line ) +{ + AssertRunningThisJob(); + + AssertMsg5( m_pWaitingOnLock == NULL, "Job (%s) at %s(%d) trying to take a lock while it was already waiting for the first one at %s(%d)", GetName(), filename, line, m_pWaitingOnLockFilename, m_waitingOnLockLine ); + + m_cLocksAttempted++; + + // Is the lock locked by this job? If so, inc the ref count. + if ( pLock->GetJobLocking() == this ) + { + pLock->IncrementReference(); + return true; + } + + if ( !pLock->BIsLocked() ) + { + // unused, take it for ourself + pLock->IncrementReference(); + _SetLock( pLock, filename, line ); + return true; + } + + return false; +} + + +//----------------------------------------------------------------------------- +// Purpose: Releases the specified lock, passing it on to the next job if necessary +//----------------------------------------------------------------------------- +void CJob::_ReleaseLock( CLock *pLock, bool bForce, const char *filename, int line ) +{ + Assert( pLock ); + if ( !pLock ) + return; + + Assert( m_vecLocks.HasElement( pLock ) ); + if ( !m_vecLocks.HasElement( pLock ) ) + { + EmitError( SPEW_JOB, "Job %s trying to release lock %s at %s(%d) it's not holding\n", GetName(), pLock->GetName(), filename, line ); + return; + } + + if ( pLock->GetJobLocking() != this ) + { + EmitError( SPEW_JOB, "Job %s trying to release lock %s at %s(%d) though the lock is held by %s\n", GetName(), pLock->GetName(), filename, line, pLock->GetJobLocking()->GetName() ); + return; + } + + if ( bForce ) + { + // Force clear reference count + pLock->ClearReference(); + } + else + { + // Dec the reference count. If it is not yet zero, don't fully unlock + if ( pLock->DecrementReference() > 0 ) + { + return; + } + } + + if ( pLock->m_pJobToNotifyOnLockRelease ) + { + // post a message to the main system to wakeup the next lock + PassLockToJob( pLock->m_pJobToNotifyOnLockRelease, pLock ); + m_pJobToNotifyOnLockRelease = NULL; + + Assert( this != pLock->m_pJobWaitingQueueTail ); + } + else + { + // just release + UnsetLock( pLock ); + Assert( NULL == pLock->m_pJobWaitingQueueTail || this == pLock->m_pJobWaitingQueueTail ); + pLock->m_pJobWaitingQueueTail = NULL; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Release all locks this job holds. This is only to be used by long lived +// jobs that don't destruct. +//----------------------------------------------------------------------------- +void CJob::ReleaseLocks() +{ + // release any locks - do this in reverse order because they're being removed from the vector in the loop + FOR_EACH_VEC_BACK( m_vecLocks, nLock ) + { + _ReleaseLock( m_vecLocks[nLock], true, __FILE__, __LINE__ ); + } + m_vecLocks.RemoveAll(); +} + +//----------------------------------------------------------------------------- +// Purpose: Assert that we don't hold any locks, and if we hold them, release them +//----------------------------------------------------------------------------- +void CJob::ShouldNotHoldAnyLocks() +{ + if ( m_vecLocks.Count() == 0 ) + return; + + CUtlString sErrMsg; + sErrMsg.Format( "Job %s detected and cleaned up leak of %d lock(s):\n", GetName(), m_vecLocks.Count() ); + FOR_EACH_VEC_BACK( m_vecLocks, nLock ) + { + CLock *pLock = m_vecLocks[nLock]; + sErrMsg.Append( CFmtStr( " Lock %s, acquired %s(%d)\n", pLock->GetName(), pLock->m_pFilename, pLock->m_line ).Access() ); + } + + AssertMsg1( false, "%s", sErrMsg.String() ); + + // Now release them + ReleaseLocks(); +} + + +//----------------------------------------------------------------------------- +// Purpose: sets up the job to notify when we've release our locks +//----------------------------------------------------------------------------- +void CJob::AddJobToNotifyOnLockRelease( CJob *pJob ) +{ + // if we already are going to be notifying someone, then have them notify the new requester + if ( m_pJobToNotifyOnLockRelease ) + { + AssertMsg( false, "AddJobToNotifyOnLockRelease attempting to walk the linked list. We've optimized this out." ); + m_pJobToNotifyOnLockRelease->AddJobToNotifyOnLockRelease( pJob ); + } + else + { + m_pJobToNotifyOnLockRelease = pJob; + } +} + +//----------------------------------------------------------------------------- +// Purpose: Sets the lock +//----------------------------------------------------------------------------- +void CJob::_SetLock( CLock *pLock, const char *filename, int line ) +{ + Assert( !m_vecLocks.HasElement( pLock ) ); + Assert( !pLock->BIsLocked() ); + + pLock->m_pJob = this; + pLock->m_sTimeAcquired.SetToJobTime(); + pLock->m_pFilename = filename; + pLock->m_line = line; + m_vecLocks.AddToTail( pLock ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Removes the lock +//----------------------------------------------------------------------------- +void CJob::UnsetLock( CLock *pLock ) +{ + Assert( pLock->GetJobLocking() == this ); + + pLock->m_pJob = NULL; + // if we've held the lock for more than a few seconds, make noise. + if ( /*!BIsTest() &&*/ pLock->m_sTimeAcquired.CServerMicroSecsPassed() >= 10 * k_nMillion ) + { + m_flags.m_bits.m_bLocksLongHeld = true; + if ( pLock->m_pJobToNotifyOnLockRelease ) + { + pLock->m_pJobToNotifyOnLockRelease->m_flags.m_bits.m_bLocksLongWait = true; + EmitWarning( SPEW_JOB, 4, "Job of type %s held lock for %.2f seconds while job of type %s was waiting\n", GetName(), (double) pLock->m_sTimeAcquired.CServerMicroSecsPassed() / k_nMillion, pLock->m_pJobToNotifyOnLockRelease->GetName() ); + } + else + EmitWarning( SPEW_JOB, 4, "Job of type %s held lock for %.2f seconds\n", GetName(), (double) pLock->m_sTimeAcquired.CServerMicroSecsPassed() / k_nMillion ); + } + m_vecLocks.FindAndRemove( pLock ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Releases the lock from the old job, and immediately passes it on to the waiting job +//----------------------------------------------------------------------------- +void CJob::PassLockToJob( CJob *pNewJob, CLock *pLock ) +{ + Assert( pNewJob->GetPauseReason() == k_EJobPauseReasonWaitingForLock ); + Assert( pNewJob->m_pWaitingOnLock == pLock ); + + pLock->m_pJobToNotifyOnLockRelease = pNewJob->m_pJobToNotifyOnLockRelease; + if ( NULL == pLock->m_pJobToNotifyOnLockRelease ) + { + pLock->m_pJobWaitingQueueTail = NULL; + } + + pNewJob->m_pJobToNotifyOnLockRelease = NULL; + Assert( pLock->m_nWaitingCount > 0 ); + pLock->m_nWaitingCount--; + + // release the lock + UnsetLock( pLock ); + + // If the other job isn't waiting on a lock, then we certainly don't + // want to call SetLock() on it + if ( pNewJob->GetPauseReason() == k_EJobPauseReasonWaitingForLock && pNewJob->m_pWaitingOnLock == pLock ) + { + // give the job the lock + pNewJob->_SetLock( pLock, pNewJob->m_pWaitingOnLockFilename, m_waitingOnLockLine ); + + // set the job with the newly acquired lock to wakeup + pNewJob->GetJobMgr().WakeupLockedJob( *pNewJob ); + } + else + { + EmitError( SPEW_JOB, "Job passed lock it wasn't waiting for. Job: %s, Lock: %s %s(%d), Paused for %s, Waiting on %s\n", + pNewJob->GetName(), pLock->GetName(), pLock->m_pFilename, pLock->m_line, pNewJob->GetPauseReasonDescription(), m_pWaitingOnLock ? m_pWaitingOnLock->GetName() : "none" ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: a lock is letting us know it's been deleted +// fail all jobs trying to get the lock +//----------------------------------------------------------------------------- +void CJob::OnLockDeleted( CLock *pLock ) +{ + //EmitWarning( SPEW_JOB, SPEW_ALWAYS, "Deleting lock %s\n", GetName() ); + + Assert( pLock->BIsLocked() ); + Assert( pLock->m_pJob == this ); + + // fail all the jobs waiting on the lock + CJob *pJob = pLock->m_pJobToNotifyOnLockRelease; + while ( pJob ) + { + // insert the job into the sleep list with 0 time, so it wakes up immediately + // it will see it doesn't have the desired lock and suicide + pJob->GetJobMgr().WakeupLockedJob( *pJob ); + + // move to the next job + CJob *pJobT = pJob; + pJob = pJob->m_pJobToNotifyOnLockRelease; + pJobT->m_pJobToNotifyOnLockRelease = NULL; + } + + m_pJobToNotifyOnLockRelease = NULL; + pLock->m_pJobToNotifyOnLockRelease = NULL; + pLock->m_pJobWaitingQueueTail = NULL; + + // remove the lock + UnsetLock( pLock ); +} + +//----------------------------------------------------------------------------- +// Purpose: Reports how many Do Not Yield guards the job currently has +//----------------------------------------------------------------------------- +int32 CJob::GetDoNotYieldDepth() const +{ + return m_stackDoNotYieldGuards.Count(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a Do Not Yield guard to the job +//----------------------------------------------------------------------------- +void CJob::PushDoNotYield( const char *pchFileAndLine ) +{ + m_stackDoNotYieldGuards.AddToHead( pchFileAndLine ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Removes the last-added Do Not Yield guard from the job +//----------------------------------------------------------------------------- +void CJob::PopDoNotYield() +{ + AssertMsg( m_stackDoNotYieldGuards.Count() > 0, "Could not pop a Do Not Yield guard when the job's stack is empty" ); + if ( m_stackDoNotYieldGuards.Count() > 0 ) + { + m_stackDoNotYieldGuards.Remove( m_stackDoNotYieldGuards.Head() ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Implementation of the stack-scope Do Not Yield guard +//----------------------------------------------------------------------------- +CDoNotYieldScope::CDoNotYieldScope( const char *pchFileAndLine ) +{ + AssertRunningJob(); + + GJobCur().PushDoNotYield( pchFileAndLine ); +} + +CDoNotYieldScope::~CDoNotYieldScope() +{ + AssertRunningJob(); + + GJobCur().PopDoNotYield(); +} + + +#ifdef DBGFLAG_VALIDATE +//----------------------------------------------------------------------------- +// Purpose: Run a global validation pass on all of our data structures and memory +// allocations. +// Input: validator - Our global validator object +// pchName - Our name (typically a member var in our container) +//----------------------------------------------------------------------------- +void CJob::Validate( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE(); + + ValidateObj( m_stackDoNotYieldGuards ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Run a global validation pass on all of our static data structures and memory +// allocations. +// Input: validator - Our global validator object +// pchName - Our name (typically a member var in our container) +//----------------------------------------------------------------------------- +void CJob::ValidateStatics( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE_STATIC( "CJob class statics" ); +} +#endif // DBGFLAG_VALIDATE + + +CLock::CLock( ) +: m_pJob( NULL ), +m_pJobToNotifyOnLockRelease( NULL ), +m_pJobWaitingQueueTail( NULL ), +m_nWaitingCount(0), +m_nsLockType(0), +m_nsNameType( k_ENameTypeNone ), +m_ulID( 0 ), +m_pchConstStr( NULL ), +m_unLockSubType ( 0 ), +m_nRefCount( 0 ), +m_pFilename( "unknown" ), +m_line( 0 ) +{ +} + + +CLock::~CLock() +{ + if ( m_pJob ) + { + m_pJob->OnLockDeleted( this ); + } +} + + +void CLock::AddToWaitingQueue( CJob *pJob ) +{ + if ( m_pJobWaitingQueueTail ) + { + Assert( NULL == m_pJobWaitingQueueTail->m_pJobToNotifyOnLockRelease ); + m_pJobWaitingQueueTail->AddJobToNotifyOnLockRelease( pJob ); + } + else + { + Assert( NULL == m_pJobToNotifyOnLockRelease ); + m_pJobToNotifyOnLockRelease = pJob; + } + + m_pJobWaitingQueueTail = pJob; + m_nWaitingCount++; +} + + +void CLock::SetName( const char *pchName ) +{ + m_nsNameType = k_ENameTypeConstStr; + m_pchConstStr = pchName; +} + + +void CLock::SetName( const char *pchPrefix, uint64 ulID ) +{ + m_nsNameType = k_ENameTypeConcat; + m_pchConstStr = pchPrefix; + m_ulID = ulID; +} + + +void CLock::SetName( const CSteamID &steamID ) +{ + m_nsNameType = k_ENameTypeSteamID; + m_ulID = steamID.ConvertToUint64(); +} + + +const char *CLock::GetName() const +{ + switch ( m_nsNameType ) + { + case k_ENameTypeNone: + return "None"; + case k_ENameTypeSteamID: + return CSteamID::Render( m_ulID ); + case k_ENameTypeConstStr: + return m_pchConstStr; + case k_ENameTypeConcat: + if ( !m_strName.Length() ) + m_strName.Format( "%s %llu", m_pchConstStr, m_ulID ); + return m_strName.Get(); + default: + AssertMsg1( false, "Invalid lock name type %d", m_nsNameType ); + return "(Unknown)"; + } +} + +#define REF_COUNT_ASSERT 1000 + +void CLock::IncrementReference() +{ + m_nRefCount++; + Assert( m_nRefCount != REF_COUNT_ASSERT ); +} + +int CLock::DecrementReference() +{ + Assert( m_nRefCount > 0 ); + if ( m_nRefCount > 0 ) + { + m_nRefCount--; + } + return m_nRefCount; +} + +void CLock::Dump( const char *pszPrefix, int nPrintMax, bool bPrintWaiting ) const +{ + if ( m_pJob != NULL ) + { + EmitInfo( SPEW_JOB, SPEW_ALWAYS, LOG_ALWAYS, "%s%s: Lock owner: %s, Type: %d, %d Waiting\n", pszPrefix, GetName(), CFmtStr( "%s (%llu), Reason: %s", m_pJob->GetName(), m_pJob->GetJobID(), m_pJob->GetPauseReasonDescription() ).Access(), (int32)m_nsLockType, m_nWaitingCount ); + EmitInfo( SPEW_JOB, SPEW_ALWAYS, LOG_ALWAYS, "%sLock acquired: %s:%d\n", pszPrefix, m_pFilename, m_line ); + } + else + { + EmitInfo( SPEW_JOB, SPEW_ALWAYS, LOG_ALWAYS, "%s%s: Lock owner: None, Type: %d, %d Waiting\n", pszPrefix, GetName(), (int32)m_nsLockType, m_nWaitingCount ); + } + + CJob *pCurrWaiting = m_pJobToNotifyOnLockRelease; + int nPrinted = 0; + int nTotal = 0; + while( pCurrWaiting != NULL && nPrinted < nPrintMax ) + { + bool bPrint = false; + if ( nPrinted < nPrintMax ) + { + bPrint = true; + } + if ( pCurrWaiting->GetPauseReason() == k_EJobPauseReasonWaitingForLock && pCurrWaiting->m_pWaitingOnLock == this ) + { + bPrint = true; + } + + if ( bPrint && bPrintWaiting ) + { + EmitInfo( SPEW_JOB, SPEW_ALWAYS, LOG_ALWAYS, "%s\tOther jobs waiting for this lock: %s (%llu)\n", pszPrefix, pCurrWaiting->GetName(), pCurrWaiting->GetJobID() ); + if ( pCurrWaiting->GetPauseReason() == k_EJobPauseReasonWaitingForLock && pCurrWaiting->m_pWaitingOnLock == this ) + { + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, 3, "%s\tAt: %s:%d\n", pszPrefix, pCurrWaiting->m_pWaitingOnLockFilename, pCurrWaiting->m_waitingOnLockLine ); + } + nPrinted++; + } + pCurrWaiting = pCurrWaiting->m_pJobToNotifyOnLockRelease; + nTotal++; + } + if ( bPrintWaiting || nTotal != 0 ) + { + EmitInfo( SPEW_JOB, SPEW_ALWAYS, LOG_ALWAYS, "%s%d out of %d waiting jobs printed.\n", pszPrefix, nPrinted, nTotal ); + } +} + +} // namespace GCSDK diff --git a/gcsdk/jobmgr.cpp b/gcsdk/jobmgr.cpp new file mode 100644 index 0000000..1821740 --- /dev/null +++ b/gcsdk/jobmgr.cpp @@ -0,0 +1,1626 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//============================================================================= + + +#include "stdafx.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ +#ifdef DEBUG_JOB_LIST +CUtlLinkedList<CJob *,int> CJobMgr::sm_listAllJobs; +#endif + +typedef int (__cdecl *QSortCompareFuncCtx_t)(void *, const void *, const void *); + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CJobMgr::CJobMgr() +: m_MapJob( 0, 0, DefLessFunc( GID_t ) ), + m_QueueJobSleeping( 0, 0, &JobSleepingLessFunc ), + m_unNextJobID( 0 ), + m_mapStatsBucket( 0, 0, DefLessFunc(uint32) ), + m_WorkThreadPool( "CJobMgr::m_WorkThreadPool" ), + m_bDebugDisallowPause( false ) +{ + SetDefLessFunc( m_MapJobTimeoutsIndexByJobID ); + SetDefLessFunc( m_mapOrphanMessages ); + m_bJobTimedOut = false; + m_nCurrentYieldIterationRegPri = 0; + m_bProfiling = false; + m_bIsShuttingDown = false; + m_cErrorsToReport = 0; + m_unFrameFuncThreadID = 0; + m_WorkThreadPool.SetWorkThreadAutoConstruct( 1, NULL ); + + if( MemAlloc_GetDebugInfoSize() > 0 ) + { + g_memMainDebugInfo.Init( 0, MemAlloc_GetDebugInfoSize() ); + } + + if( MemAlloc_GetDebugInfoSize() > 0 ) + { + g_memMainDebugInfo.EnsureCapacity( MemAlloc_GetDebugInfoSize() ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CJobMgr::~CJobMgr() +{ + m_WorkThreadPool.StopWorkThreads(); +} + + +//----------------------------------------------------------------------------- +// Purpose: limit the size of our thread pool +//----------------------------------------------------------------------------- +void CJobMgr::SetThreadPoolSize( uint cThreads ) +{ + m_WorkThreadPool.SetWorkThreadAutoConstruct( cThreads, NULL ); +} + + +//----------------------------------------------------------------------------- +// Purpose: gets the next available job ID +//----------------------------------------------------------------------------- +JobID_t CJobMgr::GetNewJobID() +{ +#ifdef GC + return GGCHost()->GenerateGID(); +#else + return ++m_unNextJobID; +#endif +} + + +//----------------------------------------------------------------------------- +// Purpose: Run jobs +// Runs once per frame and resumes any sleeping jobs that are scheduled +// to run again, also checks for jobs which have timed out. +// +// Input: limitTimer - limit timer not to exceed +// Output: true if there is still work remaining to do, false otherwise +//----------------------------------------------------------------------------- +bool CJobMgr::BFrameFuncRunSleepingJobs( CLimitTimer &limitTimer ) +{ + CheckThreadID(); // make sure frame function is called from correct thread + + bool bWorkRemaining = false; + + { + VPROF_BUDGET( "CJobMgr::BResumeSleepingJobs", VPROF_BUDGETGROUP_JOBS_COROUTINES ); + bWorkRemaining |= BResumeSleepingJobs( limitTimer ); + } + + { + VPROF_BUDGET( "CJobMgr::CheckForJobTimeouts", VPROF_BUDGETGROUP_JOBS_COROUTINES ); + CheckForJobTimeouts( limitTimer ); + } + + m_JobStats.m_cJobsCurrent = CountJobs(); + + return bWorkRemaining; +} + + + +//----------------------------------------------------------------------------- +// Purpose: Run jobs +// This function is called repeatedly in a single frame if time is left +// and will first run any yielding jobs +// Input: limitTimer - limit timer not to exceed +// Output: true if there is still work remaining to do, false otherwise +//----------------------------------------------------------------------------- +bool CJobMgr::BFrameFuncRunYieldingJobs( CLimitTimer &limitTimer ) +{ + CheckThreadID(); // make sure frame function is called from correct thread + + bool bWorkRemaining = false; + + { + VPROF_BUDGET( "CJobMgr::BResumeYieldingJobs", VPROF_BUDGETGROUP_JOBS_COROUTINES ); + bWorkRemaining |= BResumeYieldingJobs( limitTimer ); + } + + { + VPROF_BUDGET( "CJobMgr -- Dispatch completed work items", VPROF_BUDGETGROUP_JOBS_COROUTINES ); + bWorkRemaining |= m_WorkThreadPool.BDispatchCompletedWorkItems( limitTimer, this ); + } + + m_JobStats.m_cJobsCurrent = CountJobs(); + + return bWorkRemaining; +} + + +//----------------------------------------------------------------------------- +// Purpose: Registers a new job for us to keep track of. +// Input: job - The job in question +//----------------------------------------------------------------------------- +void CJobMgr::InsertJob( CJob &job ) +{ + Assert( m_MapJob.Find( job.GetJobID() ) == m_MapJob.InvalidIndex() ); + m_MapJob.Insert( job.GetJobID(), &job ); +#ifdef DEBUG_JOB_LIST + sm_listAllJobs.AddToTail( &job ); +#endif +} + + +//----------------------------------------------------------------------------- +// purpose: This job is done, accumulate its stats +//----------------------------------------------------------------------------- +void CJobMgr::AccumulateStatsofJob( CJob &job ) +{ + // if we are not profiling, but the job experienced some kind of failure + // record it anyway - we will issue a consolidated spew about it + if ( !m_bProfiling && job.m_flags.m_uFlags == 0 ) + return; + if ( job.m_flags.m_uFlags ) + m_cErrorsToReport++; + + job.m_FastTimerDelta.End(); + job.m_cyclecountTotal += job.m_FastTimerDelta.GetDuration(); + + uint32 eBucket = 0; + // the pointer to the name is a pointer to a constant string + // so use this dirty trick to make lookups fast + eBucket = (uint32)job.GetName(); + int iBucket = m_mapStatsBucket.Find( eBucket ); + if ( iBucket == m_mapStatsBucket.InvalidIndex() ) + { + iBucket = m_mapStatsBucket.Insert( eBucket ); + V_strcpy_safe( m_mapStatsBucket[iBucket].m_rgchName, job.GetName() ); + } + + JobStatsBucket_t *pJobStatsBucket = &m_mapStatsBucket[iBucket]; + pJobStatsBucket->m_cCompletes++; + pJobStatsBucket->m_cLocksAttempted += job.m_cLocksAttempted; + pJobStatsBucket->m_cLocksWaitedFor += job.m_cLocksWaitedFor; + pJobStatsBucket->m_cLocksFailed += job.m_flags.m_bits.m_bLocksFailed ? 1 : 0; + pJobStatsBucket->m_cLocksLongHeld += job.m_flags.m_bits.m_bLocksLongHeld ? 1 : 0; + pJobStatsBucket->m_cLocksLongWait += job.m_flags.m_bits.m_bLocksLongWait ? 1 : 0; + pJobStatsBucket->m_cWaitTimeout += job.m_flags.m_bits.m_bWaitTimeout ? 1 : 0; + pJobStatsBucket->m_cJobsFailed += job.m_flags.m_bits.m_bJobFailed ? 1 : 0; + pJobStatsBucket->m_cLongInterYieldTime += job.m_flags.m_bits.m_bLongInterYield ? 1 : 0; + pJobStatsBucket->m_cTimeoutNetMsg += job.m_flags.m_bits.m_bTimeoutNetMsg ? 1 : 0; + + pJobStatsBucket->m_u64RunTime += job.m_cyclecountTotal.GetLongCycles(); + if ( (uint64)job.m_cyclecountTotal.GetLongCycles() > pJobStatsBucket->m_u64RunTimeMax ) + pJobStatsBucket->m_u64RunTimeMax = job.m_cyclecountTotal.GetLongCycles(); + if ( job.m_STimeSwitched != job.m_STimeStarted ) + { + pJobStatsBucket->m_cJobsPaused++; + pJobStatsBucket->m_u64JobDuration += job.m_STimeStarted.CServerMicroSecsPassed(); + } + else + { + pJobStatsBucket->m_u64JobDuration += job.m_cyclecountTotal.GetMicroseconds(); + } +} + + +//----------------------------------------------------------------------------- +// purpose: This message was orphaned, accumulate for stats +//----------------------------------------------------------------------------- +void CJobMgr::RecordOrphanedMessage( MsgType_t eMsg, JobID_t jobIDTarget ) +{ + EG_MSG( SPEW_JOB, "Message %s arrived responding to job %lld which no longer exists, dropping message\n", PchMsgNameFromEMsg( eMsg ), jobIDTarget ); + int iBucket = m_mapOrphanMessages.Find( eMsg ); + if ( iBucket == m_mapOrphanMessages.InvalidIndex() ) + { + int ct = 0; + iBucket = m_mapOrphanMessages.Insert( eMsg, ct ); + } + m_mapOrphanMessages[iBucket]++; +} + + +//----------------------------------------------------------------------------- +// Purpose: Removes a job from the manager. Note that we don't free it. +// Input: job - The job in question +//----------------------------------------------------------------------------- +void CJobMgr::RemoveJob( CJob &job ) +{ + m_MapJob.Remove( job.GetJobID() ); + + AccumulateStatsofJob( job ); + m_JobStats.m_cJobsTotal++; + if ( job.m_flags.m_bits.m_bJobFailed ) + m_JobStats.m_cJobsFailed++; + + uint64 u64JobDuration = job.m_STimeStarted.CServerMicroSecsPassed(); + m_JobStats.m_flSumJobTimeMicrosec += u64JobDuration; + m_JobStats.m_flSumSqJobTimeMicrosec += ((double)u64JobDuration * (double)u64JobDuration); + if ( u64JobDuration > m_JobStats.m_unMaxJobTimeMicrosec ) + { + m_JobStats.m_unMaxJobTimeMicrosec = u64JobDuration; + } + +#ifdef DEBUG_JOB_LIST + sm_listAllJobs.FindAndRemove( &job ); +#endif +} + + +#ifdef GC +//----------------------------------------------------------------------------- +// Purpose: resumes the specified job if it is, in fact, waiting for a SQL query +// to return +//----------------------------------------------------------------------------- +bool CJobMgr::BResumeSQLJob( JobID_t jobID ) +{ + int iMap = m_mapSQLQueriesInFlight.Find( jobID ); + if ( m_mapSQLQueriesInFlight.IsValidIndex( iMap ) ) + { + if ( m_bSQLProfiling && m_dictSQLBuckets.IsValidIndex( m_mapSQLQueriesInFlight[iMap].m_iBucket ) ) + { + SQLProfileBucket_t &bucket = m_dictSQLBuckets[ m_mapSQLQueriesInFlight[iMap].m_iBucket ]; + bucket.m_unCount++; + bucket.m_nTotalMicrosec += (int64)m_sqlTimer.GetDurationInProgress().GetUlMicroseconds() - m_mapSQLQueriesInFlight[iMap].m_nStartMicrosec; + } + + m_mapSQLQueriesInFlight.RemoveAt( iMap ); + } + + int iJob; + if ( !BGetIJob( jobID, k_EJobPauseReasonSQL, true, &iJob ) ) + { + EG_MSG( SPEW_JOB, "BResumeSQLJob called for a job that could not be found!\n" ); + return false; + } + + // Just change the job's pause reason and add it to the yield list + // it will wake up on the next heartbeat + m_MapJob[iJob]->EndPause( k_EJobPauseReasonSQL ); + AddToYieldList( *m_MapJob[iJob] ); + + return true; +} +#endif + +//----------------------------------------------------------------------------- +// Purpose: returns true if we're running any jobs of the specified name +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CJobMgr::BIsJobRunning( const char *pchJobName ) +{ + FOR_EACH_MAP_FAST( m_MapJob, i ) + { + if ( !Q_stricmp( m_MapJob[i]->GetName(), pchJobName ) ) + return true; + } + return false; +} + + +//----------------------------------------------------------------------------- +// Purpose: returns true if there is a job active with the specified ID +//----------------------------------------------------------------------------- +bool CJobMgr::BJobExists( JobID_t jobID ) const +{ + return ( m_MapJob.Find( jobID ) != m_MapJob.InvalidIndex() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: returns a job pointer by id +//----------------------------------------------------------------------------- +const CJob *CJobMgr::GetPJob( JobID_t jobID ) const +{ + int iMap = m_MapJob.Find( jobID ); + if ( iMap != m_MapJob.InvalidIndex() ) + { + return m_MapJob[iMap]; + } + return NULL; +} + +CJob *CJobMgr::GetPJob( JobID_t jobID ) +{ + int iMap = m_MapJob.Find( jobID ); + if ( iMap != m_MapJob.InvalidIndex() ) + { + return m_MapJob[iMap]; + } + return NULL; +} + +//----------------------------------------------------------------------------- +// Purpose: Examines an incoming message to see if it belongs to an active job, +// and if so, sends it to that job. Creates a new job if necessary. +// Output: true if the message was routed to a job +//----------------------------------------------------------------------------- +bool CJobMgr::BRouteMsgToJob( void *pParent, IMsgNetPacket *pNetPacket, const JobMsgInfo_t &jobMsgInfo ) +{ + if ( pNetPacket == NULL ) + { + AssertMsg(pNetPacket, "CJobMgr::BRouteMsgToJob received NULL packet."); + return false; + } + + if ( jobMsgInfo.m_JobIDTarget != k_GIDNil ) + { + // This message is a reply to a running job + VPROF_BUDGET( "CJobMgr::BRouteMsgToJob() - continue job", VPROF_BUDGETGROUP_JOBS_COROUTINES ); + + // Find the job that this packet is destined for + int iJob = m_MapJob.Find( jobMsgInfo.m_JobIDTarget ); + if ( m_MapJob.InvalidIndex() != iJob ) + { + // found the right job, pass it off + PassMsgToJob( *(m_MapJob[iJob]), pNetPacket, jobMsgInfo ); + return true; + } + + // The job is no longer running, it most likely timed out before the response arrived. + // Continue and see if a job is registered to launch from this message + } + + // no job, so try creating a job that can handle the msg + // We pass in a pointer to m_JobIDTarget so that it gets set to the new Job's ID. This ensures + // that anyone replying to this message from within the new job has the right JobIDSource. + VPROF_BUDGET( "CJobMgr::BRouteMsgToJob() - job", VPROF_BUDGETGROUP_JOBS_COROUTINES ); + bool bRet = BLaunchJobFromNetworkMsg( pParent, jobMsgInfo, pNetPacket ); + + if ( !bRet && jobMsgInfo.m_JobIDTarget != k_GIDNil ) + { + RecordOrphanedMessage( jobMsgInfo.m_eMsg, jobMsgInfo.m_JobIDTarget ); + // return that we've handled this message (as much as it possibly can be) -- was intended for a job that has + // timed out, no one else can do anything with it + return true; + } + + return bRet; +} + + +//----------------------------------------------------------------------------- +// Purpose: Routes a message directly to the specified job +//----------------------------------------------------------------------------- +void CJobMgr::PassMsgToJob( CJob &job, IMsgNetPacket *pNetPacket, const JobMsgInfo_t &jobMsgInfo ) +{ + // Check if this job previously failed to wait for this message type, + // then this is probably a late reply. Discard it + if ( job.BHasFailedToReceivedMsgType( jobMsgInfo.m_eMsg ) ) + { + EmitInfo( SPEW_JOB, 2, LOG_ALWAYS, "Reply msg type %s to job %s is too late; discarding\n", PchMsgNameFromEMsg( jobMsgInfo.m_eMsg ), job.GetName() ); + return; + } + + // make sure it's what we're waiting for + if ( job.GetPauseReason() != k_EJobPauseReasonNetworkMsg ) + { + AssertMsg3( false, "CJobMgr::PassMsgToJob() job %s received unexpected message %s when paused for %s\n", job.GetName(), PchMsgNameFromEMsg( jobMsgInfo.m_eMsg ), job.GetPauseReasonDescription() ); + } + + // In case of error, we need to throw this message away + if ( job.GetPauseReason() != k_EJobPauseReasonNetworkMsg ) + return; + + // Add the packet and resume the job + job.AddPacketToList( pNetPacket, jobMsgInfo.m_JobIDSource ); + job.EndPause( k_EJobPauseReasonNetworkMsg ); + AddToYieldList( job ); + + return; +} + + +//----------------------------------------------------------------------------- +// Purpose: pauses the job until a network msg for the specified job arrives +//----------------------------------------------------------------------------- +bool CJobMgr::BYieldingWaitForMsg( CJob &job ) +{ + // wait until we're woken up by a networking callback, or a timeout + PauseJob( job, k_EJobPauseReasonNetworkMsg ); + return !m_bJobTimedOut; +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns IJob matching a JobID, if it is paused for the given reason +// Input: jobID - The job that should be paused for the given reason +// eJobPauseReason - Pause reason +// bShouldExist - If true, job should exist, so asserts on not finding it ok +// pIJob - IJob to fill in +// Output: true if job paused for matching reason found +//----------------------------------------------------------------------------- +bool CJobMgr::BGetIJob( JobID_t jobID, EJobPauseReason eJobPauseReason, bool bShouldExist, int *pIJob ) +{ + // If this isn't owned by a job, we don't handle it + if ( k_GIDNil == jobID ) + return false; + + // Figure out which job the msg belongs to + int iJob = m_MapJob.Find( jobID ); + Assert( m_MapJob.InvalidIndex() != iJob || !bShouldExist ); + + // If it's not one of ours, ignore it + if ( m_MapJob.InvalidIndex() == iJob ) + return false; + + // make sure it's what we're waiting for + if ( m_MapJob[iJob]->GetPauseReason() != eJobPauseReason ) + return false; + + *pIJob = iJob; + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: yields for a set amount of time +// Input : &job - job that is yielding +// m_cMicrosecondsToSleep - number of microseconds to wait for before resuming job +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CJobMgr::BYieldingWaitTime( CJob &job, uint32 cMicrosecondsToSleep ) +{ + Assert( cMicrosecondsToSleep < k_cMicroSecJobPausedTimeout ); + // sleep of zero causes an infinite loop + Assert( 0 != cMicrosecondsToSleep ); + +#ifdef _DEBUG + for ( int i = 0; i < m_QueueJobSleeping.Count(); i++ ) + { + Assert( m_QueueJobSleeping.Element(i).m_JobID != job.GetJobID() ); + } +#endif + + // insert the job into the sleep list + JobSleeping_t jobSleeping; + jobSleeping.m_JobID = job.GetJobID(); + jobSleeping.m_SWakeupTime.SetFromJobTime( cMicrosecondsToSleep ); + jobSleeping.m_STimeTouched.SetToJobTime(); + m_QueueJobSleeping.Insert( jobSleeping ); + + // yield + PauseJob( job, k_EJobPauseReasonSleepForTime ); + if ( m_bJobTimedOut ) + return false; + + return true; +} + + +#ifdef GC +//----------------------------------------------------------------------------- +// Purpose: yields waiting for a query response +// Input : &job - job that is yielding +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +// yields waiting for a query response +bool CJobMgr::BYieldingRunQuery( CJob &job, CGCSQLQueryGroup *pQueryGroup, ESchemaCatalog eSchemaCatalog ) +{ + // clear the existing results pointer, if any, to make space for the results + // for this query + pQueryGroup->SetResults( NULL ); + + if ( m_bSQLProfiling ) + { + const char *pchName = pQueryGroup->PchName(); + if ( !pchName || !pchName[0] ) + { + if ( pQueryGroup->GetStatementCount() == 1 ) + { + pchName = pQueryGroup->PchCommand( 0 ); + } + + if ( !pchName || !pchName[0] ) + { + pchName = job.GetName(); + } + } + + PendingSQLJob_t sqlJob; + sqlJob.m_nStartMicrosec = (int64)m_sqlTimer.GetDurationInProgress().GetUlMicroseconds(); + sqlJob.m_iBucket = m_dictSQLBuckets.Find( pchName ); + if ( !m_dictSQLBuckets.IsValidIndex( sqlJob.m_iBucket ) ) + { + SQLProfileBucket_t bucket = { 0, 0 }; + sqlJob.m_iBucket = m_dictSQLBuckets.Insert( pchName, bucket ); + } + m_mapSQLQueriesInFlight.Insert( job.GetJobID(), sqlJob ); + } + + VPROF_BUDGET( "GCHost", VPROF_BUDGETGROUP_STEAM ); + { + VPROF_BUDGET( "GCHost - SQLQuery", VPROF_BUDGETGROUP_STEAM ); + GGCHost()->SQLQuery( job.GetJobID(), pQueryGroup, eSchemaCatalog ); + } + PauseJob( job, k_EJobPauseReasonSQL ); + return pQueryGroup->GetResults() && pQueryGroup->GetResults()->GetError() == k_EGCSQLErrorNone; +} + + +//----------------------------------------------------------------------------- +// Purpose: turns on sql profiling +//----------------------------------------------------------------------------- +void CJobMgr::StartSQLProfiling() +{ + if ( m_bSQLProfiling ) + return; + + m_mapSQLQueriesInFlight.RemoveAll(); + m_dictSQLBuckets.RemoveAll(); + m_sqlTimer.Start(); + m_bSQLProfiling = true; +} + + +//----------------------------------------------------------------------------- +// Purpose: turns off sql profiling +//----------------------------------------------------------------------------- +void CJobMgr::StopSQLProfiling() +{ + if ( !m_bSQLProfiling ) + return; + + m_mapSQLQueriesInFlight.RemoveAll(); + m_sqlTimer.End(); + m_bSQLProfiling = false; +} + + + +//----------------------------------------------------------------------------- +// Purpose: sql profile sort func +//----------------------------------------------------------------------------- +int CJobMgr::SQLProfileSortFunc( void *pCtx, const int *lhs, const int *rhs ) +{ + SQLProfileCtx_t *pSQLProfileCtx = (SQLProfileCtx_t *)pCtx; + CUtlDict<SQLProfileBucket_t> *pDictBuckets = pSQLProfileCtx->pdictBuckets; + SQLProfileBucket_t &lhsBucket = pDictBuckets->Element( *lhs ); + SQLProfileBucket_t &rhsBucket = pDictBuckets->Element( *rhs ); + + switch ( pSQLProfileCtx->m_eSort ) + { + default: + case k_ESQLProfileSortTotalTime: return rhsBucket.m_nTotalMicrosec - lhsBucket.m_nTotalMicrosec; + case k_ESQLProfileSortTotalCount: return rhsBucket.m_unCount - lhsBucket.m_unCount; + case k_ESQLProfileSortAvgTime: return ( rhsBucket.m_nTotalMicrosec / rhsBucket.m_unCount ) - ( lhsBucket.m_nTotalMicrosec / lhsBucket.m_unCount ); + case k_ESQLProfileSortName: return Q_stricmp( pDictBuckets->GetElementName( *lhs ), pDictBuckets->GetElementName( *rhs ) ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: dumps the current sql profile +//----------------------------------------------------------------------------- +void CJobMgr::DumpSQLProfile( ESQLProfileSort eSort ) +{ + CUtlVector<int> vecSort; + for ( int iDict = 0; iDict < m_dictSQLBuckets.MaxElement(); iDict++ ) + { + if ( !m_dictSQLBuckets.IsValidIndex( iDict ) ) + continue; + + if ( m_dictSQLBuckets[iDict].m_unCount > 0 ) + { + vecSort.AddToTail( iDict ); + } + } + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "SQL statement stats:\n" ); + if ( 0 == vecSort.Count() ) + { + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\tNo SQL stats collected; use sql_profile_on / sql_profile_off to collect stats first\n" ); + return; + } + + // sort + SQLProfileCtx_t ctx; + ctx.m_eSort = eSort; + ctx.pdictBuckets = &m_dictSQLBuckets; + + V_qsort_s( vecSort.Base(), vecSort.Count(), sizeof(int), (QSortCompareFuncCtx_t)SQLProfileSortFunc, &ctx ); + + // display + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%8s %8s %8s\n", "count", "time", "avg" ); + FOR_EACH_VEC( vecSort, i ) + { + SQLProfileBucket_t &bucket = m_dictSQLBuckets[ vecSort[i] ]; + const char *pchStatement = m_dictSQLBuckets.GetElementName( vecSort[i] ); + + // cleanup the statement text + char rgchCleaned[140]; + V_strcpy_safe( rgchCleaned, pchStatement ); + for ( int i = 0; NULL != rgchCleaned[i]; i++ ) + { + if ( '\n' == rgchCleaned[i] || '\t' == rgchCleaned[i] ) + { + rgchCleaned[i] = ' '; + } + } + + bool bSeconds = bucket.m_nTotalMicrosec > k_nMillion; + float fTime = bucket.m_nTotalMicrosec / 1000.0f / ( bSeconds ? 1000.0f : 1.0f ); + + // render + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%8d %8.2f%s %8.2f %s\n", + bucket.m_unCount, + fTime, + bSeconds ? "s " : "ms", + (float)bucket.m_nTotalMicrosec / (float)bucket.m_unCount / 1000.0f, + rgchCleaned ); + } +} +#endif + + +//----------------------------------------------------------------------------- +// Purpose: pauses job until a work item completes +//----------------------------------------------------------------------------- +bool CJobMgr::BYieldingWaitForWorkItem( CJob &job, const char *pszWorkItemName ) +{ + // wait until we're woken up by a work item completed, or a timeout + PauseJob( job, k_EJobPauseReasonWorkItem ); + if ( m_bJobTimedOut || job.m_bWorkItemCanceled ) + return false; + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: adds a job work item to the thread pool +//----------------------------------------------------------------------------- +void CJobMgr::AddThreadedJobWorkItem( CWorkItem *pWorkItem ) +{ + m_WorkThreadPool.AddWorkItem( pWorkItem ); +} + + +//----------------------------------------------------------------------------- +// Purpose: returns true if we're still working +//----------------------------------------------------------------------------- +bool CJobMgr::HasOutstandingThreadPoolWorkItems() +{ + return m_WorkThreadPool.HasWorkItemsToProcess(); +} + +//----------------------------------------------------------------------------- +// Purpose: Mark that we're shutting down +//----------------------------------------------------------------------------- +void CJobMgr::SetIsShuttingDown() +{ + m_WorkThreadPool.AllowTimeouts( true ); // during shutdown, we might abort jobs before waiting for the work item to complete + m_bIsShuttingDown = true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Wakes up the specified waiting job. +// Input: jobID - The job that owns this work item +// bWorkItemCanceled - true if this job +// bShouldExist - Do we assert if the job doesn't exist? +// Output: true if the message was routed to a job +//----------------------------------------------------------------------------- +bool CJobMgr::BRouteWorkItemCompletedInternal( JobID_t jobID, bool bWorkItemCanceled, bool bShouldExist, bool bResumeImmediately ) +{ + int iJob; + + // this can resume jobs, make sure we didn't switch threads + CheckThreadID(); + + if ( !BGetIJob( jobID, k_EJobPauseReasonWorkItem, bShouldExist, &iJob ) ) + { + EG_MSG( SPEW_JOB, "BRouteWorkItemCompleted called for a job that could not be found!\n" ); + return false; + } + + // continue the job + m_MapJob[iJob]->m_bWorkItemCanceled = bWorkItemCanceled; + if ( bResumeImmediately ) + { + m_MapJob[iJob]->Continue(); + } + else + { + AddToYieldList( *m_MapJob[iJob] ); + + // reset the sleep reason + m_MapJob[iJob]->m_ePauseReason = k_EJobPauseReasonYield; + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds job to yield list (without actually pausing it) - internal +// Input : &job - job that is yielding +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +void CJobMgr::AddToYieldList( CJob &job ) +{ +#ifdef _DEBUG + FOR_EACH_LL( m_ListJobsYieldingRegPri, i ) + { + Assert( m_ListJobsYieldingRegPri[i].m_JobID != job.GetJobID() ); + } +#endif + + // insert the job into the sleep list + JobYielding_t jobYielding; + jobYielding.m_JobID = job.GetJobID(); + jobYielding.m_nIteration = m_nCurrentYieldIterationRegPri; + m_ListJobsYieldingRegPri.AddToTail( jobYielding ); +} + +//----------------------------------------------------------------------------- +// called by a job that has just been started to place itself on the yield queue instead of running +//----------------------------------------------------------------------------- +void CJobMgr::AddDelayedJobToYieldList( CJob &job ) +{ + //make sure that this job is setup to be yielded at this point, otherwise it will not resume properly + AssertMsg1( job.GetPauseReason() == k_EJobPauseReasonYield, "Delayed job %s was added to yield list but was not in expected yield state\n", job.GetName() ); + AddToYieldList( job ); +} + + +//----------------------------------------------------------------------------- +// Purpose: yields until the next Run() +// Input : &job - job that is yielding +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CJobMgr::BYield( CJob &job ) +{ + AddToYieldList( job ); + + // yield + PauseJob( job, k_EJobPauseReasonYield ); + if ( m_bJobTimedOut ) + return false; + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: yields IF NEEDED until the next Run() +// Input : &job - job that is possibly yielding +// pbYielded - optional, set to true if we did yield +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CJobMgr::BYieldIfNeeded( CJob &job, bool *pbYielded ) +{ + if ( pbYielded ) + *pbYielded = false; + + if ( job.GetMicrosecondsRun() > ( k_cMicroSecTaskGranularity / 2 ) ) + { + bool bRet = BYield( job ); + if ( pbYielded ) + *pbYielded = bRet; + return bRet; + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Resumes jobs in list passed in that are ready to be awakened +//----------------------------------------------------------------------------- +bool CJobMgr::BResumeYieldingJobsFromList( CUtlLinkedList<JobYielding_t, int> &listJobsYielding, uint nCurrentIteration, + CLimitTimer &limitTimer ) +{ + while ( listJobsYielding.Count() ) + { + int iJobYielding = listJobsYielding.Head(); + const JobYielding_t &jobYielding = listJobsYielding[ iJobYielding ]; + + if ( jobYielding.m_nIteration > nCurrentIteration ) + break; + + // pop the sleep off the top of the queue + int iJob = m_MapJob.Find( jobYielding.m_JobID ); + listJobsYielding.Remove( iJobYielding ); + + if ( m_MapJob.InvalidIndex() == iJob ) + continue; + + Assert( m_MapJob[iJob]->GetPauseReason() == k_EJobPauseReasonYield ); + + // Should never be false, but if it is we + // don't want to do anything to this job + if ( m_MapJob[iJob]->GetPauseReason() == k_EJobPauseReasonYield ) + { + // resume the job + m_MapJob[iJob]->Continue(); + } + + if ( limitTimer.BLimitReached() ) + break; + } + + return ( listJobsYielding.Count() > 0 ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Resumes any jobs that have are ready to be awaken +// Input: limitTimer - limit timer not to exceed +// Output: true if there is still work remaining to do, false otherwise +//----------------------------------------------------------------------------- +bool CJobMgr::BResumeYieldingJobs( CLimitTimer &limitTimer ) +{ + return BResumeYieldingJobsFromList( m_ListJobsYieldingRegPri, m_nCurrentYieldIterationRegPri++, limitTimer ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Resumes any jobs that have are ready to be awaken +// Input: limitTimer - limit timer not to exceed +// Output: true if there is still work remaining to do, false otherwise +//----------------------------------------------------------------------------- +bool CJobMgr::BResumeSleepingJobs( CLimitTimer &limitTimer ) +{ + while ( m_QueueJobSleeping.Count() ) + { + const JobSleeping_t &jobSleeping = m_QueueJobSleeping.ElementAtHead(); + if ( jobSleeping.m_SWakeupTime.LTime() > CJobTime::LJobTimeCur() ) + { + // Check if we need to heartbeat + if ( jobSleeping.m_STimeTouched.CServerMicroSecsPassed() >= k_cMicroSecJobHeartbeat ) + { + int iJob = m_MapJob.Find( jobSleeping.m_JobID ); + if ( m_MapJob.InvalidIndex() != iJob ) + { + m_MapJob[iJob]->Heartbeat(); + } + } + + return false; + } + + // pop the sleep off the top of the queue + int iJob = m_MapJob.Find( jobSleeping.m_JobID ); + m_QueueJobSleeping.RemoveAtHead(); + + if ( m_MapJob.InvalidIndex() == iJob ) + continue; + + Assert( m_MapJob[iJob]->GetPauseReason() == k_EJobPauseReasonSleepForTime ); + + // should never be false, but if it is we don't want to do anything to this job + if ( m_MapJob[iJob]->GetPauseReason() == k_EJobPauseReasonSleepForTime ) + { + // resume the job + m_MapJob[iJob]->Continue(); + } + + if ( limitTimer.BLimitReached() ) + break; + } + + return ( m_QueueJobSleeping.Count() > 0 ); +} + + +//----------------------------------------------------------------------------- +// Purpose: comparison function for sorting sleeping jobs list by time +// Output : Returns true on if lhs is greater than the rhs +//----------------------------------------------------------------------------- +bool CJobMgr::JobSleepingLessFunc( JobSleeping_t const &lhs, JobSleeping_t const &rhs ) +{ + // a lower time is a higher priority + return ( lhs.m_SWakeupTime.LTime() > rhs.m_SWakeupTime.LTime() ); +} + +JobID_t g_DebugJob = k_GIDNil; + +//----------------------------------------------------------------------------- +// Purpose: quickly iterates the list of jobs to make sure none have been paused +// for too long +//----------------------------------------------------------------------------- +void CJobMgr::CheckForJobTimeouts( CLimitTimer &limitTimer ) +{ + // look through each active jobs + // remove from the list any job that has successfully received it's I/O + // send a failure msg to any job that has timed out + // since the timeout time is constant, we only have to check until we find a job + + int cIter = 0; + while ( m_ListJobTimeouts.Head() != m_ListJobTimeouts.InvalidIndex() ) + { + cIter ++; + + // Break if limit timer is reached and we've already processed at least one item. + if ( cIter > 1 && limitTimer.BLimitReached() ) + break; + + JobTimeout_t &jobtimeout = m_ListJobTimeouts[ m_ListJobTimeouts.Head() ]; + + // see if it's timed out + if ( !m_bIsShuttingDown && jobtimeout.m_STimeTouched.CServerMicroSecsPassed() < k_cMicroSecJobHeartbeat ) + { + // we haven't reached our recycle or timeout limit, which means none of the jobs passed us in the queue would have either + break; + } + + // get the first job in the list, which is the most likely to have timed out + int iJob = m_MapJob.Find( jobtimeout.m_JobID ); + if ( m_MapJob.InvalidIndex() == iJob ) + { + m_MapJobTimeoutsIndexByJobID.Remove( jobtimeout.m_JobID ); + m_ListJobTimeouts.Remove( m_ListJobTimeouts.Head() ); + continue; + } + + // job still exists, make sure it is still paused at the same point + CJob *pJob = m_MapJob[iJob]; + + if ( pJob->GetTimeSwitched().LTime() == jobtimeout.m_STimePaused.LTime() ) + { + jobtimeout.m_cHeartbeatsBeforeTimeout--; + + if ( pJob->GetJobID() == g_DebugJob ) + { + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "Heartbeat!\n" ); + } + + // Always heartbeat so anyone waiting on the job (say on another server) will know it is still alive + // Note that we even do this right before we timeout, since the job will actually be continued and may just loop itself right back into this waiting state + // Note also that we do NOT check pJob->GetNextHeartbeatTime() since we've already been watching our own timer + pJob->Heartbeat(); + + if ( m_bIsShuttingDown || jobtimeout.m_cHeartbeatsBeforeTimeout <= 0 ) + { + // Job finished all its available heartbeats before its timeout limit, timeout if appropriate and remove from the list + + m_MapJobTimeoutsIndexByJobID.Remove( jobtimeout.m_JobID ); + m_ListJobTimeouts.Remove( m_ListJobTimeouts.Head() ); + + bool bShouldTimeout = true; + switch ( pJob->m_ePauseReason ) + { + case k_EJobPauseReasonWaitingForLock: + case k_EJobPauseReasonYield: + case k_EJobPauseReasonSQL: + bShouldTimeout = false; + break; + case k_EJobPauseReasonSleepForTime: + bShouldTimeout = m_bIsShuttingDown; + break; + } // switch + + // If the job WAS waiting on IO but now is waiting on a Lock, Sleeping, + // or Yielding, don't time it out. + // BUGBUG taylor we should fix things so that we can timeout Jobs waiting on + // Locks and have them properly unlink themselves from the Lock chain + if ( bShouldTimeout ) + { + TimeoutJob( *( pJob ) ); + } + } + else + { + // Job has not yet used up all its available heartbeats before its timeout limit + // We've already decremented its m_cHeartbeatsBeforeTimeout, now Reset its touched time too + jobtimeout.m_STimeTouched.SetToJobTime(); + // Move it back to the end of the queue so it can come back up to the top for either another heartbeat or a timeout + m_ListJobTimeouts.LinkToTail( m_ListJobTimeouts.Head() ); + int iIndexMap = m_MapJobTimeoutsIndexByJobID.Find( jobtimeout.m_JobID ); + if ( iIndexMap != m_MapJobTimeoutsIndexByJobID.InvalidIndex() ) + { + int &iListIndex = m_MapJobTimeoutsIndexByJobID.Element( iIndexMap ); + iListIndex = m_ListJobTimeouts.Tail(); + } + else + { + AssertMsg( false, "Map of jobs to timeout is corrupted" ); + } + } + + continue; + } + else + { + // This is really the common heartbeating case, where the job waited a short while without ever reaching the k_cMicroSecJobHeartbeat limit + // Thus, we need to heartbeat before removing it from the list IF the job has gone too long without heartbeating + if ( pJob->BJobNeedsToHeartbeat() ) + { + pJob->Heartbeat(); + } + + // Since the job didn't actually time out, clear this timeout event + m_MapJobTimeoutsIndexByJobID.Remove( jobtimeout.m_JobID ); + m_ListJobTimeouts.Remove( m_ListJobTimeouts.Head() ); + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Continues a job in a timed out state +//----------------------------------------------------------------------------- +void CJobMgr::TimeoutJob( CJob &job ) +{ + + if ( job.GetPauseReason() == k_EJobPauseReasonNetworkMsg ) + job.m_flags.m_bits.m_bTimeoutNetMsg = true; + else + { + // these are so rare I dont want to add a column for them in the rollup + EG_WARNING( SPEW_JOB, "Resuming job '%s (id: %lld)' due to timeout while paused for %s\n", job.GetName(), + job.GetJobID(), job.GetPauseReasonDescription() ); + job.m_flags.m_bits.m_bTimeoutOther = true; + } + + m_JobStats.m_cJobsTimedOut++; + m_bJobTimedOut = true; + job.Continue(); + m_bJobTimedOut = false; +} + + +//----------------------------------------------------------------------------- +// Purpose: wakes up a job that was waiting on a lock +//----------------------------------------------------------------------------- +void CJobMgr::WakeupLockedJob( CJob &job ) +{ + Assert( job.m_ePauseReason == k_EJobPauseReasonWaitingForLock ); + + // in case of error, bug out now so as not + // to cause more trouble + if ( job.m_ePauseReason != k_EJobPauseReasonWaitingForLock ) + { + return; + } + + // insert the job into the yielding list so it will wakeup next Run + AddToYieldList( job ); + + // reset the sleep reason + job.m_ePauseReason = k_EJobPauseReasonYield; +} + + +//----------------------------------------------------------------------------- +// Purpose: Pauses a job, and puts it in a list to check for timeouts +//----------------------------------------------------------------------------- +void CJobMgr::PauseJob( CJob &job, EJobPauseReason eJobPauseReason ) +{ + Assert( !m_bDebugDisallowPause ); + if ( m_bDebugDisallowPause ) + { + EmitError( SPEW_GC, "Job %s attempted to pause even though pauses were disabled\n", job.GetName() ); + } + + // add to list to check for timeouts later (or update the existing entry if it is already there) + JobTimeout_t *pJobTimeout; + int iMapIndex = m_MapJobTimeoutsIndexByJobID.Find( job.GetJobID() ); + if ( iMapIndex == m_MapJobTimeoutsIndexByJobID.InvalidIndex() ) + { + pJobTimeout = &m_ListJobTimeouts[ m_ListJobTimeouts.AddToTail() ]; + m_MapJobTimeoutsIndexByJobID.Insert( job.GetJobID(), m_ListJobTimeouts.Tail() ); + } + else + { + // There was an existing entry, in addition to updating it, move it to the tail + int &iListIndex = m_MapJobTimeoutsIndexByJobID.Element( iMapIndex ); + m_ListJobTimeouts.LinkToTail( iListIndex ); + iListIndex = m_ListJobTimeouts.Tail(); + + pJobTimeout = &m_ListJobTimeouts.Element( iListIndex ); + } + + pJobTimeout->m_JobID = job.GetJobID(); + pJobTimeout->m_STimePaused.SetToJobTime(); + pJobTimeout->m_STimeTouched.SetToJobTime(); + pJobTimeout->m_cHeartbeatsBeforeTimeout = job.CHeartbeatsBeforeTimeout(); + if ( eJobPauseReason == k_EJobPauseReasonWorkItem ) + { + // work items control their own schedule - wait up to 6 hours + pJobTimeout->m_cHeartbeatsBeforeTimeout = (6 * 60 * 60 * k_nMillion) / k_cMicroSecJobHeartbeat; + } + + if ( pJobTimeout->m_cHeartbeatsBeforeTimeout <= 0 ) + { + pJobTimeout->m_cHeartbeatsBeforeTimeout = k_cJobHeartbeatsBeforeTimeoutDefault; + } + + // tell the job to pause + job.Pause( eJobPauseReason ); +} + +//----------------------------------------------------------------------------- +// Purpose: dumps a list of currently active jobs to the console +// Output : int - number of jobs listed +//----------------------------------------------------------------------------- +int CJobMgr::DumpJobSummary() +{ + CUtlMap< uint32, JobStatsBucket_t, int > mapStatsBucket( 0, 0, DefLessFunc( uint32 ) ); + + FOR_EACH_MAP_FAST( m_MapJob, i ) + { + CJob &job = *m_MapJob[i]; + + // the pointer to the name is a pointer to a constant string + // so use this dirty trick to make lookups fast + uint32 eBucket = (uint32)job.GetName(); + int iBucket = mapStatsBucket.Find( eBucket ); + if ( iBucket == mapStatsBucket.InvalidIndex() ) + { + iBucket = mapStatsBucket.Insert( eBucket ); + V_strcpy_safe( mapStatsBucket[iBucket].m_rgchName, job.GetName() ); + } + + JobStatsBucket_t *pJobStatsBucket = &mapStatsBucket[iBucket]; + pJobStatsBucket->m_cCompletes++; // overloading this to really mean "jobs running" for this spew + pJobStatsBucket->m_cLocksAttempted += job.m_vecLocks.Count(); // overloading this to really be used for "locks held" for this spew + pJobStatsBucket->m_u64JobDuration += job.m_STimeStarted.CServerMicroSecsPassed(); + + switch ( job.m_ePauseReason ) + { + case k_EJobPauseReasonNetworkMsg: pJobStatsBucket->m_cPauseReasonNetworkMsg++; break; + case k_EJobPauseReasonSleepForTime: pJobStatsBucket->m_cPauseReasonSleepForTime++; break; + case k_EJobPauseReasonWaitingForLock: pJobStatsBucket->m_cPauseReasonWaitingForLock++; break; + case k_EJobPauseReasonYield: pJobStatsBucket->m_cPauseReasonYield++; break; + case k_EJobPauseReasonSQL: pJobStatsBucket->m_cPauseReasonSQL++; break; + case k_EJobPauseReasonWorkItem: pJobStatsBucket->m_cPauseReasonWorkItem++; break; + default: break; + } + } + + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, + "%50s --- running jobs (usec)-- -- locks held -- ----- pause reasons ---------------------------------\n", " " ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, + "%50s count aveduration netmsg sql sleep waitlock yield workitem\n", "name" ); + + JobProfileStats_t jobprofilestats; + jobprofilestats.m_iJobProfileSort = k_EJobProfileSortOrder_Count; + jobprofilestats.pmapStatsBucket = &mapStatsBucket; + + CUtlVector<int> vecSort( 0, mapStatsBucket.Count() ); + FOR_EACH_MAP_FAST( mapStatsBucket, iBucket ) + { + vecSort.AddToTail( iBucket ); + } + V_qsort_s( vecSort.Base(), vecSort.Count(), sizeof(int), (QSortCompareFuncCtx_t)ProfileSortFunc, &jobprofilestats ); + + FOR_EACH_VEC( vecSort, iVec ) + { + JobStatsBucket_t &bucket = mapStatsBucket[ vecSort[iVec] ]; + + int64 msecDurationAve = bucket.m_u64JobDuration / bucket.m_cCompletes; + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%50s %8lld %16lld %13lld %11lld %8lld %8lld %8lld %8lld %8lld \n", + bucket.m_rgchName, + bucket.m_cCompletes, + msecDurationAve, + bucket.m_cLocksAttempted, + + bucket.m_cPauseReasonNetworkMsg, + bucket.m_cPauseReasonSQL, + bucket.m_cPauseReasonSleepForTime, + bucket.m_cPauseReasonWaitingForLock, + bucket.m_cPauseReasonYield, + bucket.m_cPauseReasonWorkItem + ); + } + + return m_MapJob.Count(); +} + + +//----------------------------------------------------------------------------- +// Purpose: spews details about a job by ID +//----------------------------------------------------------------------------- +void CJobMgr::DumpJob( JobID_t jobID, int nPrintLocksMax ) const +{ + const CJob *pJob = GetPJob( jobID ); + if( !pJob ) + { + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "Invalid job ID %llu\n", jobID ); + } + else + { + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%llu\t%12s %12s\n", + pJob->GetJobID(), + pJob->GetName(), + pJob->GetPauseReasonDescription() ); + + if ( pJob->GetPauseReason() == k_EJobPauseReasonWaitingForLock && pJob->m_pWaitingOnLock != NULL ) + { + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\tWaiting for lock %s from: %s line %d\n", pJob->m_pWaitingOnLock->GetName(), pJob->m_pWaitingOnLockFilename, pJob->m_waitingOnLockLine ); + pJob->m_pWaitingOnLock->Dump( "\t ", nPrintLocksMax, true ); + } + + FOR_EACH_VEC( pJob->m_vecLocks, nLock ) + { + CLock *pLock = pJob->m_vecLocks[nLock]; + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\tHolding lock %s:\n", pLock->GetName() ); + pLock->Dump( "\t ", nPrintLocksMax, true ); + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: count the number of active jobs +//----------------------------------------------------------------------------- +int CJobMgr::CountJobs() const +{ + return m_MapJob.Count(); +} + + +//----------------------------------------------------------------------------- +// Purpose: verify that current thread is correct +//----------------------------------------------------------------------------- +void CJobMgr::CheckThreadID() +{ + uint unCurrentThread = ThreadGetCurrentId(); + + if ( m_unFrameFuncThreadID == 0 ) + { + m_unFrameFuncThreadID = unCurrentThread; + } + else + { + // if this Assert goes of, you most likely tried to start + // a job from a different thread then the frame function thread + Assert( m_unFrameFuncThreadID == unCurrentThread ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: JobType_t comparer, used to sort the list of registered +// jobs into a tree by msg that creates them +//----------------------------------------------------------------------------- +bool JobTypeSortFuncByMsg( JobType_t const * const &lhs, JobType_t const * const &rhs ) +{ + if ( lhs->m_eCreationMsg == rhs->m_eCreationMsg ) + { + return ( lhs->m_eServerType < rhs->m_eServerType ); + } + + return ( lhs->m_eCreationMsg < rhs->m_eCreationMsg ); +} + + +//----------------------------------------------------------------------------- +// Purpose: JobType_t comparer, used to sort the list of registered +// jobs into a tree by job name +//----------------------------------------------------------------------------- +bool JobTypeSortFuncByName( JobType_t const * const &lhs, JobType_t const * const &rhs ) +{ + int iCompare = Q_strcmp( lhs->m_pchName, rhs->m_pchName ); + if ( iCompare == 0 ) + { + return ( lhs->m_eServerType < rhs->m_eServerType ); + } + + return ( iCompare < 0 ); +} + + +// singeton accessor to list of registered jobs +CUtlRBTree<const JobType_t *> &GMapJobTypesByMsg() +{ + static CUtlRBTree<const JobType_t *> s_MapJobTypes( 0, 0, JobTypeSortFuncByMsg ); + return s_MapJobTypes; +} + +// singeton accessor to list of registered jobs +CUtlRBTree<const JobType_t *> &GMapJobTypesByName() +{ + static CUtlRBTree<const JobType_t *> s_MapJobTypes( 0, 0, JobTypeSortFuncByName ); + return s_MapJobTypes; +} + + +//----------------------------------------------------------------------------- +// Purpose: adds a new type of job into the global list +//----------------------------------------------------------------------------- +void CJobMgr::RegisterJobType( const JobType_t *pJobType ) +{ + Assert( pJobType->m_pchName != NULL ); + Assert( pJobType->m_pJobFactory != NULL ); + GMapJobTypesByMsg().Insert( pJobType ); + GMapJobTypesByName().Insert( pJobType ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Creates a new job from the network msg +// Input : *pServerParent - server to attach job to +// msg - network msg +// Output : true if a job was created +//----------------------------------------------------------------------------- +bool CJobMgr::BLaunchJobFromNetworkMsg( void *pParent, const JobMsgInfo_t &jobMsgInfo, IMsgNetPacket *pNetPacket ) +{ + if ( pNetPacket == NULL ) + { + AssertMsg(pNetPacket, "CJobMgr::BLaunchJobFromNetworkMsg received NULL packet."); + return false; + } + + if ( pNetPacket->BHasTargetJobName() && BIsValidSystemMsg( pNetPacket->GetEMsg(), NULL ) ) + { + JobType_t jobSearch = { pNetPacket->GetTargetJobName(), k_EGCMsgInvalid, jobMsgInfo.m_eServerType }; + int iJobType = GMapJobTypesByName().Find( &jobSearch ); + + if ( GMapJobTypesByName().IsValidIndex( iJobType ) ) + { + + // Get shortcut to job info + const JobType_t *pJobType = (GMapJobTypesByName())[iJobType]; + Assert( pJobType ); + Assert( pJobType->m_pchName ); + + // Create the job + CJob *job = pJobType->m_pJobFactory( pParent, NULL ); + + // Safety check + if ( job == NULL ) + { + AssertMsg1( job, "Job factory returned NULL for job named '%s'!\n", pJobType->m_pchName ); + return false; + } + + // Start the job + job->StartJobFromNetworkMsg( pNetPacket, jobMsgInfo.m_JobIDSource ); + return true; + } + } + else + { + JobType_t jobSearch = { 0, jobMsgInfo.m_eMsg, jobMsgInfo.m_eServerType }; + int iJobType = GMapJobTypesByMsg().Find( &jobSearch ); + + if ( GMapJobTypesByMsg().IsValidIndex( iJobType ) ) + { + + // Get shortcut to job info + const JobType_t *pJobType = (GMapJobTypesByMsg())[iJobType]; + Assert( pJobType ); + Assert( pJobType->m_pchName ); + + // Create the job + CJob *job = pJobType->m_pJobFactory( pParent, NULL ); + + // Safety check + if ( job == NULL ) + { + AssertMsg3( job, "Job factory returned NULL for job msg %d, server type %d (named '%s')!\n", (int)jobMsgInfo.m_eMsg, (int)jobMsgInfo.m_eServerType, pJobType->m_pchName ); + return false; + } + + // Start the job + job->StartJobFromNetworkMsg( pNetPacket, jobMsgInfo.m_JobIDSource ); + return true; + } + } + + return false; +} + + +//----------------------------------------------------------------------------- +// Purpose: profile sort func +//----------------------------------------------------------------------------- +int CJobMgr::ProfileSortFunc( void *pCtx, const int *lhs, const int *rhs ) +{ + JobProfileStats_t *pJobprofilestats = (JobProfileStats_t *)pCtx; + int64 d = 0; + switch ( pJobprofilestats->m_iJobProfileSort ) + { + default: + case k_EJobProfileSortOrder_Alpha: + return Q_stricmp( pJobprofilestats->pmapStatsBucket->Element(*lhs).m_rgchName, + pJobprofilestats->pmapStatsBucket->Element(*rhs).m_rgchName ); + case k_EJobProfileSortOrder_Count: + d = ((int64)pJobprofilestats->pmapStatsBucket->Element(*rhs).m_cCompletes - + (int64)pJobprofilestats->pmapStatsBucket->Element(*lhs).m_cCompletes); + break; + case k_EJobProfileSortOrder_TotalRuntime: + d = ((int64)pJobprofilestats->pmapStatsBucket->Element(*rhs).m_u64RunTime - + (int64)pJobprofilestats->pmapStatsBucket->Element(*lhs).m_u64RunTime); + break; + } + if ( d < 0 ) + return -1; + if ( d > 0 ) + return 1; + return 0; +} + + +//----------------------------------------------------------------------------- +// Purpose: dump out accumulated job profile data +//----------------------------------------------------------------------------- +void CJobMgr::ProfileJobs( EJobProfileAction ejobProfileAction, EJobProfileSortOrder iSortOrder ) +{ + bool bClearBuckets = false; + + if ( ejobProfileAction == k_EJobProfileAction_Start ) + { + if ( !m_bProfiling ) + { + bClearBuckets = true; + } + m_bProfiling = true; + } + else if ( ejobProfileAction == k_EJobProfileAction_Stop ) + { + m_bProfiling = false; + } + else if ( ejobProfileAction == k_EJobProfileAction_Clear ) + { + bClearBuckets = true; + } + + if ( bClearBuckets ) + { + m_mapStatsBucket.RemoveAll(); + } + + if ( k_EJobProfileAction_Dump != ejobProfileAction ) + return; + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, + "%44s --- completed jobs (usec)---------------------------------- ------ lock counts---------------------------------- ------ failures -----------\n", " " ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, + "%44s count averuntime maxruntime aveduration #yielded attempted waited failed longheld longwait wait-t/o t/o-msg jobfailed longslice\n", "name" ); + + JobProfileStats_t jobprofilestats; + jobprofilestats.m_iJobProfileSort = iSortOrder; + jobprofilestats.pmapStatsBucket = &m_mapStatsBucket; + + CUtlVector<int> vecSort( 0, m_mapStatsBucket.Count() ); + FOR_EACH_MAP_FAST( m_mapStatsBucket, iBucket ) + { + vecSort.AddToTail( iBucket ); + } + V_qsort_s( vecSort.Base(), vecSort.Count(), sizeof(int), (QSortCompareFuncCtx_t)ProfileSortFunc, &jobprofilestats ); + + FOR_EACH_VEC( vecSort, iVec ) + { + JobStatsBucket_t &bucket = m_mapStatsBucket[ vecSort[iVec] ]; + if ( bucket.m_cCompletes ) + { + CCycleCount ccRunTime( bucket.m_u64RunTime / bucket.m_cCompletes ); + int64 usecAve = ccRunTime.GetMicroseconds(); + + CCycleCount ccRunTimeMax( bucket.m_u64RunTimeMax ); + int64 usecMax = ccRunTimeMax.GetMicroseconds(); + + int64 msecDurationAve = bucket.m_u64JobDuration / bucket.m_cCompletes; + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%44s %12lld %12lld %12lld %12lld %8lld %8lld %8lld %8lld %8lld %8lld %8lld %8lld %8lld %8lld\n", + bucket.m_rgchName, + bucket.m_cCompletes, + usecAve, + usecMax, + msecDurationAve, + bucket.m_cJobsPaused, + bucket.m_cLocksAttempted, + bucket.m_cLocksWaitedFor, + bucket.m_cLocksFailed, + bucket.m_cLocksLongHeld, + bucket.m_cLocksLongWait, + bucket.m_cWaitTimeout, + bucket.m_cTimeoutNetMsg, + bucket.m_cJobsFailed, + bucket.m_cLongInterYieldTime ); + } + } + if ( m_mapOrphanMessages.Count() ) + { + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "Messages that arrived responding to jobs that no longer exists and were dropped\n" ); + FOR_EACH_MAP_FAST( m_mapOrphanMessages, iBucket ) + { + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%44s %12d\n", PchMsgNameFromEMsg( m_mapOrphanMessages.Key(iBucket) ), m_mapOrphanMessages[iBucket] ); + } + m_mapOrphanMessages.RemoveAll(); + } +} + +//----------------------------------------------------------------------------- +// Purpose: Dump a list of all jobs to the console +// Each job is indexed, and that index can be used with +// DebugJob() to cause a debug break in that job. +//----------------------------------------------------------------------------- +void CJobMgr::DumpJobs( const char *pszJobName, int nMax, int nPrintLocksMax ) const +{ + FOR_EACH_MAP_FAST( m_MapJob, iJob ) + { + if ( nMax <= 0 ) + break; + nMax--; + + if ( pszJobName == NULL || V_strcmp( pszJobName, m_MapJob[iJob]->GetName() ) == 0 ) + { + DumpJob( m_MapJob.Key(iJob), nPrintLocksMax ); + } + } + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "Total job count: %d\n", m_MapJob.Count() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: cause a debug break in the given job +//----------------------------------------------------------------------------- +void CJobMgr::DebugJob( int iJob ) +{ +#ifdef DEBUG_JOB_LIST + if ( sm_listAllJobs.IsValidIndex( iJob ) ) + { + sm_listAllJobs[iJob]->Debug(); + } + else + { + EmitInfo( SPEW_CONSOLE, 1, 1, "Job not found\n" ); + } +#else + EmitInfo( SPEW_CONSOLE, 1, 1, "Job debugging disabled\n" ); +#endif +} + + +#ifdef DBGFLAG_VALIDATE +//----------------------------------------------------------------------------- +// Purpose: Run a global validation pass on all of our data structures and memory +// allocations. +// Input: validator - Our global validator object +// pchName - Our name (typically a member var in our container) +//----------------------------------------------------------------------------- +void CJobMgr::Validate( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE(); + + ValidateObj( m_MapJob ); + FOR_EACH_MAP_FAST( m_MapJob, iJob ) + { + ValidatePtr( m_MapJob[iJob] ); + } + + ValidateObj( m_mapStatsBucket ); + FOR_EACH_MAP_FAST( m_mapStatsBucket, iBucket ) + { + ValidateObj( m_mapStatsBucket[iBucket] ); + } + + ValidateObj( m_ListJobsYieldingRegPri ); + ValidateObj( m_ListJobTimeouts ); + ValidateObj( m_MapJobTimeoutsIndexByJobID ); + ValidateObj( m_QueueJobSleeping ); + ValidateObj( m_WorkThreadPool ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Run a global validation pass on all of our global data +// Input: validator - Our global validator object +//----------------------------------------------------------------------------- +void CJobMgr::ValidateStatics( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE_STATIC( "CJobMgr class statics" ); + + ValidateObj( GMapJobTypesByMsg() ); + ValidateObj( GMapJobTypesByName() ); +#ifdef DEBUG_JOB_LIST + ValidateObj( sm_listAllJobs ); +#endif +} +#endif // DBGFLAG_VALIDATE + +} // namespace GCSDK diff --git a/gcsdk/jobtime.cpp b/gcsdk/jobtime.cpp new file mode 100644 index 0000000..c2dd039 --- /dev/null +++ b/gcsdk/jobtime.cpp @@ -0,0 +1,78 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Encapsultes the job system's version of time (which is != to wall clock time) +// +//============================================================================= + +#include "stdafx.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + +uint64 CJobTime::sm_lTimeCur = 0; + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CJobTime::CJobTime() +{ + m_lTime = sm_lTimeCur; +} + + +//---------------------------------------------------------------------------- +// Purpose: Sets our value to the current time +//----------------------------------------------------------------------------- +void CJobTime::SetToJobTime() +{ + m_lTime = sm_lTimeCur; +} + + +//----------------------------------------------------------------------------- +// Purpose: Sets our value as an offset from the current time +//----------------------------------------------------------------------------- +void CJobTime::SetFromJobTime( int64 dMicroSecOffset ) +{ + m_lTime = sm_lTimeCur + dMicroSecOffset; +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns the amount of time that's passed between our time and the +// current time. +// Output: Time that's passed between our time and the current time +//----------------------------------------------------------------------------- +int64 CJobTime::CServerMicroSecsPassed() const +{ + return( sm_lTimeCur - m_lTime ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Updates our current time value. We only +// update the time once per frame-- the rest of the time, we just +// access a cached copy of the time. On the server, we use an arbitrary +// time value beginning at 0, and incremented by 50 milliseconds every +// frame. +// NOTE: This should only be called once per frame. +//----------------------------------------------------------------------------- +void CJobTime::UpdateJobTime( int cMicroSecPerShellFrame ) +{ + sm_lTimeCur += cMicroSecPerShellFrame; // force a 50 msec clock +} + + +//----------------------------------------------------------------------------- +// Purpose: Stomps the current clock with the specified time +// Input : lCurrentTime - new current time +//----------------------------------------------------------------------------- +void CJobTime::SetCurrentJobTime( uint64 lCurrentTime ) +{ + sm_lTimeCur = lCurrentTime; +} + +} // namespace GCSDK
\ No newline at end of file diff --git a/gcsdk/messagelist.cpp b/gcsdk/messagelist.cpp new file mode 100644 index 0000000..f1d9732 --- /dev/null +++ b/gcsdk/messagelist.cpp @@ -0,0 +1,673 @@ +//====== Copyright , Valve Corporation, All rights reserved. ======= +// +// Purpose: Provides names for GC message types +// +//============================================================================= + + +#include "stdafx.h" + + + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + + +namespace GCSDK +{ + +//----------------------------------------------------------------------------- +// Purpose: allow global initializers for a list of message infos. The message +// list will assemble them all into a single list when it initializes. +//----------------------------------------------------------------------------- +class CMessageListRegistration +{ +public: + CMessageListRegistration( MsgInfo_t *pMsgInfo, int cMsgInfo, void *pExtra = NULL ); + + static CMessageListRegistration *sm_pFirst; + CMessageListRegistration *m_pNext; + MsgInfo_t *m_pMsgInfo; + int m_cMsgInfo; +}; + + +DECLARE_GC_EMIT_GROUP( g_EGMessages, messages ); + +//function called when a message is tallied but not registered properly. Used to help consolidate behavior for untracked messages +static void HandleUntrackedMsg( const char* pszMsgOperation, uint32 nMsgID ) +{ + //for now output this as verbose so that we can clean up the initial spam. Then promote this to warnings in the future + EG_VERBOSE( g_EGMessages, "Found %s message with ID %d that was not properly registered. Unable to tally information\n", pszMsgOperation, nMsgID ); +} + + +CMessageListRegistration::CMessageListRegistration( MsgInfo_t *pMsgInfo, int cMsgInfo, void *pExtra ) +: m_pMsgInfo( pMsgInfo ), + m_cMsgInfo( cMsgInfo ) +{ + m_pNext = sm_pFirst; + sm_pFirst = this; +} + +CMessageListRegistration *CMessageListRegistration::sm_pFirst = NULL; + +//----------------------------------------------------------------------------- +// Purpose: Returns the name of a message type +//----------------------------------------------------------------------------- +const char *PchMsgNameFromEMsg( MsgType_t eMsg ) +{ + const char *pchMsgName = k_rgchUnknown; + g_theMessageList.GetMessage( eMsg, &pchMsgName, MT_GC ); + return pchMsgName; +} + + +//----------------------------------------------------------------------------- +void MsgRegistrationFromEnumDescriptor( const ::google::protobuf::EnumDescriptor *pEnumDescriptor, int nTypeMask ) +{ + // build the struct list for messages + MsgInfo_t *pMsgInfos = new MsgInfo_t[ pEnumDescriptor->value_count() ]; + memset( pMsgInfos, 0, sizeof( MsgInfo_t ) * pEnumDescriptor->value_count() ); + + for ( int i = 0; i < pEnumDescriptor->value_count(); ++i ) + { + const ::google::protobuf::EnumValueDescriptor *pEnumValueDescriptor = pEnumDescriptor->value(i); + pMsgInfos[ i ].eMsg = pEnumValueDescriptor->number(); + pMsgInfos[ i ].pchMsgName = pEnumValueDescriptor->name().c_str(); + pMsgInfos[ i ].nFlags = nTypeMask; + } + + new CMessageListRegistration( pMsgInfos, pEnumDescriptor->value_count() ); +} + +//----------------------------------------------------------------------------- + +CMessageList g_theMessageList; + +//----------------------------------------------------------------------------- +// CMessageList +// +// builds a hash of the MsgInfo_t table so that information about messages +// can be found without searching. +//----------------------------------------------------------------------------- + +CMessageList::CMessageList( ) : m_bProfiling( false ), m_ulProfileMicrosecs( 0 ) +{ +} + + +//----------------------------------------------------------------------------- +// CMessageList +// +// builds a hash of the MsgInfo_t table so that information about messages +// can be found without searching. +//----------------------------------------------------------------------------- +bool CMessageList::BInit( ) +{ + m_bProfiling = false; + m_ulProfileMicrosecs = 0; + + // figure out our message count + int cMessageInfos = 0; + for( CMessageListRegistration *pReg = CMessageListRegistration::sm_pFirst; pReg != NULL; pReg = pReg->m_pNext) + { + cMessageInfos += pReg->m_cMsgInfo; + } + + // message indexes should fit in a short + Assert( cMessageInfos < SHRT_MAX ); + + m_vecMsgInfo.EnsureCapacity( cMessageInfos ); + m_vecMsgInfo.RemoveAll(); + m_vecMessageInfoBuckets.RemoveAll(); + + int nIndex = 0; + for( CMessageListRegistration *pReg = CMessageListRegistration::sm_pFirst; pReg != NULL; pReg = pReg->m_pNext) + { + for ( int nRegIndex = 0; nRegIndex < pReg->m_cMsgInfo; nRegIndex++ ) + { + nIndex = m_vecMsgInfo.AddToTail( pReg->m_pMsgInfo[nRegIndex] ); + + int nSlot; + int nBucket = HashMessage( pReg->m_pMsgInfo[nRegIndex].eMsg, nSlot ); + + AssureBucket( nBucket ); + + if ( m_vecMessageInfoBuckets[nBucket][nSlot] != -1 ) + { + int otherIndex = m_vecMessageInfoBuckets[nBucket][nSlot]; + MsgInfo_t &otherMsg = m_vecMsgInfo[ otherIndex ]; + AssertFatalMsg2( false, "Message collision: %s redefined as %s", + otherMsg.pchMsgName, + pReg->m_pMsgInfo[nRegIndex].pchMsgName); + } + else + { + m_vecMessageInfoBuckets[nBucket][nSlot] = (short) nIndex; + } + } + } + + //start our window + ResetWindow(); + //and start our global timer + m_sCollectTime[ MsgInfo_t::k_EStatsGroupGlobal ].SetToJobTime(); + + return true; +} + + +//----------------------------------------------------------------------------- +// Tally the sending of a message +//----------------------------------------------------------------------------- +void CMessageList::TallySendMessage( MsgType_t eMsgType, uint32 unMsgSize, uint32 nSourceMask ) +{ + short nIndex = GetMessageIndex( eMsgType ); + if ( nIndex == - 1 ) + { + HandleUntrackedMsg( "send", eMsgType ); + return; + } + + TallyMessageInternal( m_vecMsgInfo[nIndex],MsgInfo_t::k_EStatsTypeSent, unMsgSize, nSourceMask ); +} + + +//----------------------------------------------------------------------------- +// Tally the receiving of a message +//----------------------------------------------------------------------------- +void CMessageList::TallyReceiveMessage( MsgType_t eMsgType, uint32 unMsgSize, uint32 nSourceMask ) +{ + short nIndex = GetMessageIndex( eMsgType ); + if ( nIndex == - 1 ) + { + HandleUntrackedMsg( "receive", eMsgType ); + return; + } + + TallyMessageInternal( m_vecMsgInfo[nIndex], MsgInfo_t::k_EStatsTypeReceived, unMsgSize, nSourceMask ); +} + + +//----------------------------------------------------------------------------- +// Tally the receiving of a message +//----------------------------------------------------------------------------- +void CMessageList::TallyMultiplexedMessage( MsgType_t eMsgType, uint32 uSent, uint32 cRecipients, uint32 uMsgSize, uint32 nSourceMask ) +{ + short nIndex = GetMessageIndex( eMsgType ); + if ( nIndex == - 1 ) + { + HandleUntrackedMsg( "multiplex", eMsgType ); + return; + } + + MsgInfo_t &msgInfo = m_vecMsgInfo[nIndex]; + TallyMessageInternal( msgInfo, MsgInfo_t::k_EStatsTypeMultiplexedSends, uSent, nSourceMask, 1 ); + TallyMessageInternal( msgInfo, MsgInfo_t::k_EStatsTypeMultiplexedSendsRaw, uMsgSize * cRecipients, nSourceMask, cRecipients ); +} + + +//----------------------------------------------------------------------------- +// Tally the receiving of a message +//----------------------------------------------------------------------------- +void CMessageList::TallyMessageInternal( MsgInfo_t &msgInfo, MsgInfo_t::EStatsType eBucket, uint32 unMsgSize, uint32 nSourceMask, uint32 cMessages ) +{ + //log this for global stats + msgInfo.stats[ MsgInfo_t::k_EStatsGroupGlobal ][ eBucket ].nCount += cMessages; + msgInfo.stats[ MsgInfo_t::k_EStatsGroupGlobal ][ eBucket ].uBytes += unMsgSize; + msgInfo.stats[ MsgInfo_t::k_EStatsGroupGlobal ][ eBucket ].nSourceMask |= nSourceMask; + + //track this in our current timing window + msgInfo.stats[ MsgInfo_t::k_EStatsGroupWindow ][ eBucket ].nCount += cMessages; + msgInfo.stats[ MsgInfo_t::k_EStatsGroupWindow ][ eBucket ].uBytes += unMsgSize; + msgInfo.stats[ MsgInfo_t::k_EStatsGroupWindow ][ eBucket ].nSourceMask |= nSourceMask; + + //track our window totals as well + m_WindowTotals[ eBucket ].nCount += cMessages; + m_WindowTotals[ eBucket ].uBytes += unMsgSize; + m_WindowTotals[ eBucket ].nSourceMask |= nSourceMask; + + //and if we are profiling, track the data into our profiling window + if ( m_bProfiling ) + { + msgInfo.stats[ MsgInfo_t::k_EStatsGroupProfile ][ eBucket ].nCount += cMessages; + msgInfo.stats[ MsgInfo_t::k_EStatsGroupProfile ][ eBucket ].uBytes += unMsgSize; + msgInfo.stats[ MsgInfo_t::k_EStatsGroupProfile ][ eBucket ].nSourceMask |= nSourceMask; + } +} + +//called to obtain the totals for the timing window +const MsgInfo_t::Stats_t& CMessageList::GetWindowTotal( MsgInfo_t::EStatsType eType ) const +{ + return m_WindowTotals[ eType ]; +} + +//called to reset the window timings +void CMessageList::ResetWindow() +{ + //reset when our window started + m_sCollectTime[ MsgInfo_t::k_EStatsGroupWindow ].SetToJobTime(); + + // starting a new window... clear everything + FOR_EACH_VEC( m_vecMsgInfo, nIndex ) + { + for( uint32 nType = 0; nType < MsgInfo_t::k_EStatsType_Count; nType++ ) + { + m_vecMsgInfo[ nIndex ].stats[ MsgInfo_t::k_EStatsGroupWindow ][ nType ].nCount = 0; + m_vecMsgInfo[ nIndex ].stats[ MsgInfo_t::k_EStatsGroupWindow ][ nType ].uBytes = 0; + m_vecMsgInfo[ nIndex ].stats[ MsgInfo_t::k_EStatsGroupWindow ][ nType ].nSourceMask = 0; + } + } + + //and clear our totals + for( uint32 nType = 0; nType < MsgInfo_t::k_EStatsType_Count; nType++ ) + { + m_WindowTotals[ nType ].nCount = 0; + m_WindowTotals[ nType ].uBytes = 0; + } +} + + +//----------------------------------------------------------------------------- +// Turns snapshot profiling on and off +//----------------------------------------------------------------------------- +void CMessageList::EnableProfiling( bool bEnableProfiling ) +{ + m_bProfiling = bEnableProfiling; + if ( m_bProfiling ) + { + m_sCollectTime[ MsgInfo_t::k_EStatsGroupProfile ].SetToJobTime(); + + // starting a new profile... clear everything + FOR_EACH_VEC( m_vecMsgInfo, nIndex ) + { + for( uint32 nType = 0; nType < MsgInfo_t::k_EStatsType_Count; nType++ ) + { + m_vecMsgInfo[ nIndex ].stats[ MsgInfo_t::k_EStatsGroupProfile ][ nType ].nCount = 0; + m_vecMsgInfo[ nIndex ].stats[ MsgInfo_t::k_EStatsGroupProfile ][ nType ].uBytes = 0; + m_vecMsgInfo[ nIndex ].stats[ MsgInfo_t::k_EStatsGroupProfile ][ nType ].nSourceMask = 0; + } + } + } + else + { + m_ulProfileMicrosecs = m_sCollectTime[ MsgInfo_t::k_EStatsGroupProfile ].CServerMicroSecsPassed(); + } +} + + +uint64 CMessageList::GetGroupDuration( MsgInfo_t::EStatsGroup eGroup ) const +{ + //handle the special case of it being a profile, where if we are no longer profiling, we want to use our cached value + if( ( eGroup == MsgInfo_t::k_EStatsGroupProfile ) && !m_bProfiling ) + { + return m_ulProfileMicrosecs; + } + + //otherwise we can just use the timer directly + return m_sCollectTime[ eGroup ].CServerMicroSecsPassed(); +} + +//----------------------------------------------------------------------------- +// print statistics bout each message we handle +//----------------------------------------------------------------------------- +void CMessageList::PrintStats( bool bShowAll, bool bSortByFrequency, MsgInfo_t::EStatsGroup eGroup, MsgInfo_t::EStatsType eType, uint32 nSourceMask ) const +{ + // Figure out which time value we should use for rate calcs + uint64 ulMicroseconds = GetGroupDuration( eGroup ); + + // work out a sorted list + CUtlMap<uint64, uint32> mapValueToMsg( DefLessFunc( uint64 ) ); + uint64 unTotalBytes = 0; + uint32 unTotalMessages = 0; + + FOR_EACH_VEC( m_vecMsgInfo, n ) + { + //see if we are looking for a particular source, and if so, if this has that source set + if( nSourceMask && ( ( m_vecMsgInfo[n].stats[ eGroup ][ eType ].nSourceMask & nSourceMask ) == 0 ) ) + continue; + + uint32 nCount = m_vecMsgInfo[n].stats[ eGroup ][ eType ].nCount; + uint64 uBytes = m_vecMsgInfo[n].stats[ eGroup ][ eType ].uBytes; + + unTotalMessages += nCount; + unTotalBytes += uBytes; + + if ( nCount == 0 && !bShowAll ) + continue; + + mapValueToMsg.Insert( bSortByFrequency ? (uint64)nCount : uBytes, n ); + } + + double fSeconds = ulMicroseconds / ( 1000.0 * 1000.0 ); + + // 5, 46, 10, 6, 14, 6, 10, 10 + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%s", "EMsg MessageName Count % Bytes % MsgPS Avg/Msg KBPS\n" ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%s", "----- ---------------------------------------------- ---------- ------ -------------- ------ --------- ---------- ----------\n" ); + + for ( uint16 iValue = mapValueToMsg.LastInorder(); iValue != mapValueToMsg.InvalidIndex(); iValue = mapValueToMsg.PrevInorder( iValue ) ) + { + uint32 n = mapValueToMsg[iValue]; + + //see if we are looking for a particular source, and if so, if this has that source set + if( nSourceMask && ( ( m_vecMsgInfo[n].stats[ eGroup ][ eType ].nSourceMask & nSourceMask ) == 0 ) ) + continue; + + uint32 nCount = m_vecMsgInfo[n].stats[ eGroup ][ eType].nCount; + uint64 uBytes = m_vecMsgInfo[n].stats[ eGroup ][ eType].uBytes; + + uint32 uAvgMsg = 0; + if ( nCount > 0 ) + uAvgMsg = uBytes / nCount; + + float flCountPerc = 0; + if ( unTotalMessages > 0 ) + flCountPerc = nCount * 100.0f / unTotalMessages; + float flBytesPerc = 0; + if ( unTotalBytes > 0 ) + flBytesPerc = uBytes * 100.0f / unTotalBytes; + double fMsgPS = 0.0; + if ( ulMicroseconds > 0 ) + fMsgPS = nCount / fSeconds; + double fKBPS = 0.0f; + if ( ulMicroseconds > 0 ) + fKBPS = uBytes / ( fSeconds * 1000.0 ); + + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%5u %-46s %10u %5.1f%% %14llu %5.1f%% %9.1f %10u %10.2f\n", + m_vecMsgInfo[n].eMsg, + m_vecMsgInfo[n].pchMsgName, + nCount, + flCountPerc, + uBytes, + flBytesPerc, + fMsgPS, + uAvgMsg, + fKBPS ); + } + + uint32 uAvgMsgTotal = 0; + if ( unTotalMessages > 0 ) + uAvgMsgTotal = unTotalBytes / unTotalMessages; + double fMsgPS = 0.0; + if ( ulMicroseconds > 0 ) + fMsgPS = unTotalMessages / fSeconds; + double fKBPSTotal = 0.0; + if ( ulMicroseconds > 0 ) + fKBPSTotal = unTotalBytes / ( fSeconds * 1000.0 ); + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "----- ---------------------------------------------- ---------- ------ -------------- ------ --------- ---------- ----------\n" ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "Totals %10u 100.0%% %14llu 100.0%% %9.1f %10u %10.2f\n", + unTotalMessages, + unTotalBytes, + fMsgPS, + uAvgMsgTotal, + fKBPSTotal ); +} + + +//----------------------------------------------------------------------------- +// print statistics bout each message we handle +//----------------------------------------------------------------------------- +void CMessageList::PrintMultiplexStats( MsgInfo_t::EStatsGroup eGroup, bool bSortByFrequency, uint32 nSourceMask ) const +{ + // Figure out which time value we should use for rate calcs + uint64 ulMicroseconds = GetGroupDuration( eGroup ); + + MsgInfo_t::Stats_t rgTotals[MsgInfo_t::k_EStatsType_Count]; + + const MsgInfo_t::EStatsType eMultiType = MsgInfo_t::k_EStatsTypeMultiplexedSends; + const MsgInfo_t::EStatsType eRawType = MsgInfo_t::k_EStatsTypeMultiplexedSendsRaw; + + // work out a sorted list + CUtlMap<uint64, uint32> mapValueToMsg( DefLessFunc( uint64 ) ); + FOR_EACH_VEC( m_vecMsgInfo, n ) + { + const MsgInfo_t &msgInfo = m_vecMsgInfo[n]; + + //see if we are looking for a particular source, and if so, if this has that source set + if( nSourceMask && ( msgInfo.stats[ eGroup ][ eMultiType ].nSourceMask & nSourceMask ) == 0 ) + continue; + + rgTotals[eMultiType].nCount += msgInfo.stats[ eGroup ][ eMultiType ].nCount; + rgTotals[eMultiType].uBytes += msgInfo.stats[ eGroup ][ eMultiType ].uBytes; + rgTotals[eRawType].nCount += msgInfo.stats[ eGroup ][ eRawType ].nCount; + rgTotals[eRawType].uBytes += msgInfo.stats[ eGroup ][ eRawType ].uBytes; + + if ( msgInfo.stats[ eGroup ][ eMultiType ].nCount == 0 ) + continue; + + uint32 nCount = msgInfo.stats[ eGroup ][ eMultiType ].nCount; + uint64 uBytes = msgInfo.stats[ eGroup ][ eMultiType ].uBytes; + mapValueToMsg.Insert( bSortByFrequency ? (uint64)nCount : uBytes, n ); + } + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%s", "EMsg MessageName Count % KB % Avg/Msg KBPS Msg Saved % KB Saved %\n" ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%s", "----- ---------------------------------------------- ---------- ------ -------------- ------ ---------- ---------- ---------- ------ -------------- ------\n" ); + + uint32 unTotalMessages = 0; + uint64 unTotalBytes = 0; + + for ( uint16 iValue = mapValueToMsg.LastInorder(); iValue != mapValueToMsg.InvalidIndex(); iValue = mapValueToMsg.PrevInorder( iValue ) ) + { + const MsgInfo_t &msgInfo = m_vecMsgInfo[ mapValueToMsg[iValue] ]; + + //see if we are looking for a particular source, and if so, if this has that source set + if( nSourceMask && ( msgInfo.stats[ eGroup ][ eMultiType ].nSourceMask & nSourceMask ) == 0 ) + continue; + + uint32 nCount = msgInfo.stats[ eGroup ][ eMultiType ].nCount; + uint64 uBytes = msgInfo.stats[ eGroup ][ eMultiType ].uBytes; + + unTotalMessages += nCount; + unTotalBytes += uBytes; + + int32 nMessagesSaved = msgInfo.stats[ eGroup ][ eRawType ].nCount - msgInfo.stats[ eGroup ][ eMultiType ].nCount; + int64 uBytesSaved = msgInfo.stats[ eGroup ][ eRawType ].uBytes - msgInfo.stats[ eGroup ][ eMultiType ].uBytes; + + uint32 nMessagesIfNotMultiplex = msgInfo.stats[ eGroup ][ eRawType ].nCount; + uint64 uBytesIfNotMultiplex = msgInfo.stats[ eGroup ][ eRawType ].uBytes; + + uint32 uAvgMsg = 0; + if ( nCount > 0 ) + uAvgMsg = uBytes / nCount; + + float flCountPerc = 0; + if ( rgTotals[eMultiType].nCount > 0 ) + flCountPerc = nCount * 100.0f / rgTotals[eMultiType].nCount; + float flBytesPerc = 0; + if ( rgTotals[eMultiType].uBytes > 0 ) + flBytesPerc = uBytes * 100.0f / rgTotals[eMultiType].uBytes; + float fKBPS = 0.0f; + if ( ulMicroseconds > 0 ) + fKBPS = uBytes * 1000.0f / ulMicroseconds; + + float flMessagesSavedPerc = 0.0f; + if ( nMessagesIfNotMultiplex > 0 ) + flMessagesSavedPerc = nMessagesSaved * 100.0f / nMessagesIfNotMultiplex; + float flBytesSavedPerc = 0.0f; + if ( uBytesIfNotMultiplex > 0 ) + flBytesSavedPerc = uBytesSaved * 100.0f / uBytesIfNotMultiplex; + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%5u %-46s %10u %5.1f%% %14llu %5.1f%% %10u %10.2f %10d %5.1f%% %14lld %5.1f%%\n", + msgInfo.eMsg, + msgInfo.pchMsgName, + nCount, + flCountPerc, + uBytes / 1024, + flBytesPerc, + uAvgMsg, + fKBPS, + nMessagesSaved, + flMessagesSavedPerc, + uBytesSaved / 1024, + flBytesSavedPerc ); + } + + uint32 uAvgMsgTotal = 0; + if ( unTotalMessages > 0 ) + uAvgMsgTotal = unTotalBytes / unTotalMessages; + + float fKBPSTotal = 0.0f; + if ( ulMicroseconds > 0 ) + fKBPSTotal = unTotalBytes * 1000.0f / ulMicroseconds; + + float flTotalMessagesPct = 0.0f; + if ( rgTotals[eMultiType].nCount > 0 ) + flTotalMessagesPct = unTotalMessages * 100.0f / rgTotals[eMultiType].nCount; + + float flTotalBytesPct = 0.0f; + if ( rgTotals[eMultiType].uBytes > 0 ) + flTotalBytesPct = unTotalBytes * 100.0f / rgTotals[eMultiType].uBytes; + + int32 nTotalMessagesSaved = rgTotals[eRawType].nCount - rgTotals[eMultiType].nCount; + int64 nTotalBytesSaved = rgTotals[eRawType].uBytes - rgTotals[eMultiType].uBytes; + + int32 nTotalMessagesIfNotMultiplex = unTotalMessages - rgTotals[eMultiType].nCount + rgTotals[eRawType].nCount; + int64 uTotalBytesIfNotMultiplex = unTotalBytes - rgTotals[eMultiType].uBytes + rgTotals[eRawType].uBytes; + + float flTotalMessagesSavedPct = 0.0f; + if ( nTotalMessagesIfNotMultiplex > 0 ) + flTotalMessagesSavedPct = nTotalMessagesSaved * 100.0f / nTotalMessagesIfNotMultiplex; + + float flTotalBytesSavedPct = 0.0f; + if ( uTotalBytesIfNotMultiplex > 0 ) + flTotalBytesSavedPct = nTotalBytesSaved * 100.0f / uTotalBytesIfNotMultiplex; + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "----- ---------------------------------------------- ---------- ------ -------------- ------ ---------- ---------- ---------- ------ -------------- ------\n" ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "Totals %10u %5.1f%% %14llu %5.1f%% %10u %10.2f %10d %5.1f%% %14lld %5.1f%%\n", + unTotalMessages, + flTotalMessagesPct, + unTotalBytes / 1024, + flTotalBytesPct, + uAvgMsgTotal, + fKBPSTotal, + nTotalMessagesSaved, + flTotalMessagesSavedPct, + nTotalBytesSaved / 1024, + flTotalBytesSavedPct ); +} + + +//----------------------------------------------------------------------------- +// Destroys a message list by deallocating everything it's got +//----------------------------------------------------------------------------- +CMessageList::~CMessageList() +{ + int nUsedBuckets = 0; + for ( int nIndex = 0; nIndex < m_vecMessageInfoBuckets.Count(); nIndex++ ) + { + if ( m_vecMessageInfoBuckets[nIndex] != NULL ) + { + FreePv( m_vecMessageInfoBuckets[nIndex] ); + m_vecMessageInfoBuckets[nIndex] = NULL; + nUsedBuckets++; + } + } + + m_vecMessageInfoBuckets.RemoveAll(); +} + +//----------------------------------------------------------------------------- +// assure that we've got the slots in the given bucket allocated +//----------------------------------------------------------------------------- +void CMessageList::AssureBucket( int nBucket ) +{ + // if this bucket is bigger then the array, extend the array + if ( nBucket >= m_vecMessageInfoBuckets.Count() ) + { + int nOldCount = m_vecMessageInfoBuckets.Count(); + // message ID "clumps" are usually 100 apart, so we'll try + // to grow by 100 each time. Divide 100 by the bucket size and add + // one for truncation to figure out our grow-by. + int nNewCount = nBucket + (1 + (100 / m_kcBucketSize)); + + // get that count + m_vecMessageInfoBuckets.EnsureCount( nNewCount ); + + // initialize the new ones to NULL + for ( int nIndex = nOldCount; nIndex < nNewCount; nIndex++ ) + { + m_vecMessageInfoBuckets[nIndex] = NULL; + } + } + + // is the bucket we want allocated? + if ( m_vecMessageInfoBuckets[nBucket] == NULL ) + { + // nope; get one and initialize it + m_vecMessageInfoBuckets[nBucket] = (short*) PvAlloc( sizeof(short) * m_kcBucketSize ); + for ( int nIndex = 0; nIndex < m_kcBucketSize; nIndex++) + m_vecMessageInfoBuckets[nBucket][nIndex] = -1; + } + + return; +} + + +//----------------------------------------------------------------------------- +// Purpose: Hash an MsgType_t and return the index into m_vecMsgInfo where it can be found +// returns -1 if the MsgType_t is unknown +//----------------------------------------------------------------------------- +short CMessageList::GetMessageIndex( MsgType_t eMsg ) +{ + // find the slot and bucket for this message in our hash + int nSlot; + int nBucket = HashMessage( eMsg, nSlot ); + + // taller than the hash? + if ( nBucket >= m_vecMessageInfoBuckets.Count() ) + return -1; + + // not a bucket? + if ( m_vecMessageInfoBuckets[nBucket] == NULL ) + return -1; + + // get the index back to the global array + short nIndex = m_vecMessageInfoBuckets[nBucket][nSlot]; + return nIndex; +} + + +//----------------------------------------------------------------------------- +// Tests to see if a message is valid; if so, return a pointer to its name. +// A message must match at least one of the nTypeMask flags to be considered +// valid. If provided, ppMsgName is set to k_rgchUnknown if no match, +// or the message name if matched. +//----------------------------------------------------------------------------- + +bool CMessageList::GetMessage( MsgType_t eMsg, const char **ppMsgName, int nTypeMask ) +{ + VPROF_BUDGET( "GetMessage", VPROF_BUDGETGROUP_JOBS_COROUTINES ); + + // if an out variable for the name is provided, + // initialize it with a pointer to "unknown" string + if ( ppMsgName != NULL ) + *ppMsgName = k_rgchUnknown; + + short nIndex = GetMessageIndex( eMsg ); + + // good index? + if ( nIndex == -1 ) + { + if ( ppMsgName != NULL ) + { + *ppMsgName = "Unknown MsgType - Not Found"; + } + return false; + } + + const MsgInfo_t &msgInfo = m_vecMsgInfo[nIndex]; + + // get the string out + if ( ppMsgName != NULL ) + *ppMsgName = msgInfo.pchMsgName; + + // it's good if it matches the flags + return ( 0 != ( msgInfo.nFlags & nTypeMask ) ); +} + + +} // namespace GCSDK + diff --git a/gcsdk/msgprotobuf.cpp b/gcsdk/msgprotobuf.cpp new file mode 100644 index 0000000..6cdab3d --- /dev/null +++ b/gcsdk/msgprotobuf.cpp @@ -0,0 +1,466 @@ +//====== Copyright � 1996-2004, Valve Corporation, All rights reserved. ======= +// +// Purpose: +// +//============================================================================= + + +#include "stdafx.h" +#include <limits.h> +#include "msgprotobuf.h" + +#include "tier0/memdbgoff.h" + +#ifdef GC +namespace GCSDK +{ +IMPLEMENT_CLASS_MEMPOOL( CProtoBufNetPacket, 1000, UTLMEMORYPOOL_GROW_SLOW ); +} +#endif + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ +// Global pool for protobuf header allocations +CProtoBufMsgMemoryPoolMgr *GProtoBufMsgMemoryPoolMgr() +{ + static CProtoBufMsgMemoryPoolMgr s_ProtoBufMsgMemoryPoolMgr; + return &s_ProtoBufMsgMemoryPoolMgr; +} + +CThreadMutex CProtoBufMsgBase::s_PoolRegMutex; + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CProtoBufNetPacket::CProtoBufNetPacket( CNetPacket *pNetPacket, GCProtoBufMsgSrc eMsgSrc, const CSteamID steamID, uint32 nGCDirIndex, MsgType_t msgType ) + : m_msgType( msgType ), m_steamID( steamID ), m_bIsValid( false ) +{ +#ifdef VALVE_BIG_ENDIAN + ProtoBufMsgHeader_t * pHdr = (ProtoBufMsgHeader_t *)pNetPacket->PubData(); + pHdr->m_EMsgFlagged = LittleDWord( pHdr->m_EMsgFlagged ); + pHdr->m_cubProtoBufExtHdr = LittleDWord( pHdr->m_cubProtoBufExtHdr ); +#endif + + m_pNetPacket = pNetPacket; + m_pNetPacket->AddRef(); + + m_pHeader = GProtoBufMsgMemoryPoolMgr()->AllocProtoBufHdr(); + + // Pull the length of the header out of the packet, but then validate it's not longer than the packet itself (ie, corrupt packet) + const uint32 unLenProtoBufHeader = GetFixedHeader().m_cubProtoBufExtHdr; + if ( unLenProtoBufHeader + sizeof( ProtoBufMsgHeader_t ) <= pNetPacket->CubData() ) + { + if ( !m_pHeader->ParseFromArray( pNetPacket->PubData()+sizeof(ProtoBufMsgHeader_t), unLenProtoBufHeader ) ) + { + AssertMsg4( false, "Failed parsing ProtoBuf extended header out of message: %s(%d) from %s size=%u", PchMsgNameFromEMsg( GetEMsg() ), GetEMsg(), steamID.Render(), pNetPacket->CubData() ); + } + else + { + //if this packet doesn't have a source provided, we need to set the source of this message. If one is already provided, we should never stomp that data as that is the ultimate source of the message + if( m_pHeader->gc_msg_src() == GCProtoBufMsgSrc_Unspecified ) + { + //make sure to set the steam ID to what steam provided so clients can't spoof it + m_pHeader->set_client_steam_id( steamID.ConvertToUint64() ); + //and track our original source type and which GC it was first received by + m_pHeader->set_gc_dir_index_source( nGCDirIndex ); + m_pHeader->set_gc_msg_src( eMsgSrc ); + } + + m_bIsValid = true; + } + } + else + { + AssertMsg4( false, "Unparseable protobuf message arrived: message %s(%u) from %s size=%u", PchMsgNameFromEMsg( GetEMsg() ), GetEMsg(), steamID.Render(), pNetPacket->CubData() ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CProtoBufNetPacket::~CProtoBufNetPacket() +{ + GProtoBufMsgMemoryPoolMgr()->FreeProtoBufHdr( m_pHeader ); + m_pNetPacket->Release(); +} + +//----------------------------------------------------------------------------- +// Purpose: Gets the body from the associated packet +//----------------------------------------------------------------------------- +bool CProtoBufNetPacket::GetMsgBody( const uint8*& pubData, uint32& cubData ) const +{ + if ( !IsValid() ) + { + pubData = NULL; + cubData = 0; + return false; + } + + //when it initializes, it verifies the size, so we don't need to do validation here + const uint32 nHdrOffset = sizeof( ProtoBufMsgHeader_t ) + GetFixedHeader().m_cubProtoBufExtHdr; + pubData = PubData() + nHdrOffset; + cubData = CubData() - nHdrOffset; + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CProtoBufMsgMemoryPoolBase::CProtoBufMsgMemoryPoolBase( uint32 unTargetLow, uint32 unTargetHigh ) +{ + m_unTargetCountLow = unTargetLow; + m_unTargetCountHigh = unTargetHigh; + m_unAllocHitCounter = 0; + m_unAllocMissCounter = 0; + m_unAllocated = 0; + m_pTSQueueFreeObjects = new CTSQueue< ::google::protobuf::Message * >(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CProtoBufMsgMemoryPoolBase::~CProtoBufMsgMemoryPoolBase() +{ + AssertMsg( 0 == m_pTSQueueFreeObjects->Count(), "CProtoBufMsgMemoryPoolBase destructor called with items still in the queue. They must be freed by the derived class or they will be leaked" ); + + m_pTSQueueFreeObjects->Purge(); + delete m_pTSQueueFreeObjects; +} + + +//----------------------------------------------------------------------------- +// Purpose: Allocs a message from the pool +//----------------------------------------------------------------------------- +::google::protobuf::Message *CProtoBufMsgMemoryPoolBase::Alloc() +{ + ::google::protobuf::Message *pObject; + if ( !m_pTSQueueFreeObjects->PopItem( &pObject ) || !pObject ) + { + ++m_unAllocMissCounter; + pObject = InternalAlloc(); + } + else + { + ++m_unAllocHitCounter; + uint32 unCount = m_pTSQueueFreeObjects->Count(); + + // We'll free an extra cached msg every alloc if we are over the higher limit, and every 6th if we + // are over the lower limit. This allows some elasticity to peaks in demand. + bool bFreeAnother = ( unCount > m_unTargetCountHigh ) + || ( unCount > m_unTargetCountLow && m_unAllocHitCounter % 6 == 0 ); + + if ( bFreeAnother ) + { + // Pop an extra item, so we can get down to target count over time + ::google::protobuf::Message *pThrowAway; + if ( m_pTSQueueFreeObjects->PopItem( &pThrowAway ) && pThrowAway ) + { + InternalFree( pThrowAway ); + } + } + } + + ++m_unAllocated; + return pObject; +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns a message to the pool +//----------------------------------------------------------------------------- +void CProtoBufMsgMemoryPoolBase::Free( ::google::protobuf::Message *pObject ) +{ + // We always cache for re-use on free, though we may throw out later on alloc to shrink the pool + pObject->Clear(); + + --m_unAllocated; + m_pTSQueueFreeObjects->PushItem( pObject ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Counts the size of all the objects in the pool +//----------------------------------------------------------------------------- +uint32 CProtoBufMsgMemoryPoolBase::GetEstimatedSize() +{ + // to fix a linux build break + typedef ::google::protobuf::Message ProtoMsg_t; + CUtlVector<ProtoMsg_t *> vecTemp; + vecTemp.EnsureCapacity( m_pTSQueueFreeObjects->Count() * 2 ); + + // Should be "while (pObject != NULL)" because if we were actually using threads depending on count + // would be bad. Unfortunately that gives me an infinite loop. See if this can be changed when this + // is updated + while ( m_pTSQueueFreeObjects->Count() > 0 ) + { + ::google::protobuf::Message *pObject = NULL; + m_pTSQueueFreeObjects->PopItem( &pObject ); + vecTemp.AddToTail( pObject ); + } + + uint32 unEstimate = 0; + FOR_EACH_VEC( vecTemp, i ) + { + unEstimate += vecTemp[i]->SpaceUsed(); + m_pTSQueueFreeObjects->PushItem( vecTemp[i] ); + } + + // Scale the estimate by the number of objects outstanding + float fScale = 1.0f; + if ( vecTemp.Count() > 0 ) + { + float fFree = (float)vecTemp.Count(); + fScale = ( fFree + (float)m_unAllocated ) / fFree; + unEstimate *= fScale; + } + + return unEstimate; +} + + +//----------------------------------------------------------------------------- +// Purpose: Pops an item from the queue. To be used by the derived class +// destructor to free the oustanding allocations +//----------------------------------------------------------------------------- +bool CProtoBufMsgMemoryPoolBase::PopItem( google::protobuf::Message **ppMsg ) +{ + return m_pTSQueueFreeObjects->PopItem( ppMsg ); +} + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CProtoBufMsgMemoryPoolMgr::CProtoBufMsgMemoryPoolMgr() : m_PoolHeaders() +{ + m_vecMsgPools.AddToTail( &m_PoolHeaders ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CProtoBufMsgMemoryPoolMgr::~CProtoBufMsgMemoryPoolMgr() +{ + FOR_EACH_VEC( m_vecMsgPools, i ) + { + if ( m_vecMsgPools[i] != &m_PoolHeaders ) + delete m_vecMsgPools[i]; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Takes control of a message pool registered for a specific protobuf type +//----------------------------------------------------------------------------- +void CProtoBufMsgMemoryPoolMgr::RegisterPool( CProtoBufMsgMemoryPoolBase *pPool ) +{ + m_vecMsgPools.AddToTail( pPool ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Prints info on protobuf memory usage +//----------------------------------------------------------------------------- +void CProtoBufMsgMemoryPoolMgr::DumpPoolInfo() +{ + uint32 unTotalSize = 0; + Msg( "CProtoBufMsgMemoryPoolMgr:\n" ); + Msg( " PoolName Allocated Free Est. Size Hit Rate\n" ); + Msg( " ----------------------------------------- ----------- ----------- ----------- -----------\n" ); + FOR_EACH_VEC( m_vecMsgPools, i ) + { + uint32 unHitCount = m_vecMsgPools[i]->GetAllocHitCount(); + uint32 unMissCount = m_vecMsgPools[i]->GetAllocMissCount(); + float flHitRate = 0.0f; + if ( unHitCount > 0 || unMissCount > 0 ) + { + flHitRate = (float)unHitCount / (float)(unHitCount+unMissCount); + flHitRate *= 100.0f; + } + + uint32 unEstimate = m_vecMsgPools[i]->GetEstimatedSize(); + Msg( "%43s%12d%12d%12s%12s\n", m_vecMsgPools[i]->GetName().String(), m_vecMsgPools[i]->GetAllocated(), m_vecMsgPools[i]->GetFree(), Q_pretifymem( (float)unEstimate, 2, true ), CFmtStr( "%f%%", flHitRate ).Access() ); + unTotalSize += unEstimate; + } + Msg( " -----------------------------------------------------------------------------------------\n" ); + Msg( " Total: %s\n", Q_pretifymem( (float)unTotalSize, 2, true ) ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor - no eMsg, so it's a receive constructor. Don't alloc +// a header in this case +//----------------------------------------------------------------------------- +CProtoBufMsgBase::CProtoBufMsgBase() + : m_pNetPacket( NULL ) + , m_eMsg( 0 ) + , m_pProtoBufHdr( NULL ) +{ +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor - Has an eMsg, so it's a send constructor. Alloc a header +//----------------------------------------------------------------------------- +CProtoBufMsgBase::CProtoBufMsgBase( MsgType_t eMsg ) + : m_pNetPacket( NULL ) + , m_eMsg( eMsg ) + , m_pProtoBufHdr( NULL ) +{ + m_pProtoBufHdr = GProtoBufMsgMemoryPoolMgr()->AllocProtoBufHdr(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CProtoBufMsgBase::~CProtoBufMsgBase() +{ + // If we have a net packet then it allocated the header, so we don't free it ourselves + // If we don't have a net packet then we still may have not allocated a header ( default constructor ) + if ( m_pNetPacket ) + { + m_pNetPacket->Release(); + m_pNetPacket = NULL; + } + else if ( m_pProtoBufHdr ) + { + GProtoBufMsgMemoryPoolMgr()->FreeProtoBufHdr( m_pProtoBufHdr ); + } + + m_pProtoBufHdr = NULL; +} + + +//----------------------------------------------------------------------------- +// Purpose: Inits a message from a received net packet +//----------------------------------------------------------------------------- +bool CProtoBufMsgBase::InitFromPacket( IMsgNetPacket * pNetPacket ) +{ + VPROF_BUDGET( "CProtoBufMsg::InitFromPacket( IMsgNetPacket )", VPROF_BUDGETGROUP_OTHER_NETWORKING ); + + AssertMsg( NULL == m_pProtoBufHdr, "Encountered a packet being initialized that has outstanding memory. This will leak memory, and is often caused by receiving messages using the wrong constructor which preallocates or not clearing the message between uses" ); + Assert( k_EMsgFormatTypeProtocolBuffer == pNetPacket->GetEMsgFormatType() ); + + m_pNetPacket = static_cast<CProtoBufNetPacket *>( pNetPacket ); + m_pNetPacket->AddRef(); + ProtoBufMsgHeader_t &refFixedHdr = m_pNetPacket->GetFixedHeader(); + + m_eMsg = refFixedHdr.m_EMsgFlagged; + m_pProtoBufHdr = m_pNetPacket->GetProtoHeader(); + + const uint8* pubData; + uint32 cubData; + if( !m_pNetPacket->GetMsgBody( pubData, cubData ) ) + return false; + + ::google::protobuf::Message *pBody = GetGenericBody(); + return pBody->ParseFromArray( pubData, (int)cubData ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Sends a message using the given handler +//----------------------------------------------------------------------------- +bool CProtoBufMsgBase::BAsyncSend( IProtoBufSendHandler & sender ) const +{ + return BAsyncSendProto( sender, GetEMsg(), *m_pProtoBufHdr, *GetGenericBody() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Sends a protobuf object out as a message +//----------------------------------------------------------------------------- +bool CProtoBufMsgBase::BAsyncSendProto( IProtoBufSendHandler& sender, MsgType_t eMsgType, const CMsgProtoBufHeader& hdr, const ::google::protobuf::Message& proto ) +{ + VPROF_BUDGET( "CProtoBufMsg::BAsyncSendProto", VPROF_BUDGETGROUP_OTHER_NETWORKING ); + // Compute message sizes and serialize out to buffer. Need to work on a more efficient + // way to write these to the net without so much copying. + uint32 cubBodySize = proto.ByteSize(); + + uint32 cubTotalSize = 0; + uint8* pMsgMem = AllocateMessageMemory( eMsgType, hdr, cubBodySize, &cubTotalSize ); + proto.SerializeWithCachedSizesToArray( pMsgMem + cubTotalSize - cubBodySize ); + + bool bSuccess = sender.BAsyncSend( eMsgType, pMsgMem, cubTotalSize ); + FreeMessageMemory( pMsgMem ); + + return bSuccess; +} + + +//----------------------------------------------------------------------------- +// Purpose: Sends a message using the given handler +//----------------------------------------------------------------------------- +bool CProtoBufMsgBase::BAsyncSendWithPreSerializedBody( IProtoBufSendHandler & pSender, const byte *pubBody, uint32 cubBody ) const +{ + return BAsyncSendWithPreSerializedBody( pSender, GetEMsg(), *m_pProtoBufHdr, pubBody, cubBody ); +} + +//----------------------------------------------------------------------------- +//free standing version to send a protobuff given a header and pre-serialized body. Primarily used for efficient message routing +//----------------------------------------------------------------------------- +bool CProtoBufMsgBase::BAsyncSendWithPreSerializedBody( IProtoBufSendHandler& sender, MsgType_t eMsgType, const CMsgProtoBufHeader& hdr, const byte* pubBody, uint32 cubBody ) +{ + VPROF_BUDGET( "CProtoBufMsg::BAsyncSendWithPreSerializedBody", VPROF_BUDGETGROUP_OTHER_NETWORKING ); + // Compute message sizes and serialize out to buffer. Need to work on a more efficient + // way to write these to the net without so much copying. + uint32 cubTotalSize = 0; + uint8* pMsgMem = AllocateMessageMemory( eMsgType, hdr, cubBody, &cubTotalSize ); + + V_memcpy( pMsgMem + cubTotalSize - cubBody, pubBody, cubBody ); + + bool bSuccess = sender.BAsyncSend( eMsgType, pMsgMem, cubTotalSize ); + + FreeMessageMemory( pMsgMem ); + + return bSuccess; +} + + +//----------------------------------------------------------------------------- +//utility function that handles allocating a memory pool big enough for the provided header and specified body +//size and writing the header into the pool. This will return a pointer to the memory, as well as the header size. +//----------------------------------------------------------------------------- +uint8* CProtoBufMsgBase::AllocateMessageMemory( MsgType_t eMsgType, const CMsgProtoBufHeader& hdr, uint32 cubBodySize, uint32* pCubTotalSizeOut ) +{ + uint32 cubExtHdrSize = hdr.ByteSize(); + uint32 cubTotalSize = sizeof( ProtoBufMsgHeader_t ) + cubExtHdrSize + cubBodySize; + ProtoBufMsgHeader_t fixedHdr( eMsgType, cubExtHdrSize ); + + uint8 *pubMsgBytes = (uint8*)g_MemPoolMsg.Alloc( cubTotalSize ); + + // Copy over basic fixed header + // We place bytes on the network in little endian to optimize for those platforms, even though network byte order is really big endian. + #if defined(VALVE_BIG_ENDIAN) + ProtoBufMsgHeader_t hdrByteSwapped = fixedHdr; + hdrByteSwapped.m_EMsgFlagged = DWordSwap( hdrByteSwapped.m_EMsgFlagged ); + hdrByteSwapped.m_cubProtoBufExtHdr = DWordSwap( hdrByteSwapped.m_cubProtoBufExtHdr ); + Q_memcpy( pubMsgBytes, &hdrByteSwapped, sizeof( ProtoBufMsgHeader_t ) ); + #else + Q_memcpy( pubMsgBytes, &fixedHdr, sizeof( ProtoBufMsgHeader_t ) ); + #endif + + // Serialize extended pb header, we guarantee we didn't modify the message since the last call to ByteSize above, so use the cached sizes serialize + uint8 *pEnd = hdr.SerializeWithCachedSizesToArray( pubMsgBytes + sizeof( ProtoBufMsgHeader_t ) ); + NOTE_UNUSED( pEnd ); + Assert( pEnd == pubMsgBytes + sizeof( ProtoBufMsgHeader_t ) + cubExtHdrSize ); + + if( pCubTotalSizeOut ) + *pCubTotalSizeOut = cubTotalSize; + return pubMsgBytes; +} + +void CProtoBufMsgBase::FreeMessageMemory( uint8* pMemory ) +{ + g_MemPoolMsg.Free( pMemory ); +} + +}
\ No newline at end of file diff --git a/gcsdk/netpacket.cpp b/gcsdk/netpacket.cpp new file mode 100644 index 0000000..13c3818 --- /dev/null +++ b/gcsdk/netpacket.cpp @@ -0,0 +1,106 @@ +//====== Copyright (c), Valve Corporation, All rights reserved. ======= +// +// Purpose: Holds the CNetPacket class +// +//============================================================================= + + +#include "stdafx.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + + +namespace GCSDK +{ + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CNetPacket::CNetPacket() +{ + m_cRef = 0; + m_pubData = NULL; + m_cubData = 0; +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor, validates refcount +//----------------------------------------------------------------------------- +CNetPacket::~CNetPacket() +{ + Assert( m_cRef == 0 ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Inits all members +// Input : hConnection - connection we're from +// pubData - message data +// cubData - message size +// *pubNetworkBuffer - network buffer that contains the message +// assumes control of this memory +//----------------------------------------------------------------------------- +void CNetPacket::Init( uint32 cubData, const void* pCopyData ) +{ + Assert( cubData ); + + m_pubData = (uint8 *)g_MemPoolMsg.Alloc( cubData ); + m_cubData = cubData; + + if( pCopyData ) + { + Q_memcpy( m_pubData, pCopyData, cubData ); + } + + AddRef(); +} + + +//----------------------------------------------------------------------------- +// Purpose: adds to refcount +//----------------------------------------------------------------------------- +void CNetPacket::AddRef() +{ + ++m_cRef; +} + +//----------------------------------------------------------------------------- +void CNetPacket::InitAdoptBuffer( uint32 cubData, uint8* pubData ) +{ + Assert( !m_pubData ); + + m_pubData = pubData; + m_cubData = cubData; + + AddRef(); +} + +//----------------------------------------------------------------------------- +void CNetPacket::OrphanBuffer() +{ + m_pubData = NULL; + m_cubData = 0; +} + +//----------------------------------------------------------------------------- +// Purpose: decrements refcount +//----------------------------------------------------------------------------- +void CNetPacket::Release() +{ + Assert( m_cRef > 0 ); + if ( --m_cRef == 0 ) + { + // delete the network buffer we're associated with, if we have one + if ( m_pubData ) + { + g_MemPoolMsg.Free( m_pubData ); + } + // delete ourselves + g_cNetPacket--; + CNetPacketPool::sm_MemPoolNetPacket.Free( this ); + } +} + +} // namespace GCSDK diff --git a/gcsdk/netpacketpool.cpp b/gcsdk/netpacketpool.cpp new file mode 100644 index 0000000..af61914 --- /dev/null +++ b/gcsdk/netpacketpool.cpp @@ -0,0 +1,37 @@ +//====== Copyright (c), Valve Corporation, All rights reserved. ======= +// +// Purpose: +// +//============================================================================= + + +#include "stdafx.h" +#include "netpacketpool.h" +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + + +namespace GCSDK +{ + +CClassMemoryPool<CNetPacket> CNetPacketPool::sm_MemPoolNetPacket( k_cInitialNetworkBuffers, UTLMEMORYPOOL_GROW_FAST ); + +static const CThreadSafeMultiMemoryPool::MemPoolConfig_t s_MemPoolConfigAllocSize[] = { + { 32, 32 }, + { 64, 32 }, + { 96, 32 }, + { 128, 32 }, + { 256, 32 }, + { k_cubMsgSizeSmall, 32 }, + { 1312 /* sizeof( UDPRecvBuffer_t ) + some rounding */, 32 }, + { 2048, 4 }, + { 4096, 4 }, + { 16384, 4 }, + { 64*1024, 4 } +}; +CThreadSafeMultiMemoryPool g_MemPoolMsg( s_MemPoolConfigAllocSize, Q_ARRAYSIZE(s_MemPoolConfigAllocSize), UTLMEMORYPOOL_GROW_FAST ); + +// hacky global +int g_cNetPacket = 0; + +} // namespace GCSDK diff --git a/gcsdk/protobufsharedobject.cpp b/gcsdk/protobufsharedobject.cpp new file mode 100644 index 0000000..69c8142 --- /dev/null +++ b/gcsdk/protobufsharedobject.cpp @@ -0,0 +1,692 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Shared object based on a CBaseRecord subclass +// +//============================================================================= +#include "stdafx.h" + +#include "gcsdk_gcmessages.pb.h" + +#include "google/protobuf/descriptor.h" +#include "google/protobuf/reflection_ops.h" +#include "google/protobuf/descriptor.pb.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +using ::google::protobuf::FieldDescriptor; +using ::google::protobuf::Descriptor; +using ::google::protobuf::Message; +using ::google::protobuf::Reflection; + + +namespace GCSDK +{ + +//---------------------------------------------------------------------------- +// Purpose: returns true if the field has the key_field option +//---------------------------------------------------------------------------- +inline bool IsKeyField( const FieldDescriptor *pFieldDescriptor ) +{ + return pFieldDescriptor->options().GetExtension( key_field ); +} + +//---------------------------------------------------------------------------- +// Purpose: Copies a field from one protobuf message to another +//---------------------------------------------------------------------------- +void CopyProtoBufField( ::google::protobuf::Message & msgDest, const ::google::protobuf::Message & msgSource, const ::google::protobuf::FieldDescriptor *pFieldDest, const ::google::protobuf::FieldDescriptor *pFieldSource ) +{ + Assert( pFieldDest->cpp_type() == pFieldSource->cpp_type() ); + Assert( pFieldDest->is_repeated() == pFieldSource->is_repeated() ); + if( pFieldSource->cpp_type() != pFieldDest->cpp_type() || pFieldSource->is_repeated() != pFieldDest->is_repeated() ) + return; + + if ( pFieldDest->is_repeated() ) + { + int nFieldSize = msgSource.GetReflection()->FieldSize( msgSource, pFieldSource ); + switch( pFieldDest->cpp_type() ) + { + case ::google::protobuf::FieldDescriptor::CPPTYPE_INT32: + { + for ( int i = 0; i < nFieldSize; i++ ) + { + if ( msgDest.GetReflection()->FieldSize( msgDest, pFieldDest ) <= i ) + { + msgDest.GetReflection()->AddInt32( &msgDest, pFieldDest, msgSource.GetReflection()->GetRepeatedInt32( msgSource, pFieldSource, i ) ); + } + else + { + msgDest.GetReflection()->SetRepeatedInt32( &msgDest, pFieldDest, i, msgSource.GetReflection()->GetRepeatedInt32( msgSource, pFieldSource, i ) ); + } + } + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_INT64: + { + for ( int i = 0; i < nFieldSize; i++ ) + { + if ( msgDest.GetReflection()->FieldSize( msgDest, pFieldDest ) <= i ) + { + msgDest.GetReflection()->AddInt64( &msgDest, pFieldDest, msgSource.GetReflection()->GetRepeatedInt64( msgSource, pFieldSource, i ) ); + } + else + { + msgDest.GetReflection()->SetRepeatedInt64( &msgDest, pFieldDest, i, msgSource.GetReflection()->GetRepeatedInt64( msgSource, pFieldSource, i ) ); + } + } + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_UINT32: + { + for ( int i = 0; i < nFieldSize; i++ ) + { + if ( msgDest.GetReflection()->FieldSize( msgDest, pFieldDest ) <= i ) + { + msgDest.GetReflection()->AddUInt32( &msgDest, pFieldDest, msgSource.GetReflection()->GetRepeatedUInt32( msgSource, pFieldSource, i ) ); + } + else + { + msgDest.GetReflection()->SetRepeatedUInt32( &msgDest, pFieldDest, i, msgSource.GetReflection()->GetRepeatedUInt32( msgSource, pFieldSource, i ) ); + } + } + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_UINT64: + { + for ( int i = 0; i < nFieldSize; i++ ) + { + if ( msgDest.GetReflection()->FieldSize( msgDest, pFieldDest ) <= i ) + { + msgDest.GetReflection()->AddUInt64( &msgDest, pFieldDest, msgSource.GetReflection()->GetRepeatedUInt64( msgSource, pFieldSource, i ) ); + } + else + { + msgDest.GetReflection()->SetRepeatedUInt64( &msgDest, pFieldDest, i, msgSource.GetReflection()->GetRepeatedUInt64( msgSource, pFieldSource, i ) ); + } + } + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: + { + for ( int i = 0; i < nFieldSize; i++ ) + { + if ( msgDest.GetReflection()->FieldSize( msgDest, pFieldDest ) <= i ) + { + msgDest.GetReflection()->AddDouble( &msgDest, pFieldDest, msgSource.GetReflection()->GetRepeatedDouble( msgSource, pFieldSource, i ) ); + } + else + { + msgDest.GetReflection()->SetRepeatedDouble( &msgDest, pFieldDest, i, msgSource.GetReflection()->GetRepeatedDouble( msgSource, pFieldSource, i ) ); + } + } + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: + { + for ( int i = 0; i < nFieldSize; i++ ) + { + if ( msgDest.GetReflection()->FieldSize( msgDest, pFieldDest ) <= i ) + { + msgDest.GetReflection()->AddFloat( &msgDest, pFieldDest, msgSource.GetReflection()->GetRepeatedFloat( msgSource, pFieldSource, i ) ); + } + else + { + msgDest.GetReflection()->SetRepeatedFloat( &msgDest, pFieldDest, i, msgSource.GetReflection()->GetRepeatedFloat( msgSource, pFieldSource, i ) ); + } + } + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_BOOL: + { + for ( int i = 0; i < nFieldSize; i++ ) + { + if ( msgDest.GetReflection()->FieldSize( msgDest, pFieldDest ) <= i ) + { + msgDest.GetReflection()->AddBool( &msgDest, pFieldDest, msgSource.GetReflection()->GetRepeatedBool( msgSource, pFieldSource, i ) ); + } + else + { + msgDest.GetReflection()->SetRepeatedBool( &msgDest, pFieldDest, i, msgSource.GetReflection()->GetRepeatedBool( msgSource, pFieldSource, i ) ); + } + } + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_ENUM: + { + for ( int i = 0; i < nFieldSize; i++ ) + { + if ( msgDest.GetReflection()->FieldSize( msgDest, pFieldDest ) <= i ) + { + msgDest.GetReflection()->AddEnum( &msgDest, pFieldDest, msgSource.GetReflection()->GetRepeatedEnum( msgSource, pFieldSource, i ) ); + } + else + { + msgDest.GetReflection()->SetRepeatedEnum( &msgDest, pFieldDest, i, msgSource.GetReflection()->GetRepeatedEnum( msgSource, pFieldSource, i ) ); + } + } + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_STRING: + { + for ( int i = 0; i < nFieldSize; i++ ) + { + if ( msgDest.GetReflection()->FieldSize( msgDest, pFieldDest ) <= i ) + { + msgDest.GetReflection()->AddString( &msgDest, pFieldDest, msgSource.GetReflection()->GetRepeatedString( msgSource, pFieldSource, i ) ); + } + else + { + msgDest.GetReflection()->SetRepeatedString( &msgDest, pFieldDest, i, msgSource.GetReflection()->GetRepeatedString( msgSource, pFieldSource, i ) ); + } + } + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + { + for ( int i = 0; i < nFieldSize; i++ ) + { + if ( msgDest.GetReflection()->FieldSize( msgDest, pFieldDest ) <= i ) + { + msgDest.GetReflection()->AddMessage( &msgDest, pFieldDest )->CopyFrom( msgSource.GetReflection()->GetRepeatedMessage( msgSource, pFieldSource, i ) ); + } + else + { + msgDest.GetReflection()->MutableRepeatedMessage( &msgDest, pFieldDest, i )->CopyFrom( msgSource.GetReflection()->GetRepeatedMessage( msgSource, pFieldSource, i ) ); + } + } + } + break; + + } + } + else + { + switch( pFieldDest->cpp_type() ) + { + case ::google::protobuf::FieldDescriptor::CPPTYPE_INT32: + { + msgDest.GetReflection()->SetInt32( &msgDest, pFieldDest, msgSource.GetReflection()->GetInt32( msgSource, pFieldSource ) ); + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_INT64: + { + msgDest.GetReflection()->SetInt64( &msgDest, pFieldDest, msgSource.GetReflection()->GetInt64( msgSource, pFieldSource ) ); + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_UINT32: + { + msgDest.GetReflection()->SetUInt32( &msgDest, pFieldDest, msgSource.GetReflection()->GetUInt32( msgSource, pFieldSource ) ); + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_UINT64: + { + msgDest.GetReflection()->SetUInt64( &msgDest, pFieldDest, msgSource.GetReflection()->GetUInt64( msgSource, pFieldSource ) ); + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: + { + msgDest.GetReflection()->SetDouble( &msgDest, pFieldDest, msgSource.GetReflection()->GetDouble( msgSource, pFieldSource ) ); + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: + { + msgDest.GetReflection()->SetFloat( &msgDest, pFieldDest, msgSource.GetReflection()->GetFloat( msgSource, pFieldSource ) ); + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_BOOL: + { + msgDest.GetReflection()->SetBool( &msgDest, pFieldDest, msgSource.GetReflection()->GetBool( msgSource, pFieldSource ) ); + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_ENUM: + { + msgDest.GetReflection()->SetEnum( &msgDest, pFieldDest, msgSource.GetReflection()->GetEnum( msgSource, pFieldSource ) ); + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_STRING: + { + msgDest.GetReflection()->SetString( &msgDest, pFieldDest, msgSource.GetReflection()->GetString( msgSource, pFieldSource ) ); + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + { + msgDest.GetReflection()->MutableMessage( &msgDest, pFieldDest )->CopyFrom( msgSource.GetReflection()->GetMessage( msgSource, pFieldSource ) ); + } + break; + + } + } +} + +//---------------------------------------------------------------------------- +// Purpose: compares fields in two protobuf objects +//---------------------------------------------------------------------------- +bool IsProtoBufFieldLess( const ::google::protobuf::Message & msgLHS, const ::google::protobuf::Message & msgRHS, const ::google::protobuf::FieldDescriptor *pFieldLHS, const ::google::protobuf::FieldDescriptor *pFieldRHS ) +{ + Assert( pFieldLHS->cpp_type() == pFieldRHS->cpp_type() ); + if( pFieldLHS->cpp_type() != pFieldRHS->cpp_type() ) + return false; + + switch( pFieldLHS->cpp_type() ) + { + case ::google::protobuf::FieldDescriptor::CPPTYPE_INT32: + { + return msgLHS.GetReflection()->GetInt32( msgLHS, pFieldLHS ) < msgRHS.GetReflection()->GetInt32( msgRHS, pFieldRHS ); + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_INT64: + { + return msgLHS.GetReflection()->GetInt64( msgLHS, pFieldLHS ) < msgRHS.GetReflection()->GetInt64( msgRHS, pFieldRHS ); + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_UINT32: + { + return msgLHS.GetReflection()->GetUInt32( msgLHS, pFieldLHS ) < msgRHS.GetReflection()->GetUInt32( msgRHS, pFieldRHS ); + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_UINT64: + { + return msgLHS.GetReflection()->GetUInt64( msgLHS, pFieldLHS ) < msgRHS.GetReflection()->GetUInt64( msgRHS, pFieldRHS ); + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: + { + return msgLHS.GetReflection()->GetDouble( msgLHS, pFieldLHS ) < msgRHS.GetReflection()->GetDouble( msgRHS, pFieldRHS ); + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: + { + return msgLHS.GetReflection()->GetFloat( msgLHS, pFieldLHS ) < msgRHS.GetReflection()->GetFloat( msgRHS, pFieldRHS ); + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_BOOL: + { + return msgLHS.GetReflection()->GetBool( msgLHS, pFieldLHS ) < msgRHS.GetReflection()->GetBool( msgRHS, pFieldRHS ); + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_ENUM: + { + return msgLHS.GetReflection()->GetEnum( msgLHS, pFieldLHS )->number() < msgRHS.GetReflection()->GetEnum( msgRHS, pFieldRHS )->number(); + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_STRING: + { + return Q_stricmp( msgLHS.GetReflection()->GetString( msgLHS, pFieldLHS ).c_str(), msgRHS.GetReflection()->GetString( msgRHS, pFieldRHS ).c_str() ) > 0; + } + break; + + case ::google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + { + return false; + } + break; + + } + + // unknown field types return false + return false; +} + +//---------------------------------------------------------------------------- +// Purpose: Returns true if the key fields in LHS are less than the key fields +// in RHS +//---------------------------------------------------------------------------- +bool CProtoBufSharedObjectBase::BIsKeyLess( const CSharedObject & soRHS ) const +{ + const ::google::protobuf::Message & msgLHS = *GetPObject(); + const ::google::protobuf::Message & msgRHS = *((const CProtoBufSharedObjectBase &)soRHS).GetPObject(); + + const ::google::protobuf::Descriptor *pDescriptor = msgLHS.GetDescriptor(); + + for( int nField = 0; nField < pDescriptor->field_count(); nField++ ) + { + const ::google::protobuf::FieldDescriptor *pFieldDescriptor = pDescriptor->field( nField ); + if( !IsKeyField( pFieldDescriptor ) ) + continue; + + if( IsProtoBufFieldLess( msgLHS, msgRHS, pFieldDescriptor, pFieldDescriptor ) ) + return true; + + // if the fields are less when reversed it means we know the keys aren't equal and don't + // need to keep looping + if( IsProtoBufFieldLess( msgRHS, msgLHS, pFieldDescriptor, pFieldDescriptor ) ) + return false; + } + + return false; +} + + +void CProtoBufSharedObjectBase::RecursiveAddProtoBufToKV( KeyValues *pKVDest, const ::google::protobuf::Message & msg ) +{ + using ::google::protobuf::FieldDescriptor; + const ::google::protobuf::Descriptor *pDescriptor = msg.GetDescriptor(); + const ::google::protobuf::Reflection *pReflection = msg.GetReflection(); + + for ( int iField = 0; iField < pDescriptor->field_count(); iField++ ) + { + const ::google::protobuf::FieldDescriptor *pField = pDescriptor->field( iField ); + const char *pFieldName = pField->name().c_str(); + + if ( pField->is_repeated() ) + { + KeyValues *pKVContainer = pKVDest->FindKey( pFieldName, true ); + for ( int iRepeated = 0; iRepeated < pReflection->FieldSize( msg, pField ); iRepeated++ ) + { + KeyValues *pKVNode = pKVContainer->CreateNewKey(); + switch ( pField->cpp_type() ) + { + case FieldDescriptor::CPPTYPE_INT32: pKVNode->SetInt( NULL, pReflection->GetRepeatedInt32( msg, pField, iRepeated ) ); break; + case FieldDescriptor::CPPTYPE_INT64: pKVNode->SetUint64( NULL, (uint64)pReflection->GetRepeatedInt64( msg, pField, iRepeated ) ); break; + case FieldDescriptor::CPPTYPE_UINT32: pKVNode->SetInt( NULL, (int32)pReflection->GetRepeatedUInt32( msg, pField, iRepeated ) ); break; + case FieldDescriptor::CPPTYPE_UINT64: pKVNode->SetUint64( NULL, pReflection->GetRepeatedUInt64( msg, pField, iRepeated ) ); break; + case FieldDescriptor::CPPTYPE_DOUBLE: pKVNode->SetFloat( NULL, (float)pReflection->GetRepeatedDouble( msg, pField, iRepeated ) ); break; + case FieldDescriptor::CPPTYPE_FLOAT: pKVNode->SetFloat( NULL, pReflection->GetRepeatedFloat( msg, pField, iRepeated ) ); break; + case FieldDescriptor::CPPTYPE_BOOL: pKVNode->SetBool( NULL, pReflection->GetRepeatedBool( msg, pField, iRepeated ) ); break; + case FieldDescriptor::CPPTYPE_ENUM: pKVNode->SetInt( NULL, pReflection->GetRepeatedEnum( msg, pField, iRepeated )->number() ); break; + case FieldDescriptor::CPPTYPE_STRING: pKVNode->SetString( NULL, pReflection->GetRepeatedString( msg, pField, iRepeated ).c_str() ); break; + case FieldDescriptor::CPPTYPE_MESSAGE: + { + const ::google::protobuf::Message &subMsg = pReflection->GetRepeatedMessage( msg, pField, iRepeated ); + RecursiveAddProtoBufToKV( pKVNode, subMsg ); + break; + } + default: + AssertMsg1( false, "Unknown cpp_type %d", pField->cpp_type() ); + break; + } + } + } + else + { + switch ( pField->cpp_type() ) + { + case FieldDescriptor::CPPTYPE_INT32: pKVDest->SetInt( pFieldName, pReflection->GetInt32( msg, pField ) ); break; + case FieldDescriptor::CPPTYPE_INT64: pKVDest->SetUint64( pFieldName, (uint64)pReflection->GetInt64( msg, pField ) ); break; + case FieldDescriptor::CPPTYPE_UINT32: pKVDest->SetInt( pFieldName, (int32)pReflection->GetUInt32( msg, pField ) ); break; + case FieldDescriptor::CPPTYPE_UINT64: pKVDest->SetUint64( pFieldName, pReflection->GetUInt64( msg, pField ) ); break; + case FieldDescriptor::CPPTYPE_DOUBLE: pKVDest->SetFloat( pFieldName, (float)pReflection->GetDouble( msg, pField ) ); break; + case FieldDescriptor::CPPTYPE_FLOAT: pKVDest->SetFloat( pFieldName, pReflection->GetFloat( msg, pField ) ); break; + case FieldDescriptor::CPPTYPE_BOOL: pKVDest->SetBool( pFieldName, pReflection->GetBool( msg, pField ) ); break; + case FieldDescriptor::CPPTYPE_ENUM: pKVDest->SetInt( pFieldName, pReflection->GetEnum( msg, pField )->number() );break; + case FieldDescriptor::CPPTYPE_STRING: pKVDest->SetString( pFieldName, pReflection->GetString( msg, pField ).c_str() );break; + case FieldDescriptor::CPPTYPE_MESSAGE: + { + KeyValues *pKVSub = pKVDest->FindKey( pFieldName, true ); + const ::google::protobuf::Message &subMsg = pReflection->GetMessage( msg, pField ); + RecursiveAddProtoBufToKV( pKVSub, subMsg ); + break; + } + default: + AssertMsg1( false, "Unknown cpp_type %d", pField->cpp_type() ); + break; + } + } + } +} + + +KeyValues *CProtoBufSharedObjectBase::CreateKVFromProtoBuf( const ::google::protobuf::Message & msg ) +{ + KeyValues *pKVDest = new KeyValues( msg.GetDescriptor()->name().c_str() ); + RecursiveAddProtoBufToKV( pKVDest, msg ); + return pKVDest; +} + + +//---------------------------------------------------------------------------- +// Purpose: Dumps a debug string for the object +//---------------------------------------------------------------------------- +void CProtoBufSharedObjectBase::Dump( const ::google::protobuf::Message & msg ) +{ + // print line by line to get round our limited emitinfo buffer + CUtlStringList lines; + V_SplitString( msg.DebugString().c_str(), "\n", lines ); + for ( int i = 0; i < lines.Count(); i++ ) + { + EmitInfo( SPEW_SHAREDOBJ, SPEW_ALWAYS, LOG_ALWAYS, "%s\n", lines[i] ); + } +} + +//---------------------------------------------------------------------------- +// Purpose: Dumps a debug string for the object +//---------------------------------------------------------------------------- +void CProtoBufSharedObjectBase::Dump() const +{ + Dump( *GetPObject() ); +} + +//============================================================================= + +//---------------------------------------------------------------------------- +// Purpose: Parses the message bits for creating this object from the message. +// This will be called on the client/gameserver when it first learns +// about the item. +//---------------------------------------------------------------------------- +bool CProtoBufSharedObjectBase::BParseFromMessage( const CUtlBuffer & buffer ) +{ + ::google::protobuf::Message & msg = *GetPObject(); + + return msg.ParseFromArray( buffer.Base(), buffer.TellMaxPut() ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Parses the message bits for creating this object from the message. +// This will be called on the client/gameserver when it first learns +// about the item. +//---------------------------------------------------------------------------- +bool CProtoBufSharedObjectBase::BParseFromMessage( const std::string &buffer ) +{ + ::google::protobuf::Message & msg = *GetPObject(); + + return msg.ParseFromString( buffer ); +} + +//---------------------------------------------------------------------------- +// Purpose: Overrides all the fields in msgLocal that are present in the +// network message +//---------------------------------------------------------------------------- +bool CProtoBufSharedObjectBase::BUpdateFromNetwork( const CSharedObject & objUpdate ) +{ + ::google::protobuf::Message & msg = *GetPObject(); + const CProtoBufSharedObjectBase & pbobjUpdate = (const CProtoBufSharedObjectBase &)objUpdate; + + // merge the update onto the local message + msg.CopyFrom( *pbobjUpdate.GetPObject() ); + return true; +} + + +#ifdef GC + +//---------------------------------------------------------------------------- +// Purpose: Static help class that seralizes to a buffer +//---------------------------------------------------------------------------- +bool CProtoBufSharedObjectBase::SerializeToBuffer( const ::google::protobuf::Message & msg, CUtlBuffer & bufOutput ) +{ + uint32 unSize = msg.ByteSize(); + bufOutput.Clear(); + bufOutput.EnsureCapacity( unSize ); + msg.SerializeWithCachedSizesToArray( (uint8*)bufOutput.Base() ); + bufOutput.SeekPut( CUtlBuffer::SEEK_HEAD, unSize ); + return true; +} + +//---------------------------------------------------------------------------- +// Purpose: Adds the relevant message bits to create this object to the +// message. This will be called whenever a subscriber is added. +//---------------------------------------------------------------------------- +bool CProtoBufSharedObjectBase::BAddToMessage( CUtlBuffer & bufOutput ) const +{ + const ::google::protobuf::Message & msg = *GetPObject(); + SerializeToBuffer( msg, bufOutput ); + return true; +} + + +//---------------------------------------------------------------------------- +// Purpose: Adds the relevant message bits to create this object to the +// message. This will be called whenever a subscriber is added. +//---------------------------------------------------------------------------- +bool CProtoBufSharedObjectBase::BAddToMessage( std::string *pBuffer ) const +{ + const ::google::protobuf::Message & msg = *GetPObject(); + + return msg.SerializeToString( pBuffer ); +} + +//---------------------------------------------------------------------------- +// Purpose: Parses the message bits for creating this object from the message. +// This will be called on the client/gameserver when it first learns +// about the item. +//---------------------------------------------------------------------------- +bool CProtoBufSharedObjectBase::BParseFromMemcached( CUtlBuffer & buffer ) +{ + ::google::protobuf::Message & msg = *GetPObject(); + return msg.ParseFromArray( buffer.Base(), buffer.TellMaxPut() ); +} + +//---------------------------------------------------------------------------- +// Purpose: Adds the relevant message bits to create this object to the +// message. This will be called whenever a subscriber is added. +//---------------------------------------------------------------------------- +bool CProtoBufSharedObjectBase::BAddToMemcached( CUtlBuffer & bufOutput ) const +{ + const ::google::protobuf::Message & msg = *GetPObject(); + SerializeToBuffer( msg, bufOutput ); + return true; +} + +/* + +//---------------------------------------------------------------------------- +// Purpose: Adds all the information required for this object on the client +//---------------------------------------------------------------------------- +bool CProtoBufSharedObjectBase::BAppendToMessage( CUtlBuffer & bufOutput ) const +{ + const ::google::protobuf::Message & msg = *GetPObject(); + + uint32 unSize = msg.ByteSize(); + bufOutput.EnsureCapacity( bufOutput.Size() + unSize ); + msg.SerializeWithCachedSizesToArray( (uint8*)bufOutput.Base() + bufOutput.TellPut() ); + bufOutput.SeekPut( CUtlBuffer::SEEK_HEAD, bufOutput.TellPut() + unSize ); + return true; +} + +//---------------------------------------------------------------------------- +// Purpose: Adds all the information required for this object on the client +//---------------------------------------------------------------------------- +bool CProtoBufSharedObjectBase::BAppendToMessage( std::string *pBuffer ) const +{ + const ::google::protobuf::Message & msg = *GetPObject(); + + msg.AppendToString( pBuffer ); + return true; +} +*/ + + +//---------------------------------------------------------------------------- +::google::protobuf::Message *CProtoBufSharedObjectBase::BuildDestroyToMessage( const ::google::protobuf::Message & msg ) +{ + const ::google::protobuf::Descriptor *pDescriptor = msg.GetDescriptor(); + ::google::protobuf::Message *pMessageToSend = msg.New(); + + for( int nField = 0; nField < pDescriptor->field_count(); nField++ ) + { + const ::google::protobuf::FieldDescriptor *pFieldDescriptor = pDescriptor->field( nField ); + if( !IsKeyField( pFieldDescriptor ) ) + continue; + + CopyProtoBufField( *pMessageToSend, msg, pFieldDescriptor, pFieldDescriptor ); + } + + return pMessageToSend; +} + +//---------------------------------------------------------------------------- +// Purpose: Adds just the key fields to the message +//---------------------------------------------------------------------------- +bool CProtoBufSharedObjectBase::BAddDestroyToMessage( CUtlBuffer & bufDestroy ) const +{ + const ::google::protobuf::Message & msg = *GetPObject(); + + ::google::protobuf::Message *pMessageToSend = BuildDestroyToMessage( msg ); + + SerializeToBuffer( *pMessageToSend, bufDestroy ); + delete pMessageToSend; + return true; +} + +//---------------------------------------------------------------------------- +// Purpose: Adds just the key fields to the message +//---------------------------------------------------------------------------- +bool CProtoBufSharedObjectBase::BAddDestroyToMessage( std::string *pBuffer ) const +{ + const ::google::protobuf::Message & msg = *GetPObject(); + + ::google::protobuf::Message *pMessageToSend = BuildDestroyToMessage( msg ); + pMessageToSend->SerializeToString( pBuffer ); + delete pMessageToSend; + return true; +} + +#endif //GC + +//---------------------------------------------------------------------------- +// Purpose: Copy the data from the specified schema shared object into this. +// Both objects must be of the same type. +//---------------------------------------------------------------------------- +void CProtoBufSharedObjectBase::Copy( const CSharedObject & soRHS ) +{ + Assert( GetTypeID() == soRHS.GetTypeID() ); + const CProtoBufSharedObjectBase & soRHSBase = (CProtoBufSharedObjectBase &)soRHS; + GetPObject()->CopyFrom( *soRHSBase.GetPObject() ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Claims all memory for the object. +//---------------------------------------------------------------------------- +#ifdef DBGFLAG_VALIDATE +void CProtoBufSharedObjectBase::Validate( CValidator &validator, const char *pchName ) +{ + CSharedObject::Validate( validator, pchName ); + + // these are INSIDE the function instead of outside so the interface + // doesn't change + VALIDATE_SCOPE(); +} +#endif + + +} // namespace GCSDK + + diff --git a/gcsdk/scheduledfunction.cpp b/gcsdk/scheduledfunction.cpp new file mode 100644 index 0000000..26bcb85 --- /dev/null +++ b/gcsdk/scheduledfunction.cpp @@ -0,0 +1,239 @@ +//========= Copyright 1996-2005, Valve Corporation, All rights reserved. ============// +// +// Purpose: +//=============================================================================// +#include "stdafx.h" +#include "scheduledfunction.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + + +IGCScheduledFunction::~IGCScheduledFunction() +{ + GScheduledFunctionMgr().Cancel( this ); +} + + +//------------------------------------------------------------------------- +CGlobalScheduledFunction::CGlobalScheduledFunction() : + m_pfn( NULL ) +{} + +//------------------------------------------------------------------------- +void CGlobalScheduledFunction::ScheduleMS( func_t pfn, uint32 nDelayMS ) +{ + m_pfn = pfn; + GScheduledFunctionMgr().ScheduleMS( this, nDelayMS ); +} + +void CGlobalScheduledFunction::ScheduleSecond( func_t pfn, uint32 nDelaySecond ) +{ + m_pfn = pfn; + GScheduledFunctionMgr().ScheduleSecond( this, nDelaySecond ); +} + +void CGlobalScheduledFunction::ScheduleMinute( func_t pfn, uint32 nDelayMinute ) +{ + m_pfn = pfn; + GScheduledFunctionMgr().ScheduleMinute( this, nDelayMinute ); +} + + +//------------------------------------------------------------------------- +void CGlobalScheduledFunction::Cancel() +{ + GScheduledFunctionMgr().Cancel( this ); +} + +//------------------------------------------------------------------------- +void CGlobalScheduledFunction::OnEvent() +{ + m_pfn(); +} + + +//------------------------------------------------------------------------- +CScheduledFunctionMgr::CScheduledFunctionMgr() +{ + const uint32 knSecond = 1000000; + //frame rate resolution - 30s at 20Hz + m_Resolutions[ 0 ].Init( m_ScheduleList, 600, k_cMicroSecPerShellFrame ); + //second resolution - 15 minutes + m_Resolutions[ 1 ].Init( m_ScheduleList, 15 * 60, knSecond ); + //minute resolution - 5 hours + m_Resolutions[ 2 ].Init( m_ScheduleList, 5 * 60, 60 * knSecond ); +} + +//------------------------------------------------------------------------- +void CScheduledFunctionMgr::InitStartingTime() +{ + uint64 nCurrTime = CJobTime::LJobTimeCur(); + for( uint32 nCurrBucket = 0; nCurrBucket < ARRAYSIZE( m_Resolutions ); nCurrBucket++ ) + { + m_Resolutions[ nCurrBucket ].m_nAbsLastScheduleBucket = m_Resolutions[ nCurrBucket ].GetAbsScheduleBucketIndex( nCurrTime ); + } +} + +//------------------------------------------------------------------------- + +void CScheduledFunctionMgr::InternalSchedule( uint32 nResolution, IGCScheduledFunction* pEvent, uint32 nMSDelay ) +{ + //if the event is already registered, deregister it, double registration would be a very bad thing + if( pEvent->BIsScheduled() ) + { + Cancel( pEvent ); + } + + //determine which bucket this belongs in + uint32 nAbsBucket = m_Resolutions[ nResolution ].GetAbsScheduleBucketIndex( CJobTime::LJobTimeCur() + ( uint64 )nMSDelay * 1000 ); + //so we can remove it, and deal with wrapping + pEvent->m_nAbsScheduleBucket = nAbsBucket; + + //add it to our list + uint32 nInsertAfter = m_Resolutions[ nResolution ].m_pBuckets[ nAbsBucket % m_Resolutions[ nResolution ].m_nNumBuckets ]; + pEvent->m_nLLIndex = m_ScheduleList.InsertAfter( nInsertAfter, pEvent ); +} + +void CScheduledFunctionMgr::ScheduleMS( IGCScheduledFunction* pEvent, uint32 nMSDelay ) +{ + InternalSchedule( 0, pEvent, nMSDelay ); +} + +void CScheduledFunctionMgr::ScheduleSecond( IGCScheduledFunction* pEvent, uint32 nSDelay ) +{ + InternalSchedule( 1, pEvent, nSDelay * 1000 ); +} + +void CScheduledFunctionMgr::ScheduleMinute( IGCScheduledFunction* pEvent, uint32 nMinuteDelay ) +{ + InternalSchedule( 2, pEvent, nMinuteDelay * 1000 * 60 ); +} + +//------------------------------------------------------------------------- +void CScheduledFunctionMgr::Cancel( IGCScheduledFunction* pEvent ) +{ + //ignore it if not already registered + if( pEvent->m_nAbsScheduleBucket == IGCScheduledFunction::knInvalidBucket ) + return; + + if( m_ScheduleList.IsValidIndex( pEvent->m_nLLIndex ) ) + { + m_ScheduleList.Remove( pEvent->m_nLLIndex ); + } + else + { + AssertMsg( false, "Warning: Ecountered a remove request for a scheduled event but was unable to find it in the bucket that it was supposed to be in" ); + } + + pEvent->m_nAbsScheduleBucket = IGCScheduledFunction::knInvalidBucket; +} + +//------------------------------------------------------------------------- +void CScheduledFunctionMgr::RunFunctions() +{ + VPROF_BUDGET( "CGCBase::CallScheduledEvents", VPROF_BUDGETGROUP_STEAM ); + + uint64 nCurrTime = CJobTime::LJobTimeCur(); + for( uint32 nCurrBucket = 0; nCurrBucket < ARRAYSIZE( m_Resolutions ); nCurrBucket++ ) + { + m_Resolutions[ nCurrBucket ].RunFunctions( m_ScheduleList, nCurrTime ); + } +} + + +//------------------------------------------------------------------------- +CScheduledFunctionMgr& GScheduledFunctionMgr() +{ + static CScheduledFunctionMgr s_Singleton; + return s_Singleton; +} + +//------------------------------------------------------------------------- +CScheduledFunctionMgr::CScheduleBucket::CScheduleBucket() : + m_pBuckets( NULL ), + m_nNumBuckets( 0 ), + m_nMicroSPerBucket( 0 ), + m_nAbsLastScheduleBucket( 0 ) +{} + +CScheduledFunctionMgr::CScheduleBucket::~CScheduleBucket() +{ + delete [] m_pBuckets; +} + +void CScheduledFunctionMgr::CScheduleBucket::Init( TScheduleList& MasterList, uint32 nNumBuckets, uint32 nMicroSPerBucket ) +{ + //init should never happen more than once + AssertMsg( !m_pBuckets, "Error: Schedule buckets should never be initialized multiple times" ); + + m_pBuckets = new uint32[ nNumBuckets ]; + m_nNumBuckets = nNumBuckets; + m_nMicroSPerBucket = nMicroSPerBucket; + + for( uint32 nCurrBucket = 0; nCurrBucket < nNumBuckets; nCurrBucket++ ) + { + m_pBuckets[ nCurrBucket ] = MasterList.AddToTail( NULL ); + } +} + +void CScheduledFunctionMgr::CScheduleBucket::RunFunctions( TScheduleList& MasterList, uint64 nMicroSTime ) +{ + //determine the absolute bucket index of our current frame that we want to run to + uint32 nCurrFrameBucket = GetAbsScheduleBucketIndex( nMicroSTime ); + + //note that we include the starting bucket, but not the ending bucket. This addresses two issues: it avoids checking buckets multiple times, and since we may not have completely spent + //all the time for the frame, events will not fire early + for( uint32 nAbsBucket = m_nAbsLastScheduleBucket; nAbsBucket < nCurrFrameBucket; nAbsBucket++ ) + { + //map this to a relative bucket + uint32 nStartIndex = m_pBuckets[ nAbsBucket % m_nNumBuckets ]; + AssertMsg2( nStartIndex != MasterList.InvalidIndex(), "Error: Detected list corruption when accessing list bucket %d (%d micro s per bucket)", nAbsBucket % m_nNumBuckets, m_nMicroSPerBucket ); + for( uint32 nCurrEvent = MasterList.Next( nStartIndex ); nCurrEvent != MasterList.InvalidIndex(); ) + { + //make sure to advance to the next item immediately since we'll be removing this element from our list + uint32 nEventToRemove = nCurrEvent; + IGCScheduledFunction* pEvent = MasterList[ nCurrEvent ]; + nCurrEvent = MasterList.Next( nCurrEvent ); + + //see if we have hit the end of our bucket + if( !pEvent ) + { + break; + } + + //skip any event that happens in the future (can occur since we wrap the frame list over itself to map to an actual bucket) + if( pEvent->m_nAbsScheduleBucket > nAbsBucket ) + { + //just skip over this element + continue; + } + + //sanity check that this bucket matches exactly. We'll process stale ones, but this warns us that we missed an event and cycled through the bucket, causing it be very delayed + AssertMsg2( pEvent->m_nAbsScheduleBucket == nAbsBucket, "Warning: Encountered a scheduled event that was intended to be fired on bucket %d but wasn't until bucket %d", pEvent->m_nAbsScheduleBucket, nAbsBucket ); + + //mark the object has having been removed + pEvent->m_nAbsScheduleBucket = IGCScheduledFunction::knInvalidBucket; + pEvent->m_nLLIndex = MasterList.InvalidIndex(); + + //remove it from our list + MasterList.Remove( nEventToRemove ); + + //call into the object + pEvent->OnEvent(); + //NOTE: You cannot use the object any more, it may have destroyed itself!!!! + pEvent = NULL; + } + } + + //and update the event range we need to process + m_nAbsLastScheduleBucket = nCurrFrameBucket; +} + + + +} //namespace GCSDK + diff --git a/gcsdk/schemasharedobject.cpp b/gcsdk/schemasharedobject.cpp new file mode 100644 index 0000000..9ab65d1 --- /dev/null +++ b/gcsdk/schemasharedobject.cpp @@ -0,0 +1,274 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Shared object based on a CBaseRecord subclass +// +//============================================================================= +#include "stdafx.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + +#ifdef GC + +//---------------------------------------------------------------------------- +// Purpose: Returns true if the object needs to be saved to the database +//---------------------------------------------------------------------------- +bool CSchemaSharedObjectHelper::BYieldingAddToDatabase( CRecordBase *pRecordBase ) +{ + if( pRecordBase->GetPSchema()->GetRecordInfo()->BHasIdentity() ) + { + CSQLAccess sqlAccess; + return sqlAccess.BYieldingInsertWithIdentity( pRecordBase ); + } + return false; +} + + +//---------------------------------------------------------------------------- +// Purpose: Returns true if the object needs to be saved to the database +//---------------------------------------------------------------------------- +bool CSchemaSharedObjectHelper::BYieldingAddInsertToTransaction( CSQLAccess & sqlAccess, CRecordBase *pRecordBase ) +{ + Assert( !pRecordBase->GetPSchema()->GetRecordInfo()->BHasIdentity() ); + return sqlAccess.BYieldingInsertRecord( pRecordBase ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Reads the object from the database via its primary key. Initialize +// the primary key fields before you call this. +//---------------------------------------------------------------------------- +bool CSchemaSharedObjectHelper::BYieldingReadFromDatabase( CRecordBase *pRecordBase ) +{ + CSQLAccess sqlAccess; + CColumnSet csPK( pRecordBase->GetPSchema()->GetRecordInfo() ); + csPK.MakePrimaryKey(); + CColumnSet csReadFields( pRecordBase->GetPSchema()->GetRecordInfo() ); + csReadFields.MakeInverse( csPK ); + if( k_EResultOK != sqlAccess.YieldingReadRecordWithWhereColumns( pRecordBase, csReadFields, csPK ) ) + return false; + + return true; +} + + +//---------------------------------------------------------------------------- +// Purpose: Writes the non-PK fields on the object to the database +//---------------------------------------------------------------------------- +bool CSchemaSharedObjectHelper::BYieldingAddWriteToTransaction( CSQLAccess & sqlAccess, CRecordBase *pRecordBase, const CColumnSet & csDatabaseDirty ) +{ + CColumnSet csPK( pRecordBase->GetPSchema()->GetRecordInfo() ); + csPK.MakePrimaryKey(); + + if( !sqlAccess.BYieldingUpdateRecord( *pRecordBase, csPK, csDatabaseDirty ) ) + return false; + return true; +} + + +//---------------------------------------------------------------------------- +// Purpose: Adds the primary key for this object to the network message +//---------------------------------------------------------------------------- +bool CSchemaSharedObjectHelper::BYieldingAddRemoveToTransaction( CSQLAccess & sqlAccess, CRecordBase *pRecordBase ) +{ + CColumnSet csPK( pRecordBase->GetPSchema()->GetRecordInfo() ); + csPK.MakePrimaryKey(); + + if( !sqlAccess.BYieldingDeleteRecords( *pRecordBase, csPK ) ) + return false; + + return true; +} +#endif // GC + + +//---------------------------------------------------------------------------- +// Purpose: Dumps diagnostic information about the shared object +//---------------------------------------------------------------------------- +void CSchemaSharedObjectHelper::Dump( const CRecordBase *pRecordBase ) +{ + CFmtStr1024 sOutput; + const CRecordInfo *pRecordInfo = pRecordBase->GetPRecordInfo(); + for( uint32 unColumn = 0; unColumn < (uint32)pRecordInfo->GetNumColumns(); unColumn++ ) + { + if( unColumn > 0 ) + { + sOutput += ", "; + } + + sOutput += pRecordInfo->GetColumnInfo( unColumn ).GetName(); + sOutput += "="; + + char pchFieldData[128]; + pRecordBase->RenderField( unColumn, sizeof(pchFieldData), pchFieldData ); + sOutput += pchFieldData; + } + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\t\t%s\n", sOutput.Access() ); +} + + +//============================================================================= + + + +#ifdef GC + +//---------------------------------------------------------------------------- +// Purpose: Returns true if the object needs to be saved to the database +//---------------------------------------------------------------------------- +bool CSchemaSharedObjectBase::BYieldingAddToDatabase() +{ + if( GetPObject()->GetPSchema()->GetRecordInfo()->BHasIdentity() ) + { + return CSchemaSharedObjectHelper::BYieldingAddToDatabase( GetPObject() ); + } + else + return CSharedObject::BYieldingAddToDatabase(); +} + + +//---------------------------------------------------------------------------- +// Purpose: Returns true if the object needs to be saved to the database +//---------------------------------------------------------------------------- +bool CSchemaSharedObjectBase::BYieldingAddInsertToTransaction( CSQLAccess & sqlAccess ) +{ + return CSchemaSharedObjectHelper::BYieldingAddInsertToTransaction( sqlAccess, GetPObject() ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Reads the object from the database via its primary key. Initialize +// the primary key fields before you call this. +//---------------------------------------------------------------------------- +bool CSchemaSharedObjectBase::BYieldingReadFromDatabase() +{ + return CSchemaSharedObjectHelper::BYieldingReadFromDatabase( GetPObject() ); +} + +CColumnSet CSchemaSharedObjectBase::GetDatabaseDirtyColumnSet( const CUtlVector< int > &fields ) const +{ + CColumnSet cs( GetPObject()->GetPRecordInfo() ); + FOR_EACH_VEC( fields, i ) + { + cs.BAddColumn( fields[i] ); + } + + return cs; +} + +//---------------------------------------------------------------------------- +// Purpose: Writes the non-PK fields on the object to the database +//---------------------------------------------------------------------------- +bool CSchemaSharedObjectBase::BYieldingAddWriteToTransaction( CSQLAccess & sqlAccess, const CUtlVector< int > &fields ) +{ + CColumnSet csDatabaseDirty = GetDatabaseDirtyColumnSet(fields); + return CSchemaSharedObjectHelper::BYieldingAddWriteToTransaction( sqlAccess, GetPObject(), csDatabaseDirty ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Adds the primary key for this object to the network message +//---------------------------------------------------------------------------- +bool CSchemaSharedObjectBase::BYieldingAddRemoveToTransaction( CSQLAccess & sqlAccess ) +{ + if ( CSchemaSharedObjectHelper::BYieldingAddRemoveToTransaction( sqlAccess, GetPObject() ) ) + { + return true; + } + return false; +} + + +#endif // GC + +//---------------------------------------------------------------------------- +// Purpose: Returns true if this is less than than the object in soRHS. This +// comparison is deterministic, but it may not be pleasing to a user +// since it is just going to compare raw memory. If you need a sort +// that is user-visible you will need to do it at a higher level that +// actually knows what the data in these objects means. +//---------------------------------------------------------------------------- +bool CSchemaSharedObjectBase::BIsKeyLess( const CSharedObject & soRHS ) const +{ + Assert( GetTypeID() == soRHS.GetTypeID() ); + const CSchemaSharedObjectBase & soSchemaRHS = (const CSchemaSharedObjectBase &)soRHS; + + CColumnSet csPK( GetPObject()->GetPSchema()->GetRecordInfo() ); + csPK.MakePrimaryKey(); + + FOR_EACH_COLUMN_IN_SET( csPK, unColumnIndex ) + { + uint8 *pubMyData, *pubTheirData; + uint32 cubMyData, cubTheirData; + int nColumn = csPK.GetColumn( unColumnIndex ); + DbgVerify( GetPObject()->BGetField( nColumn, &pubMyData, &cubMyData ) ); + DbgVerify( soSchemaRHS.GetPObject()->BGetField( nColumn, &pubTheirData, &cubTheirData ) ); + + int nCmp = Q_memcmp( pubMyData, pubTheirData, MIN( cubMyData, cubTheirData ) ); + + // longer chunks of memory are greater + if( nCmp == 0 ) + { + nCmp = cubTheirData > cubMyData; + } + + if( nCmp != 0 ) + return nCmp < 0; + } + + return false; +} + + +//---------------------------------------------------------------------------- +// Purpose: Copy the data from the specified schema shared object into this. +// Both objects must be of the same type. +//---------------------------------------------------------------------------- +void CSchemaSharedObjectBase::Copy( const CSharedObject & soRHS ) +{ + Assert( GetTypeID() == soRHS.GetTypeID() ); + CSchemaSharedObjectBase & soRHSBase = (CSchemaSharedObjectBase &)soRHS; + *GetPObject() = *(soRHSBase.GetPObject()); +} + + +//---------------------------------------------------------------------------- +// Purpose: Dumps diagnostic information about the shared object +//---------------------------------------------------------------------------- +void CSchemaSharedObjectBase::Dump() const +{ + CSchemaSharedObjectHelper::Dump( GetPObject() ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Returns the record info object for this type of schema object +//---------------------------------------------------------------------------- +const CRecordInfo * CSchemaSharedObjectBase::GetRecordInfo() const +{ + return GetPObject()->GetPRecordInfo(); +} + + +//---------------------------------------------------------------------------- +// Purpose: Claims all memory for the object. +//---------------------------------------------------------------------------- +#ifdef DBGFLAG_VALIDATE +void CSchemaSharedObjectBase::Validate( CValidator &validator, const char *pchName ) +{ + CSharedObject::Validate( validator, pchName ); + + // these are INSIDE the function instead of outside so the interface + // doesn't change + VALIDATE_SCOPE(); +} +#endif + + +} // namespace GCSDK + + diff --git a/gcsdk/sdocache.cpp b/gcsdk/sdocache.cpp new file mode 100644 index 0000000..bc6235f --- /dev/null +++ b/gcsdk/sdocache.cpp @@ -0,0 +1,999 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Database Backed Object caching and manipulation +// +//============================================================================= +#include "stdafx.h" +#include "sdocache.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +GCConVar s_ConVarSDOCacheLRULimitMB( "@SDOCacheLRULimitMB", "400", FCVAR_REPLICATED ); +GCConVar s_ConVarSDOCacheMaxMemcachedReadJobs( "@SDOCacheMaxMemcachedReadJobs", "4", FCVAR_REPLICATED ); +GCConVar s_ConVarSDOCacheMaxMemcachedReadBatchSize( "@SDOCacheMaxMemcachedReadBatchSize", "100", FCVAR_REPLICATED ); +GCConVar s_ConVarSDOCacheMaxSQLReadJobsPerSDOType( "@SDOCacheMaxSQLReadJobsPerSDOType", "4", FCVAR_REPLICATED ); +GCConVar s_ConVarSDOCacheMaxSQLReadBatchSize( "@SDOCacheMaxSQLReadBatchSize", "100", FCVAR_REPLICATED ); +GCConVar s_ConVarSDOCacheMaxPendingSQLReads( "@SDOCacheMaxPendingSQLReads", "2000", FCVAR_REPLICATED ); +GCConVar s_ConVarSDOCacheMaxPendingMemcachedReads( "@SDOCacheMaxPendingMemcachedReads", "25000", FCVAR_REPLICATED ); + +namespace GCSDK +{ +// A string used to tell the difference between nil objects and actual objects in memcached +const char k_rgchNilObjSerializedValue[] = "nilobj"; + + +// Global instance +CSDOCache &GSDOCache() +{ + static CSDOCache s_SDOCache; + return s_SDOCache; +} + + +//----------------------------------------------------------------------------- +// Purpose: Creates a key name that looks like "Prefix_%u" but faster +//----------------------------------------------------------------------------- +void CSDOCache::CreateSimpleMemcachedName( CFmtStr &strDest, const char *pchPrefix, uint32 unPrefixLen, uint32 unSuffix ) +{ + Assert( FMTSTR_STD_LEN - unPrefixLen > 10 + 1 ); + V_memcpy( strDest.Access(), pchPrefix, unPrefixLen ); + _i64toa( unSuffix, strDest.Access() + unPrefixLen, 10 ); + strDest.Access()[ FMTSTR_STD_LEN - 1 ] = NULL; +} + + +//----------------------------------------------------------------------------- +// Purpose: constructor +//----------------------------------------------------------------------------- +CSDOCache::CSDOCache() : + m_cubLRUItems( 0 ), + m_mapTypeStats( DefLessFunc( int ) ) +{ + memset( &m_StatsSDOCache, 0, sizeof( m_StatsSDOCache ) ); +} + + +//----------------------------------------------------------------------------- +// Purpose: destructor +//----------------------------------------------------------------------------- +CSDOCache::~CSDOCache() +{ + // delete all the entries from our cache + FOR_EACH_MAP_FAST( m_mapISDOLoaded, iMap ) + { + delete m_mapISDOLoaded.Key( iMap ); + } + + FOR_EACH_MAP_FAST( m_mapQueuedRequests, iMap ) + { + delete m_mapQueuedRequests.Key( iMap ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: registers an SDO type +//----------------------------------------------------------------------------- +void CSDOCache::RegisterSDO( int nType, const char *pchName ) +{ + if ( !m_mapQueueSQLRequests.HasElement( nType ) ) + { + m_mapQueueSQLRequests.Insert( nType, new SQLRequestManager_t ); + int iMap = m_mapTypeStats.Insert( nType ); + m_mapTypeStats[iMap].m_strName = pchName; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: a SDO object has been referenced +//----------------------------------------------------------------------------- +void CSDOCache::OnSDOReferenced( ISDO *pSDO ) +{ + // lookup where the SDO is, it has where we are in the LRU + int iMap = m_mapISDOLoaded.Find( pSDO ); + Assert( m_mapISDOLoaded.IsValidIndex( iMap ) ); + if ( !m_mapISDOLoaded.IsValidIndex( iMap ) ) + return; + + // move us out of the LRU + if ( m_listLRU.IsValidIndex( m_mapISDOLoaded[iMap] ) ) + { + RemoveSDOFromLRU( iMap ); + } + else + { + // we may not have been in the LRU if it's a custom insert + } + + // Update stats + int iMapStats = m_mapTypeStats.Find( pSDO->GetType() ); + if ( m_mapTypeStats.IsValidIndex( iMapStats ) ) + { + m_mapTypeStats[iMapStats].m_nRefed++; + } +} + +//----------------------------------------------------------------------------- +// Purpose: a SDO object has gone released +//----------------------------------------------------------------------------- +void CSDOCache::OnSDOReleased( ISDO *pSDO ) +{ + // Update stats + int iMapStats = m_mapTypeStats.Find( pSDO->GetType() ); + if ( m_mapTypeStats.IsValidIndex( iMapStats ) ) + { + m_mapTypeStats[iMapStats].m_nRefed--; + } + + // lookup where the SDO is, it has where we are in the LRU + int iMap = m_mapISDOLoaded.Find( pSDO ); + if ( m_mapISDOLoaded.IsValidIndex( iMap ) ) + { + // we shouldn't be in the LRU + bool bInLRU = m_listLRU.IsValidIndex( m_mapISDOLoaded[iMap] ); + Assert( !bInLRU ); + if ( bInLRU ) + return; + + // count the bytes and move us to the head of the LRU + ISDO *pSDO = m_mapISDOLoaded.Key( iMap ); + LRUItem_t item = { pSDO, pSDO->CubBytesUsed() }; + m_mapISDOLoaded[iMap] = m_listLRU.AddToTail( item ); + m_cubLRUItems += item.m_cub; + + if ( m_mapTypeStats.IsValidIndex( iMapStats ) ) + { + m_mapTypeStats[iMapStats].m_cubUnrefed += item.m_cub; + } + } + else + { + // it's actually valid it's not in the SDO cache anymore - could be an orphaned pointer + if ( pSDO->GetRefCount() == 0 ) + { + delete pSDO; + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: pulls a cached SDO from the right maps +//----------------------------------------------------------------------------- +void CSDOCache::RemoveSDOFromLRU( int iMap ) +{ + int iLRU = m_mapISDOLoaded[iMap]; + + Assert( m_listLRU.IsValidIndex( iLRU ) ); + if ( !m_listLRU.IsValidIndex( iLRU ) ) + return; + + int iMapStats = m_mapTypeStats.Find( m_listLRU[iLRU].m_pSDO->GetType() ); + if ( m_mapTypeStats.IsValidIndex( iMapStats ) ) + { + m_mapTypeStats[iMapStats].m_cubUnrefed -= m_listLRU[iLRU].m_cub; + } + + m_cubLRUItems -= m_listLRU[iLRU].m_cub; + m_listLRU.Remove( iLRU ); + m_mapISDOLoaded[iMap] = m_listLRU.InvalidIndex(); + + // Update stats +} + + +//----------------------------------------------------------------------------- +// Purpose: writes a SDO to memcached +//----------------------------------------------------------------------------- +bool CSDOCache::WriteSDOToMemcached( ISDO *pSDO ) +{ + CFmtStr strKey; + CUtlBuffer buf; + + pSDO->GetMemcachedKeyName( strKey ); + pSDO->WriteToBuffer( buf ); + return GGCBase()->BMemcachedSet( strKey.Access(), buf ); +} + + +//----------------------------------------------------------------------------- +// Purpose: writes a SDO to memcached +//----------------------------------------------------------------------------- +bool CSDOCache::DeleteSDOFromMemcached( ISDO *pSDO ) +{ + CFmtStr strKey; + pSDO->GetMemcachedKeyName( strKey ); + return GGCBase()->BMemcachedDelete( strKey.Access() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: runs the queued batch loader +//----------------------------------------------------------------------------- +class CGCJobLoadSDOSetFromMemcached : public CGCJob +{ +public: + CGCJobLoadSDOSetFromMemcached( CGCBase *pGC ) + : CGCJob( pGC ) + { + } + + void AddSDOToLoad( ISDO *pSDO, int iRequestID ) + { + int i = m_vecRequests.AddToTail(); + m_vecRequests[i].m_pSDO = pSDO; + m_vecRequests[i].m_iRequestID = iRequestID; + } + + bool BYieldingRunJob( void * ) + { + Assert( m_vecRequests.Count() > 0 ); + if ( 0 == m_vecRequests.Count() ) + { + GSDOCache().OnMemcachedLoadJobComplete( GetJobID() ); + return false; + } + + // get the names of all the items + CUtlVector<CUtlString> vecKeys; + vecKeys.AddMultipleToTail( m_vecRequests.Count() ); + FOR_EACH_VEC( m_vecRequests, i ) + { + CFmtStr strKey; + m_vecRequests[i].m_pSDO->GetMemcachedKeyName( strKey ); + vecKeys[i] = strKey; + } + + // ask in a batch from memcached + CUtlVector<CGCBase::GCMemcachedGetResult_t> vecGetResults; + bool bGetSuccess = m_pGC->BYieldingMemcachedGet( vecKeys, vecGetResults ); + + // go through each request looking up the results + FOR_EACH_VEC( m_vecRequests, i ) + { + if ( bGetSuccess && vecGetResults.IsValidIndex( i ) && vecGetResults[i].m_bKeyFound ) + { + const CGCBase::GCMemcachedGetResult_t &result = vecGetResults[i]; + bool bNilObj = ( result.m_bufValue.Count() == sizeof( k_rgchNilObjSerializedValue ) + && 0 == V_memcmp( result.m_bufValue.Base(), k_rgchNilObjSerializedValue, sizeof( k_rgchNilObjSerializedValue ) ) ); + + // we've loaded OK + if ( bNilObj || ( result.m_bufValue.Count() && m_vecRequests[i].m_pSDO->BReadFromBuffer( result.m_bufValue.Base(), result.m_bufValue.Count() ) ) ) + { + GSDOCache().OnSDOLoadSuccess( m_vecRequests[i].m_pSDO->GetType(), m_vecRequests[i].m_iRequestID, bNilObj, &m_vecRequests[i].m_pSDO ); + GSDOCache().GetStats().m_cItemsLoadedFromMemcached += 1; + GSDOCache().GetStats().m_cNilItemsLoadedFromMemcached += bNilObj ? 1 : 0; + } + else + { + // couldn't load; delete the entry + m_pGC->BMemcachedDelete( vecKeys[i] ); + GSDOCache().OnMemcachedSDOLoadFailure( m_vecRequests[i].m_pSDO->GetType(), m_vecRequests[i].m_iRequestID ); + } + } + else + { + // post back failure + GSDOCache().OnMemcachedSDOLoadFailure( m_vecRequests[i].m_pSDO->GetType(), m_vecRequests[i].m_iRequestID ); + } + } + + GSDOCache().OnMemcachedLoadJobComplete( GetJobID() ); + return true; + } + +private: + struct Request_t + { + int m_iRequestID; + ISDO *m_pSDO; + }; + + CUtlVector<Request_t> m_vecRequests; +}; + + +//----------------------------------------------------------------------------- +// Purpose: runs the queued batch loader +//----------------------------------------------------------------------------- +class CGCJobLoadSDOSetFromSQL : public CGCJob +{ +public: + CGCJobLoadSDOSetFromSQL( CGCBase *pGC, int eSDOType ) + : CGCJob( pGC ), m_eSDOType( eSDOType ) + { + } + + void AddSDOToLoad( ISDO *pSDO, int iRequestID ) + { + m_vecPSDO.AddToTail( pSDO ); + m_vecRequestID.AddToTail( iRequestID ); + m_vecResults.AddToTail( false ); + } + + bool BYieldingRunJob( void * ) + { + Assert( m_vecPSDO.Count() == m_vecRequestID.Count() && m_vecPSDO.Count() == m_vecResults.Count() ); + Assert( m_vecPSDO.Count() > 0 ); + if ( 0 == m_vecPSDO.Count() + || m_vecPSDO.Count() != m_vecRequestID.Count() + || m_vecPSDO.Count() != m_vecResults.Count() ) + { + GSDOCache().OnSQLLoadJobComplete( m_eSDOType, GetJobID() ); + return false; + } + + // use the first item to load the rest + bool bSQLLayerSucceeded = m_vecPSDO[0]->BYldLoadFromSQL( m_vecPSDO, m_vecResults ); + + Assert( m_vecPSDO.Count() == m_vecRequestID.Count() && m_vecPSDO.Count() == m_vecResults.Count() ); + Assert( m_vecPSDO.Count() > 0 ); + if ( 0 == m_vecPSDO.Count() + || m_vecPSDO.Count() != m_vecRequestID.Count() + || m_vecPSDO.Count() != m_vecResults.Count() ) + { + GSDOCache().OnSQLLoadJobComplete( m_eSDOType, GetJobID() ); + return false; + } + + // walk each result + FOR_EACH_VEC( m_vecRequestID, i ) + { + if ( bSQLLayerSucceeded ) + { + // loaded, great + GSDOCache().OnSDOLoadSuccess( m_eSDOType, m_vecRequestID[i], !m_vecResults[i], &m_vecPSDO[i] ); + GSDOCache().WriteSDOToMemcached( m_vecPSDO[i] ); + + GSDOCache().GetStats().m_cItemsLoadedFromSQL += 1; + GSDOCache().GetStats().m_cNilItemsLoadedFromSQL += m_vecResults[i] ? 0 : 1; + } + else + { + // no good, item couldn't load + GSDOCache().OnSQLSDOLoadFailure( m_eSDOType, m_vecRequestID[i], bSQLLayerSucceeded ); + GSDOCache().GetStats().m_cItemsFailedLoadFromSQL += 1; + } + } + + GSDOCache().OnSQLLoadJobComplete( m_eSDOType, GetJobID() ); + return true; + } + +private: + int m_eSDOType; + + // these objects all stay in sync + // they need to be separate vectors so that they can be easily passed into other API's + CUtlVector<ISDO *> m_vecPSDO; + CUtlVector<int> m_vecRequestID; + CUtlVector<bool> m_vecResults; +}; + + +//----------------------------------------------------------------------------- +// Purpose: continues any jobs that were waiting on items +//----------------------------------------------------------------------------- +bool CSDOCache::BFrameFuncRunJobsUntilCompleted( CLimitTimer &limitTimer ) +{ + bool bDoneWork = false; + + // continue any jobs + while ( m_queueJobsToContinue.Count() && !limitTimer.BLimitReached() ) + { + // pop the item off the head + JobToWake_t jobToWake = m_queueJobsToContinue[ m_queueJobsToContinue.Head() ]; + m_queueJobsToContinue.Remove( m_queueJobsToContinue.Head() ); + GGCBase()->GetJobMgr().BRouteWorkItemCompletedIfExists( jobToWake.m_jobID, !jobToWake.m_bLoadLayerSuccess ); + } + + // if we're over the limit, LRU an item + while ( !limitTimer.BLimitReached() && m_cubLRUItems > (uint32)( s_ConVarSDOCacheLRULimitMB.GetInt() * k_nMegabyte ) && m_listLRU.Count() ) + { + // pull off the last item in the LRU + LRUItem_t item = m_listLRU[ m_listLRU.Head() ]; + + // kill the item + int iMap = m_mapISDOLoaded.Find( item.m_pSDO ); + Assert( m_mapISDOLoaded.IsValidIndex( iMap ) ); + if ( m_mapISDOLoaded.IsValidIndex( iMap ) ) + { + Assert( 0 == item.m_pSDO->GetRefCount() ); + if ( 0 == item.m_pSDO->GetRefCount() ) + { + RemoveSDOFromLRU( iMap ); + + int iMapStats = m_mapTypeStats.Find( item.m_pSDO->GetType() ); + if ( m_mapTypeStats.IsValidIndex( iMapStats ) ) + { + if ( item.m_pSDO->BNilObject() ) + { + m_mapTypeStats[iMapStats].m_nNilObjects--; + } + else + { + m_mapTypeStats[iMapStats].m_nLoaded--; + } + } + + m_mapISDOLoaded.RemoveAt( iMap ); + delete item.m_pSDO; + } + + m_StatsSDOCache.m_cItemsLRUd += 1; + m_StatsSDOCache.m_cBytesLRUd += item.m_cub; + } + + bDoneWork = true; + } + + // return true if there is still work remaining + return bDoneWork || m_queueJobsToContinue.Count() > 0; +} + + +//----------------------------------------------------------------------------- +// Purpose: runs the queued batch loader +//----------------------------------------------------------------------------- +bool CSDOCache::BFrameFuncRunMemcachedQueriesUntilCompleted( CLimitTimer &limitTimer ) +{ + bool bDoneWork = false; + + // batch load the set + if ( m_queueMemcachedRequests.Count() ) + { + // pass these off to a job + if ( m_vecMemcachedJobs.Count() < s_ConVarSDOCacheMaxMemcachedReadJobs.GetInt() ) + { + CGCJobLoadSDOSetFromMemcached *pJob = new CGCJobLoadSDOSetFromMemcached( GGCBase() ); + + // add a full batch to the job + int cItemsInBatch = 0; + while ( m_queueMemcachedRequests.Count() && cItemsInBatch < s_ConVarSDOCacheMaxMemcachedReadBatchSize.GetInt() ) + { + m_StatsSDOCache.m_cQueuedMemcachedRequests--; + int iMapQueuedRequest = m_queueMemcachedRequests[ m_queueMemcachedRequests.Head() ]; + m_queueMemcachedRequests.Remove( m_queueMemcachedRequests.Head() ); + Assert( m_mapQueuedRequests.IsValidIndex( iMapQueuedRequest ) ); + if ( m_mapQueuedRequests.IsValidIndex( iMapQueuedRequest ) ) + { + cItemsInBatch++; + pJob->AddSDOToLoad( m_mapQueuedRequests.Key( iMapQueuedRequest ), iMapQueuedRequest ); + } + } + + if ( cItemsInBatch > 0 ) + { + m_StatMemcachedBatchSize.AddSample( cItemsInBatch * 100 ); + m_StatsSDOCache.m_nMemcachedBatchSizeAvgx100 = (int)m_StatMemcachedBatchSize.GetAveragedSample(); + + // add to the list + m_vecMemcachedJobs.AddToTail( pJob->GetJobID() ); + + // start the job + pJob->StartJob( NULL ); + + // mark that we should be ran again + bDoneWork = true; + } + else + { + delete pJob; + } + } + } + + // return if we still have work to do + return bDoneWork; +} + + +//----------------------------------------------------------------------------- +// Purpose: runs the queued batch loader +//----------------------------------------------------------------------------- +bool CSDOCache::BFrameFuncRunSQLQueriesUntilCompleted( CLimitTimer &limitTimer ) +{ + bool bDoneWork = false; + + // loop through all items looking for batches to load + FOR_EACH_MAP_FAST( m_mapQueueSQLRequests, iMapType ) + { + // batch load the set + SQLRequestManager_t *sqlRequestManager = m_mapQueueSQLRequests[iMapType]; + if ( sqlRequestManager->m_queueRequestIDsToLoadFromSQL.Count() ) + { + // pass these off to a job + if ( sqlRequestManager->m_vecSQLJobs.Count() < s_ConVarSDOCacheMaxSQLReadJobsPerSDOType.GetInt() ) + { + int nSDOType = m_mapQueueSQLRequests.Key( iMapType ); + CGCJobLoadSDOSetFromSQL *pJob = new CGCJobLoadSDOSetFromSQL( GGCBase(), nSDOType ); + + // add a full batch to the job + int cItemsInBatch = 0; + while ( sqlRequestManager->m_queueRequestIDsToLoadFromSQL.Count() && cItemsInBatch < s_ConVarSDOCacheMaxSQLReadBatchSize.GetInt() ) + { + m_StatsSDOCache.m_cQueuedSQLRequests--; + int iMapQueuedRequest = sqlRequestManager->m_queueRequestIDsToLoadFromSQL[ sqlRequestManager->m_queueRequestIDsToLoadFromSQL.Head() ]; + sqlRequestManager->m_queueRequestIDsToLoadFromSQL.Remove( sqlRequestManager->m_queueRequestIDsToLoadFromSQL.Head() ); + Assert( m_mapQueuedRequests.IsValidIndex( iMapQueuedRequest ) ); + if ( m_mapQueuedRequests.IsValidIndex( iMapQueuedRequest ) ) + { + pJob->AddSDOToLoad( m_mapQueuedRequests.Key( iMapQueuedRequest ), iMapQueuedRequest ); + cItemsInBatch++; + } + } + + if ( cItemsInBatch > 0 ) + { + m_StatSQLBatchSize.AddSample( cItemsInBatch * 100 ); + m_StatsSDOCache.m_nSQLBatchSizeAvgx100 = (int)m_StatSQLBatchSize.GetAveragedSample(); + + // add to the list + sqlRequestManager->m_vecSQLJobs.AddToTail( pJob->GetJobID() ); + + // start the job + pJob->StartJob( NULL ); + bDoneWork = true; + } + else + { + delete pJob; + } + } + } + } + + // update stats + m_StatsSDOCache.m_cItemsInCache = m_mapISDOLoaded.Count(); + m_StatsSDOCache.m_cItemsQueuedToLoad = m_mapQueuedRequests.Count(); + m_StatsSDOCache.m_cBytesUnreferenced = m_cubLRUItems; + m_StatsSDOCache.m_cItemsUnreferenced = m_listLRU.Count(); + + if ( m_StatsSDOCache.m_cItemsUnreferenced ) + { + // estimate the total bytes from the size of the average size of an unreferenced item + m_StatsSDOCache.m_cBytesInCacheEst = (m_StatsSDOCache.m_cItemsInCache * m_StatsSDOCache.m_cBytesUnreferenced) / m_StatsSDOCache.m_cItemsUnreferenced; + } + + // return true if we still have work to do + return bDoneWork; +} + + +//----------------------------------------------------------------------------- +// Purpose: queues a new item to try load from memcached +//----------------------------------------------------------------------------- +int CSDOCache::QueueMemcachedLoad( ISDO *pSDO ) +{ + int iMap = -1; + if ( m_queueMemcachedRequests.Count() < s_ConVarSDOCacheMaxPendingMemcachedReads.GetInt() ) + { + // insert a fresh item into the list + iMap = m_mapQueuedRequests.Insert( pSDO ); + + // add the key to the queue + m_queueMemcachedRequests.AddToTail( iMap ); + m_StatsSDOCache.m_cQueuedMemcachedRequests++; + } + else + { + m_StatsSDOCache.m_cMemcachedRequestsRejectedTooBusy++; + delete pSDO; + } + + return iMap; +} + + +//----------------------------------------------------------------------------- +// Purpose: job results +//----------------------------------------------------------------------------- +void CSDOCache::OnSDOLoadSuccess( int eSDOType, int iRequestID, bool bNilObj, ISDO **ppSDO ) +{ + Assert( m_mapQueuedRequests.IsValidIndex( iRequestID ) ); + if ( !m_mapQueuedRequests.IsValidIndex( iRequestID ) ) + return; + + // set jobs waiting for the data to wake up + CCopyableUtlVector<JobID_t> &vecJobs = m_mapQueuedRequests[iRequestID]; + FOR_EACH_VEC( vecJobs, i ) + { + JobToWake_t jobToWake = { vecJobs[i], true /* success */ }; + m_queueJobsToContinue.AddToTail( jobToWake ); + } + + // move from requested to loaded + ISDO *pSDO = m_mapQueuedRequests.Key( iRequestID ); + m_mapQueuedRequests.RemoveAt( iRequestID ); + + // If the query succeeded but the object doesn't exist then we need to cache that + if ( bNilObj ) + { + ISDO *pInvalidSDO = pSDO->AllocNilObject(); + delete pSDO; + pSDO = pInvalidSDO; + *ppSDO = pSDO; + } + else + { + // do any extra initialization on the SDO + pSDO->PostLoadInit(); + } + + Assert( !m_mapISDOLoaded.HasElement( pSDO ) ); + int iMap = m_mapISDOLoaded.Insert( pSDO, m_listLRU.InvalidIndex() ); + + // update stats + int iMapStats = m_mapTypeStats.Find( pSDO->GetType() ); + if ( m_mapTypeStats.IsValidIndex( iMapStats ) ) + { + if ( pSDO->BNilObject() ) + { + m_mapTypeStats[iMapStats].m_nNilObjects++; + } + else + { + m_mapTypeStats[iMapStats].m_nLoaded++; + } + } + + // put us in the LRU - if it's just a hint, we may not be referenced immediately + Assert( m_mapISDOLoaded.IsValidIndex( iMap ) ); + if ( m_mapISDOLoaded.IsValidIndex( iMap ) ) + { + LRUItem_t item = { pSDO, pSDO->CubBytesUsed() }; + m_mapISDOLoaded[iMap] = m_listLRU.AddToTail( item ); + m_cubLRUItems += item.m_cub; + + if ( m_mapTypeStats.IsValidIndex( iMapStats ) ) + { + m_mapTypeStats[iMapStats].m_cubUnrefed += item.m_cub; + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: job results +//----------------------------------------------------------------------------- +void CSDOCache::OnMemcachedSDOLoadFailure( int eSDOType, int iRequestID ) +{ + // we've failed to load an item from memcached - mark as needing load from SQL + Assert( m_mapQueuedRequests.IsValidIndex( iRequestID ) ); + if ( !m_mapQueuedRequests.IsValidIndex( iRequestID ) ) + return; + + Assert( eSDOType == m_mapQueuedRequests.Key( iRequestID )->GetType() ); + if ( eSDOType != m_mapQueuedRequests.Key( iRequestID )->GetType() ) + return; + + int iSQLIndex = m_mapQueueSQLRequests.Find( eSDOType ); + AssertMsg1( m_mapQueueSQLRequests.IsValidIndex( iSQLIndex ), "Error, could not load SDO from SQL for unregistered type %d", eSDOType ); + if ( !m_mapQueueSQLRequests.IsValidIndex( iSQLIndex ) ) + return; + + SQLRequestManager_t *sqlRequestManager = m_mapQueueSQLRequests[iSQLIndex]; + + if ( sqlRequestManager->m_queueRequestIDsToLoadFromSQL.Count() < s_ConVarSDOCacheMaxPendingSQLReads.GetInt() ) + { + sqlRequestManager->m_queueRequestIDsToLoadFromSQL.AddToTail( iRequestID ); + m_StatsSDOCache.m_cQueuedSQLRequests++; + } + else + { + // too many outstanding items, reject and fail immediately + m_StatsSDOCache.m_cSQLRequestsRejectedTooBusy++; + OnSQLSDOLoadFailure( eSDOType, iRequestID, false /* loader failure */ ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: job results +//----------------------------------------------------------------------------- +void CSDOCache::OnSQLSDOLoadFailure( int eSDOType, int iRequestID, bool bSQLLayerSucceeded ) +{ + Assert( m_mapQueuedRequests.IsValidIndex( iRequestID ) ); + if ( !m_mapQueuedRequests.IsValidIndex( iRequestID ) ) + return; + + // failed to load from SQL + // set jobs waiting for the data to wake up + CCopyableUtlVector<JobID_t> &vecJobs = m_mapQueuedRequests[iRequestID]; + FOR_EACH_VEC( vecJobs, i ) + { + JobToWake_t jobToWake = { vecJobs[i], bSQLLayerSucceeded }; + m_queueJobsToContinue.AddToTail( jobToWake ); + } + + // kill the object - no one should have references to it, since it's only in the request list + ISDO *pSDO = m_mapQueuedRequests.Key( iRequestID ); + m_mapQueuedRequests.RemoveAt( iRequestID ); + + Assert( 0 == pSDO->GetRefCount() ); + if ( 0 == pSDO->GetRefCount() ) + { + delete pSDO; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: job results +//----------------------------------------------------------------------------- +void CSDOCache::OnMemcachedLoadJobComplete( JobID_t jobID ) +{ + m_vecMemcachedJobs.FindAndRemove( jobID ); +} + + +//----------------------------------------------------------------------------- +// Purpose: job results +//----------------------------------------------------------------------------- +void CSDOCache::OnSQLLoadJobComplete( int eSDOType, JobID_t jobID ) +{ + int iSQLIndex = m_mapQueueSQLRequests.Find( eSDOType ); + AssertMsg1( m_mapQueueSQLRequests.IsValidIndex( iSQLIndex ), "Error, could not load SDO from SQL for unregistered type %d", eSDOType ); + if ( !m_mapQueueSQLRequests.IsValidIndex( iSQLIndex ) ) + return; + + SQLRequestManager_t *sqlRequestManager = m_mapQueueSQLRequests[iSQLIndex]; + sqlRequestManager->m_vecSQLJobs.FindAndRemove( jobID ); +} + + +//----------------------------------------------------------------------------- +// Purpose: deletes all unreferenced objects +//----------------------------------------------------------------------------- +void CSDOCache::Flush() +{ + int cReferencedObjects = 0; + FOR_EACH_MAP_FAST( m_mapISDOLoaded, iMap ) + { + if ( m_mapISDOLoaded.Key( iMap )->GetRefCount() == 0 ) + { + int iMapStats = m_mapTypeStats.Find( m_mapISDOLoaded.Key( iMap )->GetType() ); + if ( m_mapTypeStats.IsValidIndex( iMapStats ) ) + { + if ( m_mapISDOLoaded.Key( iMap )->BNilObject() ) + { + m_mapTypeStats[iMapStats].m_nNilObjects--; + } + else + { + m_mapTypeStats[iMapStats].m_nLoaded--; + } + } + + RemoveSDOFromLRU( iMap ); + DeleteSDOFromMemcached( m_mapISDOLoaded.Key( iMap ) ); + delete m_mapISDOLoaded.Key( iMap ); + m_mapISDOLoaded.RemoveAt( iMap ); + } + else + { + cReferencedObjects++; + } + } + + Assert( cReferencedObjects == 0 ); + Assert( m_cubLRUItems == 0 ); +} + + +//----------------------------------------------------------------------------- +// Purpose: returns the number of bytes we estimate we have referenced +//----------------------------------------------------------------------------- +int CSDOCache::CubReferencedEst() +{ + return MAX( 0, (int)m_StatsSDOCache.m_cBytesInCacheEst - (int)m_StatsSDOCache.m_cBytesUnreferenced ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Prints information about the cache +//----------------------------------------------------------------------------- +void CSDOCache::Dump() +{ + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "SDO cache:\n" ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " Items Cached: %llu\n", m_StatsSDOCache.m_cItemsInCache ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " Estimated Bytes Cached: %s\n", V_pretifymem( m_StatsSDOCache.m_cBytesInCacheEst, 2, true ) ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " Loads Queued: %llu\n", m_StatsSDOCache.m_cItemsQueuedToLoad ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " Memcached Loads Queued: %llu\n", m_StatsSDOCache.m_cQueuedMemcachedRequests ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " SQL Loads Queued: %llu\n", m_StatsSDOCache.m_cQueuedSQLRequests ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " Items Unreferenced: %llu\n", m_StatsSDOCache.m_cItemsUnreferenced ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " Bytes Unreferenced: %s\n", V_pretifymem( m_StatsSDOCache.m_cBytesUnreferenced, 2, true ) ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " Items LRU'd: %llu\n", m_StatsSDOCache.m_cItemsLRUd ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " Bytes LRU'd: %s\n", V_pretifymem( m_StatsSDOCache.m_cBytesLRUd, 2, true ) ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " Items Loaded From Memcached: %llu\n", m_StatsSDOCache.m_cItemsLoadedFromMemcached ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " Items Loaded From SQL: %llu\n", m_StatsSDOCache.m_cItemsLoadedFromSQL ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " Nils Loaded From Memcached: %llu\n", m_StatsSDOCache.m_cNilItemsLoadedFromMemcached ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " Nils Loaded From SQL: %llu\n", m_StatsSDOCache.m_cNilItemsLoadedFromSQL ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " Average Memcached Batch: %f\n", (float)m_StatsSDOCache.m_nMemcachedBatchSizeAvgx100 / 100.0f ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " Average SQL Batch: %f\n", (float)m_StatsSDOCache.m_nSQLBatchSizeAvgx100 / 100.0f ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " Memcached Requests Rejected: %llu\n", m_StatsSDOCache.m_cMemcachedRequestsRejectedTooBusy ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " SQL Requests Rejected: %llu\n", m_StatsSDOCache.m_cSQLRequestsRejectedTooBusy ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " Failed SQL Loads: %llu\n", m_StatsSDOCache.m_cItemsFailedLoadFromSQL ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\n" ); + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "Per-type stats\n" ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, + "%-35s --- loaded --- referenced --- nil objects --- size (est)\n", "name" ); + + FOR_EACH_MAP_FAST( m_mapTypeStats, i ) + { + int nLoaded = m_mapTypeStats[i].m_nLoaded + m_mapTypeStats[i].m_nNilObjects; + char *pchRefed = "unknown"; + if ( m_mapTypeStats[i].m_nRefed < nLoaded && m_mapTypeStats[i].m_cubUnrefed > 0 ) + { + pchRefed = V_pretifymem( ( ((int64)nLoaded * (int64)m_mapTypeStats[i].m_cubUnrefed) / ( nLoaded - m_mapTypeStats[i].m_nRefed ) ), 2, true ); + } + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%-35s %11d %14d %15d %14s\n", m_mapTypeStats[i].m_strName.String(), m_mapTypeStats[i].m_nLoaded, m_mapTypeStats[i].m_nRefed, m_mapTypeStats[i].m_nNilObjects, pchRefed ); + } +} + + +//**tempcomment** - This is good code. We'll hook it back up later +// +//static ConVar s_ConVarVerifyMemcacheDeletesBadEntries( "@VerifyMemcacheDeletesBadEntries", "1", FCVAR_REPLICATED ); +// +////----------------------------------------------------------------------------- +//// Purpose: verifies a set of memcached data against what's in SQL +////----------------------------------------------------------------------------- +//void CSDOCache::YldVerifyMemcachedData( CreateSDOFunc_t pCreateSDOFunc, CUtlVector<uint32> &vecIDs, int *pcMatches, int *pcMismatches ) +//{ +// // create the objects +// CUtlVector<ISDO *> vecPSDOSQL; +// CUtlVector<bool> vecSQLResults; +// FOR_EACH_VEC( vecIDs, i ) +// { +// vecPSDOSQL.AddToTail( pCreateSDOFunc( *this, vecIDs[i] ) ); +// vecSQLResults.AddToTail( false ); +// } +// +// CUtlVector<CUtlString> vecSKeysToDelete; +// +// // load them in a batch +// bool bSQLLayerSucceeded = vecPSDOSQL[0]->BYldLoadFromSQL( vecIDs, vecPSDOSQL, vecSQLResults ); +// if ( bSQLLayerSucceeded ) +// { +// // retrieve the memcache data +// CUtlVector<CUtlString> vecSKeys; +// FOR_EACH_VEC( vecPSDOSQL, i ) +// { +// int iKey = vecSKeys.AddToTail(); +// vecPSDOSQL[i]->GetMemcachedKeyName( vecSKeys[iKey] ); +// } +// CMemcachedAccess memcachedAccess( m_pServer->GetPMemcachedMgr() ); +// CUtlVector<MemcachedGetResult_t> vecMemcacheResults; +// if ( memcachedAccess.BYldGetMulti( vecSKeys, vecMemcacheResults ) ) +// { +// Assert( vecMemcacheResults.Count() == vecSQLResults.Count() ); +// +// FOR_EACH_VEC( vecSQLResults, i ) +// { +// bool bClearKey = false; +// if ( vecSQLResults[i] && vecMemcacheResults[i].m_bKeyFound ) +// { +// // compare the results +// ISDO *pSDOCached = pCreateSDOFunc( *this, vecIDs[i] ); +// const CUtlAllocation &allocMemcache = vecMemcacheResults[i].m_bufValue; +// if ( pSDOCached->BReadFromBuffer( allocMemcache.Base(), allocMemcache.Count() ) ) +// { +// if ( pSDOCached->IsEqual( vecPSDOSQL[i] ) ) +// { +// // cool +// *pcMatches += 1; +// } +// else +// { +// // boo +// *pcMismatches += 1; +// +// // print the differing bytes +// /* +// Msg( "key %s differs:\n", vecSKeys[i] ); +// for ( int n = 0; n < allocMemcache.Count(); n++ ) +// { +// const byte *p1 = (byte*)bufSQL.Base() + n; +// const byte *p2 = (byte*)bufCached.Base() + n; +// if ( *p1 != *p2 ) +// { +// Msg( " %3d: %2x %2x\n", n, *p1, *p2 ); +// } +// } +// */ +// +// if ( s_ConVarVerifyMemcacheDeletesBadEntries.GetBool() ) +// bClearKey = true; +// } +// } +// else +// { +// // data failed to parse, clear +// bClearKey = true; +// } +// delete pSDOCached; +// +// if ( bClearKey ) +// vecSKeysToDelete.AddToTail( vecSKeys[i] ); +// } +// } +// } +// } +// else +// { +// // SQL failure, ignore +// } +// +// // clear any suspect keys +// if ( vecSKeysToDelete.Count() ) +// { +// CMemcachedAccess memcachedAccess( m_pServer->GetPMemcachedMgr() ); +// memcachedAccess.BAsyncDeleteMulti( vecSKeysToDelete ); +// } +// +// // delete all the SDO objects +// FOR_EACH_VEC( vecPSDOSQL, i ) +// { +// delete vecPSDOSQL[i]; +// } +//} +// +//</**tempcomment**> - This is good code. We'll hook it back up later + + +//----------------------------------------------------------------------------- +// Purpose: default comparison function - compares serialized versions +//----------------------------------------------------------------------------- +bool CompareSDOObjects( ISDO *pSDO1, ISDO *pSDO2 ) +{ + CUtlBuffer b1, b2; + pSDO1->WriteToBuffer( b1 ); + pSDO2->WriteToBuffer( b2 ); + + return ( b1.TellPut() == b2.TellPut() && 0 == Q_memcmp( b1.Base(), b2.Base(), b1.TellPut() ) ); +} + + +#ifdef DBGFLAG_VALIDATE +//----------------------------------------------------------------------------- +// Purpose: validates memory +//----------------------------------------------------------------------------- +void CSDOCache::Validate( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE(); + + for ( int i = k_ESDOTypeInvalid+1; i < k_ESDOTypeMax; i++ ) + { + SDOSet_t &SDOSet = m_rgSDOSet[i]; + ValidateObj( SDOSet.m_mapISDOLoaded ); + ValidateObj( SDOSet.m_mapQueuedRequests ); + ValidateObj( SDOSet.m_queueRequestIDsToLoadFromSQL ); + ValidateObj( SDOSet.m_vecSQLJobs ); + + FOR_EACH_MAP_FAST( SDOSet.m_mapISDOLoaded, iMap ) + { + ValidatePtr( SDOSet.m_mapISDOLoaded[iMap].m_pSDO ); + } + + FOR_EACH_MAP_FAST( SDOSet.m_mapQueuedRequests, iMap ) + { + ValidatePtr( SDOSet.m_mapQueuedRequests[iMap].m_pSDO ); + ValidateObj( SDOSet.m_mapQueuedRequests[iMap].m_vecJobsWaiting ); + } + } + + ValidateObj( m_queueMemcachedRequests ); + ValidateObj( m_vecMemcachedJobs ); + ValidateObj( m_queueJobsToContinue ); + ValidateObj( m_listLRU ); + ValidateObj( m_StatMemcachedBatchSize ); + ValidateObj( m_StatSQLBatchSize ); +} +#endif // DBGFLAG_VALIDATE + +} // namespace GCSDK diff --git a/gcsdk/sharedobject.cpp b/gcsdk/sharedobject.cpp new file mode 100644 index 0000000..5b9c4e2 --- /dev/null +++ b/gcsdk/sharedobject.cpp @@ -0,0 +1,560 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Base class for objects that are kept in synch between client and server +// +//============================================================================= +#include "stdafx.h" +#include "gcsdk_gcmessages.pb.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + +//---------------------------------------------------------------------------- +// Purpose: Map of all the factory functions for all CSharedObject classes +//---------------------------------------------------------------------------- + CUtlMap<int, CSharedObject::SharedObjectInfo_t> CSharedObject::sm_mapFactories(DefLessFunc(int)); + + +//---------------------------------------------------------------------------- +// Purpose: Registers a new CSharedObject class +//---------------------------------------------------------------------------- +void CSharedObject::RegisterFactory( int nTypeID, SOCreationFunc_t fnFactory, uint32 unFlags, const char *pchClassName ) +{ + SharedObjectInfo_t info; + info.m_pFactoryFunction = fnFactory; + info.m_unFlags = unFlags; + info.m_pchClassName = pchClassName; + info.m_sBuildCacheSubNodeName.Format( "BuildCacheSubscribed(%s)", pchClassName ); + info.m_sCreateNodeName.Format( "Create(%s)", pchClassName ); + info.m_sUpdateNodeName.Format( "Update(%s)", pchClassName ); + sm_mapFactories.InsertOrReplace( nTypeID, info ); + + //register this class with our SO stats as well + #ifdef GC + g_SharedObjectStats.RegisterSharedObjectType( nTypeID, pchClassName ); + #endif //GC +} + + +//---------------------------------------------------------------------------- +// Purpose: Creates a new CSharedObject instance of the specified type ID +//---------------------------------------------------------------------------- +CSharedObject *CSharedObject::Create( int nTypeID ) +{ + int nIndex = sm_mapFactories.Find( nTypeID ); + AssertMsg1( sm_mapFactories.IsValidIndex( nIndex ), "Probably failed to set object type (%d) on the server/client.\n", nTypeID ); + if( sm_mapFactories.IsValidIndex( nIndex ) ) + { + return sm_mapFactories[nIndex].m_pFactoryFunction(); + } + else + { + return NULL; + } +} + +//---------------------------------------------------------------------------- +// Purpose: Various accessors for static class data +//---------------------------------------------------------------------------- +uint32 CSharedObject::GetTypeFlags( int nTypeID ) +{ + int nIndex = sm_mapFactories.Find( nTypeID ); + if( !sm_mapFactories.IsValidIndex( nIndex ) ) + return 0; + else + return sm_mapFactories[nIndex].m_unFlags; +} + +const char *CSharedObject::PchClassName( int nTypeID ) +{ + int nIndex = sm_mapFactories.Find( nTypeID ); + if( !sm_mapFactories.IsValidIndex( nIndex ) ) + return 0; + else + return sm_mapFactories[nIndex].m_pchClassName; + +} + +const char *CSharedObject::PchClassBuildCacheNodeName( int nTypeID ) +{ + int nIndex = sm_mapFactories.Find( nTypeID ); + if( !sm_mapFactories.IsValidIndex( nIndex ) ) + return 0; + else + return sm_mapFactories[nIndex].m_sBuildCacheSubNodeName.Get(); +} + +const char *CSharedObject::PchClassCreateNodeName( int nTypeID ) +{ + int nIndex = sm_mapFactories.Find( nTypeID ); + if( !sm_mapFactories.IsValidIndex( nIndex ) ) + return 0; + else + return sm_mapFactories[nIndex].m_sCreateNodeName.Get(); +} + +const char *CSharedObject::PchClassUpdateNodeName( int nTypeID ) +{ + int nIndex = sm_mapFactories.Find( nTypeID ); + if( !sm_mapFactories.IsValidIndex( nIndex ) ) + return 0; + else + return sm_mapFactories[nIndex].m_sUpdateNodeName.Get(); +} + + +//---------------------------------------------------------------------------- +// Purpose: Figures out if the primary keys on these two objects are the same +// using the subclass-defined less function +//---------------------------------------------------------------------------- +bool CSharedObject::BIsKeyEqual( const CSharedObject & soRHS ) const +{ + // Make sure they are the same type. + if ( GetTypeID() != soRHS.GetTypeID() ) + return false; + + return !BIsKeyLess( soRHS ) && !soRHS.BIsKeyLess( *this ); +} + + + +#ifdef GC + +//---------------------------------------------------------------------------- +// Purpose: Sends a create message for this shared object to the specified +// steam ID. +//---------------------------------------------------------------------------- +bool CSharedObject::BSendCreateToSteamID( const CSteamID & steamID, const CSteamID & steamIDOwner, uint64 ulVersion ) const +{ + // Don't send these while shutting down. We'll use higher-level + // connection-oriented messages instead + if ( GGCBase()->GetIsShuttingDown() ) + return false; + + CProtoBufMsg< CMsgSOSingleObject > msg( k_ESOMsg_Create ); + msg.Body().set_owner( steamIDOwner.ConvertToUint64() ); + msg.Body().set_type_id( GetTypeID() ); + msg.Body().set_version( ulVersion ); + if( !BAddToMessage( msg.Body().mutable_object_data() ) ) + { + EmitWarning( SPEW_SHAREDOBJ, SPEW_ALWAYS, "Failed to add create fields to message in create" ); + return false; + } + + return GGCBase()->BSendGCMsgToClient( steamID, msg ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Sends a destroy message for this object to the specified steam ID +//---------------------------------------------------------------------------- +bool CSharedObject::BSendDestroyToSteamID( const CSteamID & steamID, const CSteamID & steamIDOwner, uint64 ulVersion ) const +{ + // Don't send these while shutting down. We'll use higher-level + // connection-oriented messages instead + if ( GGCBase()->GetIsShuttingDown() ) + return false; + + CProtoBufMsg< CMsgSOSingleObject > msg( k_ESOMsg_Destroy ); + msg.Body().set_owner( steamIDOwner.ConvertToUint64() ); + msg.Body().set_type_id( GetTypeID() ); + msg.Body().set_version( ulVersion ); + if( !BAddDestroyToMessage( msg.Body().mutable_object_data() ) ) + { + EmitWarning( SPEW_SHAREDOBJ, SPEW_ALWAYS, "Failed to add key to message in create" ); + return false; + } + return GGCBase()->BSendGCMsgToClient( steamID, msg ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Wraps BYieldingAddInsertToTransaction with a transaction and a +// commit. +//---------------------------------------------------------------------------- +bool CSharedObject::BYieldingAddToDatabase() +{ + CSQLAccess sqlAccess; + sqlAccess.BBeginTransaction( CFmtStr( "CSharedObject::BYieldingAddToDatabase Type %d", GetTypeID() ) ); + if( !BYieldingAddInsertToTransaction( sqlAccess ) ) + { + sqlAccess.RollbackTransaction(); + return false; + } + + return sqlAccess.BCommitTransaction(); +} + + +//---------------------------------------------------------------------------- +// Purpose: Wraps BYieldingAddWriteToTransaction with a transaction and a +// commit. +//---------------------------------------------------------------------------- +bool CSharedObject::BYieldingWriteToDatabase( const CUtlVector< int > &fields ) +{ + CSQLAccess sqlAccess; + sqlAccess.BBeginTransaction( CFmtStr( "CSharedObject::BYieldingWriteToDatabase Type %d", GetTypeID() ) ); + if( !BYieldingAddWriteToTransaction( sqlAccess, fields ) ) + { + sqlAccess.RollbackTransaction(); + return false; + } + + if( sqlAccess.BCommitTransaction() ) + { + return true; + } + else + { + return false; + } +} + + +//---------------------------------------------------------------------------- +// Purpose: Wraps BYieldingAddRemoveToTransaction with a transaction and a +// commit. +//---------------------------------------------------------------------------- +bool CSharedObject::BYieldingRemoveFromDatabase() +{ + CSQLAccess sqlAccess; + sqlAccess.BBeginTransaction( CFmtStr( "CSharedObject::BYieldingRemoveFromDatabase Type %d", GetTypeID() ) ); + if( !BYieldingAddRemoveToTransaction( sqlAccess ) ) + { + sqlAccess.RollbackTransaction(); + return false; + } + + return sqlAccess.BCommitTransaction(); +} + +//-------------------------------------------------------------------------------------------------------------------------- +// CSharedObjectStats +//-------------------------------------------------------------------------------------------------------------------------- + +//our global stats for our SO objects +CSharedObjectStats g_SharedObjectStats; + + +CSharedObjectStats::CSharedObjectStats() + : m_bCollectingStats( false ) + , m_nMicroSElapsed( 0 ) +{ +} + +CSharedObjectStats::SOStats_t::SOStats_t() + : m_nNumActive( 0 ) +{ + ResetStats(); +} + +void CSharedObjectStats::SOStats_t::ResetStats() +{ + m_nNumCreated = 0; + m_nNumDestroyed = 0; + m_nNumSends = 0; + m_nRawBytesSent = 0; + m_nMultiplexedBytesSent = 0; + m_nNumSubOwner = 0; + m_nNumSubOtherUsers = 0; + m_nNumSubGameServer = 0; +} + +void CSharedObjectStats::StartCollectingStats() +{ + //do nothing if already collecting + if( m_bCollectingStats ) + return; + + m_bCollectingStats = true; + m_CollectTime.SetToJobTime(); +} + +void CSharedObjectStats::StopCollectingStats() +{ + if( !m_bCollectingStats ) + return; + + m_bCollectingStats = false; + m_nMicroSElapsed = m_CollectTime.CServerMicroSecsPassed(); +} + +void CSharedObjectStats::RegisterSharedObjectType( int nTypeID, const char* pszTypeName ) +{ + if( nTypeID < 0 ) + { + AssertMsg2( false, "Error registering shared object type %d (%s), negative type ID's are not supported", nTypeID, pszTypeName ); + return; + } + + //see if we need to grow our list to accommodate this type id + if( nTypeID >= m_vTypeToIndex.Count() ) + { + //make sure people aren't getting carried away with index ranges + AssertMsg( nTypeID < 8 * 1024, "Warning: Using a very large type ID which can be inefficient for the shared object stats. Try to keep values to a smaller range" ); + int nOldSize = m_vTypeToIndex.Count(); + m_vTypeToIndex.AddMultipleToTail( nTypeID + 1 - nOldSize ); + for( int nCurrFill = nOldSize; nCurrFill < nTypeID; nCurrFill++ ) + { + m_vTypeToIndex[ nCurrFill ] = knInvalidIndex; + } + } + else if( m_vTypeToIndex[ nTypeID ] != knInvalidIndex ) + { + //we have already registered something in this slot + AssertMsg2( false, "Error registering shared object type %d (%s), may have multiple registrations of this type, check for conflicts", nTypeID, pszTypeName ); + return; + } + + //we need to register this type by adding a new record, and updating our index + int nNewIndex = m_Stats.AddToTail(); + m_Stats[ nNewIndex ].m_sName = pszTypeName; + m_Stats[ nNewIndex ].m_nTypeID = nTypeID; + m_vTypeToIndex[ nTypeID ] = nNewIndex; +} + +void CSharedObjectStats::TrackSharedObjectLifetime( int nTypeID, int32 nCount ) +{ + uint16 nIndex = ( m_vTypeToIndex.IsValidIndex( nTypeID ) ) ? m_vTypeToIndex[ nTypeID ] : knInvalidIndex; + if( nIndex != knInvalidIndex ) + { + m_Stats[ nIndex ].m_nNumActive += nCount; + } +} + +void CSharedObjectStats::TrackSharedObjectSendCreate( int nTypeID, uint32 nCount ) +{ + uint16 nIndex = ( m_vTypeToIndex.IsValidIndex( nTypeID ) ) ? m_vTypeToIndex[ nTypeID ] : knInvalidIndex; + if( nIndex != knInvalidIndex ) + { + m_Stats[ nIndex ].m_nNumCreated += nCount; + } +} + +void CSharedObjectStats::TrackSharedObjectSendDestroy( int nTypeID, uint32 nCount ) +{ + uint16 nIndex = (m_vTypeToIndex.IsValidIndex( nTypeID ) ) ? m_vTypeToIndex[ nTypeID ] : knInvalidIndex; + if( nIndex != knInvalidIndex ) + { + m_Stats[ nIndex ].m_nNumDestroyed += nCount; + } +} + +void CSharedObjectStats::TrackSubscription( int nTypeID, uint32 nFlags, uint32 nNumSubscriptions ) +{ + if( !m_bCollectingStats ) + return; + + uint16 nIndex = ( m_vTypeToIndex.IsValidIndex( nTypeID ) ) ? m_vTypeToIndex[ nTypeID ] : knInvalidIndex; + if( nIndex != knInvalidIndex ) + { + if( nFlags & k_ESOFlag_SendToOwner ) + m_Stats[ nIndex ].m_nNumSubOwner += nNumSubscriptions; + if( nFlags & k_ESOFlag_SendToOtherUsers ) + m_Stats[ nIndex ].m_nNumSubOtherUsers += nNumSubscriptions; + if( nFlags & k_ESOFlag_SendToOtherGameservers ) + m_Stats[ nIndex ].m_nNumSubGameServer += nNumSubscriptions; + } + +} + +void CSharedObjectStats::TrackSharedObjectSend( int nTypeID, uint32 nNumClients, uint32 nMsgSize ) +{ + if( !m_bCollectingStats ) + return; + + uint16 nIndex = ( m_vTypeToIndex.IsValidIndex( nTypeID ) ) ? m_vTypeToIndex[ nTypeID ] : knInvalidIndex; + if( nIndex != knInvalidIndex ) + { + m_Stats[ nIndex ].m_nNumSends++; + m_Stats[ nIndex ].m_nRawBytesSent += nMsgSize; + m_Stats[ nIndex ].m_nMultiplexedBytesSent += nMsgSize * nNumClients; + } +} + +void CSharedObjectStats::ResetStats() +{ + FOR_EACH_VEC( m_Stats, nCurrSO ) + { + m_Stats[ nCurrSO ].ResetStats(); + } +} + +bool CSharedObjectStats::SortSOStatsSent( const SOStats_t* pLhs, const SOStats_t* pRhs ) +{ + //highest bandwidth ones go up top + if( pLhs->m_nMultiplexedBytesSent != pRhs->m_nMultiplexedBytesSent ) + return pLhs->m_nMultiplexedBytesSent > pRhs->m_nMultiplexedBytesSent; + + //otherwise sort by the name + return pLhs->m_nTypeID < pRhs->m_nTypeID; +} + +bool CSharedObjectStats::SortSOStatsSubscribe( const SOStats_t* pLhs, const SOStats_t* pRhs ) +{ + //sort based on total number of subscriptions + uint32 nLhsSub = pLhs->m_nNumSubOwner + pLhs->m_nNumSubOtherUsers + pLhs->m_nNumSubGameServer; + uint32 nRhsSub = pRhs->m_nNumSubOwner + pRhs->m_nNumSubOtherUsers + pRhs->m_nNumSubGameServer; + if( nLhsSub != nRhsSub ) + return nLhsSub > nRhsSub; + + //otherwise sort by the name + return pLhs->m_nTypeID < pRhs->m_nTypeID; +} + +void CSharedObjectStats::ReportCollectedStats() const +{ + //build up a list of our stats, so we can sort them appropriately (also total how many bytes we send) + uint64 nTotalRawBytes = 0; + uint64 nTotalMultiplexBytes = 0; + uint32 nTotalActive = 0; + uint32 nTotalCreates = 0; + uint32 nTotalFrees = 0; + uint32 nTotalSends = 0; + uint32 nTotalSubOwner = 0; + uint32 nTotalSubOtherUsers = 0; + uint32 nTotalSubGameServer = 0; + + CUtlVector< const SOStats_t* > sortedStats( 0, m_Stats.Count() ); + FOR_EACH_VEC( m_Stats, nCurrSO ) + { + sortedStats.AddToTail( &( m_Stats[ nCurrSO ] ) ); + + nTotalRawBytes += m_Stats[ nCurrSO ].m_nRawBytesSent; + nTotalMultiplexBytes += m_Stats[ nCurrSO ].m_nMultiplexedBytesSent; + nTotalActive += m_Stats[ nCurrSO ].m_nNumActive; + nTotalCreates += m_Stats[ nCurrSO ].m_nNumCreated; + nTotalFrees += m_Stats[ nCurrSO ].m_nNumDestroyed; + nTotalSends += m_Stats[ nCurrSO ].m_nNumSends; + nTotalSubOwner += m_Stats[ nCurrSO ].m_nNumSubOwner; + nTotalSubOtherUsers += m_Stats[ nCurrSO ].m_nNumSubOtherUsers; + nTotalSubGameServer += m_Stats[ nCurrSO ].m_nNumSubGameServer; + } + + //determine the time scale to normalize this into per second measurements + uint64 nMicroSElapsed = ( m_bCollectingStats ) ? m_CollectTime.CServerMicroSecsPassed() : m_nMicroSElapsed; + double fSeconds = nMicroSElapsed / 1000000.0; + double fToS = ( nMicroSElapsed > 0 ) ? 1000000.0 / nMicroSElapsed : 1.0; + + //----------------------------------------- + // Update stats + + std::sort( sortedStats.begin(), sortedStats.end(), SortSOStatsSent ); + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\n" ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "SO Cache Transmits - %.2f second capture\n", fSeconds ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\n" ); + + //now run through and display our report + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%s", "Type Name SendKB % SendKB/S MPlex Create C/S Destroy D/S Update U/S Active U/S (%)\n" ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%s", "---- ------------------------------ ------- ------ -------- ------ ------- ------- ------- ------- ------- ------- --------- -------\n" ); + + FOR_EACH_VEC( sortedStats, nCurrSO ) + { + const SOStats_t& stats = *sortedStats[ nCurrSO ]; + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%4d %-30s %7u %5.1f%% %8.1f %6.2f %7u %7.0f %7u %7.0f %7u %7.0f %9u %6.3f%%\n", + stats.m_nTypeID, + stats.m_sName.Get(), + ( uint32 )( stats.m_nRawBytesSent / 1024 ), + 100.0 * ( double )stats.m_nRawBytesSent / ( double )MAX( 1, nTotalRawBytes ), + ( double )( stats.m_nRawBytesSent / 1024 ) * fToS, + ( double )stats.m_nMultiplexedBytesSent / MAX( 1, stats.m_nRawBytesSent ), + stats.m_nNumCreated, + stats.m_nNumCreated * fToS, + stats.m_nNumDestroyed, + stats.m_nNumDestroyed * fToS, + stats.m_nNumSends, + stats.m_nNumSends * fToS, + stats.m_nNumActive, + 100.0 * stats.m_nNumSends * fToS / MAX( 1, stats.m_nNumActive ) ); + } + + //close it out with the totals + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%s", "---- ------------------------------ ------- ------ -------- ------ ------- ------- ------- ------- ------- ------- --------- -------\n" ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " Totals %7u %5.1f%% %8.1f %6.2f %7u %7.0f %7u %7.0f %7u %7.0f %9u %6.3f%%\n", + ( uint32 )( nTotalRawBytes / 1024 ), + 100.0, + ( double )( nTotalRawBytes / 1024 ) * fToS, + ( double )nTotalMultiplexBytes / ( double )MAX( 1, nTotalRawBytes ), + nTotalCreates, + nTotalCreates * fToS, + nTotalFrees, + nTotalFrees * fToS, + nTotalSends, + nTotalSends * fToS, + nTotalActive, + 100.0 * nTotalSends * fToS / MAX( 1, nTotalActive ) ); + + //----------------------------------------- + // Subscription stats + + std::sort( sortedStats.begin(), sortedStats.end(), SortSOStatsSubscribe ); + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\n" ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "SO Cache Subscriptions Sends - %.2f second capture\n", fSeconds ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\n" ); + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%s", "Type Name Owner Owner/S Other Other/S Server Server/S\n" ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%s", "---- ------------------------------ -------- -------- -------- -------- -------- --------\n" ); + + FOR_EACH_VEC( sortedStats, nCurrSO ) + { + const SOStats_t& stats = *sortedStats[ nCurrSO ]; + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%4d %-30s %8u %8.0f %8u %8.0f %8u %8.0f\n", + stats.m_nTypeID, + stats.m_sName.Get(), + stats.m_nNumSubOwner, + stats.m_nNumSubOwner * fToS, + stats.m_nNumSubOtherUsers, + stats.m_nNumSubOtherUsers * fToS, + stats.m_nNumSubGameServer, + stats.m_nNumSubGameServer * fToS ); + } + + //close it out with the totals + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "%s", "---- ------------------------------ -------- -------- -------- -------- -------- --------\n" ); + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, " Totals %8u %8.0f %8u %8.0f %8u %8.0f\n", + nTotalSubOwner, + nTotalSubOwner * fToS, + nTotalSubOtherUsers, + nTotalSubOtherUsers * fToS, + nTotalSubGameServer, + nTotalSubGameServer * fToS ); + +} + +#endif // GC + + +//---------------------------------------------------------------------------- +// Purpose: Claims all the memory for the shared object +//---------------------------------------------------------------------------- +#ifdef DBGFLAG_VALIDATE +void CSharedObject::Validate( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE(); +} + + +//---------------------------------------------------------------------------- +// Purpose: Claims all the static memory for the shared object (i.e. the +// factory function map. +//---------------------------------------------------------------------------- +void CSharedObject::ValidateStatics( CValidator & validator ) +{ + CValidateAutoPushPop validatorAutoPushPop( validator, NULL, "CSharedObject::ValidateStatics", "CSharedObject::ValidateStatics" ); + ValidateObj( sm_mapFactories ); +} + +#endif + + +} // namespace GCSDK + + + diff --git a/gcsdk/sharedobjectcache.cpp b/gcsdk/sharedobjectcache.cpp new file mode 100644 index 0000000..c2bf736 --- /dev/null +++ b/gcsdk/sharedobjectcache.cpp @@ -0,0 +1,397 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: A cache of a bunch of CSharedObjects +// +//============================================================================= + +#include "stdafx.h" +#include <time.h> + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + +#ifdef GC +static GCConVar add_object_clean_do_has_element( "add_object_clean_do_has_element", "0", 0, "Enables AddObjectClean() checking that the cache doesn't already have this pointer" ); +#endif + +//---------------------------------------------------------------------------- +// Purpose: Constructor +//---------------------------------------------------------------------------- +CSharedObjectTypeCache::CSharedObjectTypeCache( int nTypeID ) +: m_nTypeID( nTypeID ) +{ + +} + + +//---------------------------------------------------------------------------- +// Purpose: Destructor +//---------------------------------------------------------------------------- +CSharedObjectTypeCache::~CSharedObjectTypeCache() +{ + for ( int i = 0; i < m_vecObjects.Count(); i++ ) + { + // NULL the entry so that this SO isn't found during + // cleanup assertion checking. + CSharedObject *pObj = m_vecObjects[ i ]; + m_vecObjects[ i ] = NULL; + +#ifdef GC + if ( pObj->BShouldDeleteByCache() ) + { +#if ENABLE_SO_CONSTRUCT_DESTRUCT_PARANOIA + --pObj->m_nRefCount; + AssertMsg1( pObj->m_nRefCount == 0, "Destroying shared object %s that's still in use!", pObj->GetDebugString().String() ); +#endif // ENABLE_SO_CONSTRUCT_DESTRUCT_PARANOIA + delete pObj; + } +#else + delete pObj; +#endif + } + m_vecObjects.Purge(); +} + +//---------------------------------------------------------------------------- +// Purpose: Common shared add-to-cache code shared between AddObject() and +// AddObjectClean(). +//---------------------------------------------------------------------------- +void CSharedObjectTypeCache::AddObjectInternal( CSharedObject *pObject ) +{ + Assert( pObject ); + + m_vecObjects.AddToTail( pObject ); +#ifdef GC +#if ENABLE_SO_CONSTRUCT_DESTRUCT_PARANOIA + AssertMsg1( pObject->m_nRefCount >= 0, "AddObjectInternal(): Invalid ref count for shared object %s", pObject->GetDebugString().String() ); + ++pObject->m_nRefCount; +#endif // ENABLE_SO_CONSTRUCT_DESTRUCT_PARANOIA +#endif +} + +//---------------------------------------------------------------------------- +// Purpose: Adds a shared object of the appropriate type to this type cache. +//---------------------------------------------------------------------------- +bool CSharedObjectTypeCache::AddObject( CSharedObject *pObject ) +{ + Assert( pObject ); + Assert( m_nTypeID == pObject->GetTypeID() ); + if( m_vecObjects.HasElement( pObject ) ) + return false; + + AddObjectInternal( pObject ); + return true; +} + +//---------------------------------------------------------------------------- +// Purpose: Adds an object without dirtying. This is done when the object +// is just being loaded from SQL or memcached, so it's safe not to do the +// has element check. +//---------------------------------------------------------------------------- +bool CSharedObjectTypeCache::AddObjectClean( CSharedObject *pObject ) +{ + Assert( m_nTypeID == pObject->GetTypeID() ); + +#ifdef GC + if ( add_object_clean_do_has_element.GetBool() ) + { + Assert( !m_vecObjects.HasElement( pObject ) ); + if( m_vecObjects.HasElement( pObject ) ) + return false; + } +#endif + + AddObjectInternal( pObject ); + return true; +} + + +//---------------------------------------------------------------------------- +// Purpose: Destroys the object matching the one passed in. This could be the +// same one or simply one with matching index fields. +//---------------------------------------------------------------------------- +CSharedObject *CSharedObjectTypeCache::RemoveObject( const CSharedObject & soIndex ) +{ + Assert( m_nTypeID == soIndex.GetTypeID() ); // This is probably harmless, but it's most likely a bug + int nIndex = FindSharedObjectIndex( soIndex ); + if( m_vecObjects.IsValidIndex( nIndex ) ) + { + return RemoveObjectByIndex( nIndex ); + } + else + { + return NULL; + } +} + +CSharedObject *CSharedObjectTypeCache::RemoveObjectByIndex( uint32 nObj ) +{ + CSharedObject *pObj = m_vecObjects[nObj]; +#ifdef GC +#if ENABLE_SO_CONSTRUCT_DESTRUCT_PARANOIA + AssertMsg1( pObj->m_nRefCount > 0, "Invalid ref count for shared object %s", pObj->GetDebugString().String() ); + --pObj->m_nRefCount; +#endif // ENABLE_SO_CONSTRUCT_DESTRUCT_PARANOIA +#endif // GC + m_vecObjects.Remove( nObj ); + return pObj; +} + +//---------------------------------------------------------------------------- +// Purpose: Empties the object lists and deletes all elements +//---------------------------------------------------------------------------- +void CSharedObjectTypeCache::DestroyAllObjects() +{ + for ( int i = 0; i < m_vecObjects.Count(); i++ ) + { +#ifdef GC + if ( m_vecObjects[i]->BShouldDeleteByCache() ) + { +#if ENABLE_SO_CONSTRUCT_DESTRUCT_PARANOIA + --m_vecObjects[i]->m_nRefCount; + AssertMsg1( m_vecObjects[i]->m_nRefCount == 0, "Destroying shared object %s that's still in use!", m_vecObjects[i]->GetDebugString().String() ); +#endif // ENABLE_SO_CONSTRUCT_DESTRUCT_PARANOIA + delete m_vecObjects[i]; + } +#else + delete m_vecObjects[i]; +#endif + } + m_vecObjects.Purge(); +} + + +//---------------------------------------------------------------------------- +// Purpose: Empties the object lists but doesn't delete any of the objects +//---------------------------------------------------------------------------- +void CSharedObjectTypeCache::RemoveAllObjectsWithoutDeleting() +{ + m_vecObjects.RemoveAll(); +} + + +//---------------------------------------------------------------------------- +// Purpose: Makes sure there's room in the object vector for the suggested +// number of items +//---------------------------------------------------------------------------- +void CSharedObjectTypeCache::EnsureCapacity( uint32 nItems ) +{ + m_vecObjects.EnsureCapacity( nItems ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Searches the object list for an object that matches the provided +// object on its index fields. +//---------------------------------------------------------------------------- +CSharedObject *CSharedObjectTypeCache::FindSharedObject( const CSharedObject & soIndex ) +{ + int nIndex = FindSharedObjectIndex( soIndex ); + if( m_vecObjects.IsValidIndex( nIndex ) ) + return m_vecObjects[nIndex]; + else + return NULL; +} + + +//---------------------------------------------------------------------------- +// Purpose: Searches the object list for an object that matches the provided +// object on its index fields. +//---------------------------------------------------------------------------- +int CSharedObjectTypeCache::FindSharedObjectIndex( const CSharedObject & soIndex ) const +{ + FOR_EACH_VEC( m_vecObjects, nObj ) + { + if( m_vecObjects[nObj]->BIsKeyEqual( soIndex ) ) + return nObj; + } + + return -1; +} + + +//---------------------------------------------------------------------------- +// Purpose: Dumps all the objects in the type cache +//---------------------------------------------------------------------------- +void CSharedObjectTypeCache::Dump() const +{ + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "\tTypeCache for %d (%d objects):\n", GetTypeID(), m_vecObjects.Count() ); + FOR_EACH_VEC( m_vecObjects, nObj ) + { + m_vecObjects[nObj]->Dump(); + } +} + + +//---------------------------------------------------------------------------- +// Purpose: Claims all the memory for the cache and its objects +//---------------------------------------------------------------------------- +#ifdef DBGFLAG_VALIDATE +void CSharedObjectTypeCache::Validate( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE(); + ValidateObj( m_vecObjects ); + + FOR_EACH_VEC( m_vecObjects, nIndex ) + { + m_vecObjects[nIndex]->Validate( validator, "m_vecObjects[n]" ); + } +} +#endif + + +//---------------------------------------------------------------------------- +// Purpose: Constructor +//---------------------------------------------------------------------------- +CSharedObjectCache::CSharedObjectCache( ) +: m_mapObjects( DefLessFunc(int) ) +, m_ulVersion( 0 ) +{ + +} + + +//---------------------------------------------------------------------------- +// Purpose: Destructor +//---------------------------------------------------------------------------- +CSharedObjectCache::~CSharedObjectCache() +{ + FOR_EACH_MAP( m_mapObjects, nTypeIndex ) + { + delete m_mapObjects[nTypeIndex]; + } + m_mapObjects.Purge(); +} + + +//---------------------------------------------------------------------------- +// Purpose: Returns the type cache for the specified type ID, returning NULL +// if the cache didn't previously exist. +//---------------------------------------------------------------------------- +CSharedObjectTypeCache *CSharedObjectCache::FindBaseTypeCache( int nClassID ) const +{ + int nIndex = m_mapObjects.Find( nClassID ); + CSharedObjectTypeCache *pTypeCache = NULL; + if( m_mapObjects.IsValidIndex( nIndex ) ) + { + pTypeCache = m_mapObjects[nIndex]; + } + return pTypeCache; +} + +//---------------------------------------------------------------------------- +// Purpose: Returns the type cache for the specified type ID, creating a new +// cache and returning it if one didn't previously exist. Never intended +// to return NULL. +//---------------------------------------------------------------------------- +CSharedObjectTypeCache *CSharedObjectCache::CreateBaseTypeCache( int nClassID ) +{ + //see if we already have an existing one + CSharedObjectTypeCache *pCache = FindBaseTypeCache( nClassID ); + if( pCache ) + return pCache; + + //nope, need to create one + CSharedObjectTypeCache* pTypeCache = AllocateTypeCache( nClassID ); + m_mapObjects.Insert( nClassID, pTypeCache ); +#if 0 + // Kyle says: this is the newer way of managing caches on Dota but we haven't + // brought any of it over yet + m_CacheObjects.AddToTail( pTypeCache ); + //sort this cache for faster access + std::sort( m_CacheObjects.begin(), m_CacheObjects.end(), SortCacheByTypeID ); +#endif + return pTypeCache; +} + + +//---------------------------------------------------------------------------- +// Purpose: Adds a shared object to the cache. +//---------------------------------------------------------------------------- +bool CSharedObjectCache::AddObject( CSharedObject *pSharedObject ) +{ + CSharedObjectTypeCache *pTypeCache = CreateBaseTypeCache( pSharedObject->GetTypeID() ); + if ( !pTypeCache->AddObject( pSharedObject ) ) + return false; + + MarkDirty(); + return true; +} + + +//---------------------------------------------------------------------------- +// Purpose: Removes the object matching the one passed in from this cache, +// without destroying the actual object. +//---------------------------------------------------------------------------- +CSharedObject *CSharedObjectCache::RemoveObject( const CSharedObject & soIndex ) +{ + CSharedObjectTypeCache *pTypeCache = FindBaseTypeCache( soIndex.GetTypeID() ); + if( !pTypeCache ) + return NULL; + + MarkDirty(); + + return pTypeCache->RemoveObject( soIndex ); +} + + +//---------------------------------------------------------------------------- +// Purpose: Empties the object lists but doesn't delete any of the objects +//---------------------------------------------------------------------------- +void CSharedObjectCache::RemoveAllObjectsWithoutDeleting() +{ + FOR_EACH_MAP_FAST( m_mapObjects, nType ) + { + m_mapObjects[nType]->RemoveAllObjectsWithoutDeleting(); + } + MarkDirty(); +} + + +//---------------------------------------------------------------------------- +// Purpose: Searches the object list for an object that matches the provided +// object on its index fields. +//---------------------------------------------------------------------------- +CSharedObject *CSharedObjectCache::FindSharedObject( const CSharedObject & soIndex ) +{ + CSharedObjectTypeCache *pTypeCache = FindBaseTypeCache( soIndex.GetTypeID() ); + if( pTypeCache ) + return pTypeCache->FindSharedObject( soIndex ); + else + return NULL; +} + +//---------------------------------------------------------------------------- +// Purpose: Dumps all the objects in the type cache +//---------------------------------------------------------------------------- +void CSharedObjectCache::Dump() const +{ + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "SharedObjectCache for %s (%d types):\n", GetOwner().Render(), m_mapObjects.Count() ); + FOR_EACH_MAP( m_mapObjects, nTypeIndex ) + { + m_mapObjects[nTypeIndex]->Dump(); + } +} + + +//---------------------------------------------------------------------------- +// Purpose: Claims all the memory for the cache +//---------------------------------------------------------------------------- +#ifdef DBGFLAG_VALIDATE +void CSharedObjectCache::Validate( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE(); + + ValidateObj( m_mapObjects ); + FOR_EACH_MAP( m_mapObjects, nTypeIndex ) + { + m_mapObjects[nTypeIndex]->Validate( validator, "m_mapObjects[n]" ); + } +} +#endif + + +} // namespace GCSDK diff --git a/gcsdk/sharedobjecttransaction.cpp b/gcsdk/sharedobjecttransaction.cpp new file mode 100644 index 0000000..38d9ca7 --- /dev/null +++ b/gcsdk/sharedobjecttransaction.cpp @@ -0,0 +1,819 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +//============================================================================= +#include "stdafx.h" + +#include "sharedobjecttransaction.h" + +#include "sqlaccess/sqlaccess.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + + //----------------------------------------------------------------------------- + // Purpose: + //----------------------------------------------------------------------------- + class CTrustedHelper_OutputAndSetErrorState + { + public: + CTrustedHelper_OutputAndSetErrorState( CSharedObjectTransactionEx& SharedObjectTransaction, const char *pszFunctionContext, const CSteamID& CacheOwnerSteamID, const CSharedObject *pObject ) + : m_SharedObjectTransaction( SharedObjectTransaction ) + , m_pszFunctionContext( pszFunctionContext ) + , m_CacheOwnerSteamID( CacheOwnerSteamID ) + , m_pObject( pObject ) + { + } + + void operator()( const bool bExpResult, const char *pszExp ) const + { + if ( !bExpResult ) + { + AssertMsg4( bExpResult, "Failed verification: %s (context '%s'; owner '%s'; object '%s')", pszExp, m_pszFunctionContext, m_CacheOwnerSteamID.Render(), m_pObject ? m_pObject->GetDebugString().String() : "[none]" ); + m_SharedObjectTransaction.SetErrorState(); + } + } + + private: + CSharedObjectTransactionEx& m_SharedObjectTransaction; + const char *m_pszFunctionContext; + const CSteamID& m_CacheOwnerSteamID; + const CSharedObject *m_pObject; + }; + + #define CSOTVerifyBase( exp_, obj_ ) \ + CVerifyIfTrustedHelper( CTrustedHelper_OutputAndSetErrorState( *this, __FUNCTION__, m_pLockedSOCache->GetOwner(), obj_ ), (exp_), #exp_ ).GetResult() + + #define CSOTVerify( exp_ ) \ + CSOTVerifyBase( exp_, NULL ) + + #define CSOTVerifyObj( exp_ ) \ + CSOTVerifyBase( exp_, pObject ) + + //----------------------------------------------------------------------------- + // Purpose: + //----------------------------------------------------------------------------- + CSharedObjectTransactionEx::CSharedObjectTransactionEx( CGCSharedObjectCache *pLockedSOCache, const char *pszTransactionName ) + : m_pLockedSOCache( pLockedSOCache ) + , m_pSQLAccess( &m_sqlAccessInternal ) + { + Assert( pszTransactionName ); + Assert( pszTransactionName[0] ); + Assert( m_pLockedSOCache ); + + // We have to start a SQL transaction no matter what. If we don't do this, then + // any code internally or externally that tries to add operations to the open transaction + // will instead run immediately. We Verify() here because we know the only thing that + // can fail beginning a new transaction is to already be in a transaction, and that + // can't be the case because we just made this object. + m_bTransactionBuildSuccess = m_pSQLAccess->BBeginTransaction( pszTransactionName ); + Verify( m_bTransactionBuildSuccess ); + + // Grab another lock on our user that owns the cache we got passed in. This means that, + // barring any maliscious or really terrible code happening on the outside, even if our + // calling code unlocks *their* lock, we'll still have ours and can access the cache safely. + Verify( GGCBase()->BLockSteamIDImmediate( m_pLockedSOCache->GetOwner() ) ); + + // Because we just constructed a fresh object, we want to guarantee that our internal state + // is consistent. This way, either we're guaranteed to fail construction or we know moving + // forward that we started in a good place so anything that's wrong since then is some kind + // of real programmer error. + Verify( BIsValidInternalState() ); + } + + //----------------------------------------------------------------------------- + // Purpose: + //----------------------------------------------------------------------------- + CSharedObjectTransactionEx::~CSharedObjectTransactionEx() + { + // If we fall off the stack and we haven't been submitted, manually rollback whatever work + // we queued up in SQL and free up our local storage. This isn't an error. + if ( m_pSQLAccess->BInTransaction() ) + { + Rollback(); + } + + // We're finally done manipulating this user's cache. If we're practicing best + // practices for locking, this will still leave us locked in our outer scope. If we're + // not, at least we'll be safe. + GGCBase()->UnlockSteamID( m_pLockedSOCache->GetOwner() ); + } + + //----------------------------------------------------------------------------- + // Purpose: + //----------------------------------------------------------------------------- + bool CSharedObjectTransactionEx::BIsValidInternalState() + { + // Sanity check basic internal data. + Assert( m_pSQLAccess ); + + // If we've done something bad with our transaction (ie., committed from outside, or + // tried to submit through this object) and we're still doing work, that's a case we + // can't handle. + if ( !CSOTVerify( m_pSQLAccess->BInTransaction() ) ) + return false; + + // Verify we have a cache and it's a cache we understand how to manipulate. + if ( !CSOTVerify( m_pLockedSOCache ) || !CSOTVerify( m_pLockedSOCache->GetOwner().IsValid() ) || !CSOTVerify( m_pLockedSOCache->GetOwner().BIndividualAccount() ) ) + return false; + + // Transactions can yield when trying to commit, so we need to be running a job. We also want + // to be paranoid and make sure that the lock is actively held by our current job. + AssertRunningJob(); + + if ( !CSOTVerify( GGCBase()->IsSteamIDLockedByJob( m_pLockedSOCache->GetOwner(), &GJobCur() ) ) ) + return false; + + // Internal state is such that we can at least try to perform operations, at least. + return true; + } + + //----------------------------------------------------------------------------- + // Purpose: + //----------------------------------------------------------------------------- + bool CSharedObjectTransactionEx::BIsValidInput( const CSharedObject *pObject ) + { + if ( !BIsValidInternalState() ) + return false; + + if ( !CSOTVerifyObj( pObject ) ) + return false; + + // Make sure we have a valid type ID for this object. There aren't any objects that return negative + // IDs, but if we pass in a deleted or bogus pointer, we'll probably crash here when we hit the vtable. + if ( !CSOTVerifyObj( pObject->GetTypeID() > 0 ) ) + return false; + + return true; + } + + //----------------------------------------------------------------------------- + // Purpose: + //----------------------------------------------------------------------------- + bool CSharedObjectTransactionEx::BTrackModifiedObjectInternal( CSharedObject *pObject, CSharedObject **out_ppWritableObject ) + { + if ( !BIsValidInput( pObject ) ) + return false; + + if ( !CSOTVerifyObj( out_ppWritableObject ) ) + return false; + + // If an object is in the added list, we're going to do a full write so modifying it at this point adds nothing + // new. We'll return the current only version of it as our writable object. + { + const CreateOrDestroyCommitInfo_t *pInfo = InternalFindCommitInfo( pObject, m_vecObjects_Added ); + if ( pInfo ) + { + *out_ppWritableObject = pInfo->m_pObject; + return true; + } + } + + // If an object is in the modified list, we've already made a copy that we can make modifications to, so we'll + // return that copy. + { + const ModifyCommitInfo_t *pInfo = InternalFindCommitInfo( pObject, m_vecObjects_Modified ); + if ( pInfo ) + { + *out_ppWritableObject = pInfo->m_pWriteableObject; + return true; + } + } + + // We aren't already tracking this object. We're acting as if we've never seen it before, so first make sure that + // we're in the cache we think we're in. + if ( !CSOTVerify( m_pLockedSOCache->FindSharedObject( *pObject ) ) ) + return false; + + // Make a copy of our current state that we can make modifications to and track the association. + *out_ppWritableObject = CSharedObject::Create( pObject->GetTypeID() ); + (*out_ppWritableObject)->Copy( *pObject ); + m_vecObjects_Modified.AddToTail( ModifyCommitInfo_t( *out_ppWritableObject, pObject ) ); + return true; + } + + //----------------------------------------------------------------------------- + // Purpose: + //----------------------------------------------------------------------------- + bool CSharedObjectTransactionEx::BAddNewObjectInternal( CSharedObject *pObject ) + { + // Make sure we pass basic sanity check measures for our inputs (ie., we have inputs, we have appropriate + // locks to manipulate those inputs, etc.). + if ( !BIsValidInput( pObject ) ) + return false; + + // Make sure this object isn't already in this cache. This can cause problems during rollback if we didn't + // really add it as part of the transaction and remove it when a transaction fails. + if ( !CSOTVerifyObj( m_pLockedSOCache->FindSharedObject( *pObject ) == NULL ) ) + return false; + + // Make sure this object isn't already in the list of objects we're adding as part of this transaction. + // Having the object in the list multiple times is potentially harmless, but it probably indicates some + // calling code is doing something we don't expect. + if ( !CSOTVerifyObj( InternalFindCommitInfo( pObject, m_vecObjects_Added ) == NULL ) ) + return false; + + // Success. + m_vecObjects_Added.AddToTail( CreateOrDestroyCommitInfo_t( pObject ) ); + return true; + } + + //----------------------------------------------------------------------------- + // Purpose: + //----------------------------------------------------------------------------- + bool CSharedObjectTransactionEx::BRemoveObjectInternal( CSharedObject *pObject ) + { + // Make sure we pass basic sanity check measures for our inputs (ie., we have inputs, we have appropriate + // locks to manipulate those inputs, etc.). + if ( !BIsValidInput( pObject ) ) + return false; + + // Make sure the object we're removing is in the cache we're trying to remove it from. + if ( !CSOTVerifyObj( m_pLockedSOCache->FindSharedObject( *pObject ) ) ) + return false; + + // Look through our lists of objects that we're adding and objects that we're modifying. If we're + // removing an object that we're adding in the same transaction through the same pointer, this will + // result in a broken SO cache. Removing a modified object may or may not be safe but it's probably + // indicative of a higher-level logic bug regardless. + if ( !CSOTVerifyObj( InternalFindCommitInfo( pObject, m_vecObjects_Added ) == NULL ) ) + return false; + + if ( !CSOTVerifyObj( InternalFindCommitInfo( pObject, m_vecObjects_Modified ) == NULL ) ) + return false; + + // Success. + m_vecObjects_Removed.AddToTail( CreateOrDestroyCommitInfo_t( pObject ) ); + return true; + } + + //----------------------------------------------------------------------------- + // Purpose: + //----------------------------------------------------------------------------- + const CSharedObject *CSharedObjectTransactionEx::InternalFindSharedObject( CGCSharedObjectCache *pSOCache, const CSharedObject& soIndex ) const + { + FOR_EACH_VEC( m_vecObjects_Modified, i ) + { + auto& info = m_vecObjects_Modified[i]; + if ( info.m_pObject->BIsKeyEqual( soIndex ) ) + return info.m_pObject; + } + + FOR_EACH_VEC( m_vecObjects_Added, i ) + { + auto& info = m_vecObjects_Added[i]; + if ( info.m_pObject->BIsKeyEqual( soIndex ) ) + return info.m_pObject; + } + + return NULL; + } + + //----------------------------------------------------------------------------- + // Purpose: + //----------------------------------------------------------------------------- + const char *CSharedObjectTransactionEx::InternalPreCommit() + { + // ... + if ( !BIsValidInternalState() ) + return "invalid internal state"; + + // ... + if ( !CSOTVerify( m_pSQLAccess->BInTransaction() ) ) + return "transaction closed before commit"; + + // Did we run into some error internally building this transaction? This doesn't assert because it + // could be a SQL error or something else that doesn't necessarily indicate a problem with the way + // we're using this class. It *probably* indicates the calling code has a problem, but it isn't + // guaranteed so we don't assert. + if ( !m_bTransactionBuildSuccess ) + return "error(s) building transaction"; + + // Nothing wrong so far. + return NULL; + } + + //----------------------------------------------------------------------------- + // Purpose: + //----------------------------------------------------------------------------- + bool CSharedObjectTransactionEx::BYieldingCommit() + { + const char *pszPreCommitFailureDesc = InternalPreCommit(); + if ( pszPreCommitFailureDesc ) + { + // We can't spit out information about which cache we're dealing with here because it's possible at + // this point that the error we're displaying is "we don't have the cache anymore!" which means pulling + // memory is probably unsafe. + EmitError( SPEW_GC, "Failed to commit CSharedObjectTransaction '%s': %s!\n", GetInternalTransactionDesc(), pszPreCommitFailureDesc ); + + Rollback(); + return false; + } + + bool bNetworkRelevantObjectsChanged = false; + bool bDBRelevantObjectsChanged = false; + bool bGeneratedCommitSQL = true; + + // Add insert statements to SQL access instance for everything that needs to be written to the + // database. Some types only exist in memory and have no database backing. + FOR_EACH_VEC( m_vecObjects_Added, i ) + { + CreateOrDestroyCommitInfo_t& info = m_vecObjects_Added[i]; + + if ( info.m_pObject->BIsDatabaseBacked() ) + { + bGeneratedCommitSQL &= info.m_pObject->BYieldingAddInsertToTransaction( *m_pSQLAccess ); + bDBRelevantObjectsChanged |= true; + } + + bNetworkRelevantObjectsChanged |= info.m_pObject->BIsNetworked(); + } + + // For every object that we've changed state on, we've also been tracking information on which + // fields we modified. Here we ask each of the modified objects (which aren't currently in the + // SO cache, but are clones living outside of it) to queue up their updates to the SQL transaction. + { + CUtlVector< int > vecDirtyFields; + + FOR_EACH_VEC( m_vecObjects_Modified, i ) + { + const ModifyCommitInfo_t& info = m_vecObjects_Modified[i]; + + Assert( info.m_pObject ); + Assert( info.m_pWriteableObject ); + Assert( info.m_pObject != info.m_pWriteableObject ); + + if ( info.m_pWriteableObject->BIsDatabaseBacked() ) + { + vecDirtyFields.RemoveAll(); + m_SODirtyList.GetDirtyFieldSetByObj( info.m_pWriteableObject, vecDirtyFields ); + + bGeneratedCommitSQL &= info.m_pWriteableObject->BYieldingAddWriteToTransaction( *m_pSQLAccess, vecDirtyFields ); + bDBRelevantObjectsChanged |= true; + } + + bNetworkRelevantObjectsChanged |= info.m_pWriteableObject->BIsNetworked(); + + AssertMsg4( info.m_pWriteableObject->BIsDatabaseBacked() == info.m_pObject->BIsDatabaseBacked(), + "Disagreement over DB backing state between SOs '%s' and '%s' in transaction '%s' for user '%s'!", + info.m_pObject->GetDebugString().String(), + info.m_pWriteableObject->GetDebugString().String(), + GetInternalTransactionDesc(), + m_pLockedSOCache->GetOwner().Render() ); + AssertMsg4( info.m_pWriteableObject->BIsNetworked() == info.m_pObject->BIsNetworked(), + "Disagreement over network state between SOs '%s' and '%s' in transaction '%s' for user '%s'!", + info.m_pObject->GetDebugString().String(), + info.m_pWriteableObject->GetDebugString().String(), + GetInternalTransactionDesc(), + m_pLockedSOCache->GetOwner().Render() ); + } + } + + // Have each object that we'd like to remove queue up the SQL work necessary to do so. + FOR_EACH_VEC( m_vecObjects_Removed, i ) + { + CreateOrDestroyCommitInfo_t& info = m_vecObjects_Removed[i]; + + if ( info.m_pObject->BIsDatabaseBacked() ) + { + bGeneratedCommitSQL &= info.m_pObject->BYieldingAddRemoveToTransaction( *m_pSQLAccess ); + bDBRelevantObjectsChanged |= true; + } + + // We don't have to update network state here as removes are sent immediately to the client + // before we've even flushed our dirty updates. + } + + // Our "did we generate the SQL to do this work in the DB" variable starts off true, so the only + // way we'd expect it to be false here is if we attempted to do work above and ran into some errors. + // If we don't have any SQL work to do at all, for example if we're only adding/modifying/removing + // memory-only items, then "bGeneratedCommitSQL" will be true and "bDBRelevantObjectsChanged" will + // be false. + if ( !bGeneratedCommitSQL ) + { + EmitError( SPEW_GC, "Failed to commit CSharedObjectTransaction '%s' for user '%s': failed to add inserts/writes.\n", GetInternalTransactionDesc(), m_pLockedSOCache->GetOwner().Render() ); + + Rollback(); + return false; + } + + // Try to commit DB transaction. We don't know for sure whether we're the only code that's adding commands + // to this transaction, so we can't completely skip this if we didn't do any SQL work internally. We can + // say "did we do anything internally?; if not, maybe we'll be empty". + if ( !m_pSQLAccess->BCommitTransaction( !bDBRelevantObjectsChanged ) ) + { + EmitError( SPEW_GC, "Failed to commit CSharedObjectTransaction '%s' for user '%s': SQL transaction failure.\n", GetInternalTransactionDesc(), m_pLockedSOCache->GetOwner().Render() ); + + Rollback(); + return false; + } + + // The database work committed successfully, so we update our memory state to match the state we just wrote + // to the DB. + FOR_EACH_VEC( m_vecObjects_Added, i ) + { + CreateOrDestroyCommitInfo_t& info = m_vecObjects_Added[i]; + m_pLockedSOCache->AddObject( info.m_pObject ); // internally will assert if cache isn't locked + } + + FOR_EACH_VEC( m_vecObjects_Modified, i ) + { + ModifyCommitInfo_t& info = m_vecObjects_Modified[i]; + info.m_pObject->Copy( *info.m_pWriteableObject ); // stomp the version that's already in the cache with the properties from where we've been writing + delete info.m_pWriteableObject; + } + + FOR_EACH_VEC( m_vecObjects_Removed, i ) + { + CreateOrDestroyCommitInfo_t &info = m_vecObjects_Removed[i]; + Verify( m_pLockedSOCache->BDestroyObject( *info.m_pObject, false ) ); // internally will assert if cache isn't locked + } + + // Did we change anything that affected network state? If so, tell our cache to flush the dirty + // state list. + if ( bNetworkRelevantObjectsChanged ) + { + // Our cache is responsible for sending all the network updates, including creates from the + // AddObject() calls above, so now that our commit has succeeded we copy over our list of + // dirty objects from inside this transaction. + for ( const ModifyCommitInfo_t& info : m_vecObjects_Modified ) + { + m_pLockedSOCache->DirtyNetworkObject( info.m_pObject ); + } + + m_pLockedSOCache->SendAllNetworkUpdates(); + } + + // Cleanup. + m_vecObjects_Added.RemoveAll(); + m_vecObjects_Modified.RemoveAll(); + m_vecObjects_Removed.RemoveAll(); + + return true; + } + + void CSharedObjectTransactionEx::Rollback() + { + // Clean up any memory allocated to handle any database work that is outstanding but not yet + // committed. + m_pSQLAccess->RollbackTransaction(); + Assert( !m_pSQLAccess->BInTransaction() ); + + // Clean up any in-memory changes that are currently outstanding. + + // We made new objects for our adds, so we need to free up that memory. + FOR_EACH_VEC( m_vecObjects_Added, i ) + { + delete m_vecObjects_Added[i].m_pObject; + } + m_vecObjects_Added.RemoveAll(); + + // For our modifies, we haven't done any work on the versions that are in the cache, so all we have + // to do is delete the temp memory that we allocated for a writable version. + FOR_EACH_VEC( m_vecObjects_Modified, i ) + { + ModifyCommitInfo_t& info = m_vecObjects_Modified[i]; + delete info.m_pWriteableObject; + } + m_vecObjects_Modified.RemoveAll(); + + // We didn't actually do any in-memory work for our items that were set to be deleted, so we + // can just free the memory for our tracking state. + m_vecObjects_Removed.RemoveAll(); + } + + + + + + + + + + + + + + + + CSharedObjectTransaction::CSharedObjectTransaction( CSQLAccess &sqlAccess, const char *pName ) + : m_sqlAccess( sqlAccess ) + { + if ( m_sqlAccess.BInTransaction() == false ) + { + m_sqlAccess.BBeginTransaction( pName ); + } + } + + CSharedObjectTransaction::~CSharedObjectTransaction() + { + Rollback(); + } + + CSharedObjectTransaction::undoinfo_t *CSharedObjectTransaction::FindObjectInVector( const CSharedObject *pObject, CUtlVector<undoinfo_t> &vec ) const + { + FOR_EACH_VEC( vec, i ) + { + if ( vec[i].pObject == pObject ) + return &vec[i]; + } + + return NULL; + } + + bool CSharedObjectTransaction::AssertValidInput( const CGCSharedObjectCache *pSOCache, const CSharedObject *pObject, const char *pszContext ) + { + Assert( pszContext ); + + Assert( pSOCache ); + Assert( pObject ); + if ( pSOCache == NULL || pObject == NULL ) + { + SetError( CFmtStr( "%s: attempt to manipulate invalid SO cache %s, object %s", pszContext, pSOCache ? pSOCache->GetOwner().Render() : "[none]", pObject ? pObject->GetDebugString().String() : "[none]" ) ); + return false; + } + + const bool bSOCachedLocked = GGCBase()->IsSteamIDLockedByCurJob( pSOCache->GetOwner() ); + Assert( bSOCachedLocked ); + if ( !bSOCachedLocked ) + { + SetError( CFmtStr( "%s: attempt to manipulate non-locked SO cache %s to add object %s", pszContext, pSOCache->GetOwner().Render(), pObject->GetDebugString().String() ) ); + return false; + } + + return pSOCache != NULL && pObject != NULL && bSOCachedLocked; + } + + void CSharedObjectTransaction::AddManagedObject( CGCSharedObjectCache *pSOCache, CSharedObject *pObject ) + { + if ( !AssertValidInput( pSOCache, pObject, "AddManagedObject()" ) ) + return; + + // if an object is in the added list, we're going to do a full write so modifying it at this point adds nothing + // new; if an object is in the modified list, we already tracked the initial state; either way we have no new + // data to track here + if ( FindObjectInVector( pObject, m_vecObjects_Added ) || FindObjectInVector( pObject, m_vecObjects_Modified ) ) + return; + + undoinfo_t info = { pObject, pSOCache, NULL }; + info.pOriginalCopy = CSharedObject::Create( pObject->GetTypeID() ); + info.pOriginalCopy->Copy( *pObject ); + m_vecObjects_Modified.AddToTail( info ); + } + + void CSharedObjectTransaction::AddNewObject( CGCSharedObjectCache *pSOCache, CSharedObject *pObject ) + { + if ( !AssertValidInput( pSOCache, pObject, "AddNewObject()" ) ) + return; + + if ( FindObjectInVector( pObject, m_vecObjects_Added ) ) + return; + + undoinfo_t info = { pObject, pSOCache, NULL }; + m_vecObjects_Added.AddToTail( info ); + } + + void CSharedObjectTransaction::RemoveObject( CGCSharedObjectCache *pSOCache, CSharedObject *pObject ) + { + if ( !AssertValidInput( pSOCache, pObject, "RemoveObject()" ) ) + return; + + // make sure the object we're removing is in the cache we're trying to remove it from. + AssertMsg1( pSOCache->FindSharedObject( *pObject ), "Attempting to remove object '%s' from non-owning cache!", pObject->GetDebugString().Get() ); + + // look through our lists of objects that we're adding and objects that we're modifying. If we're + // removing an object that we're adding in the same transaction through the same pointer, this will + // result in a broken SO cache. Removing a modified object may or may not be safe but it's probably + // indicative of a higher-level logic bug regardless. + if ( undoinfo_t *pInfo = FindObjectInVector( pObject, m_vecObjects_Added ) ) + { + EmitError( SPEW_GC, "Attempting to add and remove the same object from the same CSharedObjectTransaction! Object: %s", pInfo->pObject->GetDebugString().Get() ); + Assert( !"Attempting to add and remove the same object from the same CSharedObjectTransaction!" ); + m_vecObjects_Added.FindAndFastRemove( *pInfo ); + } + if ( undoinfo_t *pInfo = FindObjectInVector( pObject, m_vecObjects_Modified ) ) + { + EmitError( SPEW_GC, "Attempting to modify and remove the same object from the same CSharedObjectTransaction! Object: %s", pInfo->pObject->GetDebugString().Get() ); + Assert( !"Attempting to modify and remove the same object from the same CSharedObjectTransaction!" ); + m_vecObjects_Modified.FindAndFastRemove( *pInfo ); + } + + // @note Tom Bui: the act of removing an item may change the object, so to roll that back, + // we need the original version + undoinfo_t info = { pObject, pSOCache, NULL }; + info.pOriginalCopy = CSharedObject::Create( pObject->GetTypeID() ); + info.pOriginalCopy->Copy( *pObject ); + m_vecObjects_Removed.AddToTail( info ); + + if ( !pObject->BYieldingAddRemoveToTransaction( m_sqlAccess ) ) + { + SetError( "RemoveObject(): BYieldingAddRemoveToTransaction() failed" ); + return; + } + } + + void CSharedObjectTransaction::ModifiedObject( CGCSharedObjectCache *pSOCache, CSharedObject *pObject, uint32 unFieldIdx ) + { + if ( !AssertValidInput( pSOCache, pObject, "ModifiedObject()" ) ) + return; + + // look for an object in the transaction -- this might be a new object created for this + // transaction or it might be an object we've already tagged for modification; we don't + // use FindSharedObject() for this because we might not be in an SO cache yet + + // if we're in the add list, we aren't intended to be in a cache yet, and so we also don't + // have to dirty any fields -- we don't exist for real yet so when we finalize this transaction, + // effectively *everything* is dirty + if ( FindObjectInVector( pObject, m_vecObjects_Added ) ) + return; + + // make sure the object we're removing is in the cache we think it is. This check has to happen + // after the "is in the added list?" check above because we won't actually put items in the cache + // for real until after the SQL transaction succeeds + AssertMsg1( pSOCache->FindSharedObject( *pObject ), "Attempting to modify object '%s' in non-owning cache!", pObject->GetDebugString().Get() ); + + undoinfo_t *pInfo = FindObjectInVector( pObject, m_vecObjects_Modified ); + if ( pInfo ) + { + pInfo->pSOCache->DirtyObjectField( pObject, unFieldIdx ); + return; + } + + Assert( !"Attempt to modify an unmanaged object in CSharedObjectTransaction!" ); + SetError( CFmtStr( "ModifiedObject(): attempt to modify an unmanaged object %s", pObject->GetDebugString().String() ) ); + } + + CSharedObject *CSharedObjectTransaction::FindSharedObject( CGCSharedObjectCache *pSOCache, const CSharedObject &soIndex ) + { + // search in modified objects + FOR_EACH_VEC( m_vecObjects_Modified, i ) + { + undoinfo_t &info = m_vecObjects_Modified[i]; + if ( info.pSOCache == pSOCache && info.pObject->BIsKeyEqual( soIndex ) ) + return info.pObject; + } + + // search in new objects + FOR_EACH_VEC( m_vecObjects_Added, i ) + { + undoinfo_t &info = m_vecObjects_Added[i]; + if ( info.pSOCache == pSOCache && info.pObject->BIsKeyEqual( soIndex ) ) + return info.pObject; + } + + return NULL; + } + + void CSharedObjectTransaction::Rollback() + { + if ( m_sqlAccess.BInTransaction() ) + { + m_sqlAccess.RollbackTransaction(); + } + Undo(); + } + + bool CSharedObjectTransaction::BYieldingCommit( bool bAllowEmpty ) + { + const char *pszPreExistingError = GetError(); + if ( pszPreExistingError ) + { + EmitError( SPEW_GC, "Failed to commit CSharedObjectTransaction '%s': %s.\n", PchName(), pszPreExistingError ); + + Undo(); + return false; + } + + Assert( m_sqlAccess.BInTransaction() ); + if ( !m_sqlAccess.BInTransaction() ) + { + EmitError( SPEW_GC, "Failed to commit CSharedObjectTransaction '%s': transaction closed before commit!\n", PchName() ); + + Undo(); + return false; + } + + bool bSuccess = true; + bool bDBRelevantObjectsChanged = false; + + // add insert statements to sql access + FOR_EACH_VEC( m_vecObjects_Added, i ) + { + undoinfo_t &info = m_vecObjects_Added[i]; + if ( info.pObject->BIsDatabaseBacked() ) + { + bSuccess &= info.pObject->BYieldingAddInsertToTransaction( m_sqlAccess ); + bDBRelevantObjectsChanged = true; + } + } + + // add update statements to sql access + FOR_EACH_VEC( m_vecObjects_Modified, i ) + { + undoinfo_t &info = m_vecObjects_Modified[i]; + if ( info.pObject->BIsDatabaseBacked() ) + { + bSuccess &= info.pSOCache->BYieldingAddWriteToTransaction( info.pObject, m_sqlAccess ); + bDBRelevantObjectsChanged = true; + } + } + + if ( bSuccess == false ) + { + EmitError( SPEW_GC, "Failed to commit CSharedObjectTransaction '%s': failed to add inserts/writes.\n", PchName() ); + + Undo(); + return false; + } + + // try to commit db transaction. + if ( m_sqlAccess.BCommitTransaction( bAllowEmpty && !bDBRelevantObjectsChanged ) == false ) + { + EmitError( SPEW_GC, "Failed to commit CSharedObjectTransaction '%s': SQL transaction failure.\n", PchName() ); + + Undo(); + return false; + } + + // remove objects from SO cache + FOR_EACH_VEC( m_vecObjects_Removed, i ) + { + undoinfo_t &info = m_vecObjects_Removed[i]; + DbgVerify( info.pSOCache->BDestroyObject( *info.pObject, false ) ); // internally will assert if cache isn't locked + delete info.pOriginalCopy; + } + m_vecObjects_Removed.RemoveAll(); + + // add new objects to SO cache + FOR_EACH_VEC( m_vecObjects_Added, i ) + { + undoinfo_t &info = m_vecObjects_Added[i]; + info.pSOCache->AddObject( info.pObject ); // internally will assert if cache isn't locked + Assert( info.pOriginalCopy == NULL ); + } + + // free up memory for original state of modified objects + FOR_EACH_VEC( m_vecObjects_Modified, i ) + { + undoinfo_t &info = m_vecObjects_Modified[i]; + info.pSOCache->DirtyNetworkObject( info.pObject ); + delete info.pOriginalCopy; + } + + // send network updates for objects that were added or modified + // this is OK to call more than once on a CGCSharedObjectCache, because internally + // it keeps a list of things that were marked dirty + FOR_EACH_VEC( m_vecObjects_Added, i ) + { + m_vecObjects_Added[i].pSOCache->SendAllNetworkUpdates(); + } + FOR_EACH_VEC( m_vecObjects_Modified, i ) + { + m_vecObjects_Modified[i].pSOCache->SendAllNetworkUpdates(); + } + + m_vecObjects_Added.RemoveAll(); + m_vecObjects_Modified.RemoveAll(); + + return true; + } + + void CSharedObjectTransaction::Undo() + { + FOR_EACH_VEC( m_vecObjects_Added, i ) + { + undoinfo_t &info = m_vecObjects_Added[i]; + delete info.pObject; + } + m_vecObjects_Added.RemoveAll(); + + FOR_EACH_VEC( m_vecObjects_Removed, i ) + { + undoinfo_t &info = m_vecObjects_Removed[i]; + AssertMsg1( GGCBase()->IsSteamIDLockedByCurJob( info.pSOCache->GetOwner() ), "Attempt to modify in-memory object '%s' during CSharedObjectTransaction removal rollback.", info.pObject->GetDebugString().Get() ); + info.pObject->Copy( *info.pOriginalCopy ); + delete info.pOriginalCopy; + } + m_vecObjects_Removed.RemoveAll(); + + FOR_EACH_VEC( m_vecObjects_Modified, i ) + { + undoinfo_t &info = m_vecObjects_Modified[i]; + AssertMsg1( GGCBase()->IsSteamIDLockedByCurJob( info.pSOCache->GetOwner() ), "Attempt to modify in-memory object '%s' during CSharedObjectTransaction modify rollback.", info.pObject->GetDebugString().Get() ); + info.pObject->Copy( *info.pOriginalCopy ); + delete info.pOriginalCopy; + } + m_vecObjects_Modified.RemoveAll(); + + ClearError(); + } + + const char *CSharedObjectTransaction::PchName() const + { + return m_sqlAccess.PchTransactionName(); + } +}; diff --git a/gcsdk/sqlaccess/columnset.cpp b/gcsdk/sqlaccess/columnset.cpp new file mode 100644 index 0000000..4798aa1 --- /dev/null +++ b/gcsdk/sqlaccess/columnset.cpp @@ -0,0 +1,368 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Sets of columns in SQL queries +// +// $NoKeywords: $ +//============================================================================= + +#include "stdafx.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + +//----------------------------------------------------------------------------- +// Purpose: Constructs a column set with no columns in it +//----------------------------------------------------------------------------- +CColumnSet::CColumnSet( const CRecordInfo *pRecordInfo ) +: m_pRecordInfo( pRecordInfo ) +{ +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructs a column set with a single column in it +// Inputs: nColumn - the column to add +//----------------------------------------------------------------------------- + +CColumnSet::CColumnSet( const CRecordInfo *pRecordInfo, int col1 ) + : m_pRecordInfo( pRecordInfo ) +{ + m_vecColumns.AddToTail( col1 ); +} + +CColumnSet::CColumnSet( const CRecordInfo *pRecordInfo, int col1, int col2 ) + : m_pRecordInfo( pRecordInfo ) +{ + m_vecColumns.EnsureCapacity( 2 ); + m_vecColumns.AddToTail( col1 ); + m_vecColumns.AddToTail( col2 ); +} + +CColumnSet::CColumnSet( const CRecordInfo *pRecordInfo, int col1, int col2, int col3 ) + : m_pRecordInfo( pRecordInfo ) +{ + m_vecColumns.EnsureCapacity( 3 ); + m_vecColumns.AddToTail( col1 ); + m_vecColumns.AddToTail( col2 ); + m_vecColumns.AddToTail( col3 ); +} + +CColumnSet::CColumnSet( const CRecordInfo *pRecordInfo, int col1, int col2, int col3, int col4 ) + : m_pRecordInfo( pRecordInfo ) +{ + m_vecColumns.EnsureCapacity( 4 ); + m_vecColumns.AddToTail( col1 ); + m_vecColumns.AddToTail( col2 ); + m_vecColumns.AddToTail( col3 ); + m_vecColumns.AddToTail( col4 ); +} + +CColumnSet::CColumnSet( const CRecordInfo *pRecordInfo, int col1, int col2, int col3, int col4, int col5 ) + : m_pRecordInfo( pRecordInfo ) +{ + m_vecColumns.EnsureCapacity( 5 ); + m_vecColumns.AddToTail( col1 ); + m_vecColumns.AddToTail( col2 ); + m_vecColumns.AddToTail( col3 ); + m_vecColumns.AddToTail( col4 ); + m_vecColumns.AddToTail( col5 ); +} + +CColumnSet::CColumnSet( const CRecordInfo *pRecordInfo, int col1, int col2, int col3, int col4, int col5, int col6 ) + : m_pRecordInfo( pRecordInfo ) +{ + m_vecColumns.EnsureCapacity( 6 ); + m_vecColumns.AddToTail( col1 ); + m_vecColumns.AddToTail( col2 ); + m_vecColumns.AddToTail( col3 ); + m_vecColumns.AddToTail( col4 ); + m_vecColumns.AddToTail( col5 ); + m_vecColumns.AddToTail( col6 ); +} + +CColumnSet::CColumnSet( const CRecordInfo *pRecordInfo, int col1, int col2, int col3, int col4, int col5, int col6, int col7 ) + : m_pRecordInfo( pRecordInfo ) +{ + m_vecColumns.EnsureCapacity( 7 ); + m_vecColumns.AddToTail( col1 ); + m_vecColumns.AddToTail( col2 ); + m_vecColumns.AddToTail( col3 ); + m_vecColumns.AddToTail( col4 ); + m_vecColumns.AddToTail( col5 ); + m_vecColumns.AddToTail( col6 ); + m_vecColumns.AddToTail( col7 ); +} + +CColumnSet::CColumnSet( const CRecordInfo *pRecordInfo, int col1, int col2, int col3, int col4, int col5, int col6, int col7, int col8 ) +: m_pRecordInfo( pRecordInfo ) +{ + m_vecColumns.EnsureCapacity( 8 ); + m_vecColumns.AddToTail( col1 ); + m_vecColumns.AddToTail( col2 ); + m_vecColumns.AddToTail( col3 ); + m_vecColumns.AddToTail( col4 ); + m_vecColumns.AddToTail( col5 ); + m_vecColumns.AddToTail( col6 ); + m_vecColumns.AddToTail( col7 ); + m_vecColumns.AddToTail( col8 ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Copy constructor +//----------------------------------------------------------------------------- +CColumnSet::CColumnSet( const CColumnSet & rhs ) +{ + MEM_ALLOC_CREDIT_("CColumnSet"); + m_vecColumns.CopyArray( rhs.m_vecColumns.Base(), rhs.m_vecColumns.Count() ); + m_pRecordInfo = rhs.m_pRecordInfo; +} + + +//----------------------------------------------------------------------------- +// Purpose: Assignment operator +//----------------------------------------------------------------------------- +CColumnSet & CColumnSet::operator=( const CColumnSet & rhs ) +{ + MEM_ALLOC_CREDIT_("CColumnSet"); + m_vecColumns.CopyArray( rhs.m_vecColumns.Base(), rhs.m_vecColumns.Count() ); + m_pRecordInfo = rhs.m_pRecordInfo; + return *this; +} + + +//----------------------------------------------------------------------------- +// Purpose: Addition operator. lhs ColumnSet will be a union of the two +// ColumnSets +//----------------------------------------------------------------------------- +CColumnSet & CColumnSet::operator+=( const CColumnSet & rhs ) +{ + Assert( this->GetRecordInfo() == rhs.GetRecordInfo() ); + FOR_EACH_COLUMN_IN_SET( rhs, i ) + { + BAddColumn( rhs.GetColumn( i ) ); + } + + return *this; +} + +//----------------------------------------------------------------------------- +// Purpose: Addition operator. Returns a union of lhs and rhs +//----------------------------------------------------------------------------- +const CColumnSet CColumnSet::operator+( const CColumnSet & rhs ) const +{ + return CColumnSet( *this ) += rhs; +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a column to the set if it is +// Inputs: nColumn - THe column to add +//----------------------------------------------------------------------------- +void CColumnSet::BAddColumn( int nColumn ) +{ + if( nColumn >= 0 && nColumn < m_pRecordInfo->GetNumColumns() ) + { + //not sure best way to handle the 'is already set case' + if( !IsSet( nColumn ) ) + m_vecColumns.AddToTail( nColumn ); + } + else + { + AssertMsg3( false, "Attempting to set an out of range column on schema type %s, %d (of %d)", GetRecordInfo()->GetName(), nColumn, m_pRecordInfo->GetNumColumns() ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Removes a column from the set +// Inputs: nColumn - THe column to remove +//----------------------------------------------------------------------------- +void CColumnSet::BRemoveColumn( int nColumn ) +{ + m_vecColumns.FindAndRemove( nColumn ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns true if a column is in the set +// Inputs: nColumn - THe column to test +//----------------------------------------------------------------------------- +bool CColumnSet::IsSet( int nColumn ) const +{ + int nIndex = m_vecColumns.Find( nColumn ); + return m_vecColumns.IsValidIndex( nIndex ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns the number of columns in the set +//----------------------------------------------------------------------------- +uint32 CColumnSet::GetColumnCount() const +{ + return m_vecColumns.Count(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns the column index of the Nth column in the set +// Inputs: nIndex - the position in the set to return a column index for. +//----------------------------------------------------------------------------- +int CColumnSet::GetColumn( int nIndex ) const +{ + return m_vecColumns[nIndex]; +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns a CColumnInfo object for the nth column in the set +// Inputs: nIndex - the position in the set to return a column info for. +//----------------------------------------------------------------------------- +const CColumnInfo & CColumnSet::GetColumnInfo( int nIndex ) const +{ + return m_pRecordInfo->GetColumnInfo( GetColumn( nIndex ) ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Empties the column set +//----------------------------------------------------------------------------- +void CColumnSet::MakeEmpty() +{ + m_vecColumns.RemoveAll(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Makes the column set be the full set of all columns in the record info +//----------------------------------------------------------------------------- +void CColumnSet::MakeFull() +{ + MakeEmpty(); + const int nNumColumns = m_pRecordInfo->GetNumColumns(); + m_vecColumns.EnsureCapacity( nNumColumns ); + for( int nColumn = 0; nColumn < m_pRecordInfo->GetNumColumns(); nColumn++ ) + { + //do a direct add to avoid the exponential cost since we know we won't have conflicts + m_vecColumns.AddToTail( nColumn ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Makes the column set be the full set of all insertable columns in +// the record info +//----------------------------------------------------------------------------- +void CColumnSet::MakeInsertable() +{ + MakeEmpty(); + for( int nColumn = 0; nColumn < m_pRecordInfo->GetNumColumns(); nColumn++ ) + { + const CColumnInfo & columnInfo = m_pRecordInfo->GetColumnInfo( nColumn ); + if( columnInfo.BIsInsertable() ) + { + //do a direct add to avoid the exponential cost since we know we won't have conflicts + m_vecColumns.AddToTail( nColumn ); + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Makes the column set be the full set of all noninsertable columns in +// the record info +//----------------------------------------------------------------------------- +void CColumnSet::MakeNoninsertable() +{ + MakeEmpty(); + for( int nColumn = 0; nColumn < m_pRecordInfo->GetNumColumns(); nColumn++ ) + { + const CColumnInfo & columnInfo = m_pRecordInfo->GetColumnInfo( nColumn ); + if( !columnInfo.BIsInsertable() ) + { + //do a direct add to avoid the exponential cost since we know we won't have conflicts + m_vecColumns.AddToTail( nColumn ); + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Makes the column set be the full set of all primary key columns in +// the record info +//----------------------------------------------------------------------------- +void CColumnSet::MakePrimaryKey() +{ + MakeEmpty(); + for( int nColumn = 0; nColumn < m_pRecordInfo->GetNumColumns(); nColumn++ ) + { + const CColumnInfo & columnInfo = m_pRecordInfo->GetColumnInfo( nColumn ); + if( columnInfo.BIsPrimaryKey() ) + { + //do a direct add to avoid the exponential cost since we know we won't have conflicts + m_vecColumns.AddToTail( nColumn ); + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Makes the column set be the full set of all primary key columns in +// the record info +//----------------------------------------------------------------------------- +void CColumnSet::MakeInverse( const CColumnSet & columnSet ) +{ + MakeEmpty(); + for( int nColumn = 0; nColumn < m_pRecordInfo->GetNumColumns(); nColumn++ ) + { + if( !columnSet.IsSet( nColumn ) ) + { + //do a direct add to avoid the exponential cost since we know we won't have conflicts + m_vecColumns.AddToTail( nColumn ); + } + } +} + +//----------------------------------------------------------------------------- +// determines if the current column set has all fields set. Useful for detection of new columns being added to the schema +//----------------------------------------------------------------------------- +bool CColumnSet::BAreAllFieldsSet() const +{ + for( int nColumn = 0; nColumn < m_pRecordInfo->GetNumColumns(); nColumn++ ) + { + if( !IsSet( nColumn ) ) + return false; + } + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Returns a Column Set which is the inverse of the given column set +// STATIC - Difference from MakeInverse is that it has a return value +//----------------------------------------------------------------------------- +CColumnSet CColumnSet::Inverse( const CColumnSet & columnSet ) +{ + CColumnSet set( columnSet.GetRecordInfo() ); + set.MakeInverse( columnSet ); + return set; +} + +//----------------------------------------------------------------------------- +// Purpose: Claims the memory for CColumnSet +//----------------------------------------------------------------------------- +#ifdef DBGFLAG_VALIDATE +void CColumnSet::Validate( CValidator &validator, const char *pchName ) +{ + // these are INSIDE the function instead of outside so the interface + // doesn't change + VALIDATE_SCOPE(); + + ValidateObj( m_vecColumns ); +} +#endif + + +} // namespace GCSDK diff --git a/gcsdk/sqlaccess/record.cpp b/gcsdk/sqlaccess/record.cpp new file mode 100644 index 0000000..6812b50 --- /dev/null +++ b/gcsdk/sqlaccess/record.cpp @@ -0,0 +1,856 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//============================================================================= + +#include "stdafx.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CRecordBase::~CRecordBase() +{ + Cleanup(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Copy constructor +// Input: that - CRecord to copy from +//----------------------------------------------------------------------------- +CRecordBase::CRecordBase( const CRecordBase &that ) +{ + *this = that; +} + + +//----------------------------------------------------------------------------- +// Purpose: Assignment operator - COPIES the record data +// Input: that - CRecord to copy from +//----------------------------------------------------------------------------- +CRecordBase& CRecordBase::operator = ( const CRecordBase & that ) +{ + Assert( GetITable() == that.GetITable() ); + + // COPY that record + Copy( that ); + + return *this; +} + + +//----------------------------------------------------------------------------- +// Purpose: Copies the data in the record. This is overridden by CRecordVar and +// CRecordExternal +// Input: that - CRecord to copy from +//----------------------------------------------------------------------------- +void CRecordBase::Copy( const CRecordBase & that ) +{ + Cleanup(); + Q_memcpy( PubRecordFixed(), that.PubRecordFixed(), GetPSchema()->CubRecordFixed() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Return the record info for this record's schema +//----------------------------------------------------------------------------- +const CRecordInfo *CRecordBase::GetPRecordInfo() const +{ + return GetPSchema()->GetRecordInfo(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Copies the data in the var record. +// Input: that - CRecord to copy from +//----------------------------------------------------------------------------- +void CRecordVar::Copy( const CRecordBase & baseThat ) +{ + const CRecordVar & that = (const CRecordVar &)baseThat; + + // COPY that record + Cleanup(); + m_pSchema = that.m_pSchema; + Q_memcpy( PubRecordFixed(), that.PubRecordFixed(), GetPSchema()->CubRecordFixed() ); + + SetFlag( k_EAllocatedVarBlock, false ); + if ( VarFieldBlockInfo_t *pVarBlockInfo = GetPSchema()->PVarFieldBlockInfoFromRecord( PubRecordFixed() ) ) + { + if ( pVarBlockInfo->m_cubBlock ) + { + void *pvNewBlock = malloc( pVarBlockInfo->m_cubBlock ); + Q_memcpy( pvNewBlock, pVarBlockInfo->m_pubBlock, pVarBlockInfo->m_cubBlock ); + pVarBlockInfo->m_pubBlock = ( uint8 * )pvNewBlock; + SetFlag( k_EAllocatedVarBlock, true ); + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Copies the data in the var record. +// Input: that - CRecord to copy from +//----------------------------------------------------------------------------- +void CRecordExternal::Copy( const CRecordBase & baseThat ) +{ + const CRecordExternal & that = (const CRecordExternal &)baseThat; + + Cleanup(); + m_pSchema = that.m_pSchema; + + m_pubRecordFixedExternal = ( uint8 * )malloc( m_pSchema->CubRecordFixed() ); + Q_memcpy( m_pubRecordFixedExternal, that.PubRecordFixed(), m_pSchema->CubRecordFixed() ); + SetFlag( k_EAllocatedFixed, true ); + + SetFlag( k_EAllocatedVarBlock, false ); + if ( VarFieldBlockInfo_t *pVarBlockInfo = m_pSchema->PVarFieldBlockInfoFromRecord( PubRecordFixed() ) ) + { + if ( pVarBlockInfo->m_cubBlock ) + { + void *pvNewBlock = malloc( pVarBlockInfo->m_cubBlock ); + Q_memcpy( pvNewBlock, pVarBlockInfo->m_pubBlock, pVarBlockInfo->m_cubBlock ); + pVarBlockInfo->m_pubBlock = ( uint8 * )pvNewBlock; + SetFlag( k_EAllocatedVarBlock, true ); + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Initialize to an empty record +// Input: pSchema - Schema for the record this will hold +//----------------------------------------------------------------------------- +void CRecordExternal::Init( CSchema *pSchema ) +{ + Cleanup(); + m_pSchema = pSchema; + m_pubRecordFixedExternal = ( uint8 * )malloc( m_pSchema->CubRecordFixed() ); + Q_memset( m_pubRecordFixedExternal, 0, m_pSchema->CubRecordFixed() ); + SetFlag( k_EAllocatedFixed, true ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Initialize pointing to a record expanded in memory +// Input: pSchema - Schema for the record this will hold +// pubRecord - Pointer to fixed record data +// bTakeOwnership - Should we delete the record when destroyed +// Output: Size of the record's data +//----------------------------------------------------------------------------- +int CRecordBase::InitFromBytes( uint8 *pubRecord ) +{ + Cleanup(); + + Q_memcpy( PubRecordFixed(), pubRecord, GetPSchema()->CubRecordFixed() ); + int cubRead = GetPSchema()->CubRecordFixed(); + return cubRead; +} + +int CRecordVar::InitFromBytes( uint8 *pubRecord ) +{ + Cleanup(); + + Q_memcpy( PubRecordFixed(), pubRecord, GetPSchema()->CubRecordFixed() ); + int cubRead = GetPSchema()->CubRecordFixed(); + if ( VarFieldBlockInfo_t *pVarBlockInfo = GetPSchema()->PVarFieldBlockInfoFromRecord( PubRecordFixed() ) ) + { + if ( pVarBlockInfo->m_cubBlock ) + { + void *pvNewBlock = malloc( pVarBlockInfo->m_cubBlock ); + Q_memcpy( pvNewBlock, pVarBlockInfo->m_pubBlock, pVarBlockInfo->m_cubBlock ); + pVarBlockInfo->m_pubBlock = ( uint8 * )pvNewBlock; + SetFlag( k_EAllocatedVarBlock, true ); + cubRead += pVarBlockInfo->m_cubBlock; + } + } + + return cubRead; +} + + +int CRecordExternal::Init( CSchema *pSchema, uint8 *pubRecord, bool bTakeOwnership ) +{ + m_pSchema = pSchema; + m_pubRecordFixedExternal = pubRecord; + SetFlag( k_EAllocatedFixed, bTakeOwnership ); + SetFlag( k_EAllocatedVarBlock, bTakeOwnership ); + int cubRead = m_pSchema->CubRecordFixed() + CubRecordVarBlock(); + + return cubRead; +} + +CSchema *CRecordBase::GetPSchema() +{ + return GetPSchemaImpl(); +} + +CSchema *CRecordBase::GetPSchemaImpl() +{ + CSchema *pSchema = NULL; + int i = GetITable(); + if ( i != -1 ) + pSchema = &GSchemaFull().GetSchema( i ); + return pSchema; +} + +//----------------------------------------------------------------------------- +// Purpose: Render a field to a buffer +// Input: unColumn - field to render +// cchBuffer - size of render buffer +// pchBuffer - buffer to render into +//----------------------------------------------------------------------------- +void CRecordBase::RenderField( uint32 unColumn, int cchBuffer, char *pchBuffer ) const +{ + Q_strncpy( pchBuffer, "", cchBuffer ); + + uint8 *pubData; + uint32 cubData; + if ( !BGetField( unColumn, &pubData, &cubData ) ) + return; + + // Get the column info and figure out how to interpret the data + ConvertFieldToText( GetPRecordInfo()->GetColumnInfo( unColumn ).GetType(), pubData, cubData, pchBuffer, cchBuffer, false ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Reset to base state, freeing any memory we are responsible for +//----------------------------------------------------------------------------- +void CRecordBase::Cleanup() +{ +} + +void CRecordVar::Cleanup() +{ + // Must do this before freeing memory that encloses it + // (eg releasing the net packet) + if ( BFlagSet( k_EAllocatedVarBlock ) ) + { + void *pvVarBlock = m_pSchema->PVarFieldBlockInfoFromRecord( PubRecordFixed() )->m_pubBlock; + free( pvVarBlock ); + m_pSchema->PVarFieldBlockInfoFromRecord( PubRecordFixed() )->m_pubBlock = NULL; + SetFlag( k_EAllocatedVarBlock, false ); + } +} + +void CRecordExternal::Cleanup() +{ + // clean up the variable-length memory we might have allocated + if ( BFlagSet( k_EAllocatedVarBlock ) ) + { + void *pvVarBlock = m_pSchema->PVarFieldBlockInfoFromRecord( PubRecordFixed() )->m_pubBlock; + free( pvVarBlock ); + m_pSchema->PVarFieldBlockInfoFromRecord( PubRecordFixed() )->m_pubBlock = NULL; + SetFlag( k_EAllocatedVarBlock, false ); + } + + // clean up the external memory we might have allocated + if ( BFlagSet( k_EAllocatedFixed ) ) + free( m_pubRecordFixedExternal ); + SetFlag( k_EAllocatedFixed, false ); + m_pubRecordFixedExternal = NULL; + + // clean up the lowest layer, not calling CRecordVar + CRecordBase::Cleanup(); +} + +//----------------------------------------------------------------------------- +// Purpose: Deserializes a block of memory into this record +// Input: pubData - Memory block to deserialize from +//----------------------------------------------------------------------------- +void CRecordExternal::DeSerialize( uint8 *pubData ) +{ + InitFromBytes( pubData ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Calculates the size of this record when serialized +// Output: Size of serialized message +//----------------------------------------------------------------------------- +uint32 CRecordBase::CubSerialized() +{ + return CubRecordFixed() + CubRecordVarBlock(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get pointer to fixed part of record +// Output: pubRecordFixed +//----------------------------------------------------------------------------- +uint8* CRecordBase::PubRecordFixed() +{ + return ( uint8 * )( this + 1 ); +} + +uint8* CRecordExternal::PubRecordFixed() +{ + Assert( m_pubRecordFixedExternal ); + return m_pubRecordFixedExternal; +} + +uint8* CRecordVar::PubRecordFixed() +{ + return ( uint8 * )( this + 1 ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get pointer to fixed part of record +// Output: pubRecordFixed +//----------------------------------------------------------------------------- +const uint8* CRecordBase::PubRecordFixed() const +{ + return const_cast<CRecordBase *>( this )->PubRecordFixed(); +} + +const uint8* CRecordVar::PubRecordFixed() const +{ + return const_cast<CRecordVar *>( this )->PubRecordFixed(); +} + +const uint8* CRecordExternal::PubRecordFixed() const +{ + return const_cast<CRecordExternal *>( this )->PubRecordFixed(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get size of fixed part of record +// Output: size in bytes of fixed part +//----------------------------------------------------------------------------- +uint32 CRecordBase::CubRecordFixed() const +{ + return GetPSchema()->CubRecordFixed(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get pointer to variable part of record +// Output: Pointer to variable-length block -- may be NULL if this record +// has no var-length fields or they are all empty +//----------------------------------------------------------------------------- +uint8* CRecordBase::PubRecordVarBlock() +{ + VarFieldBlockInfo_t *pVarFieldBlockInfo = GetPSchema()->PVarFieldBlockInfoFromRecord( PubRecordFixed() ); + if ( pVarFieldBlockInfo ) + { + return pVarFieldBlockInfo->m_pubBlock; + } + else + { + return NULL; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Get pointer to variable part of record +// Output: Pointer to variable-length block -- may be NULL if this record +// has no var-length fields or they are all empty +//----------------------------------------------------------------------------- +const uint8* CRecordBase::PubRecordVarBlock() const +{ + return const_cast<CRecordBase *>( this )->PubRecordVarBlock(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get size of variable part of record +// Output: Size in bytes of var-length block - may be zero if this record +// has no var-length fields or they are all empty +//----------------------------------------------------------------------------- +uint32 CRecordBase::CubRecordVarBlock() const +{ + VarFieldBlockInfo_t *pVarFieldBlockInfo = GetPSchema()->PVarFieldBlockInfoFromRecord( PubRecordFixed() ); + if ( pVarFieldBlockInfo ) + { + return pVarFieldBlockInfo->m_cubBlock; + } + else + { + return 0; + } +} + +//----------------------------------------------------------------------------- +// Purpose: Get size of variable part of record +// Output: Size in bytes of var-length block - may be zero if this record +// has no var-length fields or they are all empty +//----------------------------------------------------------------------------- +bool CRecordBase::BAssureRecordVarStorage( uint32 cVariableBytes ) +{ + // get the variable field block + VarFieldBlockInfo_t *pVarFieldBlockInfo = GetPSchema()->PVarFieldBlockInfoFromRecord( PubRecordFixed() ); + if ( pVarFieldBlockInfo ) + { + // if we have it, see if it's got enough storage + if ( pVarFieldBlockInfo->m_cubBlock >= cVariableBytes ) + { + // already there + return true; + } + + // allocate it + uint8* pubData = (uint8*) malloc( cVariableBytes ); + if ( pubData == NULL ) + return false; + + // do we have something right now? + if ( pVarFieldBlockInfo->m_cubBlock != 0 ) + { + // sure do. copy it over. + Q_memcpy( pubData, pVarFieldBlockInfo->m_pubBlock, pVarFieldBlockInfo->m_cubBlock ); + + // free what was there + free( pVarFieldBlockInfo->m_pubBlock ); + } + + // hook up our buffer + pVarFieldBlockInfo->m_cubBlockFree = cVariableBytes - pVarFieldBlockInfo->m_cubBlock; + pVarFieldBlockInfo->m_cubBlock = cVariableBytes; + pVarFieldBlockInfo->m_pubBlock = pubData; + + return true; + } + else + { + // we don't have one; + // we've got no variable length fields, and so can't preallocate for them! + return false; + } +} + +//----------------------------------------------------------------------------- +// Purpose: Initialize this whole record to random data +// Input: unPrimaryIndex - Primary index to set +//----------------------------------------------------------------------------- +void CRecordExternal::InitRecordRandom( uint32 unPrimaryIndex ) +{ + bool bRealloced = false; + GetPSchema()->InitRecordRandom( PubRecordFixed(), unPrimaryIndex, &bRealloced, BFlagSet( k_EAllocatedVarBlock ) ); + + if ( bRealloced ) + SetFlag( k_EAllocatedVarBlock, true ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Set a field in this record to random bits +// Input: iField - Field to set +//----------------------------------------------------------------------------- +void CRecordExternal::SetFieldRandom( int iField ) +{ + bool bRealloced = false; + GetPSchema()->SetFieldRandom( PubRecordFixed(), iField, &bRealloced, BFlagSet( k_EAllocatedVarBlock ) ); + + if ( bRealloced ) + SetFlag( k_EAllocatedVarBlock, true ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get a field (var or fixed) from this record +// Input: iField - Field to set +// ppubData - Receives pointer to fields data +// pcubField - Receives count of bytes of data (will count the null for strings) +// Output: true if succeeds +//----------------------------------------------------------------------------- +bool CRecordBase::BGetField( int iField, uint8 **ppubData, uint32 *pcubField ) const +{ + return GetPSchema()->BGetFieldData( PubRecordFixed(), iField, ppubData, pcubField ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Sets the data for a field, whether fixed or variable length +// Input: iField - index of field to set +// pubData - pointer to field data to copy from +// cubData - size in bytes of that data +// Output: true if successful +//----------------------------------------------------------------------------- +bool CRecordBase::BSetField( int iField, void *pvData, uint32 cubData ) +{ + bool bRealloced = false; + bool bResult = BSetField( iField, pvData, cubData, &bRealloced ); + Assert( !bRealloced ); + + return bResult; +} + +bool CRecordBase::BSetField( int iField, void *pvData, uint32 cubData, bool *pbRealloced ) +{ + uint8 *pubData = reinterpret_cast<uint8 *>( pvData ); + + if ( !GetPSchema()->BSetFieldData( PubRecordFixed(), iField, pubData, cubData, pbRealloced ) ) + return false; + + return true; +} + +bool CRecordVar::BSetField( int iField, void *pvData, uint32 cubData ) +{ + bool bRealloced = false; + bool bResult = CRecordBase::BSetField( iField, pvData, cubData, &bRealloced ); + + if ( bRealloced ) + SetFlag( k_EAllocatedVarBlock, true ); + return bResult; +} + + +//----------------------------------------------------------------------------- +// Purpose: Erases a field, setting it to 0 length (if possible) and filling with nulls +// Input: iField - index of field to wipe +// NOTE: This relies on CSchema::BSetFieldData nulling out the rest of a field when it is set to 0 length! +//----------------------------------------------------------------------------- +void CRecordBase::WipeField( int iField ) +{ + bool bRealloced = false; + + // Empty Data + uint32 un = 0; + + // Length should be 0, except for non-variable length strings where length should be 1 (for an empty string "") + int cub = 0; + Field_t &field = GetPSchema()->GetField( iField ); + Assert( !field.BIsVariableLength() ); + if ( field.BIsStringType() ) + cub = 1; + + GetPSchema()->BSetFieldData( PubRecordFixed(), iField, ( uint8 * ) &un, cub, &bRealloced ); + + Assert( !bRealloced ); +} + +void CRecordVar::WipeField( int iField ) +{ + bool bRealloced = false; + + // Empty Data + uint32 un = 0; + + // Length should be 0, except for non-variable length strings where length should be 1 (for an empty string "") + int cub = 0; + Field_t &field = GetPSchema()->GetField( iField ); + if ( field.BIsStringType() && !field.BIsVariableLength() ) + cub = 1; + + GetPSchema()->BSetFieldData( PubRecordFixed(), iField, ( uint8 * ) &un, cub, &bRealloced ); + + if ( bRealloced ) + SetFlag( k_EAllocatedVarBlock, true ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get a string field - will return empty string instead of NULL if field has no datas +// Input: iField - Field to get +// pcubField - Receives count of bytes of data (will count the null for strings) +// Output: const pointer to string data (to an empty string if no data) +//----------------------------------------------------------------------------- +const char * CRecordBase::GetStringField( int iField, uint32 *pcubField ) +{ + uint8 * pubData = NULL; + *pcubField = 0; + + if ( BGetField( iField, &pubData, pcubField ) && *pcubField > 0 ) + return ( const char * ) pubData; + else + return ""; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get an int field +// Input: iField - Field to get +// Output: Int (0 if no data) +//----------------------------------------------------------------------------- +int CRecordBase::GetInt( int iField ) +{ + return ( int ) GetUint32( iField ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get a uint16 field +// Input: iField - Field to get +// Output: uint16 (0 if no data) +//----------------------------------------------------------------------------- +uint16 CRecordBase::GetUint16( int iField ) +{ + uint8 * pubData = NULL; + uint32 cubField = 0; + + DbgVerify( BGetField( iField, &pubData, &cubField ) ); + Assert( 0 < cubField ); + + if ( NULL != pubData ) + return *( uint16 * ) pubData; + else + return 0; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get a uint32 field +// Input: iField - Field to get +// Output: uint32 (0 if no data) +//----------------------------------------------------------------------------- +uint32 CRecordBase::GetUint32( int iField ) +{ + uint8 * pubData = NULL; + uint32 cubField = 0; + + DbgVerify( BGetField( iField, &pubData, &cubField ) ); + Assert( 0 < cubField ); + + if ( NULL != pubData ) + return *( uint32 * ) pubData; + else + return 0; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get a uint64 field +// Input: iField - Field to get +// Output: uint64 (0 if no data) +//----------------------------------------------------------------------------- +uint64 CRecordBase::GetUint64( int iField ) +{ + uint8 * pubData = NULL; + uint32 cubField = 0; + + DbgVerify( BGetField( iField, &pubData, &cubField ) ); + Assert( 0 < cubField ); + + if ( NULL != pubData ) + return *( uint64 * ) pubData; + else + return 0; +} + + + +const char * CRecordBase::ReadVarCharField( const CVarCharField &field ) const +{ + Assert( false ); + return NULL; +} + +const uint8 * CRecordBase::ReadVarDataField( const CVarField &field, uint32 *pcubField ) const +{ + Assert( false ); + return NULL; +} + +// These may cause a realloc +bool CRecordBase::SetVarCharField( CVarCharField &field, const char *pchString, bool bTruncate, int32 iField ) +{ + Assert( false ); + return false ; +} + +void CRecordBase::SetVarDataField( CVarField &field, const void *pvData, uint32 cubData ) +{ + Assert( false ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Read data from a varchar field +// Input: field - opaque field object to read from +// Output: pointer to data - may be NULL if that field is empty. +//----------------------------------------------------------------------------- +const char * CRecordVar::ReadVarCharField( const CVarCharField &field ) const +{ + Assert ( GetPSchema()->BHasVariableFields() ); + + uint8 *pubData; + uint32 cubData; + if ( GetPSchema()->BGetVarField( PubRecordFixed(), &field, &pubData, &cubData ) ) + return (const char *)pubData; + else + return ""; +} + + +//----------------------------------------------------------------------------- +// Purpose: Read data from a vardata field +// Input: field - opaque field object to read from +// Output: pointer to data - may be NULL if that field is empty. +//----------------------------------------------------------------------------- +const uint8 *CRecordVar::ReadVarDataField( const CVarField &field, uint32 *pcubField ) const +{ + Assert ( GetPSchema()->BHasVariableFields() ); + + uint8 *pubData; + *pcubField = 0; + if ( GetPSchema()->BGetVarField( PubRecordFixed(), &field, &pubData, pcubField ) ) + return pubData; + else + return NULL; +} + + +//----------------------------------------------------------------------------- +// Purpose: Update (in memory) a varchar field +// Input: field - opaque field object to update +// pchString - string data to set +//----------------------------------------------------------------------------- +bool CRecordVar::SetVarCharField( CVarCharField &field, const char *pchString, bool bTruncate, int32 iField ) +{ + Assert ( GetPSchema()->BHasVariableFields() ); + if( iField < 0 ) + { + AssertMsg1( false, "Encountered a bad call to SetVarCharField with an invalid field specified: %d", iField ); + return false; + } + + bool bTruncated = false; + int cchLen = Q_strlen( pchString ) + 1; + + // since we're a VARCHAR field, cbMaxLength is the length in characters + const int cchMaxLength = m_pSchema->GetField( iField ).m_cchMaxLength; + if ( ( cchMaxLength > 0 ) && ( cchLen > cchMaxLength ) ) + { + if( bTruncate ) + { + bTruncated = true; + cchLen = cchMaxLength; + } + else + { + // caller should check his data and not pass stuff that wont fit + AssertMsg4( false, "Overflow in SetVarCharField (%u > %u) for column %s in table %s", cchLen, cchMaxLength, m_pSchema->GetField( iField ).m_rgchName, m_pSchema->GetPchName() ); + return false; + } + } + + bool bRealloced = false; + bool fSuccess = GetPSchema()->BSetVarField( PubRecordFixed(), &field, pchString, cchLen, &bRealloced, BFlagSet( k_EAllocatedVarBlock ) ); + if( fSuccess && bTruncated ) + { + //make sure the last character is NULL if we truncated + VarFieldBlockInfo_t *pBlock = GetPSchema()->PVarFieldBlockInfoFromRecord( PubRecordFixed() ); + uint8 *pubVarBlock = pBlock->m_pubBlock; + char* pField = ( char* )( pubVarBlock + field.m_dubOffset ); + pField[ cchLen - 1 ] = '\0'; + } + + if ( bRealloced ) + SetFlag( k_EAllocatedVarBlock, true ); + return fSuccess; +} + + +//----------------------------------------------------------------------------- +// Purpose: Update (in memory) a vardata field +// Input: field - opaque field object to update +// pvData - pointer to data to put there +// cubData - size in bytes of the data +//----------------------------------------------------------------------------- +void CRecordVar::SetVarDataField( CVarField &field, const void *pvData, uint32 cubData ) +{ + Assert ( GetPSchema()->BHasVariableFields() ); + + bool bRealloced = false; + GetPSchema()->BSetVarField( PubRecordFixed(), &field, pvData, cubData, &bRealloced, BFlagSet( k_EAllocatedVarBlock ) ); + + if ( bRealloced ) + SetFlag( k_EAllocatedVarBlock, true ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Set or clear the specified flag in m_nFlags +// Input: eFlag - flag (single bit) to change +// bSet - Set it, else clear it +//----------------------------------------------------------------------------- +void CRecordVar::SetFlag( int eFlag, bool bSet ) +{ + if ( bSet ) + m_nFlags |= eFlag; + else + m_nFlags &= ~eFlag; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the state of the specified flag +// Input: eFlag - flag (single bit) to check +//----------------------------------------------------------------------------- +bool CRecordVar::BFlagSet( int eFlag ) const +{ + return 0 != ( m_nFlags & eFlag ); +} + + +#ifdef DBGFLAG_VALIDATE +//----------------------------------------------------------------------------- +// Purpose: Run a global validation pass on all of our data structures and memory +// allocations. +// Input: validator - Our global validator object +// pchName - Our name (typically a member var in our container) +//----------------------------------------------------------------------------- +void CRecordBase::Validate( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE(); +} + +void CRecordVar::Validate( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE(); + + if ( BFlagSet( k_EAllocatedVarBlock ) ) + { + validator.ClaimMemory( GetPSchema()->PVarFieldBlockInfoFromRecord( PubRecordFixed() )->m_pubBlock ); + } + +} + +void CRecordExternal::Validate( CValidator &validator, const char *pchName ) +{ + if ( BFlagSet( k_EAllocatedFixed ) ) + { + validator.ClaimMemory( m_pubRecordFixedExternal ); + } + + CRecordBase::Validate( validator, pchName ); +} + +void CRecordBase::ValidateStatics( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE_STATIC( "CRecordBase class statics" ); +} + +#endif // DBGFLAG_VALIDATE + + +//----------------------------------------------------------------------------- +// Purpose: Return the schema for this record type +//----------------------------------------------------------------------------- +CSchema *CRecordType::GetSchema() const +{ + return &GSchemaFull().GetSchema( GetITable() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Return the CRecordInfo for this record type +//----------------------------------------------------------------------------- +CRecordInfo *CRecordType::GetRecordInfo() const +{ + return GetSchema()->GetRecordInfo(); +} + +} // namespace GCSDK diff --git a/gcsdk/sqlaccess/recordinfo.cpp b/gcsdk/sqlaccess/recordinfo.cpp new file mode 100644 index 0000000..57fb435 --- /dev/null +++ b/gcsdk/sqlaccess/recordinfo.cpp @@ -0,0 +1,915 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//============================================================================= + + +#include "stdafx.h" + +//#include "sqlaccess.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ +// Memory pool for CRecordInfo +CThreadSafeClassMemoryPool<CRecordInfo> CRecordInfo::sm_MemPoolRecordInfo( 10, UTLMEMORYPOOL_GROW_FAST ); + +#ifdef _DEBUG +// validation tracking +CUtlRBTree<CRecordInfo *, int > CRecordInfo::sm_mapPMemPoolRecordInfo( DefLessFunc( CRecordInfo *) ); +CThreadMutex CRecordInfo::sm_mutexMemPoolRecordInfo; +#endif + + +//----------------------------------------------------------------------------- +// determine if this fieldset is equal to the other one +//----------------------------------------------------------------------------- +/* static */ +bool FieldSet_t::CompareFieldSets( const FieldSet_t& refThis, CRecordInfo* pRecordInfoThis, + const FieldSet_t& refOther, CRecordInfo* pRecordInfoOther ) +{ + // same number of columns? + int cColumns = refThis.GetCount(); + if ( refOther.GetCount() != cColumns ) + return false; + + int cIncludedColumns = refThis.GetIncludedCount(); + if ( refOther.GetIncludedCount() != cIncludedColumns ) + return false; + + // do the regular columns first; this is order-dependent + for ( int m = 0; m < cColumns; m++ ) + { + int nThisField = refThis.GetField( m ); + const CColumnInfo& refThisColumn = pRecordInfoThis->GetColumnInfo( nThisField ); + + int nOtherField = refOther.GetField( m ); + const CColumnInfo& refOtherColumn = pRecordInfoOther->GetColumnInfo( nOtherField ); + + if ( refOtherColumn != refThisColumn ) + { + return false; + } + } + + // do the included columns now; order independent + for ( int m = 0; m < cIncludedColumns; m++ ) + { + int nThisField = refThis.GetIncludedField( m ); + const CColumnInfo& refThisColumn = pRecordInfoThis->GetColumnInfo( nThisField ); + bool bFoundMatch = false; + + for ( int n = 0; n < cIncludedColumns; n++ ) + { + int nOtherField = refOther.GetIncludedField( n ); + const CColumnInfo& refOtherColumn = pRecordInfoOther->GetColumnInfo( nOtherField ); + + if ( refOtherColumn == refThisColumn ) + { + bFoundMatch = true; + break; + } + } + + if ( !bFoundMatch ) + return false; + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CRecordInfo::CRecordInfo() +: m_MapIColumnInfo( 0, 0, CaselessStringLessThan ) +{ + m_rgchName[0] = 0; + m_bPreparedForUse = false; + m_bAllColumnsAdded = false; + m_bHaveChecksum = false; + m_bHaveColumnNameIndex = false; + m_nHasPrimaryKey = k_EPrimaryKeyTypeNone; + m_iPKIndex = -1; + m_cubFixedSize = 0; + m_nChecksum = 0; + m_eSchemaCatalog = k_ESchemaCatalogInvalid; + m_nTableID = 0; +} + +//----------------------------------------------------------------------------- +// Purpose: Initializes this record info from DS equivalent information +//----------------------------------------------------------------------------- +void CRecordInfo::InitFromDSSchema( CSchema *pSchema ) +{ + // copy the name over + SetName( pSchema->GetPchName() ); + + // copy each of the fields, preallocating capacity + int cFields = pSchema->GetCField(); + m_VecColumnInfo.EnsureCapacity( cFields ); + for ( int iField = 0; iField < cFields; iField++ ) + { + Field_t &field = pSchema->GetField( iField ); + AddColumn( field.m_rgchSQLName, iField+1, field.m_EType, field.m_cubLength, field.m_nColFlags, field.m_cchMaxLength ); + } + + m_nTableID = pSchema->GetITable(); + + // copy the list of PK index fields + m_iPKIndex = pSchema->GetPKIndex( ); + + // copy the list of Indexes + m_VecIndexes = pSchema->GetIndexes( ); + + // which schema? + m_eSchemaCatalog = pSchema->GetESchemaCatalog(); + + // copy full-text column list + // and the index of the catalog it will create on + m_vecFTSFields = pSchema->GetFTSColumns(); + m_nFullTextCatalogIndex = pSchema->GetFTSIndexCatalog(); + + // Copy over the FK data + int cFKs = pSchema->GetFKCount(); + for ( int i = 0; i < cFKs; ++i ) + { + FKData_t &fkData = pSchema->GetFKData( i ); + AddFK( fkData ); + } + + // prepare for use + PrepareForUse( ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a new column to this record info +// Input: pchName - column name +// nSQLColumn - column index in SQL to bind to (1-based) +// eType - data type of column +// cubFixedSize - for fixed-size fields, the size +// nColFlags - attributes +//----------------------------------------------------------------------------- +void CRecordInfo::AddColumn( const char *pchName, int nSQLColumn, EGCSQLType eType, int cubFixedSize, int nColFlags, int cchMaxSize ) +{ + Assert( !m_bPreparedForUse ); + if ( m_bPreparedForUse ) + return; + uint32 unColumn = m_VecColumnInfo.AddToTail(); + CColumnInfo &columnInfo = m_VecColumnInfo[unColumn]; + + columnInfo.Set( pchName, nSQLColumn, eType, cubFixedSize, nColFlags, cchMaxSize ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a new FK to this record info +//----------------------------------------------------------------------------- +void CRecordInfo::AddFK( const FKData_t &fkData ) +{ + m_VecFKData.AddToTail( fkData ); +} + + +//----------------------------------------------------------------------------- +// Purpose: compare function to sort by column name +//----------------------------------------------------------------------------- +int __cdecl CompareColumnInfo( const CColumnInfo *pColumnInfoLeft, const CColumnInfo *pColumnInfoRight ) +{ + const char *pchLeft = ( (CColumnInfo *) pColumnInfoLeft )->GetName(); + const char *pchRight = ( (CColumnInfo *) pColumnInfoRight )->GetName(); + Assert( pchLeft && pchLeft[0] ); + Assert( pchRight && pchRight[0] ); + return Q_stricmp( pchLeft, pchRight ); +} + + +//----------------------------------------------------------------------------- +// Purpose: compares this record info to another record info +//----------------------------------------------------------------------------- +bool CRecordInfo::EqualTo( CRecordInfo* pOther ) +{ + int nOurs = GetChecksum(); + int nTheirs = pOther->GetChecksum(); + + // if this much isn't equal, we're no good + if (nOurs != nTheirs) + return false; + + if ( !CompareIndexLists( pOther ) ) + return false; + + if ( !CompareFKs( pOther ) ) + return false; + + return CompareFTSIndexLists( pOther ); +} + +//----------------------------------------------------------------------------- +// Purpose: format the index list into a string +//----------------------------------------------------------------------------- +void CRecordInfo::GetIndexFieldList( CFmtStr1024 *pstr, int nIndents ) const +{ + // table name at first + pstr->sprintf( "Table %s:\n", this->GetName() ); + + // for each of the indexes ... + for ( int n = 0; n < m_VecIndexes.Count(); n++ ) + { + const FieldSet_t& fs = m_VecIndexes[n]; + + // indent enough + for ( int x = 0; x < nIndents; x++ ) + { + pstr->Append( "\t" ); + } + + // show if it is clustered or not + pstr->AppendFormat( "Index %d (%s): %sclustered, %sunique {", n, + fs.GetIndexName(), + fs.IsClustered() ? "" : "non-", + fs.IsUnique() ? "" : "non-" ); + + // then show all the columns + for (int m = 0; m < fs.GetCount(); m++ ) + { + int x = fs.GetField( m ); + const char* pstrName = m_VecColumnInfo[x].GetName(); + pstr->AppendFormat( "%s %s", ( m == 0 ) ? "" : ",", pstrName ); + } + + // then the included columns, too + for ( int m = 0; m < fs.GetIncludedCount(); m++ ) + { + int x = fs.GetIncludedField( m ); + const char* pstrName = m_VecColumnInfo[x].GetName(); + pstr->AppendFormat( ", *%s", pstrName ); + } + pstr->Append( " }\n" ); + } + + return; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the number of foreign key constraints defined for the table +//----------------------------------------------------------------------------- +int CRecordInfo::GetFKCount() +{ + return m_VecFKData.Count(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get data for a foreign key by index (valid for 0...GetFKCount()-1) +//----------------------------------------------------------------------------- +FKData_t &CRecordInfo::GetFKData( int iIndex ) +{ + return m_VecFKData[iIndex]; +} + + +//----------------------------------------------------------------------------- +// Purpose: format the FK list into a string +//----------------------------------------------------------------------------- +void CRecordInfo::GetFKListString( CFmtStr1024 *pstr, int nIndents ) +{ + // table name at first + pstr->sprintf( "Table %s Foreign Keys: \n", this->GetName() ); + + + + if ( m_VecFKData.Count() == 0 ) + { + // indent enough + pstr->AppendIndent( nIndents ); + pstr->Append( "No foreign keys for table\n" ); + } + else + { + for ( int n = 0; n < m_VecFKData.Count(); n++ ) + { + // indent enough + pstr->AppendIndent( nIndents ); + + FKData_t &fkData = m_VecFKData[n]; + CFmtStr sColumns, sParentColumns; + FOR_EACH_VEC( fkData.m_VecColumnRelations, i ) + { + FKColumnRelation_t &colRelation = fkData.m_VecColumnRelations[i]; + if ( i > 0) + { + sColumns += ","; + sParentColumns += ","; + } + sColumns += colRelation.m_rgchCol; + sParentColumns += colRelation.m_rgchParentCol; + } + + pstr->AppendFormat( "CONSTRAINT %s FOREIGN KEY (%s) REFERENCES %s(%s) ON DELETE %s ON UPDATE %s\n", + fkData.m_rgchName, sColumns.Access(), fkData.m_rgchParentTableName, sParentColumns.Access(), + PchNameFromEForeignKeyAction( fkData.m_eOnDeleteAction ), PchNameFromEForeignKeyAction( fkData.m_eOnUpdateAction ) ); + } + } + + return; +} + + + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +void CRecordInfo::AddFTSFields( CUtlVector< int > &vecFields ) +{ + AssertMsg( m_vecFTSFields.Count() == 0, "Only one FTS index per table" ); + FOR_EACH_VEC( vecFields, n ) + { + int nField = vecFields[n]; + m_vecFTSFields.AddToTail( nField ); + } + + return; +} + +//----------------------------------------------------------------------------- +// Purpose: compares FK lists in this record with those of another +//----------------------------------------------------------------------------- +bool CRecordInfo::CompareFKs( CRecordInfo *pOther ) +{ + if ( pOther->m_VecFKData.Count() != m_VecFKData.Count() ) + return false; + + for( int i=0; i < m_VecFKData.Count(); ++i ) + { + FKData_t &fkDataMine = m_VecFKData[i]; + + bool bFoundInOther = false; + for ( int j=0; j < pOther->m_VecFKData.Count(); ++j ) + { + FKData_t &fkDataOther = pOther->m_VecFKData[j]; + + if ( fkDataMine == fkDataOther ) + { + bFoundInOther = true; + break; + } + } + + if ( !bFoundInOther ) + return false; + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Locate an index by its properties (ignoring the name). +// Returns position of index in the index array, or -1 if not found. +//----------------------------------------------------------------------------- +int CRecordInfo::FindIndex( CRecordInfo *pRec, const FieldSet_t& fieldSet ) +{ + for ( int i = 0; i < m_VecIndexes.Count(); i++ ) + { + if ( FieldSet_t::CompareFieldSets( m_VecIndexes[i], this, fieldSet, pRec ) ) + return i; + } + + // Not found + return -1; +} + +//----------------------------------------------------------------------------- +// Purpose: Locate an index with the given name. +// Returns position of index in the index array, or -1 if not found. +//----------------------------------------------------------------------------- +int CRecordInfo::FindIndexByName( const char *pszName ) const +{ + for ( int i = 0; i < m_VecIndexes.Count(); i++ ) + { + if ( V_stricmp( m_VecIndexes[i].GetIndexName(), pszName )== 0 ) + return i; + } + + // Not found + return -1; +} + +//----------------------------------------------------------------------------- +// Purpose: compares index lists in this record with those of another +//----------------------------------------------------------------------------- +bool CRecordInfo::CompareIndexLists( CRecordInfo* pOther ) +{ + // compare the index lists (but don't use CRCs) + + // different size? can't be the same + if ( pOther->GetIndexFieldCount() != GetIndexFieldCount() ) + { + return false; + } + + // We have to loop through both lists of indexes and try to find a match. + // We also must make sure the match is exact, and that no previous match + // can alias another attempt at a match. Pretty messy, but with no available + // identity over index objects, we're forced to a suboptimal solution. + + int nIndexes = GetIndexFieldCount(); + + // get a copy of the other index vector, which we'll remove items from as + // matches are found. + + CUtlVector<FieldSet_t> vecOtherIndexes; + vecOtherIndexes.CopyArray( pOther->GetIndexFields().Base(), nIndexes ); + + for ( int nOurs = 0; nOurs < nIndexes; nOurs++ ) + { + int nOtherMatchIndex = -1; + const FieldSet_t& refOurs = GetIndexFields()[nOurs]; + + // rip through copy of other to find one that matches + for ( int nOther = 0; nOther < vecOtherIndexes.Count(); nOther++ ) + { + const FieldSet_t& refOther = vecOtherIndexes[nOther]; + if ( FieldSet_t::CompareFieldSets( refOurs, this, refOther, pOther ) ) + { + nOtherMatchIndex = nOther; + break; + } + } + + if ( nOtherMatchIndex >= 0 ) + { + // this works! remove it from other copy + vecOtherIndexes.Remove( nOtherMatchIndex ); + } + else + { + // something didn't match, so bail out early + return false; + } + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: compares full-text indexes for this record with those of another +// column order in an FTS is irrelevant, so this is a simple match +//----------------------------------------------------------------------------- +bool CRecordInfo::CompareFTSIndexLists( CRecordInfo* pOther ) const +{ + // compare full-text index columns + if ( m_vecFTSFields.Count() != pOther->m_vecFTSFields.Count() ) + { + // counts don't match, so obviously no good + return false; + } + for ( int nColumnIndex = 0; nColumnIndex < m_vecFTSFields.Count(); nColumnIndex++ ) + { + bool bFound = false; + for ( int nInnerIndex = 0; nInnerIndex < pOther->m_vecFTSFields.Count(); nInnerIndex++ ) + { + if ( m_vecFTSFields[nInnerIndex] == pOther->m_vecFTSFields[nColumnIndex] ) + { + bFound = true; + break; + } + } + + if ( !bFound ) + return false; + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns the checksum for this record info +//----------------------------------------------------------------------------- +int CRecordInfo::GetChecksum() +{ + Assert( m_bPreparedForUse ); + + // calculate it now if we haven't already + if ( !m_bHaveChecksum ) + CalculateChecksum(); + + return m_nChecksum; +} + + +//----------------------------------------------------------------------------- +// Purpose: Prepares this object for use after all columns have been added +//----------------------------------------------------------------------------- +void CRecordInfo::PrepareForUse() +{ + Assert( !m_bPreparedForUse ); + Assert( 0 == m_cubFixedSize ); + Assert( 0 == m_nChecksum ); + + SetAllColumnsAdded(); + + FOR_EACH_VEC( m_VecColumnInfo, nColumn ) + { + CColumnInfo &columnInfo = m_VecColumnInfo[nColumn]; + + // keep track of total fixed size of all columns + if ( !columnInfo.BIsVariableLength() ) + m_cubFixedSize += columnInfo.GetFixedSize(); + + if ( columnInfo.BIsPrimaryKey() ) + { + // a PK column! if we have seen one before, + // know we have a-column PK; otherwise, a single column PK + if (m_nHasPrimaryKey == k_EPrimaryKeyTypeNone) + m_nHasPrimaryKey = k_EPrimaryKeyTypeSingle; + else + m_nHasPrimaryKey = k_EPrimaryKeyTypeMulti; + } + } + + // make sure count matches the enum + /* + Assert( ( m_nHasPrimaryKey == k_EPrimaryKeyTypeNone && m_VecPKFields.Count() == 0 ) || + ( m_nHasPrimaryKey == k_EPrimaryKeyTypeMulti && m_VecPKFields.Count() > 1) || + ( m_nHasPrimaryKey == k_EPrimaryKeyTypeSingle && m_VecPKFields.Count() == 1) ); + */ + + m_bPreparedForUse = true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns index of column with specified name +// Input: pchName - column name +// punColumn - pointer to fill in with index +// Output: return true if found, false otherwise +//----------------------------------------------------------------------------- +bool CRecordInfo::BFindColumnByName( const char *pchName, int *punColumn ) +{ + Assert( m_bAllColumnsAdded ); + Assert( pchName && *pchName ); + Assert( punColumn ); + + *punColumn = -1; + + // if we haven't already built the name index, build it now + if ( !m_bHaveColumnNameIndex ) + BuildColumnNameIndex(); + + *punColumn = m_MapIColumnInfo.Find( pchName ); + return ( m_MapIColumnInfo.InvalidIndex() != *punColumn ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Sets the name of this record info +// Input: pchName - name +// Notes: record info that describes a table will have a name (the table name); +// record info that describes a result set will not +//----------------------------------------------------------------------------- +void CRecordInfo::SetName( const char *pchName ) +{ + Assert( pchName && *pchName ); + Assert( !m_bPreparedForUse ); // don't change this after prepared for use + Q_strncpy( m_rgchName, pchName, Q_ARRAYSIZE( m_rgchName ) ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Builds the column name index for fast lookup by name +//----------------------------------------------------------------------------- +void CRecordInfo::BuildColumnNameIndex() +{ + AUTO_LOCK( m_Mutex ); + + if ( m_bHaveColumnNameIndex ) + return; + + Assert( m_bAllColumnsAdded ); + + Assert( 0 == m_MapIColumnInfo.Count() ); + + FOR_EACH_VEC( m_VecColumnInfo, nColumn ) + { + // build name->column index map + CColumnInfo &columnInfo = m_VecColumnInfo[nColumn]; + m_MapIColumnInfo.Insert( columnInfo.GetName(), nColumn ); + } + m_bHaveColumnNameIndex = true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Calculates the checksum for this record info +//----------------------------------------------------------------------------- +void CRecordInfo::CalculateChecksum() +{ + AUTO_LOCK( m_Mutex ); + + if ( m_bHaveChecksum ) + return; + + // build the column name index if necessary + if ( !m_bHaveColumnNameIndex ) + BuildColumnNameIndex(); + + CRC32_t crc32; + CRC32_Init( &crc32 ); + + FOR_EACH_MAP( m_MapIColumnInfo, iMapItem ) + { + uint32 unColumn = m_MapIColumnInfo[iMapItem]; + CColumnInfo &columnInfo = m_VecColumnInfo[unColumn]; + // calculate checksum of all of our columns + columnInfo.CalculateChecksum(); + int nChecksum = columnInfo.GetChecksum(); + CRC32_ProcessBuffer( &crc32, (void*) &nChecksum, sizeof( nChecksum ) ); + } + + // keep checksum for entire record info + CRC32_Final( &crc32 ); + m_nChecksum = crc32; + m_bHaveChecksum = true; +} + +//----------------------------------------------------------------------------- +// Purpose: add another index disallowing duplicates. If a duplicate item is +// found, we'll set the flags on the new item from the existing one. +//----------------------------------------------------------------------------- +int CRecordInfo::AddIndex( const FieldSet_t& fieldSet ) +{ + for ( int n = 0; n < m_VecIndexes.Count(); n++ ) + { + FieldSet_t& fs = m_VecIndexes[n]; + if ( FieldSet_t::CompareFieldSets( fieldSet, this, fs, this ) ) + { + fs.SetClustered( fs.IsClustered() ); + return -1; + } + } + + int nRet = m_VecIndexes.AddToTail( fieldSet ); + return nRet; +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns true if there is an IDENTITY column in the record info +//----------------------------------------------------------------------------- +bool CRecordInfo::BHasIdentity() const +{ + FOR_EACH_VEC( m_VecColumnInfo, nColumn) + { + if( m_VecColumnInfo[nColumn].BIsAutoIncrement() ) + return true; + } + return false; +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CColumnInfo::CColumnInfo() +{ + m_rgchName[0] = 0; + m_nSQLColumn = 0; + m_eType = k_EGCSQLTypeInvalid; + m_nColFlags = 0; + m_cubFixedSize = 0; + m_cchMaxSize = 0; + m_nChecksum = 0; + m_bHaveChecksum = false; +} + +//----------------------------------------------------------------------------- +// Purpose: Sets column info for this column +// Input: pchName - column name +// nSQLColumn - column index in SQL to bind to (1-based) +// eType - data type of column +// cubFixedSize - for fixed-size fields, the size +// nColFlags - attributes +//----------------------------------------------------------------------------- +void CColumnInfo::Set( const char *pchName, int nSQLColumn, EGCSQLType eType, int cubFixedSize, int nColFlags, int cchMaxSize ) +{ + Assert( !m_rgchName[0] ); + Q_strncpy( m_rgchName, pchName, Q_ARRAYSIZE( m_rgchName ) ); + m_nSQLColumn = nSQLColumn; + m_eType = eType; + + m_nColFlags = nColFlags; + ValidateColFlags(); + + if ( !BIsVariableLength() ) + { + Assert( cubFixedSize > 0 ); + m_cubFixedSize = cubFixedSize; + m_cchMaxSize = 0; + } + else + { + // it's variable length, so we need a max length + m_cchMaxSize = cchMaxSize; + m_cubFixedSize = 0; + } +} + + + +//----------------------------------------------------------------------------- +// Purpose: returns whether this column is variable length +//----------------------------------------------------------------------------- +bool CColumnInfo::BIsVariableLength() const +{ + return m_eType == k_EGCSQLType_Blob || m_eType == k_EGCSQLType_String || m_eType == k_EGCSQLType_Image; +} + +//----------------------------------------------------------------------------- +// Purpose: convert column flags to a visible representation +//----------------------------------------------------------------------------- +void CColumnInfo::GetColFlagDescription( char* pstrOut, int cubOutLength ) const +{ + if ( m_nColFlags == 0 ) + Q_strncpy( pstrOut, "(none)", cubOutLength ); + else + { + pstrOut[0] = 0; + if ( m_nColFlags & k_nColFlagIndexed ) + Q_strncat( pstrOut, "(Indexed)", cubOutLength ); + if ( m_nColFlags & k_nColFlagUnique ) + Q_strncat( pstrOut, "(Unique)", cubOutLength ); + if ( m_nColFlags & k_nColFlagPrimaryKey ) + Q_strncat( pstrOut, "(PrimaryKey)", cubOutLength ); + if ( m_nColFlags & k_nColFlagAutoIncrement ) + Q_strncat( pstrOut, "(AutoIncrement)", cubOutLength ); + if ( m_nColFlags & k_nColFlagClustered ) + Q_strncat( pstrOut, "(Clustered)", cubOutLength ); + } + + return; +} + + + +//----------------------------------------------------------------------------- +// Purpose: sets column flag bits +// Input: nColFlag - bits to set. (Other bits are not cleared.) +//----------------------------------------------------------------------------- +void CColumnInfo::SetColFlagBits( int nColFlag ) +{ + ValidateColFlags(); + m_nColFlags |= nColFlag; // set these bits + ValidateColFlags(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Calculates the checksum for this column +//----------------------------------------------------------------------------- +void CColumnInfo::CalculateChecksum() +{ + if ( m_bHaveChecksum ) + return; + + // calculate checksum of this column for easy comparsion + CRC32_t crc32; + CRC32_Init( &crc32 ); + CRC32_ProcessBuffer( &crc32, (void*) m_rgchName, Q_strlen( m_rgchName ) ); + CRC32_ProcessBuffer( &crc32, (void*) &m_nColFlags, sizeof( m_nColFlags ) ); + CRC32_ProcessBuffer( &crc32, (void*) &m_eType, sizeof( m_eType ) ); + CRC32_ProcessBuffer( &crc32, (void*) &m_cubFixedSize, sizeof( m_cubFixedSize ) ); + CRC32_ProcessBuffer( &crc32, (void*) &m_cchMaxSize, sizeof( m_cchMaxSize ) ); + CRC32_Final( &crc32 ); + + m_nChecksum = crc32; + m_bHaveChecksum = true; +} + +//----------------------------------------------------------------------------- +// determine if this CColumnInfo is the same as the referenced +//----------------------------------------------------------------------------- +bool CColumnInfo::operator==( const CColumnInfo& refOther ) const +{ + if ( m_eType != refOther.m_eType ) + return false; + if ( m_cubFixedSize != refOther.m_cubFixedSize ) + return false; + if ( m_cchMaxSize != refOther.m_cchMaxSize ) + return false; + if ( m_nColFlags != refOther.m_nColFlags ) + return false; + if ( 0 != Q_strcmp( m_rgchName, refOther.m_rgchName ) ) + return false; + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Validates that column flags are set in valid combinations +//----------------------------------------------------------------------------- +void CColumnInfo::ValidateColFlags() const +{ + // Check that column flags follow rules about how columns get expressed in SQL + + if ( m_nColFlags & k_nColFlagPrimaryKey ) + { + // a primary key must also be unique and indexed + Assert( m_nColFlags & k_nColFlagUnique ); + Assert( m_nColFlags & k_nColFlagIndexed ); + } + + // a column with uniqueness constraint must also be indexed + if ( m_nColFlags & k_nColFlagUnique ) + Assert( m_nColFlags & k_nColFlagIndexed ); +} + + +CRecordInfo *CRecordInfo::Alloc() +{ + CRecordInfo *pRecordInfo = sm_MemPoolRecordInfo.Alloc(); + +#ifdef _DEBUG + AUTO_LOCK( sm_mutexMemPoolRecordInfo ); + sm_mapPMemPoolRecordInfo.Insert( pRecordInfo ); +#endif + + return pRecordInfo; +} + + +void CRecordInfo::DestroyThis() +{ +#ifdef _DEBUG + AUTO_LOCK( sm_mutexMemPoolRecordInfo ); + sm_mapPMemPoolRecordInfo.Remove( this ); +#endif + + sm_MemPoolRecordInfo.Free( this ); +} + + +#ifdef DBGFLAG_VALIDATE + + +void CRecordInfo::ValidateStatics( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE_STATIC( "CRecordInfo class statics" ); + + ValidateObj( sm_MemPoolRecordInfo ); + +#ifdef _DEBUG + AUTO_LOCK( sm_mutexMemPoolRecordInfo ); + ValidateObj( sm_mapPMemPoolRecordInfo ); + FOR_EACH_MAP_FAST( sm_mapPMemPoolRecordInfo, i ) + { + sm_mapPMemPoolRecordInfo[i]->Validate( validator, "sm_mapPMemPoolRecordInfo[i]" ); + } +#endif +} + + +void CRecordInfo::Validate( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE(); + + m_VecIndexes.Validate( validator, "m_VecIndexes" ); + + ValidateObj( m_VecFKData ); + FOR_EACH_VEC( m_VecFKData, i ) + { + ValidateObj( m_VecFKData[i] ); + } + + for ( int iIndex = 0; iIndex < m_VecIndexes.Count(); iIndex++ ) + { + ValidateObj( m_VecIndexes[iIndex] ); + } + ValidateObj( m_vecFTSFields ); + + ValidateObj( m_VecColumnInfo ); + FOR_EACH_VEC( m_VecColumnInfo, nColumn ) + { + CColumnInfo &columnInfo = GetColumnInfo( nColumn ); + ValidateObj( columnInfo ); + } + ValidateObj( m_MapIColumnInfo ); +} + +void CColumnInfo::Validate( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE(); + +} + +#endif // DBGFLAG_VALIDATE + +} // namespace GCSDK diff --git a/gcsdk/sqlaccess/schema.cpp b/gcsdk/sqlaccess/schema.cpp new file mode 100644 index 0000000..e92be8e --- /dev/null +++ b/gcsdk/sqlaccess/schema.cpp @@ -0,0 +1,1505 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//============================================================================= + + +#include "stdafx.h" +//#include "sqlaccess/sqlaccess.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ +#ifndef STEAM +bool isspace( char ch ) +{ + return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r'; +} + +int Q_strnlen( const char *str, int count ) +{ + // can't make more meaningful checks, because this routine is used itself + // to check the NUL-terminatedness of strings + if ( !str || count < 0 ) + return -1; + + for ( const char *pch = str; pch < str + count; pch++ ) + { + if ( *pch == '\0' ) + return pch - str; + } + + return -1; +} + +#endif + +//----------------------------------------------------------------------------- +// Purpose: convert an ESchemaCatalog into a string for diagnostics and logging. +// this can't be in enum_names because of data type dependencies. +//----------------------------------------------------------------------------- +const char* PchNameFromESchemaCatalog( ESchemaCatalog e ) +{ + switch (e) + { + case k_ESchemaCatalogInvalid: + return "k_ESchemaCatalogInvalid"; + break; + + case k_ESchemaCatalogMain: + return "k_ESchemaCatalogMain"; + break; + } + + AssertMsg1( false, "unknown ESchemaCatalog (%d)", e ); + return "Unknown"; +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CSchema::CSchema() +{ + m_iTable = -1; + m_rgchName[0] = 0; + m_cubRecord = 0; + m_bTestTable = false; + m_cRecordMax = 0; + m_bHasVarFields = false; + m_nHasPrimaryKey = k_EPrimaryKeyTypeNone; + m_iPKIndex = -1; + m_pRecordInfo = NULL; + m_wipePolicy = k_EWipePolicyPreserveAlways; + m_bAllowWipeInProd = false; + m_bPrepopulatedTable = false; + m_nFullTextIndexCatalog = -1; + m_eSchemaCatalog = k_ESchemaCatalogInvalid; +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CSchema::~CSchema() +{ + SAFE_RELEASE( m_pRecordInfo ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Calculates offset of each field within a record structure, and the +// maximum length of a record structure. +//----------------------------------------------------------------------------- +void CSchema::CalcOffsets() +{ + int dubOffsetCur = 0; + + for ( int iField = 0; iField < m_VecField.Count(); iField++ ) + { + m_VecField[iField].m_dubOffset = dubOffsetCur; + dubOffsetCur += m_VecField[iField].m_cubLength; + + if ( m_VecField[iField].BIsVariableLength() ) + m_bHasVarFields = true; + } + + m_cubRecord = dubOffsetCur; + + if ( m_bHasVarFields ) + m_cubRecord += sizeof( VarFieldBlockInfo_t ); +} + + +//----------------------------------------------------------------------------- +// Purpose: called to make final calculations when all fields/indexes/etc have +// been added and the schema is ready to be used +//----------------------------------------------------------------------------- +void CSchema::PrepareForUse() +{ + // Create a record description (new form of schema information, for SQL) that corresponds to this schema object. + // This contains essentially the information as the CSchema object, we keep both for the moment to bridge the DS and SQL worlds. + Assert( !m_pRecordInfo ); + SAFE_RELEASE( m_pRecordInfo ); + m_pRecordInfo = CRecordInfo::Alloc(); + m_pRecordInfo->InitFromDSSchema( this ); +} + + + + +//----------------------------------------------------------------------------- +// Purpose: For a record that has variable-length fields, gets the info block from +// the tail end +// Input : pvRecord - Record data +//----------------------------------------------------------------------------- +VarFieldBlockInfo_t* CSchema::PVarFieldBlockInfoFromRecord( const void *pvRecord ) const +{ + if ( !m_bHasVarFields ) + return NULL; + + uint8 *pubRecord = ( uint8* )pvRecord; + return ( VarFieldBlockInfo_t * )( pubRecord + m_cubRecord ) - 1; +} + + +//----------------------------------------------------------------------------- +// Purpose: Return the total size of the variable-length block for this record +// For records that have no variable-length fields, it will return zero +// Input : pvRecord - Record data +//----------------------------------------------------------------------------- +uint32 CSchema::CubRecordVariable( const void *pvRecord ) const +{ + VarFieldBlockInfo_t *pVarFieldsBlockInfo = PVarFieldBlockInfoFromRecord( pvRecord ); + if ( pVarFieldsBlockInfo ) + return pVarFieldsBlockInfo->m_cubBlock; + else + return 0; +} + + + +void CSchema::RenderField( uint8 *pubRecord, int iField, int cchBuffer, char *pchBuffer ) +{ + Field_t *pField = &m_VecField[iField]; + + uint8 *pubData; + uint32 cubData; + char chEmpty = 0; + + if ( pField->BIsVariableLength() ) + { + if ( !BGetVarField( pubRecord, ( VarField_t * )( pubRecord + pField->m_dubOffset ), &pubData, &cubData ) ) + { + // just render a single byte + pubData = ( uint8* )&chEmpty; + cubData = 1; + } + } + else + { + pubData = pubRecord + pField->m_dubOffset; + cubData = m_VecField[iField].m_cubLength; + } + + ConvertFieldToText( pField->m_EType, pubData, cubData, pchBuffer, cchBuffer ); +} + +//----------------------------------------------------------------------------- +// Purpose: Renders a text version of a record to the console. +// Input : pubRecord - Location of the record data +//----------------------------------------------------------------------------- +void CSchema::RenderRecord( uint8 *pubRecord ) +{ + char rgchT[k_cMedBuff]; + + // First the header + EmitInfo( SPEW_CONSOLE, 2, 2, "%d\t*** Record header: # of lines ***\n", 1 + m_VecField.Count() ); + + // Render each field in turn + for ( int iField = 0; iField < m_VecField.Count(); iField++ ) + { + Field_t *pField = &m_VecField[iField]; + + RenderField( pubRecord, iField, Q_ARRAYSIZE(rgchT), rgchT ); + + EmitInfo( SPEW_CONSOLE, 2, 2, "\t%s\t\t// Field %d (%s)\n", rgchT, iField, pField->m_rgchName ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Get data and size of a field, whether fixed or variable length +// Input: pvRecord - Fixed-length part of record +// iField - index of field to get +// ppubField - receives pointer to field data +// pcubField - receives size in bytes of that data +// Output: true if successful +//----------------------------------------------------------------------------- +bool CSchema::BGetFieldData( const void *pvRecord, int iField, uint8 **ppubField, uint32 *pcubField ) const +{ + *ppubField = NULL; + *pcubField = 0; + const Field_t &field = m_VecField[iField]; + if ( field.BIsVariableLength() ) + { + return BGetVarField( pvRecord, ( VarField_t * )( ( uint8 * ) pvRecord + field.m_dubOffset ), ppubField, pcubField ); + } + else + { + *ppubField = ( ( uint8 * ) pvRecord + field.m_dubOffset ); + *pcubField = field.CubFieldUpdateSize(); + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Set data and size of a field, whether fixed or variable length +// Input: pvRecord - Fixed-length part of record +// iField - index of field to set +// pubField - pointer to field data to copy from +// cubField - size in bytes of that data +// Output: true if successful +//----------------------------------------------------------------------------- +bool CSchema::BSetFieldData( void *pvRecord, int iField, uint8 *pubField, uint32 cubField, bool *pbVarBlockRealloced ) +{ + *pbVarBlockRealloced = false; + + Field_t &field = m_VecField[iField]; + + if ( field.BIsVariableLength() ) + { + return BSetVarField( pvRecord, ( VarField_t * )( ( uint8 * ) pvRecord + field.m_dubOffset ), pubField, cubField, pbVarBlockRealloced, /*bFreeOnRealloc*/ true ); + } + else // fixed length + { + uint8 *pubFieldWrite = ( ( uint8 * ) pvRecord + field.m_dubOffset ); + + // Must fit in field and last byte for strings MUST be NULL + if ( cubField > field.m_cubLength || + ( k_EGCSQLType_String == field.m_EType && Q_strnlen( reinterpret_cast< char* >( pubField ), cubField ) == -1 ) ) + { + Assert( false ); + return false; + } + + if ( k_EGCSQLType_Blob != field.m_EType && k_EGCSQLType_Image != field.m_EType ) + { + // Copy the data (string or binary, doesn't matter) + if ( cubField > 0 ) + Q_memcpy( pubFieldWrite, pubField, cubField ); + + // Null termination and overwrite any old data + if ( cubField < field.m_cubLength ) + Q_memset( pubFieldWrite + cubField, 0, field.m_cubLength - cubField ); + } + else + { + // Only support fixed char or binary + Assert( false ); + return false; + } + } + + return true; +} + + + +//----------------------------------------------------------------------------- +// Purpose: Get data from a variable-length field +// Input: pvRecord - Fixed-length part of record +// pVarField - fixed part of field in that record +// ppubField - receives pointer to field data +// pcubField - receives size in bytes of that data +//----------------------------------------------------------------------------- +bool CSchema::BGetVarField( const void *pvRecord, const VarField_t *pVarField, uint8 **ppubField, uint32 *pcubField ) const +{ + Assert( m_bHasVarFields ); + + *ppubField = 0; + *pcubField = 0; + + VarFieldBlockInfo_t *pVarFieldBlockInfo = PVarFieldBlockInfoFromRecord( pvRecord ); + + if ( !pVarField->m_cubField ) + { + *pcubField = 0; + *ppubField = NULL; + return true; + } + + if ( pVarFieldBlockInfo->m_cubBlock ) + { + uint8 *pubVarBlock = pVarFieldBlockInfo->m_pubBlock; + + *ppubField = pubVarBlock + pVarField->m_dubOffset; + *pcubField = pVarField->m_cubField; + + // Sanity check + Assert( *pcubField <= k_cubVarFieldMax ); + + return true; + } + else + { + // Should never happen + Assert( false ); + return false; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Update a variable-length field in a record (may realloc the var block) +// Input: pvRecord - Fixed-length part of record +// pVarField - fixed part of field in that record +// pvData - data to place in the field +// cubData - size of that data +// pbRealloced - set to indicate if the var block was realloced +// bFreeOnRealloc - If we have to grow or shrink the var block, should we free the old memory? +// Usually true, unless it lives in a NetPacket or something like that +//----------------------------------------------------------------------------- +bool CSchema::BSetVarField( void *pvRecord, VarField_t *pVarField, const void *pvData, uint32 cubData, bool *pbRealloced, bool bFreeOnRealloc ) +{ + VarFieldBlockInfo_t *pVarFieldBlockInfo = PVarFieldBlockInfoFromRecord( pvRecord ); + *pbRealloced = false; + + + if ( cubData > k_cubVarFieldMax ) + { + // field size is too big + Assert( false ); + return false; + } + + // if no block exists, allocate and copy into it + if ( pVarFieldBlockInfo->m_cubBlock == 0 ) + { + // Nothing to do? + if ( !cubData ) + return true; + + // create it + void *pvBlock = PvAlloc( cubData ); + *pbRealloced = true; + + // copy the data + Q_memcpy( pvBlock, pvData, cubData ); + + // set the record's structure to point at our new data + pVarFieldBlockInfo->m_cubBlock = cubData; + pVarFieldBlockInfo->m_pubBlock = ( uint8 * )pvBlock; + + // set the field to point at its landing place + pVarField->m_cubField = cubData; + pVarField->m_dubOffset = 0; + } + else + { + // there is some block available. + + // is this field changing size? + if ( cubData != 0 && ( cubData == pVarField->m_cubField ) ) + { + // no size change - no need to reallocate anything + Q_memcpy( pVarFieldBlockInfo->m_pubBlock + pVarField->m_dubOffset, pvData, cubData ); + } + else + { + // size change - realloc + *pbRealloced = true; + uint32 cubFieldOld = pVarField->m_cubField; + uint32 dubOffsetOld = pVarField->m_dubOffset; + uint32 cubBlockNew = pVarFieldBlockInfo->m_cubBlock - pVarField->m_cubField + cubData; + + if ( ( cubBlockNew + m_cubRecord ) > k_cubRecordMax ) + { + // total record size is too big + Assert( false ); + return false; + } + + void *pvBlockNew = NULL; + + if ( cubBlockNew ) + { + // if this field has never been placed in the block (that is, it's not changing an old value) + // and we have enough space, fastest to put it at the end in the free space. + if ( pVarField->m_cubField == 0 && pVarField->m_dubOffset == 0 && cubData <= pVarFieldBlockInfo->m_cubBlockFree ) + { + uint8 *pubLastUsed = pVarFieldBlockInfo->m_pubBlock; + for ( int iField = 0; iField < m_VecField.Count(); ++iField ) + { + Field_t &field = m_VecField[iField]; + if ( field.BIsVariableLength() ) + { + // Needs updating + VarField_t *pVarFieldCur = ( VarField_t * )( ( uint8 * )pvRecord + field.m_dubOffset ); + pubLastUsed += pVarFieldCur->m_cubField; + } + } + + // copy it there + Q_memcpy( pubLastUsed, pvData, cubData ); + + // set up the field + pVarField->m_cubField = cubData; + pVarField->m_dubOffset = static_cast<int>( (pubLastUsed - pVarFieldBlockInfo->m_pubBlock) ); + + // note that we used some up + pVarFieldBlockInfo->m_cubBlockFree -= cubData; + } + else + { + // yes ... rellocate + pvBlockNew = PvAlloc( cubBlockNew ); + + uint8 *pubBlockOldCursor = pVarFieldBlockInfo->m_pubBlock; + uint8 *pubBlockNewCursor = ( uint8* )pvBlockNew; + + // copy data, skipping over this field (will put at end) + while ( pubBlockOldCursor < ( pVarFieldBlockInfo->m_pubBlock + pVarFieldBlockInfo->m_cubBlock ) ) + { + if ( pVarField->m_cubField && ( (int)pVarField->m_dubOffset == ( pubBlockOldCursor - pVarFieldBlockInfo->m_pubBlock ) ) ) + { + pubBlockOldCursor += pVarField->m_cubField; + } + else + { + *pubBlockNewCursor++ = *pubBlockOldCursor++; + } + } + + // put this field data at the end + Q_memcpy( pubBlockNewCursor, pvData, cubData ); + + // free the old block + if ( bFreeOnRealloc ) + FreePv( pVarFieldBlockInfo->m_pubBlock ); + + // Update the block info + pVarFieldBlockInfo->m_cubBlock = cubBlockNew; + pVarFieldBlockInfo->m_pubBlock = ( uint8 * )pvBlockNew; + pVarFieldBlockInfo->m_cubBlockFree = 0; + + // update this field + pVarField->m_cubField = cubData; + if ( cubData > 0 ) + pVarField->m_dubOffset = static_cast<int>( pubBlockNewCursor - pVarFieldBlockInfo->m_pubBlock ); + else + pVarField->m_dubOffset = 0; + + // update other fields + for ( int iField = 0; iField < m_VecField.Count(); ++iField ) + { + Field_t &field = m_VecField[iField]; + if ( field.BIsVariableLength() ) + { + // Needs updating + VarField_t *pVarFieldCur = ( VarField_t * )( ( uint8 * )pvRecord + field.m_dubOffset ); + + // Except the one we just changed + if ( pVarFieldCur == pVarField ) + continue; + + // And empty fields + if ( !pVarFieldCur->m_cubField ) + continue; + + if ( pVarFieldCur->m_dubOffset > dubOffsetOld ) + pVarFieldCur->m_dubOffset -= cubFieldOld; + } + } + } + } + else + { + // all of the variable data is gone, so + // free the old block + if ( bFreeOnRealloc ) + FreePv( pVarFieldBlockInfo->m_pubBlock ); + + // ... update the block info + pVarFieldBlockInfo->m_cubBlock = 0; + pVarFieldBlockInfo->m_pubBlock = NULL; + pVarFieldBlockInfo->m_cubBlockFree = 0; + + // ... and update this field + pVarField->m_dubOffset = 0; + pVarField->m_cubField = cubData; + } + } + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: If this is a variable-length record, and we just read it from +// a stream, then the var block is after the fixed-length part of the +// record. So, we need to update the record's pointer to reflect that +// Input: pvRecord - Beginning of the serialized record +//----------------------------------------------------------------------------- +void CSchema::FixupDeserializedRecord( void *pvRecord ) +{ + // Nothing to do if not a variable record + if ( !BHasVariableFields() ) + return; + + uint8 *pubRecord = ( uint8 * )pvRecord; + VarFieldBlockInfo_t *pVarBlockInfo = PVarFieldBlockInfoFromRecord( pvRecord ); + pVarBlockInfo->m_pubBlock = pubRecord + m_cubRecord; +} + + +//----------------------------------------------------------------------------- +// Purpose: Initializes a new record with random data. +// Input: pubRecord - The record's in-memory data that we'll fill out +// unPrimaryIndex - Primary index of the record +// pbRealloced - Set to 'true' if the var-fields block for this record was reallocated +// (or allocated for the first time) +// bFreeOnRealloc - If true, we should Free() the var block after reallocating a new block +// (should be false if that block lives inside a NetPacket) +//----------------------------------------------------------------------------- +void CSchema::InitRecordRandom( uint8 *pubRecord, uint32 unPrimaryIndex, bool *pbVarBlockRealloced, bool bFreeVarBlockOnRealloc ) +{ + // Fill out each field in turn + for ( int iField = 0; iField < m_VecField.Count(); iField++ ) + { + bool bRealloced = false; + SetFieldRandom( pubRecord, iField, &bRealloced, bFreeVarBlockOnRealloc ); + if ( bRealloced ) + { + *pbVarBlockRealloced = true; + // if we just allocated it, we can free it next time we need to + bFreeVarBlockOnRealloc = true; + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Sets a single field of a record to a random value. +// Input: pubRecord - Where the record lives in memory +// iField - Which field to set +// pbRealloced - Set to 'true' if the var-fields block for this record was reallocated +// (or allocated for the first time) +// bFreeOnRealloc - If true, we should Free() the var block after reallocating a new block +// (should be false if that block lives inside a NetPacket) +//----------------------------------------------------------------------------- +void CSchema::SetFieldRandom( uint8 *pubRecord, int iField, bool *pbVarBlockRealloced, bool bFreeVarBlockOnRealloc ) +{ + Field_t &field = m_VecField[iField]; + *pbVarBlockRealloced = false; + + // Strings get random text + if ( k_EGCSQLType_String == field.m_EType ) + { + // Generate up to cubLength - 1 chars + uint32 cch = UNRandFast() % field.m_cubLength; + uint32 ich = 0; + for ( ; ich < cch; ich++ ) + *( ( char * ) pubRecord + field.m_dubOffset + ich ) = CHRandFast(); + + // Null termination + for ( ; ich < field.m_cubLength; ich++ ) + *( ( char * ) pubRecord + field.m_dubOffset + ich ) = 0; + } + else if ( field.BIsVariableLength() ) + { + // Need temp buffer to randomize before setting + // kept reasonably small to prevent spamming the console + uint8 rgubBuff[512]; + uint32 cubData = UNRandFast() % sizeof(rgubBuff); + + // For strings, put in (cubData-1) random characters (each randomly in the range [32,126]) + // then a trailing NULL + if ( field.BIsStringType() ) + { + uint32 ich = 0; + for ( ; ich < (cubData-1); ich++ ) + rgubBuff[ich] = CHRandFast(); + + rgubBuff[ich] = 0; + } + else + { + // binary - just fill in random bytes + for ( uint32 iub = 0; iub < cubData; iub++ ) + { + rgubBuff[iub] = ( uint8 )( UNRandFast() % 256 ); + } + } + + // Set the variable field in the record + BSetVarField( pubRecord, ( VarField_t * ) ( pubRecord + field.m_dubOffset ), rgubBuff, cubData, pbVarBlockRealloced, bFreeVarBlockOnRealloc ); + } + // Binaries are filled with completely random bytes + else + { + for ( uint32 iub = 0; iub < field.m_cubLength; iub++ ) + { + *( pubRecord + field.m_dubOffset + iub ) = ( uint8 ) ( UNRandFast() % 256 ); + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns checksum of our contents +// Output : checksum +//----------------------------------------------------------------------------- +uint32 CSchema::CalcChecksum() +{ + CRC32_t crc32; + CRC32_Init( &crc32 ); + + FOR_EACH_VEC( m_VecField, nField ) + { + CRC32_ProcessBuffer( &crc32, &m_VecField[nField], sizeof( m_VecField[nField] ) ); + } + + // keep checksum for entire record info + CRC32_Final( &crc32 ); + + return (uint32)crc32; +} + + +void CSchema::AddIntField( char *pchName, char *pchSQLName, EGCSQLType eType, int cubSize ) +{ + int nExpectedSize = -1; + switch ( eType ) + { + case k_EGCSQLType_int8: + nExpectedSize = 1; + break; + + case k_EGCSQLType_int16: + nExpectedSize = 2; + break; + + case k_EGCSQLType_int32: + case k_EGCSQLType_float: + nExpectedSize = 4; + break; + + case k_EGCSQLType_int64: + case k_EGCSQLType_double: + nExpectedSize = 8; + break; + } + + AssertMsg2( nExpectedSize != -1, "Unexpected EType in AddIntField: %d for column %s", eType, pchSQLName ); + AssertMsg3( nExpectedSize == cubSize, "Unexpected size for in AddIntField for column %s: %d doesn't match %d ", pchSQLName, nExpectedSize, cubSize ); + + AddField( pchName, pchSQLName, eType, cubSize, 0 ); +} + +//----------------------------------------------------------------------------- +// Purpose: Adds a field from our intrinsic schema to this schema. +// Input: pchName - Name of the field +// pchSQLName - Name of the field in SQL database +// eType - Type of the field +// cubSize - Size of the field +// pfnCompare - Function used to compare fields (NULL if none specified) +//----------------------------------------------------------------------------- +void CSchema::AddField( char *pchName, char *pchSQLName, EGCSQLType eType, uint32 cubSize, int cchMaxLength ) +{ + int iFieldNew = m_VecField.AddToTail(); + Field_t &field = m_VecField[iFieldNew]; + + field.m_EType = eType; + field.m_cubLength = cubSize; + field.m_nColFlags = 0; + Q_strncpy( field.m_rgchName, pchName, Q_ARRAYSIZE( field.m_rgchName ) ); + Q_strncpy( field.m_rgchSQLName, pchSQLName, Q_ARRAYSIZE( field.m_rgchSQLName ) ); + field.m_cchMaxLength = cchMaxLength; +} + + + +//----------------------------------------------------------------------------- +// Purpose: Marks a given field as the primary index +// Input: +// bClustered - true to create a clustered index +// pchName - Name of the field to make primary index +//----------------------------------------------------------------------------- +int CSchema::PrimaryKey( bool bClustered, int nFillFactor, const char *pchName ) +{ + int iField = FindIField( pchName ); + AssertFatalMsg2( k_iFieldNil != iField, "Could not find index column \"%s\" on table \"%s\"", pchName, m_rgchName ); + + Assert( m_nHasPrimaryKey == k_EPrimaryKeyTypeNone ); // may not already have a primary key defined + Assert( m_iPKIndex == k_iFieldNil ); + + // must not have been indexed in any way before + Assert( 0 == ( m_VecField[ iField ].m_nColFlags & ( k_nColFlagPrimaryKey | k_nColFlagIndexed | k_nColFlagUnique | k_nColFlagClustered ) ) ); + + // note that we have a single-column PK + m_nHasPrimaryKey = k_EPrimaryKeyTypeSingle; + + // set our flags + m_VecField[ iField ].m_nColFlags |= ( k_nColFlagPrimaryKey | k_nColFlagIndexed | k_nColFlagUnique ); + if ( bClustered ) + m_VecField[ iField ].m_nColFlags |= k_nColFlagClustered; + + // add to list of primary keys and remember the indexID + CUtlVector<int> vecColumns; + vecColumns.AddToTail( iField ); + + FieldSet_t vecIndex( true /* unique */ , bClustered, vecColumns, NULL ); + vecIndex.SetFillFactor( nFillFactor ); + m_iPKIndex = m_VecIndexes.AddToTail( vecIndex ); + + return m_iPKIndex; +} + +//----------------------------------------------------------------------------- +// Purpose: Marks a set of fields as the primary index +// Input: +// bClustered - clustered index created if true +// nFillFactor - fill facto to use; 0 is database default +// pchName - Name of the field to make primary index +//----------------------------------------------------------------------------- +int CSchema::PrimaryKeys( bool bClustered, int nFillFactor, const char *pchNames ) +{ + Assert( pchNames != NULL ); // no bogus parameters, please + Assert( m_nHasPrimaryKey == k_EPrimaryKeyTypeNone ); // may not already have a primary key defined + Assert( m_iPKIndex == k_iFieldNil ); // no primary key defined + + int nFlags = k_nColFlagPrimaryKey | k_nColFlagIndexed | k_nColFlagUnique; + if ( bClustered ) + nFlags |= k_nColFlagClustered; + + // go add all those fields as Indexed + int nNewIndex = AddIndexToFieldList( pchNames, NULL, nFlags, nFillFactor ); + if ( nNewIndex != k_iFieldNil ) + { + m_nHasPrimaryKey = k_EPrimaryKeyTypeMulti; // remember that we have multiple keys + m_iPKIndex = nNewIndex; + } + + return nNewIndex; +} + +//----------------------------------------------------------------------------- +// Purpose: Marks a set of fields as the primary index +// Input: +// pchName - Names of the fields for the primary index; comma-separated if multiple +// pchIndexName - Name of the index object (not in SQL) +// nFlags - flags for CColumnInfo on this object +// nFillFactor - fill facto to use; 0 is database default +//----------------------------------------------------------------------------- +int CSchema::AddIndexToFieldList( const char *pchNames, const char *pchIndexName, int nFlags, int nFillFactor ) +{ + // need a copy of string list since the argument is const and read-only + int cNamesLen = Q_strlen( pchNames ) + 1; + char* pchNamesCopy = (char*) PvAlloc( cNamesLen ); + Q_strncpy( pchNamesCopy, pchNames, cNamesLen ); + + if (pchNames == NULL) + { + // not enough memory! + AssertFatal( false ); + return k_iFieldNil; + } + + CUtlVector<int> vecKeys; + + char* pchCurrent = pchNamesCopy; + do + { + // find next token + char* pchEnd = strchr(pchCurrent, ','); + if ( pchEnd != NULL ) + { + *pchEnd++ = 0; + } + + // skip whitespace + while (isspace(*pchCurrent)) + pchCurrent++; + + // find the name; we expect a C++ name, not a SQL name + int iField = FindIField( pchCurrent ); + AssertFatalMsg2( k_iFieldNil != iField, "Could not find index column \"%s\" on table \"%s\"", pchCurrent, m_rgchName ); + + // mark as a primary key + m_VecField[iField].m_nColFlags |= nFlags ; + + // add it to our collection + vecKeys.AddToTail( iField ); + + // move past the end of this token in the string + pchCurrent = pchEnd; + } while ( pchCurrent != NULL ); + + // release our copy + FreePv(pchNamesCopy); + + // create a fieldset with our list of indexes + // and add it to our indexes collection + bool bUnique = 0 != (nFlags & k_nColFlagUnique); + bool bClustered = 0 != (nFlags & k_nColFlagClustered); + FieldSet_t vecIndex( bUnique, bClustered, vecKeys, pchIndexName ); + vecIndex.SetFillFactor( nFillFactor ); + int iReturn = m_VecIndexes.AddToTail( vecIndex ); + return iReturn; +} + + +//----------------------------------------------------------------------------- +// Purpose: Marks a given field as indexed. +// Input: pchName - Name of the field to index +// pchIndexName - Name of the index object (not in SQL) +//----------------------------------------------------------------------------- +int CSchema::IndexField( const char *pchIndexName, const char *pchName ) +{ + Assert( pchName != NULL ); + Assert( pchIndexName != NULL ); + + // find the field by name + int iField = FindIField( pchName ); + AssertFatalMsg2( k_iFieldNil != iField, "Could not find index column \"%s\" on table \"%s\"", pchName, m_rgchName ); + + // add an index to it + return AddIndexToFieldNumber( iField, pchIndexName, false /* Not clustered */ ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Marks a given field as indexed. +// Input: pchIndexName - Name of the index object (not in SQL) +// pchName - Name of the fields to index; comma separated if multiple +//----------------------------------------------------------------------------- +int CSchema::IndexFields( const char *pchIndexName, const char *pchNames ) +{ + Assert( pchNames != NULL ); + Assert( pchIndexName != NULL ); + + // go add all those fields as Indexed + int nNewIndex = AddIndexToFieldList( pchNames, pchIndexName, k_nColFlagIndexed, 0 ); + return nNewIndex; +} + + +//----------------------------------------------------------------------------- +// Purpose: Includes a column (or columns) in an index for the INCLUDE clause +// Input: pchIndexName - Name of the index object (not in SQL) +// pchName - Name of the fields to index; comma separated if multiple +//----------------------------------------------------------------------------- +void CSchema::AddIncludedFields( const char *pchIndexName, const char *pchNames ) +{ + Assert( pchNames != NULL ); + Assert( pchIndexName != NULL ); + + // find that index by name + int nIndexIndex = -1; + FOR_EACH_VEC( m_VecIndexes, i ) + { + FieldSet_t &refSet = m_VecIndexes.Element(i); + const char *pstrMatch = refSet.GetIndexName(); + if ( Q_stricmp( pstrMatch, pchIndexName ) == 0 ) + { + nIndexIndex = i; + break; + } + } + + // must have found it + AssertFatalMsg1( nIndexIndex != -1, "Index \"%s\" not found", pchIndexName ); + FieldSet_t &refSet = m_VecIndexes.Element( nIndexIndex ); + + // need a copy of string list since the argument is const and read-only + int cNamesLen = Q_strlen( pchNames ) + 1; + char* pchNamesCopy = (char*) PvAlloc( cNamesLen ); + Q_strncpy( pchNamesCopy, pchNames, cNamesLen ); + + char* pchCurrent = pchNamesCopy; + do + { + // find next token + char* pchEnd = strchr(pchCurrent, ','); + if ( pchEnd != NULL ) + { + *pchEnd++ = 0; + } + + // skip whitespace + while (isspace(*pchCurrent)) + pchCurrent++; + + // find the name; we expect a C++ name, not a SQL name + int iField = FindIField( pchCurrent ); + AssertFatalMsg2( k_iFieldNil != iField, "Could not find index column \"%s\" on table \"%s\"", pchCurrent, m_rgchName ); + + // add it to our collection + refSet.AddIncludedColumn( iField ); + + // move past the end of this token in the string + pchCurrent = pchEnd; + } while ( pchCurrent != NULL ); + + // release our copy + FreePv( pchNamesCopy ); + + return; +} + +//----------------------------------------------------------------------------- +// Purpose: Marks a given field as indexed. +// Input: pchIndexName - Name of the index object (not in SQL) +// pchName - Name of the fields to index; comma separated if multiple +//----------------------------------------------------------------------------- +int CSchema::UniqueFields( const char *pchIndexName, const char *pchNames ) +{ + Assert( pchNames != NULL ); + Assert( pchIndexName != NULL ); + + // go add all those fields as Indexed + int nNewIndex = AddIndexToFieldList( pchNames, pchIndexName, k_nColFlagIndexed | k_nColFlagUnique, 0 ); + return nNewIndex; +} + + +//----------------------------------------------------------------------------- +// Purpose: Marks a given field a having a clustered index. +// Input: pchName - Name of the field to index +// pchIndexName - Name of the index object (not in SQL) +//----------------------------------------------------------------------------- +int CSchema::ClusteredIndexField( int nFillFactor, const char *pchIndexName, const char *pchName ) +{ + Assert( pchName != NULL ); + Assert( pchIndexName != NULL ); + + // can't previously have some other clustered index + FOR_EACH_VEC( m_VecIndexes, iIndex ) + { + if ( m_VecIndexes[iIndex].IsClustered() ) + { + AssertFatalMsg3( false, "On table %s, can't make index %s clustered because %s is already the clustered index\n", + m_rgchName, pchIndexName, m_VecIndexes[iIndex].GetIndexName() ); + return -1; + } + } + + // find the field by name + int iField = FindIField( pchName ); + AssertFatalMsg2( k_iFieldNil != iField, "Could not find index column \"%s\" on table \"%s\"", pchName, m_rgchName ); + + // add an index to it + int iIndex = AddIndexToFieldNumber( iField, pchIndexName, true /* clustered */ ); + m_VecIndexes[iIndex].SetFillFactor( nFillFactor ); + return iIndex; +} + + +//----------------------------------------------------------------------------- +// Purpose: Marks a given set of fields as having a clustered index. +// Input: pchIndexName - Name of the index object (not in SQL) +// pchName - Name of the fields to index; comma separated if multiple +//----------------------------------------------------------------------------- +int CSchema::ClusteredIndexFields( int nFillFactor, const char *pchIndexName, const char *pchNames ) +{ + Assert( pchNames != NULL ); + Assert( pchIndexName != NULL ); + + // can't previously have some other clustered index + FOR_EACH_VEC( m_VecIndexes, iIndex ) + { + if ( m_VecIndexes[iIndex].IsClustered() ) + { + AssertFatalMsg3( false, "On table %s, can't make index %s clustered because %s is already the clustered index\n", + m_rgchName, pchIndexName, m_VecIndexes[iIndex].GetIndexName() ); + return -1; + } + } + + // go add all those fields as Indexed + int nNewIndex = AddIndexToFieldList( pchNames, pchIndexName, k_nColFlagClustered | k_nColFlagIndexed, nFillFactor ); + return nNewIndex; +} + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +void CSchema::AddFullTextIndex( CSchemaFull *pSchemaFull, const char *pchCatalogName, const char *pchColumnName ) +{ + Assert( pchCatalogName != NULL ); + Assert( pchColumnName != NULL ); + + // need a copy of string list since the argument is const and read-only + int cNamesLen = Q_strlen( pchColumnName ) + 1; + char* pchNamesCopy = (char*) PvAlloc( cNamesLen ); + Q_strncpy( pchNamesCopy, pchColumnName, cNamesLen ); + + if ( pchNamesCopy == NULL ) + { + // not enough memory! + AssertFatal( false ); + return; + } + + char* pchCurrent = pchNamesCopy; + do + { + // find next token + char* pchEnd = strchr(pchCurrent, ','); + if ( pchEnd != NULL ) + { + *pchEnd++ = 0; + } + + // skip whitespace + while (isspace(*pchCurrent)) + pchCurrent++; + + // find the name; we expect a C++ name, not a SQL name + int iField = FindIField( pchCurrent ); + AssertFatalMsg2( k_iFieldNil != iField, "Could not find index column \"%s\" on table \"%s\"", pchCurrent, m_rgchName ); + + // add it to our collection + m_VecFullTextIndexes.AddToTail( iField ); + + // move past the end of this token in the string + pchCurrent = pchEnd; + } while ( pchCurrent != NULL ); + + // release our copy + FreePv(pchNamesCopy); + + // make a note of the catalog we want + int nCatalogID = pSchemaFull->GetFTSCatalogByName( m_eSchemaCatalog, pchCatalogName ); + AssertFatalMsg2( nCatalogID != -1, "Could not find fulltext catalog \"%s\" on table \"%s\"", pchCatalogName, m_rgchName ); + m_nFullTextIndexCatalog = nCatalogID; + + return; +} + + +//----------------------------------------------------------------------------- +// Purpose: Marks a field as indexed given its field number +//----------------------------------------------------------------------------- +int CSchema::AddIndexToFieldNumber( int iField, const char *pchIndexName, bool bClustered ) +{ + // mark the field as indexed + m_VecField[iField].m_nColFlags |= k_nColFlagIndexed; + + // meant to be clustered? + if ( bClustered ) + m_VecField[iField].m_nColFlags |= k_nColFlagClustered; + + // add to list of indexes, and remember the indexID + CUtlVector<int> vecColumns; + vecColumns.AddToTail( iField ); + + // false: not unique + // false: not clustered + FieldSet_t vecIndex( false, bClustered, vecColumns, pchIndexName); + int iIndex = m_VecIndexes.AddToTail( vecIndex ); + + return iIndex; +} + + +//----------------------------------------------------------------------------- +// Purpose: Marks a given field as unique. +// Input: pchName - Name of the field to index +// pchIndexName - Name of the index object +//----------------------------------------------------------------------------- +int CSchema::UniqueField( const char *pchIndexName, const char *pchName ) +{ + Assert( pchName != NULL ); + Assert( pchIndexName != NULL ); + + int iField = FindIField( pchName ); + AssertFatalMsg2( k_iFieldNil != iField, "Could not find index column \"%s\" on table \"%s\"", pchName, m_rgchName ); + + m_VecField[iField].m_nColFlags |= ( k_nColFlagUnique | k_nColFlagIndexed ); + + // add to list of indexes, and remember the indexID + CUtlVector<int> vecColumns; + vecColumns.AddToTail( iField ); + + // true: unique + // false: not clustered + FieldSet_t vecIndex( true, false, vecColumns, pchIndexName ); + int iIndex = m_VecIndexes.AddToTail( vecIndex ); + + return iIndex; +} + + +//----------------------------------------------------------------------------- +// Purpose: Marks a given field as auto increment. +// Input: pchName - Name of the field +//----------------------------------------------------------------------------- +void CSchema::AutoIncrementField( char *pchName ) +{ + int iField = FindIField( pchName ); + AssertFatalMsg2( k_iFieldNil != iField, "Could not find index column \"%s\" on table \"%s\"", pchName, m_rgchName ); + m_VecField[iField].m_nColFlags |= k_nColFlagAutoIncrement; +} + + +//----------------------------------------------------------------------------- +// Purpose: Finds the field with a given name. +// Input: pchName - Name of the field to search for +// Output: Index of the matching field (k_iFieldNil if there isn't one) +//----------------------------------------------------------------------------- +int CSchema::FindIField( const char *pchName ) +{ + for ( int iField = 0; iField < m_VecField.Count(); iField++ ) + { + if ( 0 == Q_strncmp( pchName, m_VecField[iField].m_rgchName, k_cSQLObjectNameMax ) ) + return iField; + } + + return k_iFieldNil; +} + + +//----------------------------------------------------------------------------- +// Purpose: Finds the field with a given SQL name. +// Input: pchName - Name of the field to search for +// Output: Index of the matching field (k_iFieldNil if there isn't one) +//----------------------------------------------------------------------------- +int CSchema::FindIFieldSQL( const char *pchName ) +{ + for ( int iField = 0; iField < m_VecField.Count(); iField++ ) + { + if ( 0 == Q_strncmp( pchName, m_VecField[iField].m_rgchSQLName, k_cSQLObjectNameMax ) ) + return iField; + } + + return k_iFieldNil; +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a schema conversion instruction (for use in converting from +// a different Schema to this one). +//----------------------------------------------------------------------------- +void CSchema::AddDeleteField( const char *pchFieldName ) +{ + DeleteField_t &deleteField = m_VecDeleteField[m_VecDeleteField.AddToTail()]; + Q_strncpy( deleteField.m_rgchFieldName, pchFieldName, sizeof( deleteField.m_rgchFieldName ) ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a schema conversion instruction (for use in converting from +// a different Schema to this one). +//----------------------------------------------------------------------------- +void CSchema::AddRenameField( const char *pchFieldNameOld, const char *pchFieldNameNew ) +{ + RenameField_t &renameField = m_VecRenameField[m_VecRenameField.AddToTail()]; + Q_strncpy( renameField.m_rgchFieldNameOld, pchFieldNameOld, sizeof( renameField.m_rgchFieldNameOld ) ); + renameField.m_iFieldDst = FindIField( pchFieldNameNew ); + Assert( k_iFieldNil != renameField.m_iFieldDst ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a schema conversion instruction (for use in converting from +// a different Schema to this one). +// Input: pchFieldNameOld - Name of the field in the old schema +// pchFieldNameMew - Name of the field in the new schema +// pfnAlterField - Function to translate data from the old format to +// the new +//----------------------------------------------------------------------------- +void CSchema::AddAlterField( const char *pchFieldNameOld, const char *pchFieldNameNew, PfnAlterField_t pfnAlterField ) +{ + Assert( pfnAlterField ); + AlterField_t &alterField = m_VecAlterField[m_VecAlterField.AddToTail()]; + Q_strncpy( alterField.m_rgchFieldNameOld, pchFieldNameOld, sizeof( alterField.m_rgchFieldNameOld ) ); + alterField.m_iFieldDst = FindIField( pchFieldNameNew ); + Assert( k_iFieldNil != alterField.m_iFieldDst ); + alterField.m_pfnAlterFunc = pfnAlterField; +} + + +//----------------------------------------------------------------------------- +// Purpose: Figures out how to map a field from another Schema into us. +// First we check our conversion instructions to see if any apply, +// and then we look for a straightforward match. +// Input: pchFieldName - Name of the field we're trying to map +// piFieldDst - [Return] Index of the field to map it to +// ppfnAlterField - [Return] Optional function to convert data +// Output: true if we know what to do with this field (if false, the conversion +// is undefined and dangerous). +//----------------------------------------------------------------------------- +bool CSchema::BCanConvertField( const char *pchFieldName, int *piFieldDst, PfnAlterField_t *ppfnAlterField ) +{ + *ppfnAlterField = NULL; + + // Should this field be deleted? + for ( int iDeleteField = 0; iDeleteField < m_VecDeleteField.Count(); iDeleteField++ ) + { + if ( 0 == Q_strcmp( pchFieldName, m_VecDeleteField[iDeleteField].m_rgchFieldName ) ) + { + *piFieldDst = k_iFieldNil; + return true; + } + } + + // Should this field be renamed? + for ( int iRenameField = 0; iRenameField < m_VecRenameField.Count(); iRenameField++ ) + { + if ( 0 == Q_strcmp( pchFieldName, m_VecRenameField[iRenameField].m_rgchFieldNameOld ) ) + { + *piFieldDst = m_VecRenameField[iRenameField].m_iFieldDst; + return true; + } + } + + // Was this field altered? + for ( int iAlterField = 0; iAlterField < m_VecAlterField.Count(); iAlterField++ ) + { + if ( 0 == Q_strcmp( pchFieldName, m_VecAlterField[iAlterField].m_rgchFieldNameOld ) ) + { + *piFieldDst = m_VecAlterField[iAlterField].m_iFieldDst; + *ppfnAlterField = m_VecAlterField[iAlterField].m_pfnAlterFunc; + return true; + } + } + + // Find out which of our fields this field maps to (if it doesn't map + // to any of them, we don't know what to do with it). + *piFieldDst = FindIField( pchFieldName ); + return ( k_iFieldNil != *piFieldDst ); +} + +//----------------------------------------------------------------------------- +// Purpose: Return the size to use when writing to a field. +//----------------------------------------------------------------------------- +int Field_t::CubFieldUpdateSize() const +{ + switch ( m_EType ) + { + case k_EGCSQLType_String: + case k_EGCSQLType_Blob: + case k_EGCSQLType_Image: + // Nobody should call this function for + // var-length fields + Assert( false ); + return m_cubLength; + + case k_EGCSQLType_int8: + case k_EGCSQLType_int16: + case k_EGCSQLType_int32: + case k_EGCSQLType_int64: + case k_EGCSQLType_float: + case k_EGCSQLType_double: + return m_cubLength; + + default: + Assert(false); + return 0; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Tell whether or not a field is of string type. +//----------------------------------------------------------------------------- +bool Field_t::BIsStringType() const +{ + return ( k_EGCSQLType_String == m_EType ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Tell whether or not the field is of variable-length type. +//----------------------------------------------------------------------------- +bool Field_t::BIsVariableLength() const +{ + return ( k_EGCSQLType_String == m_EType ) || ( k_EGCSQLType_Blob == m_EType ) || ( k_EGCSQLType_Image == m_EType ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Add a foreign key +//----------------------------------------------------------------------------- +void CSchema::AddFK( const char* pchName, const char* pchColumn, const char* pchParentTable, const char* pchParentColumn, EForeignKeyAction eOnDeleteAction, EForeignKeyAction eOnUpdateAction ) +{ + int iTail = m_VecFKData.AddToTail(); + FKData_t &fkData = m_VecFKData[iTail]; + + Q_snprintf( fkData.m_rgchName, Q_ARRAYSIZE( fkData.m_rgchName ), "%s_%s", GetPchName(), pchName ); + Q_strncpy( fkData.m_rgchParentTableName, pchParentTable, Q_ARRAYSIZE( fkData.m_rgchParentTableName ) ); + fkData.m_eOnDeleteAction = eOnDeleteAction; + fkData.m_eOnUpdateAction = eOnUpdateAction; + + // Now we need to split up the column name strings and add their data + FKColumnRelation_t colRelation; + Q_memset( &colRelation, 0, sizeof( FKColumnRelation_t ) ); + uint iMyColumn = 0; + uint iParentColumn = 0; + uint iParentString = 0; + for( uint i=0; i<(uint)Q_strlen( pchColumn )+1; ++i ) + { + if ( pchColumn[i] != ',' && pchColumn[i] != 0 ) + { + colRelation.m_rgchCol[ iMyColumn++ ] = pchColumn[i]; + } + else + { + Assert( Q_strlen( colRelation.m_rgchCol ) ); + // Should have a matching column name in the parent string + while( iParentString < (uint)Q_strlen( pchParentColumn ) ) + { + if ( pchParentColumn[iParentString] != ',' ) + { + colRelation.m_rgchParentCol[ iParentColumn++ ] = pchParentColumn[iParentString]; + ++iParentString; + } + else + { + ++iParentString; + break; + } + } + + AssertMsg( Q_strlen( colRelation.m_rgchParentCol ), "Column counts for FK do not match between child/parent\n" ); + fkData.m_VecColumnRelations.AddToTail( colRelation ); + Q_memset( &colRelation, 0, sizeof( FKColumnRelation_t ) ); + // Reset positions + iMyColumn = 0; + iParentColumn = 0; + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Caches the insert statement for each table so we don't need to +// generate it for each row we insert. +//----------------------------------------------------------------------------- +const char *CSchema::GetInsertStatementText() const +{ + if ( m_sInsertStatementText.IsEmpty() ) + { + TSQLCmdStr sBuilder; + BuildInsertStatementText( &sBuilder, GetRecordInfo() ); + m_sInsertStatementText.Set( sBuilder ); + } + + return m_sInsertStatementText; +} + +//----------------------------------------------------------------------------- +// Purpose: Caches the insert via MERGE statement for each table so we don't need to +// generate it for each row we insert. +//----------------------------------------------------------------------------- +const char *CSchema::GetMergeStatementTextOnPKWhenMatchedUpdateWhenNotMatchedInsert() +{ + if ( m_sMergeStatementTextOnPKWhenMatchedUpdateWhenNotMatchedInsert.IsEmpty() ) + { + TSQLCmdStr sBuilder; + BuildMergeStatementTextOnPKWhenMatchedUpdateWhenNotMatchedInsert( &sBuilder, GetRecordInfo() ); + m_sMergeStatementTextOnPKWhenMatchedUpdateWhenNotMatchedInsert.Set( sBuilder ); + } + + return m_sMergeStatementTextOnPKWhenMatchedUpdateWhenNotMatchedInsert; +} + + +//----------------------------------------------------------------------------- +// Purpose: Caches the insert via MERGE statement for each table so we don't need to +// generate it for each row we insert. +//----------------------------------------------------------------------------- +const char *CSchema::GetMergeStatementTextOnPKWhenNotMatchedInsert() +{ + if ( m_sMergeStatementTextOnPKWhenNotMatchedInsert.IsEmpty() ) + { + TSQLCmdStr sBuilder; + BuildMergeStatementTextOnPKWhenNotMatchedInsert( &sBuilder, GetRecordInfo() ); + m_sMergeStatementTextOnPKWhenNotMatchedInsert.Set( sBuilder ); + } + + return m_sMergeStatementTextOnPKWhenNotMatchedInsert; +} + +//----------------------------------------------------------------------------- +// Purpose: Get the number of foreign key constraints defined for the table +//----------------------------------------------------------------------------- +int CSchema::GetFKCount() +{ + return m_VecFKData.Count(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get data for a foreign key by index (valid for 0...GetFKCount()-1) +//----------------------------------------------------------------------------- +FKData_t &CSchema::GetFKData( int iIndex ) +{ + return m_VecFKData[iIndex]; +} + + +#ifdef DBGFLAG_VALIDATE +//----------------------------------------------------------------------------- +// Purpose: Run a global validation pass on all of our data structures and memory +// allocations. +// Input: validator - Our global validator object +// pchName - Our name (typically a member var in our container) +//----------------------------------------------------------------------------- +void CSchema::Validate( CValidator &validator, const char *pchName ) +{ + // 1. + // Claim our memory + VALIDATE_SCOPE(); + + m_VecField.Validate( validator, "m_VecField" ); + m_VecDeleteField.Validate( validator, "m_VecDeleteField" ); + m_VecRenameField.Validate( validator, "m_VecRenameField" ); + m_VecIndexes.Validate( validator, "m_VecIndexes" ); + m_VecFullTextIndexes.Validate( validator, "m_VecFullTextIndexes" ); + ValidateObj( m_VecFKData ); + FOR_EACH_VEC( m_VecFKData, i ) + { + ValidateObj( m_VecFKData[i] ); + } + + for ( int iIndex = 0; iIndex < m_VecIndexes.Count(); iIndex++ ) + { + ValidateObj( m_VecIndexes[iIndex] ); + } + + ValidateObj( m_sInsertStatementText ); + + // 2. + // Validate that our fields make sense +#if defined(_DEBUG) + uint32 dubOffset = 0; + for ( int iField = 0; iField < m_VecField.Count(); iField++ ) + { + Assert( dubOffset == m_VecField[iField].m_dubOffset ); + dubOffset += m_VecField[iField].m_cubLength; + } +#endif // defined(_DEBUG) +} + + +//----------------------------------------------------------------------------- +// Purpose: Validates that a given record from our table is in a good state. +// Input: pubRecord - Record to validate +//----------------------------------------------------------------------------- +void CSchema::ValidateRecord( uint8 *pubRecord ) +{ + // Make sure each record is in a consistent state + for ( int iField = 0; iField < m_VecField.Count(); iField++ ) + { + Field_t &field = m_VecField[iField]; + + // Ensure that strings are null-terminated, and that everything after the terminator is 0 + if ( k_EGCSQLType_String == field.m_EType ) + { + char *pchField = ( char * ) pubRecord + field.m_dubOffset; + Assert( 0 == pchField[field.m_cubLength - 1] ); + for ( uint32 ich = 0; ich < field.m_cubLength; ich++ ) + { + if ( 0 == pchField[ich] ) + { + while ( ich < field.m_cubLength ) + { + Assert( 0 == pchField[ich] ); + ich++; + } + } + } + } + } +} +#endif // DBGFLAG_VALIDATE +} // namespace GCSDK diff --git a/gcsdk/sqlaccess/schemafull.cpp b/gcsdk/sqlaccess/schemafull.cpp new file mode 100644 index 0000000..d8e9490 --- /dev/null +++ b/gcsdk/sqlaccess/schemafull.cpp @@ -0,0 +1,409 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//============================================================================= + + +#include "stdafx.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ +CSchemaFull g_SchemaFull; +CSchemaFull & GSchemaFull() +{ + return g_SchemaFull; +} + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CSchemaFull::CSchemaFull() +{ + m_pubScratchBuffer = NULL; + m_cubScratchBuffer = 0; + m_unCheckSum = 0; + + m_mapFTSEnabled.SetLessFunc( DefLessFunc( enum ESchemaCatalog ) ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CSchemaFull::~CSchemaFull() +{ + Uninit(); +} + + + +//----------------------------------------------------------------------------- +// Purpose: Call this after you've finished setting up the SchemaFull (either +// by loading it or by in GenerateIntrinsic). It calculates our checksum +// and allocates our scratch buffer. +//----------------------------------------------------------------------------- +void CSchemaFull::FinishInit() +{ + // Calculate our checksum + m_unCheckSum = 0; + + for ( int iSchema = 0; iSchema < m_VecSchema.Count(); iSchema++ ) + m_unCheckSum += m_VecSchema[iSchema].CalcChecksum(); + + // Allocate our scratch buffer + Assert( NULL == m_pubScratchBuffer ); + // Include some slop for field IDs and sizes in a sparse record + // 2k is way overkill but still no big deal + m_cubScratchBuffer = k_cubRecordMax + 2048; + m_pubScratchBuffer = ( uint8 * ) malloc( m_cubScratchBuffer ); +} + +//----------------------------------------------------------------------------- +// Purpose: Call this after you've finished setting up the SchemaFull (either +// by loading it or by in GenerateIntrinsic). It calculates our checksum +// and allocates our scratch buffer. +//----------------------------------------------------------------------------- + +void CSchemaFull::SetITable( CSchema* pSchema, int iTable ) +{ + // make sure we don't have this schema anywhere already + for ( int iSchema = 0; iSchema < m_VecSchema.Count(); iSchema++ ) + { + if ( pSchema != &m_VecSchema[iSchema] ) + AssertFatalMsg( m_VecSchema[iSchema].GetITable() != iTable, "Duplicate iTable in schema definition.\n" ); + } + + // set the pSchema object + pSchema->SetITable( iTable ); +} + + + +//----------------------------------------------------------------------------- +// Purpose: Uninits the schema. Need to call this explicitly before app shutdown +// on static instances of this object, as the CSchema objects +// point to memory in static memory pools which may destruct +// before static instances of this object. +//----------------------------------------------------------------------------- +void CSchemaFull::Uninit() +{ + m_VecSchema.RemoveAll(); + if ( NULL != m_pubScratchBuffer ) + { + free( m_pubScratchBuffer ); + m_pubScratchBuffer = NULL; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the scratch buffer. It is large enough to handle any +// record, sparse or otherwise +// +//----------------------------------------------------------------------------- +uint8* CSchemaFull::GetPubScratchBuffer( ) +{ + return m_pubScratchBuffer; +} + + +//----------------------------------------------------------------------------- +// Purpose: This is used during the generation of our intrinsic schema. We've +// added a new schema to ourselves, and we need to make sure that it +// matches the corresponding C class. +// Input: pSchema - Schema to check +// cField - Number of fields the schema should contain. +// cubRecord - Size of a record in the schema +//----------------------------------------------------------------------------- +void CSchemaFull::CheckSchema( CSchema *pSchema, int cField, uint32 cubRecord ) +{ + // We generate our structures and our schema using macros that operate on the + // same source. We check a couple of things to make sure that they're properly in sync. + + // This will fail if the schema's definition specifies the wrong iTable + if ( pSchema != &m_VecSchema[pSchema->GetITable()] ) + { + EmitError( SPEW_SQL, "Table %s has a bad iTable\n", pSchema->GetPchName() ); + } + + // This will fail if there are missing lines in the schema definition + if ( pSchema->GetCField() != cField ) + { + EmitError( SPEW_SQL, "Badly formed table %s (blank line in schema def?)\n", pSchema->GetPchName() ); + AssertFatal( false ); + } + + // This is unlikely to fail. It indicates some kind of size mismatch (maybe a packing problem?) + if ( pSchema->CubRecordFixed() != cubRecord ) + { + // You may hit this if END_FIELDDATA_HAS_VAR_FIELDS is not used properly + EmitError( SPEW_SQL, "Table %s has an inconsistent size (class = %d, schema = %d)\n", + pSchema->GetPchName(), cubRecord, pSchema->CubRecordFixed() ); + AssertFatal( false ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Finds the table with a given name. +// Input: pchName - Name of the table to search for +// Output: Index of the matching table ( k_iTableNil if there isn't one) +//----------------------------------------------------------------------------- +int CSchemaFull::FindITable( const char *pchName ) +{ + for ( int iSchema = 0; iSchema < m_VecSchema.Count(); iSchema++ ) + { + if ( 0 == Q_strcmp( pchName, m_VecSchema[iSchema].GetPchName() ) ) + return iSchema; + } + + return k_iTableNil; +} + + +//----------------------------------------------------------------------------- +// Purpose: Finds the table with a given iTable (iSchema) +// Input: iTable - +// Output: NULL or a const char * to the name (for temporary use only) +//----------------------------------------------------------------------------- +const char * CSchemaFull::PchTableFromITable( int iTable ) +{ + if ( iTable < 0 || iTable >= m_VecSchema.Count() ) + return NULL; + else + return m_VecSchema[ iTable ].GetPchName(); +} + + + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +void CSchemaFull::AddFullTextCatalog( enum ESchemaCatalog eCatalog, const char *pstrCatalogName, int nFileGroup ) +{ + CFTSCatalogInfo info; + info.m_eCatalog = eCatalog; + info.m_nFileGroup = nFileGroup; + info.m_pstrName = strdup(pstrCatalogName); + + m_vecFTSCatalogs.AddToTail( info ); +} + + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +int CSchemaFull::GetFTSCatalogByName( enum ESchemaCatalog eCatalog, const char *pstrCatalogName ) +{ + int nIndex = -1; + FOR_EACH_VEC( m_vecFTSCatalogs, i ) + { + CFTSCatalogInfo &refInfo = m_vecFTSCatalogs[ i ]; + if ( 0 == Q_stricmp( pstrCatalogName, refInfo.m_pstrName ) ) + { + nIndex = i; + break; + } + } + + return nIndex; +} + + +//----------------------------------------------------------------------------- +// Purpose: turn on FTS for the named schema catalog. Called by the +// InitIntrinsic() function. +//----------------------------------------------------------------------------- +void CSchemaFull::EnableFTS( enum ESchemaCatalog eCatalog ) +{ + // mark it enabled in the map + m_mapFTSEnabled.Insert( eCatalog, true ); +} + + +//----------------------------------------------------------------------------- +// Purpose: is FTS enabled for the supplied schema catalog? +//----------------------------------------------------------------------------- +bool CSchemaFull::GetFTSEnabled( enum ESchemaCatalog eCatalog ) +{ + int iEntry = m_mapFTSEnabled.Find( eCatalog ); + if ( iEntry == m_mapFTSEnabled.InvalidIndex() ) + return false; + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Adds a schema conversion instruction (for use in converting from +// a different SchemaFull to this one). +//----------------------------------------------------------------------------- +void CSchemaFull::AddDeleteTable( const char *pchTableName ) +{ + DeleteTable_t &deleteTable = m_VecDeleteTable[m_VecDeleteTable.AddToTail()]; + Q_strncpy( deleteTable.m_rgchTableName, pchTableName, sizeof( deleteTable.m_rgchTableName ) ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a schema conversion instruction (for use in converting from +// a different SchemaFull to this one). +//----------------------------------------------------------------------------- +void CSchemaFull::AddRenameTable( const char *pchTableNameOld, const char *pchTableNameNew ) +{ + RenameTable_t &renameTable = m_VecRenameTable[m_VecRenameTable.AddToTail()]; + Q_strncpy( renameTable.m_rgchTableNameOld, pchTableNameOld, sizeof( renameTable.m_rgchTableNameOld ) ); + renameTable.m_iTableDst = FindITable( pchTableNameNew ); + Assert( k_iTableNil != renameTable.m_iTableDst ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a schema conversion instruction (for use in converting from +// a different SchemaFull to this one). +//----------------------------------------------------------------------------- +void CSchemaFull::AddDeleteField( const char *pchTableName, const char *pchFieldName ) +{ + int iSchema = FindITable( pchTableName ); + AssertFatal( k_iTableNil != iSchema ); + + m_VecSchema[iSchema].AddDeleteField( pchFieldName ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a schema conversion instruction (for use in converting from +// a different SchemaFull to this one). +//----------------------------------------------------------------------------- +void CSchemaFull::AddRenameField( const char *pchTableName, const char *pchFieldNameOld, const char *pchFieldNameNew ) +{ + int iSchema = FindITable( pchTableName ); + AssertFatal( k_iTableNil != iSchema ); + + m_VecSchema[iSchema].AddRenameField( pchFieldNameOld, pchFieldNameNew ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a schema conversion instruction (for use in converting from +// a different SchemaFull to this one). +//----------------------------------------------------------------------------- +void CSchemaFull::AddAlterField( const char *pchTableName, const char *pchFieldNameOld, const char *pchFieldnameNew, PfnAlterField_t pfnAlterField ) +{ + int iSchema = FindITable( pchTableName ); + AssertFatal( k_iTableNil != iSchema ); + + m_VecSchema[iSchema].AddAlterField( pchFieldNameOld, pchFieldnameNew, pfnAlterField ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Add a trigger to the desired schema +//----------------------------------------------------------------------------- +void CSchemaFull::AddTrigger( ESchemaCatalog eCatalog, const char *pchTableName, const char *pchTriggerName, ETriggerType eTriggerType, const char *pchTriggerText ) +{ + CTriggerInfo trigger; + trigger.m_eTriggerType = eTriggerType; + trigger.m_eSchemaCatalog = eCatalog; + Q_strncpy( trigger.m_szTriggerName, pchTriggerName, Q_ARRAYSIZE( trigger.m_szTriggerName ) ); + Q_strncpy( trigger.m_szTriggerTableName, pchTableName, Q_ARRAYSIZE( trigger.m_szTriggerTableName ) ); + trigger.m_strText = pchTriggerText; + + // add it to our list + m_VecTriggers.AddToTail( trigger ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Figures out how to map a table from another SchemaFull into us. +// First we check our conversion instructions to see if any apply, +// and then we look for a straightforward match. +// Input: pchTableName - Name of the table we're trying to map +// piTableDst - [Return] Index of the table to map it to +// Output: true if we know what to do with this table (if false, the conversion +// is undefined and dangerous). +//----------------------------------------------------------------------------- +bool CSchemaFull::BCanConvertTable( const char *pchTableName, int *piTableDst ) +{ + // Should this table be deleted? + for ( int iDeleteTable = 0; iDeleteTable < m_VecDeleteTable.Count(); iDeleteTable++ ) + { + if ( 0 == Q_strcmp( pchTableName, m_VecDeleteTable[iDeleteTable].m_rgchTableName ) ) + { + *piTableDst = k_iTableNil; + return true; + } + } + + // Should this table be renamed? + for ( int iRenameTable = 0; iRenameTable < m_VecRenameTable.Count(); iRenameTable++ ) + { + if ( 0 == Q_strcmp( pchTableName, m_VecRenameTable[iRenameTable].m_rgchTableNameOld ) ) + { + *piTableDst = m_VecRenameTable[iRenameTable].m_iTableDst; + return true; + } + } + + // Find out which of our tables this table maps to (if it doesn't map + // to any of them, we don't know what to do with it). + *piTableDst = FindITable( pchTableName ); + return ( k_iTableNil != *piTableDst ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets the default SQL schema name for a catalog +//----------------------------------------------------------------------------- +const char *CSchemaFull::GetDefaultSchemaNameForCatalog( ESchemaCatalog eCatalog ) +{ + // For all catalogs it's actually the same + if ( m_strDefaultSchemaName.IsEmpty() ) + { + m_strDefaultSchemaName.Set( CFmtStr( "App%u", GGCBase()->GetAppID() ) ); + } + + return m_strDefaultSchemaName.Get(); +} + + +#ifdef DBGFLAG_VALIDATE +//----------------------------------------------------------------------------- +// Purpose: Run a global validation pass on all of our data structures and memory +// allocations. +// Input: validator - Our global validator object +// pchName - Our name (typically a member var in our container) +//----------------------------------------------------------------------------- +void CSchemaFull::Validate( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE(); + + ValidateObj( m_VecSchema ); + for ( int iSchema = 0; iSchema < m_VecSchema.Count(); iSchema++ ) + { + ValidateObj( m_VecSchema[iSchema] ); + } + + ValidateObj( m_VecDeleteTable ); + ValidateObj( m_VecRenameTable ); + + ValidateObj( m_mapFTSEnabled ); + + ValidateObj( m_vecFTSCatalogs ); + FOR_EACH_VEC( m_vecFTSCatalogs, i ) + { + ValidateObj( m_vecFTSCatalogs[i] ); + } + + ValidateObj( m_VecTriggers ); + FOR_EACH_VEC( m_VecTriggers, i ) + { + ValidateObj( m_VecTriggers[i] ); + } + + validator.ClaimMemory( m_pubScratchBuffer ); +} +#endif // DBGFLAG_VALIDATE + +} // namespace GCSDK diff --git a/gcsdk/sqlaccess/schemaupdate.cpp b/gcsdk/sqlaccess/schemaupdate.cpp new file mode 100644 index 0000000..b15b5f0 --- /dev/null +++ b/gcsdk/sqlaccess/schemaupdate.cpp @@ -0,0 +1,1696 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Contains the job that's responsible for updating the database schema +// +//============================================================================= + +#include "stdafx.h" +#include "gcsdk/sqlaccess/schemaupdate.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ +#ifndef SQL_SUCCESS +#define SQL_SUCCESS 0 +#define SQL_SUCCESS_WITH_INFO 1 +#define SQL_NO_DATA 100 +#define SQL_ERROR (-1) +#endif // SQLSUCCESS + +// this comes from sql.h. The GC really shouldn't depend on the MSSQL headers +#define SQL_INDEX_CLUSTERED 1 + +inline bool SQL_OK( SQLRETURN nRet ) +{ + return ( ( SQL_SUCCESS == nRet ) || (SQL_SUCCESS_WITH_INFO == nRet ) ); +} + +#define SQL_FAILED( ret ) ( !SQL_OK( ret ) ) + +#define EXIT_ON_SQL_FAILURE( ret ) \ +{ \ + if ( !SQL_OK( ret ) ) \ +{ \ + goto Exit; \ +} \ +} + +#define EXIT_ON_BOOLSQL_FAILURE( ret ) \ +{ \ + if ( !(ret) ) \ +{ \ + nRet = SQL_ERROR; \ + goto Exit; \ +} \ +} + +#define RETURN_SQL_ERROR_ON_FALSE( ret ) \ + if( !(ret) ) \ + {\ + return SQL_ERROR;\ + } + + +//----------------------------------------------------------------------------- +// Purpose: Emits a message and appends it to a string +//----------------------------------------------------------------------------- +void EmitAndAppend( CFmtStr1024 & sTarget, const CGCEmitGroup& Group, int iLevel, int iLevelLog, PRINTF_FORMAT_STRING const char *pchMessage, ... ) +{ + va_list args; + va_start( args, pchMessage ); + + if( sTarget.Length() < 1024 ) + sTarget.AppendFormatV( pchMessage, args ); + + EmitInfoV( Group, iLevel, iLevelLog, pchMessage, args ); + + va_end( args ); +} + +//----------------------------------------------------------------------------- +// builds a command of the form: +// CREATE [CLUSTERED] [UNIQUE] INDEX <index_name> ON <table_name> +// (col1, col2, ...) +// [INCLUDE (icol1, icol2, ...)] +// [WITH (FILLFACTOR = n)] +//----------------------------------------------------------------------------- +CUtlString GetAddIndexSQL( CRecordInfo *pRecordInfo, const FieldSet_t &refFields ) +{ + CFmtStrMax sCmd; + sCmd.sprintf( "CREATE %s%sINDEX %s ON App%u.%s (", + refFields.IsClustered() ? "CLUSTERED " : "", + refFields.IsUnique() ? "UNIQUE " : "", + refFields.GetIndexName(), + GGCBase()->GetAppID(), + pRecordInfo->GetName() ); + + // add real columns + for ( int n = 0; n < refFields.GetCount(); n++ ) + { + int nField = refFields.GetField( n ); + const CColumnInfo &refInfo = pRecordInfo->GetColumnInfo( nField ); + + sCmd.AppendFormat( "%s%s", + (n > 0) ? "," : "", + refInfo.GetName() ); + } + sCmd += ")"; + + // do we have any included columns? + if ( refFields.GetIncludedCount() > 0 ) + { + // yes, add those + sCmd += "\nINCLUDE ("; + + for ( int n = 0; n < refFields.GetIncludedCount(); n++ ) + { + int nField = refFields.GetIncludedField( n ); + const CColumnInfo &refInfo = pRecordInfo->GetColumnInfo( nField ); + + sCmd.AppendFormat( "%s%s", + (n > 0) ? "," : "", + refInfo.GetName() ); + } + sCmd += ")"; + } + + // do we need a fill factor? + if ( refFields.GetFillFactor() != 0) + { + sCmd.AppendFormat("\nWITH (FILLFACTOR = %d)", + refFields.GetFillFactor() ); + } + + return CUtlString( sCmd.String() ); +} + +CUtlString GetAlterColumnText( CRecordInfo *pRecordInfo, const CColumnInfo *pColumnInfoDesired ) +{ + Assert( pRecordInfo ); + Assert( pColumnInfoDesired ); + + char rgchTmp[128]; + CUtlString sCmd; + sCmd.Format( "ALTER TABLE App%u.%s ALTER COLUMN %s %s %s", GGCBase()->GetAppID(), pRecordInfo->GetName(), pColumnInfoDesired->GetName(), + SQLTypeFromField( *pColumnInfoDesired, rgchTmp, Q_ARRAYSIZE( rgchTmp ) ), + pColumnInfoDesired->BIsPrimaryKey() ? "NOT NULL" : "" + ); + return sCmd; +} + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CSchemaUpdate::CSchemaUpdate() +{ + m_mapPRecordInfoDesired.SetLessFunc( CaselessStringLessThan ); + m_eConversionMode = k_EConversionModeInspectOnly; + + m_bConversionNeeded = false; + m_cTablesDesiredMissing = 0; + m_cTablesActualDifferent = 0; + m_cTablesActualUnknown = 0; + m_cTablesNeedingChange = 0; + m_cColumnsDesiredMissing = 0; + m_cColumnsActualDifferent = 0; + m_cColumnsActualUnknown = 0; + m_bSkippedAChange = false; +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CSchemaUpdate::~CSchemaUpdate() +{ + // release all the record info's we're holding onto + FOR_EACH_MAP_FAST( m_mapPRecordInfoDesired, iRecordInfo ) + { + SAFE_RELEASE( m_mapPRecordInfoDesired[iRecordInfo] ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a record info that describes a desired table that should +// exist in the database +// Input: pRecordInfo - pointer to record info +//----------------------------------------------------------------------------- +void CSchemaUpdate::AddRecordInfoDesired( CRecordInfo *pRecordInfo ) +{ + Assert( pRecordInfo ); + const char *pchName = pRecordInfo->GetName(); + Assert( pchName && pchName[0] ); + Assert( m_mapPRecordInfoDesired.InvalidIndex() == m_mapPRecordInfoDesired.Find( pchName ) ); + // addref the record info since we're hanging onto it + pRecordInfo->AddRef(); + // insert it in our map, indexed by name + m_mapPRecordInfoDesired.Insert( pchName, pRecordInfo ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a CFTSCatalogInfo that tells us about a FTS catalog that +// should exist in the database +//----------------------------------------------------------------------------- +void CSchemaUpdate::AddFTSInfo( const CFTSCatalogInfo &refFTSInfo ) +{ + m_listFTSCatalogInfo.AddToTail( refFTSInfo ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a CFTSCatalogInfo that tells us about a FTS catalog that +// should exist in the database +//----------------------------------------------------------------------------- +void CSchemaUpdate::AddTriggerInfos( const CUtlVector< CTriggerInfo > &refTriggerInfo ) +{ + m_vecTriggerInfo = refTriggerInfo; +} + + +//----------------------------------------------------------------------------- +// Purpose: Validates and updates the database schema +//----------------------------------------------------------------------------- +bool CJobUpdateSchema::BYieldingRunJob( void * ) +{ + // update the main schema + EmitInfo( SPEW_GC, 2, 2, "Updating main schema...\n" ); + if ( !BYieldingUpdateSchema( k_ESchemaCatalogMain ) ) + { + m_pGC->SetStartupComplete( false ); + return false; + } + + // Could fail, but we shouldn't stop from starting up if it does + BYieldingUpdateSchema( k_ESchemaCatalogOGS ); + + bool bSuccess = m_pGC->BYieldingFinishStartup(); + m_pGC->SetStartupComplete( bSuccess ); + if ( bSuccess ) + { + bSuccess = m_pGC->BYieldingPostStartup(); + } + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +bool CJobUpdateSchema::BYieldingUpdateSchema( ESchemaCatalog eSchemaCatalog ) +{ + if( !YieldingBuildTypeMap( eSchemaCatalog ) ) + return false; + + // make an object to communicate desired database schema & results + CSchemaUpdate *pSchemaUpdate = new CSchemaUpdate(); + + // do safe conversions only + // TODO - do one round of inspection only first so we can send watchdog alert about what's about + // to happen, then do conversions. Also force conversions in dev system. + pSchemaUpdate->m_eConversionMode = k_EConversionModeConvertSafe; + + // Add all the tables to desired schema + for ( int iTable = 0; iTable < m_iTableCount; iTable++ ) + { + // MERGE COMMENT: "schema" cannot be used as a variable name because it is an empty #define resulting in error C2059 + CSchema &gc_schema = GSchemaFull().GetSchema( iTable ); + + // is it in the schema we want? + if ( gc_schema.GetESchemaCatalog() == eSchemaCatalog ) + pSchemaUpdate->AddRecordInfoDesired( gc_schema.GetRecordInfo() ); + } + + // add all the FTS catalogs to the desired schema + for ( int n = 0; n < GSchemaFull().GetCFTSCatalogs(); n++ ) + { + const CFTSCatalogInfo &refInfo = GSchemaFull().GetFTSCatalogInfo( n ); + pSchemaUpdate->AddFTSInfo( refInfo ); + } + + pSchemaUpdate->AddTriggerInfos( GSchemaFull().GetTriggerInfos() ); + + SQLRETURN nRet = YieldingEnsureDatabaseSchemaCorrect( eSchemaCatalog, pSchemaUpdate ); + if( !SQL_OK( nRet ) ) + { + AssertMsg( false, "SQL Schema Update failed" ); + return false; + } + + SAFE_RELEASE( pSchemaUpdate ); + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Examines actual running schema of database, compares to specified desired +// schema, and changes the actual schema to correspond to desired schema +// Input: pSQLThread - SQL thread to execute on +// pSchemaUpdate - pointer to object with desired schema +// Output: SQL return code. +//----------------------------------------------------------------------------- +SQLRETURN CJobUpdateSchema::YieldingEnsureDatabaseSchemaCorrect( ESchemaCatalog eSchemaCatalog, CSchemaUpdate *pSchemaUpdate ) +{ + Assert( pSchemaUpdate ); + + CMapPRecordInfo &mapPRecordInfoDesired = pSchemaUpdate->m_mapPRecordInfoDesired; + const CUtlVector< CTriggerInfo > &vecTriggerInfoDesired = pSchemaUpdate->m_vecTriggerInfo; + m_eConversionMode = pSchemaUpdate->m_eConversionMode; + + bool bDoConversion = true; +// bool bDoConversion = ( ( k_EConversionModeConvertSafe == eConversionMode ) || +// ( k_EConversionModeConvertIrreversible == eConversionMode ) ); + + CMapPRecordInfo mapPRecordInfoActual; + mapPRecordInfoActual.SetLessFunc( CaselessStringLessThan ); + CUtlVector<CRecordInfo *> vecPRecordInfoDesiredMissing; + CUtlVector<CRecordInfo *> vecPRecordInfoActualDifferent; + CUtlVector<CRecordInfo *> vecPRecordInfoActualUnknown; + CUtlVector< CTriggerInfo > vecTriggerInfoActual; + CUtlVector< CTriggerInfo > vecTriggerInfoMissing; + CUtlVector< CTriggerInfo > vecTriggerInfoDifferent; + + pSchemaUpdate->m_cTablesDesiredMissing = 0; + pSchemaUpdate->m_cTablesActualDifferent = 0; + pSchemaUpdate->m_cTablesActualUnknown = 0; + pSchemaUpdate->m_cTablesNeedingChange = 0; + pSchemaUpdate->m_cColumnsDesiredMissing = 0; + pSchemaUpdate->m_cColumnsActualDifferent = 0; + pSchemaUpdate->m_cColumnsActualUnknown = 0; + pSchemaUpdate->m_sDetail.Clear(); + + CFmtStr1024 &sDetail = pSchemaUpdate->m_sDetail; + + CFastTimer tickCounterOverall; + tickCounterOverall.Start(); + + // + // Do some up-front bookkeeping to see how many tables need to be created and/or altered + // + + int nSchemaID; + SQLRETURN nRet = YieldingGetSchemaID( eSchemaCatalog, &nSchemaID ); + EXIT_ON_SQL_FAILURE( nRet ); + + // Determine the actual running schema + nRet = YieldingGetRecordInfoForAllTables( eSchemaCatalog, nSchemaID, mapPRecordInfoActual ); + EXIT_ON_SQL_FAILURE( nRet ); + + nRet = YieldingGetTriggers( eSchemaCatalog, nSchemaID, vecTriggerInfoActual ); + EXIT_ON_SQL_FAILURE( nRet ); + + // Look through the list of desired tables, find any that are missing or different from the actual schema + FOR_EACH_MAP_FAST( mapPRecordInfoDesired, iRecordInfoDesired ) + { + // is this desired table in the currently connected catalog? + CRecordInfo *pRecordInfoDesired = mapPRecordInfoDesired[iRecordInfoDesired]; + if ( pRecordInfoDesired->GetESchemaCatalog() == eSchemaCatalog ) + { + // yes. do something about it + int iRecordInfoActual = mapPRecordInfoActual.Find( pRecordInfoDesired->GetName() ); + if ( mapPRecordInfoDesired.InvalidIndex() == iRecordInfoActual ) + { + // This table is desired but does not exist + vecPRecordInfoDesiredMissing.AddToTail( pRecordInfoDesired ); + } + else + { + // Table with same name exists in desired & actual schemas; is it exactly the same? + CRecordInfo *pRecordInfoActual = mapPRecordInfoActual[iRecordInfoActual]; + if ( !pRecordInfoDesired->EqualTo( pRecordInfoActual ) ) + { + // This desired table exists but the actual table is different than desired + vecPRecordInfoActualDifferent.AddToTail( pRecordInfoActual ); + } + } + } + } + + // Now, look through the list of actual tables and find any that do not exist in the list of desired tables + FOR_EACH_MAP_FAST( mapPRecordInfoActual, iRecordInfoActual ) + { + CRecordInfo *pRecordInfoActual = mapPRecordInfoActual[iRecordInfoActual]; + int iRecordInfoDesired = mapPRecordInfoDesired.Find( pRecordInfoActual->GetName() ); + if ( !mapPRecordInfoDesired.IsValidIndex( iRecordInfoDesired ) ) + { + // This table exists but is not in the list of desired tables + // maybe it's an old table. + + vecPRecordInfoActualUnknown.AddToTail( pRecordInfoActual ); + } + } + + // find a list of missing triggers + FOR_EACH_VEC( vecTriggerInfoDesired, iDesired ) + { + // not of this catalog? skip it + if ( vecTriggerInfoDesired[ iDesired ].m_eSchemaCatalog != eSchemaCatalog ) + continue; + + // it is our catalog, so try and match + bool bMatched = false; + FOR_EACH_VEC( vecTriggerInfoActual, iActual ) + { + // is it the same table and trigger name? + if ( vecTriggerInfoActual[ iActual ] == vecTriggerInfoDesired[ iDesired ] ) + { + // yes! test the text for differences + if ( vecTriggerInfoActual[ iActual ].IsDifferent( vecTriggerInfoDesired[ iDesired ] ) ) + { + vecTriggerInfoDifferent.AddToTail( vecTriggerInfoDesired[ iDesired ] ); + } + else + { + // we have a match! + vecTriggerInfoActual[ iActual ].m_bMatched = true; + bMatched = true; + } + break; + } + } + + if ( !bMatched ) + { + vecTriggerInfoMissing.AddToTail( vecTriggerInfoDesired[ iDesired ] ); + } + } + + // + // Now do the actual conversion + // + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "Database conversion: %s\n", + bDoConversion ? "beginning" : "inspection only" ); + + + // find tables which need to be created + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "# of specified tables that do not currently exist in database: %d\n", + vecPRecordInfoDesiredMissing.Count() ); + if ( vecPRecordInfoDesiredMissing.Count() > 0 ) + pSchemaUpdate->m_bConversionNeeded = true; + + // Create any tables which need to be created + for ( int iTable = 0; iTable < vecPRecordInfoDesiredMissing.Count(); iTable++ ) + { + CRecordInfo *pRecordInfoDesired = vecPRecordInfoDesiredMissing[iTable]; + if ( bDoConversion ) + { + CFastTimer tickCounter; + tickCounter.Start(); + + // Create the table + nRet = YieldingCreateTable( eSchemaCatalog, pRecordInfoDesired ); + EXIT_ON_SQL_FAILURE( nRet ); + tickCounter.End(); + int nElapsedMilliseconds = tickCounter.GetDuration().GetMilliseconds(); + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\tCreated table: %s: %d millisec\n", pRecordInfoDesired->GetName(), nElapsedMilliseconds ); + } + else + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t%s\n", pRecordInfoDesired->GetName() ); + } + + // find tables which are different + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "[GC]: # of specified tables that differ from schema in database: %d\n", + vecPRecordInfoActualDifferent.Count() ); + +#ifdef _DEBUG + // are some different? if so, list their names only for now. + // This is in _debug only because it's useful for debugging the below loop, + // but spewey for everyday life (as long as the below loop is working). + if ( vecPRecordInfoActualDifferent.Count() > 0 ) + { + FOR_EACH_VEC( vecPRecordInfoActualDifferent, i ) + { + CRecordInfo *pRecordInfoActual = vecPRecordInfoActualDifferent[i]; + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "[GC]: table %s is different\n", + pRecordInfoActual->GetName() ); + } + } +#endif + + pSchemaUpdate->m_bSkippedAChange = false; + + // Alter any table which needs to be altered + for ( int iTable = 0; iTable < vecPRecordInfoActualDifferent.Count(); iTable++ ) + { + CRecordInfo *pRecordInfoActual = vecPRecordInfoActualDifferent[iTable]; + int iRecordInfoDesired = mapPRecordInfoDesired.Find( pRecordInfoActual->GetName() ); + Assert( mapPRecordInfoDesired.InvalidIndex() != iRecordInfoDesired ); + CRecordInfo *pRecordInfoDesired = mapPRecordInfoDesired[iRecordInfoDesired]; + + CUtlVector<const CColumnInfo *> vecPColumnInfoDesiredMissing; + CUtlVector<const CColumnInfo *> vecPColumnInfoActualDifferent; + CUtlVector<const CColumnInfo *> vecPColumnInfoActualUnknown; + + // We know something is different between the actual & desired schema for this table, but don't yet know what + + // Compare each column + for ( int iColumnDesired = 0; iColumnDesired < pRecordInfoDesired->GetNumColumns(); iColumnDesired ++ ) + { + const CColumnInfo &columnInfoDesired = pRecordInfoDesired->GetColumnInfo( iColumnDesired ); + int iColumnActual = -1; + bool bRet = pRecordInfoActual->BFindColumnByName( columnInfoDesired.GetName(), &iColumnActual ); + if ( bRet ) + { + const CColumnInfo &columnInfoActual = pRecordInfoActual->GetColumnInfo( iColumnActual ); + if ( columnInfoActual.GetChecksum() != columnInfoDesired.GetChecksum() ) + { + // The actual column is different than the desired column + vecPColumnInfoActualDifferent.AddToTail( &columnInfoActual ); + } + } + else + { + // The desired column is missing from the actual table + vecPColumnInfoDesiredMissing.AddToTail( &columnInfoDesired ); + } + } + for ( int iColumnActual = 0; iColumnActual < pRecordInfoActual->GetNumColumns(); iColumnActual ++ ) + { + const CColumnInfo &columnInfoActual = pRecordInfoActual->GetColumnInfo( iColumnActual ); + int iColumnDesired = -1; + bool bRet = pRecordInfoDesired->BFindColumnByName( columnInfoActual.GetName(), &iColumnDesired ); + if ( !bRet ) + { + // this column exists in the running schema, but not in the desired schema (e.g. old column) + vecPColumnInfoActualUnknown.AddToTail( &columnInfoActual ); + } + } + + if ( ( vecPColumnInfoDesiredMissing.Count() > 0 ) || ( vecPColumnInfoActualDifferent.Count() > 0 ) ) + { + pSchemaUpdate->m_bConversionNeeded = true; + pSchemaUpdate->m_cTablesNeedingChange++; + } + + // Add any desired columns which are missing from the actual schema + if ( vecPColumnInfoDesiredMissing.Count() > 0 ) + { + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t\tDesired columns missing in table %s:\n", pRecordInfoActual->GetName() ); + + for ( int iColumn = 0; iColumn < vecPColumnInfoDesiredMissing.Count(); iColumn++ ) + { + const CColumnInfo *pColumnInfoDesired = vecPColumnInfoDesiredMissing[iColumn]; + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t\t\t%s\n", pColumnInfoDesired->GetName() ); + if ( bDoConversion ) + { + CFastTimer tickCounter; + tickCounter.Start(); + // Add the column + nRet = YieldingAlterTableAddColumn( eSchemaCatalog, pRecordInfoActual, pColumnInfoDesired ); + EXIT_ON_SQL_FAILURE( nRet ); + tickCounter.End(); + int nElapsedMilliseconds = tickCounter.GetDuration().GetMilliseconds(); + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t\t\t\tCreated column %s: %d millisec\n", pColumnInfoDesired->GetName(), nElapsedMilliseconds ); + } + } + } + + // Check for any stray indices that aren't found in the specification? + bool bIndexMismatch = false; + for ( int idx = 0 ; idx < pRecordInfoActual->GetIndexFieldCount() ; ++idx ) + { + const FieldSet_t &fs = pRecordInfoActual->GetIndexFields()[idx]; + if ( pRecordInfoDesired->FindIndex( pRecordInfoActual, fs ) >= 0 ) + continue; + if ( pRecordInfoDesired->FindIndexByName( fs.GetIndexName() ) >= 0 ) + continue; // we already handled this above + bIndexMismatch = true; + + if ( idx == pRecordInfoActual->GetPKIndex() ) + { + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\tTable %s primary key in database differs from specification.\n", + pRecordInfoDesired->GetName() ); + nRet = YieldingProcessUnsafeConversion( eSchemaCatalog, + CFmtStr("ALTER TABLE App%u.%s DROP CONSTRAINT %s", GGCBase()->GetAppID(), pRecordInfoActual->GetName(), fs.GetIndexName() ).String(), + CFmtStr("%s: remove old primary key.", pRecordInfoDesired->GetName() ).String() ); + EXIT_ON_SQL_FAILURE( nRet ); + } + else + { + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\tTable %s index %s exists in database but is not in specification. (Possible performance problem?).\n", + pRecordInfoDesired->GetName(), fs.GetIndexName() ); + nRet = YieldingProcessUnsafeConversion( eSchemaCatalog, + CFmtStr("DROP INDEX %s ON App%u.%s", fs.GetIndexName(), GGCBase()->GetAppID(), pRecordInfoActual->GetName() ).String(), + CFmtStr("%s: cleanup stray index %s.", pRecordInfoDesired->GetName(), fs.GetIndexName() ).String() ); + EXIT_ON_SQL_FAILURE( nRet ); + } + } + + // Change any columns which are different between desired and actual schema + if ( vecPColumnInfoActualDifferent.Count() > 0 ) + { + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\tColumns that differ between specification and database in table %s:\n", + pRecordInfoActual->GetName() ); + for ( int iColumn = 0; iColumn < vecPColumnInfoActualDifferent.Count(); iColumn++ ) + { + const CColumnInfo *pColumnInfoActual = vecPColumnInfoActualDifferent[iColumn]; + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t\t%s", pColumnInfoActual->GetName() ); + int iColumnDesired = -1; + DbgVerify( pRecordInfoDesired->BFindColumnByName( pColumnInfoActual->GetName(), &iColumnDesired ) ); + const CColumnInfo *pColumnInfoDesired = &pRecordInfoDesired->GetColumnInfo( iColumnDesired ); + + // if type or size changed, alter the column + if ( ( pColumnInfoDesired->GetType() != pColumnInfoActual->GetType() ) || + // fixed length field, and the sizes differ + ( ! pColumnInfoDesired->BIsVariableLength() && + ( pColumnInfoDesired->GetFixedSize() != pColumnInfoActual->GetFixedSize() ) ) || + // variable length field, and the sizes differ + // fixed length field, and the sizes differ + ( pColumnInfoDesired->BIsVariableLength() && + ( pColumnInfoDesired->GetMaxSize() != pColumnInfoActual->GetMaxSize() ) ) ) + { + if ( k_EConversionModeConvertIrreversible != m_eConversionMode ) + { + pSchemaUpdate->m_bSkippedAChange = true; + if ( pColumnInfoDesired->GetType() != pColumnInfoActual->GetType() ) + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t(data types differ: desired=%s, actual=%s)", PchNameFromEGCSQLType( pColumnInfoDesired->GetType() ), PchNameFromEGCSQLType( pColumnInfoActual->GetType() ) ); + if ( pColumnInfoDesired->GetFixedSize() != pColumnInfoActual->GetFixedSize() ) + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t(column sizes differ: desired=%d, actual=%d)", pColumnInfoDesired->GetFixedSize(), pColumnInfoActual->GetFixedSize() ); + if ( pColumnInfoDesired->GetMaxSize() != pColumnInfoActual->GetMaxSize() ) + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t(maximum column sizes differ: desired=%d, actual=%d)", pColumnInfoDesired->GetMaxSize(), pColumnInfoActual->GetMaxSize() ); + } + nRet = YieldingChangeColumnTypeOrLength( eSchemaCatalog, pRecordInfoActual, pColumnInfoDesired ); + EXIT_ON_SQL_FAILURE( nRet ); + } + + // If column constraints/indexes are different, make appropriate adjustments + // do this second so it has a chance to succeed - otherwise, we may create an index here that + // prevents us from performing an alter table + if ( pColumnInfoDesired->GetColFlags() != pColumnInfoActual->GetColFlags() ) + { + if ( k_EConversionModeConvertIrreversible != m_eConversionMode ) + { + char szDesiredFlags[k_nKiloByte]; + char szActualFlags[k_nKiloByte]; + pColumnInfoDesired->GetColFlagDescription( szDesiredFlags, Q_ARRAYSIZE( szDesiredFlags ) ); + pColumnInfoActual->GetColFlagDescription( szActualFlags, Q_ARRAYSIZE( szActualFlags ) ); + + pSchemaUpdate->m_bSkippedAChange = true; + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t(column flags differ: desired=\"%s\", actual=\"%s\")", + szDesiredFlags, szActualFlags ); + } + nRet = YieldingChangeColumnProperties( eSchemaCatalog, pRecordInfoActual, pColumnInfoActual, pColumnInfoDesired ); + EXIT_ON_SQL_FAILURE( nRet ); + } + + if ( pSchemaUpdate->m_bSkippedAChange ) + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t(Not attempting unsafe change).\n" ); + } + } + + // Scan for any new / changed indices + for ( int idx = 0 ; idx < pRecordInfoDesired->GetIndexFieldCount() ; ++idx ) + { + const FieldSet_t &fs = pRecordInfoDesired->GetIndexFields()[idx]; + int iActualIdx = pRecordInfoActual->FindIndex( pRecordInfoDesired, fs ); + if ( iActualIdx >= 0 ) + continue; + bIndexMismatch = true; + + // The exact index we want doesn't exist. Is it the primary key? + CUtlString sCommand; + CUtlString sComment; + if ( idx == pRecordInfoDesired->GetPKIndex() ) + { + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\tTable %s primary key in specification differs from database.\n", + pRecordInfoDesired->GetName() ); + sComment.Format( "%s: create new primary key constraint", pRecordInfoDesired->GetName() ); + TSQLCmdStr cmd; + BuildTablePKConstraintText( &cmd, pRecordInfoDesired ); + for ( int i = 0 ; i < fs.GetCount() ; ++i ) // make sure they are non-NULL + { + int idxField = fs.GetField(i); + const CColumnInfo *pColInfo = &pRecordInfoDesired->GetColumnInfo( idxField ); + Assert( pColInfo->BIsPrimaryKey() ); + sCommand += GetAlterColumnText( pRecordInfoDesired, pColInfo ); + sCommand += ";\n"; + } + + CUtlString sCreatePK; + sCreatePK.Format( "GO\nALTER TABLE App%u.%s ADD %s\n", GGCBase()->GetAppID(), pRecordInfoDesired->GetName(), cmd.String() ); + sCommand += sCreatePK; + } + else + { + + // Another common thing that could happen is that an index is changed. + // Look for an existing index with the same name as a way to try to + // detect this common case and provide a more specific message. (Otherwise, + // we will report it as a missing index, and an extra unwanted index --- which + // is correct but a bit more confusing.) + + iActualIdx = pRecordInfoActual->FindIndexByName( fs.GetIndexName() ); + if ( iActualIdx < 0 ) + { + sComment.Format("%s: add index %s", pRecordInfoDesired->GetName(), fs.GetIndexName() ); + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\tTable %s index %s is specified but not present in database.\n", + pRecordInfoDesired->GetName(), fs.GetIndexName() ); + } + else + { + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\tTable %s index %s differs between specification and database.\n", + pRecordInfoDesired->GetName(), fs.GetIndexName() ); + sComment.Format( "%s: fix index %s", pRecordInfoDesired->GetName(), fs.GetIndexName() ); + sCommand.Format( "DROP INDEX %s ON App%u.%s;\n", fs.GetIndexName(), GGCBase()->GetAppID(), pRecordInfoDesired->GetName() ); + } + sCommand += GetAddIndexSQL( pRecordInfoDesired, fs ); + } + nRet = YieldingProcessUnsafeConversion( eSchemaCatalog, sCommand, sComment ); + EXIT_ON_SQL_FAILURE( nRet ); + } + + // Just to be safe, let's run the old code, too. + if ( !bIndexMismatch && !pRecordInfoActual->CompareIndexLists( pRecordInfoDesired ) ) + { + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\tIndex sets in table %s differ between specification and database [GC]:\n", + pRecordInfoDesired->GetName() ); + CFmtStr1024 sTemp; + + pRecordInfoDesired->GetIndexFieldList( &sTemp, 3 ); + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t\tDesired %s", sTemp.Access() ); + + pRecordInfoActual->GetIndexFieldList( &sTemp, 3 ); + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t\tActual %s", sTemp.Access() ); + } + + // what about foreign key constraints? + if ( ! pRecordInfoActual->CompareFKs( pRecordInfoDesired ) ) + { + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\tForeign Key constraints in table %s differs between specification and database [GC]:\n", + pRecordInfoDesired->GetName() ); + + CFmtStr1024 sTemp; + pRecordInfoDesired->GetFKListString( &sTemp, 3 ); + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t\tDesired %s", sTemp.Access() ); + + pRecordInfoActual->GetFKListString( &sTemp, 3 ); + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t\tActual %s", sTemp.Access() ); + } + + + + if ( vecPColumnInfoActualUnknown.Count() > 0 ) + { + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\tColumns in database [GC] table %s that are not in specification (ignored):\n", + pRecordInfoActual->GetName() ); + + // Since this can actually destroy data, let's not ever, ever run it automatically. + CUtlString sCommand; + sCommand.Format( "ALTER TABLE App%u.%s DROP COLUMN ", GGCBase()->GetAppID(), pRecordInfoActual->GetName() ); + + for ( int iColumn = 0; iColumn < vecPColumnInfoActualUnknown.Count(); iColumn++ ) + { + const CColumnInfo *pColumnInfo = vecPColumnInfoActualUnknown[iColumn]; + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t\t%s\n", pColumnInfo->GetName() ); + if ( iColumn != 0 ) + sCommand += ", "; + sCommand += pColumnInfo->GetName(); + } + AddDataDestroyingConversion( sCommand, + CFmtStr( "-- Drop extra column(s) in %s\n", pRecordInfoActual->GetName() ) ); + } + + pSchemaUpdate->m_cColumnsDesiredMissing += vecPColumnInfoDesiredMissing.Count(); + pSchemaUpdate->m_cColumnsActualDifferent += vecPColumnInfoActualDifferent.Count(); + pSchemaUpdate->m_cColumnsActualUnknown += vecPColumnInfoActualUnknown.Count(); + } + + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "[GC]: # of tables that currently exist in database but were unspecified: %d\n", + vecPRecordInfoActualUnknown.Count() ); + for ( int iTable = 0; iTable < vecPRecordInfoActualUnknown.Count(); iTable++ ) + { + CRecordInfo *pRecordInfo = vecPRecordInfoActualUnknown[iTable]; + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t%s\n", pRecordInfo->GetName() ); + AddDataDestroyingConversion( + CFmtStr( "DROP TABLE App%u.%s\n", GGCBase()->GetAppID(), pRecordInfo->GetName() ), + CFmtStr( "-- Drop extra table %s\n", pRecordInfo->GetName() ) ); + } + + // then, the triggers + if ( vecTriggerInfoMissing.Count() > 0 ) + { + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "[GC]: # of specified triggers that do not currently exist: %d\n", + vecTriggerInfoMissing.Count() ); + + FOR_EACH_VEC( vecTriggerInfoMissing, iMissing ) + { + CFastTimer tickCounter; + tickCounter.Start(); + + // Create the trigger + nRet = YieldingCreateTrigger( eSchemaCatalog, vecTriggerInfoMissing[ iMissing] ); + EXIT_ON_SQL_FAILURE( nRet ); + tickCounter.End(); + int nElapsedMilliseconds = tickCounter.GetDuration().GetMilliseconds(); + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "[GC]: Created trigger %s on table %s: %d millisec\n", + vecTriggerInfoMissing[ iMissing ].m_szTriggerName, + vecTriggerInfoMissing[ iMissing ].m_szTriggerTableName, nElapsedMilliseconds ); + } + } + + // different triggers + FOR_EACH_VEC( vecTriggerInfoDifferent, iDifferent ) + { + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "[GC]: Trigger %s on table %s differs from the desired trigger\n", vecTriggerInfoMissing[ iDifferent ].m_szTriggerName, + vecTriggerInfoMissing[ iDifferent ].m_szTriggerTableName); + + // a different trigger text is a forced failure. + nRet = SQL_ERROR; + } + + // extra triggers + FOR_EACH_VEC( vecTriggerInfoActual, iActual ) + { + // if it was never matched, it isn't in the schema anywhere + if ( ! vecTriggerInfoActual[ iActual ].m_bMatched ) + { + SQLRETURN nSQLReturn = YieldingDropTrigger( eSchemaCatalog, vecTriggerInfoActual[ iActual ] ); + + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "[GC]: Trigger %s on table %s is not in the declared schema ... Drop %s", + vecTriggerInfoActual[ iActual ].m_szTriggerName, + vecTriggerInfoActual[ iActual ].m_szTriggerTableName, + SQL_OK( nSQLReturn ) ? "OK" : "FAILED!" ) ; + + if ( !SQL_OK ( nSQLReturn ) ) + { + // it broke; latch the failure + nRet = nSQLReturn; + } + } + } + + tickCounterOverall.End(); + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "[GC]: Total time: %d milliseconds\n", + tickCounterOverall.GetDuration().GetMilliseconds() ); + +Exit: + + // Spew suggested SQL to clean up stuff + if ( !m_sRecommendedSQL.IsEmpty() || !m_sDataDestroyingCleanupSQL.IsEmpty() ) + { + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\tThe following SQL might work to fixup schema differences.\n" ); + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t** \n" ); + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t** DISCLAIMER ** This conversion code is not well tested. Review the SQL and use at your own risk.\n" ); + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t** \n" ); + { + CUtlVectorAutoPurge<char*> vecLines; + V_SplitString( m_sRecommendedSQL, "\n", vecLines ); + FOR_EACH_VEC( vecLines, i ) + { + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t\t%s\n", vecLines[i] ); + } + m_sRecommendedSQL.Clear(); + } + if ( !m_sDataDestroyingCleanupSQL.IsEmpty() ) + { + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t\t-- WARNING: The following operations will *destroy* data that is in the database but not in the specification.\n" ); + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t\t-- If you have manually created extra tables or columns in your database, it will appear here!\n" ); + CUtlVectorAutoPurge<char*> vecLines; + V_SplitString( m_sDataDestroyingCleanupSQL, "\n", vecLines ); + FOR_EACH_VEC( vecLines, i ) + { + EmitAndAppend( sDetail, SPEW_GC, 2, 2, "\t\t%s\n", vecLines[i] ); + } + m_sDataDestroyingCleanupSQL.Clear(); + } + } + + pSchemaUpdate->m_cTablesDesiredMissing = vecPRecordInfoDesiredMissing.Count(); + pSchemaUpdate->m_cTablesActualDifferent = vecPRecordInfoActualDifferent.Count(); + pSchemaUpdate->m_cTablesActualUnknown = vecPRecordInfoActualUnknown.Count(); + + FOR_EACH_MAP_FAST( mapPRecordInfoActual, iRecordInfoActual ) + SAFE_RELEASE( mapPRecordInfoActual[iRecordInfoActual] ); + + return nRet; +} + + +//----------------------------------------------------------------------------- +// Purpose: Builds a record info for each table in database that describes the +// columns in that table +// Input: pSQLConnection - SQL connection to execute on +// mapPRecordInfo - map to add the table record infos to +// Output: SQL return code. +//----------------------------------------------------------------------------- +SQLRETURN CJobUpdateSchema::YieldingGetSchemaID( ESchemaCatalog eSchemaCatalog, int *pSchemaID ) +{ + CSQLAccess sqlAccess( eSchemaCatalog ); + CFmtStr1024 sDefaultSchema; + if( !sqlAccess.BYieldingExecuteString( FILE_AND_LINE, "SELECT default_schema_name FROM sys.database_principals WHERE name=CURRENT_USER", + &sDefaultSchema ) ) + return SQL_ERROR; + + CFmtStr sExpectedDefaultSchema( "App%u", GGCBase()->GetAppID() ); + if ( 0 != Q_stricmp( sDefaultSchema, sExpectedDefaultSchema ) ) + { + AssertMsg2( false, "SQL connection has the wrong default schema. Expected: %s. Actual %s", sExpectedDefaultSchema.Get(), sDefaultSchema.Get() ); + return SQL_ERROR; + } + + sqlAccess.AddBindParam( sDefaultSchema ); + if( !sqlAccess.BYieldingExecuteScalarInt( FILE_AND_LINE, "SELECT SCHEMA_ID(?)", pSchemaID ) ) + return SQL_ERROR; + + return SQL_SUCCESS; +} + + +//----------------------------------------------------------------------------- +// Purpose: Builds a record info for each table in database that describes the +// columns in that table +// Input: pSQLConnection - SQL connection to execute on +// mapPRecordInfo - map to add the table record infos to +// Output: SQL return code. +//----------------------------------------------------------------------------- +SQLRETURN CJobUpdateSchema::YieldingGetRecordInfoForAllTables( ESchemaCatalog eSchemaCatalog, int nSchemaID, CMapPRecordInfo &mapPRecordInfo ) +{ + CSQLAccess sqlAccess( eSchemaCatalog ); + + // create a query that returns all tables in the database + sqlAccess.AddBindParam( nSchemaID ); + RETURN_SQL_ERROR_ON_FALSE( sqlAccess.BYieldingExecute( FILE_AND_LINE, "SELECT object_id, name FROM sys.objects WHERE type_desc = 'USER_TABLE' AND is_ms_shipped = 0 AND schema_id = ?" ) ); + + FOR_EACH_SQL_RESULT( sqlAccess, 0, tableIDRecord ) + { + int nTableID; + RETURN_SQL_ERROR_ON_FALSE( tableIDRecord.BGetIntValue( 0, &nTableID ) ); + + CFmtStr1024 sTableName; + RETURN_SQL_ERROR_ON_FALSE( tableIDRecord.BGetStringValue( 1, &sTableName) ); + + YieldingGetColumnInfoForTable( eSchemaCatalog, mapPRecordInfo, nTableID, sTableName ); + } + + + // now, get the column indexes and constraints for each table + FOR_EACH_MAP_FAST( mapPRecordInfo, iRecordInfo ) + { + CRecordInfo *pRecordInfo = mapPRecordInfo[iRecordInfo]; + pRecordInfo->SetAllColumnsAdded(); + + // determine indexes and constraints + YieldingGetColumnIndexes( eSchemaCatalog, pRecordInfo ); + + // determine FK constraints + YieldingGetTableFKConstraints( eSchemaCatalog, pRecordInfo ); + + // do final calculations then ready to use + pRecordInfo->PrepareForUse(); + } + + return SQL_SUCCESS; +} + +SQLRETURN CJobUpdateSchema::YieldingGetColumnInfoForTable( ESchemaCatalog eSchemaCatalog, CMapPRecordInfo &mapPRecordInfo, int nTableID, const char *pchTableName ) +{ + CSQLAccess sqlAccess( eSchemaCatalog ); + + sqlAccess.AddBindParam( nTableID ); + RETURN_SQL_ERROR_ON_FALSE( sqlAccess.BYieldingExecute( FILE_AND_LINE, "SELECT name, column_id, user_type_id, max_length, is_identity FROM sys.columns WHERE object_id = ?") ); + + CRecordInfo *pRecordInfo = CRecordInfo::Alloc(); + pRecordInfo->SetName( pchTableName ); + pRecordInfo->SetTableID( nTableID ); + FOR_EACH_SQL_RESULT( sqlAccess, 0, columnInfo ) + { + CFmtStr1024 sColumnName; + int nType; + int nColumnID; + int nMaxLength; + bool bIdentity; + RETURN_SQL_ERROR_ON_FALSE( columnInfo.BGetStringValue( 0, &sColumnName) ); + RETURN_SQL_ERROR_ON_FALSE( columnInfo.BGetIntValue( 1, &nColumnID ) ); + RETURN_SQL_ERROR_ON_FALSE( columnInfo.BGetIntValue( 2, &nType ) ); + RETURN_SQL_ERROR_ON_FALSE( columnInfo.BGetIntValue( 3, &nMaxLength ) ); + RETURN_SQL_ERROR_ON_FALSE( columnInfo.BGetBoolValue( 4, &bIdentity) ); + + int nColFlags = 0; + if( bIdentity ) + nColFlags |= k_nColFlagAutoIncrement; + + pRecordInfo->AddColumn( sColumnName, nColumnID, GetEGCSQLTypeForMSSQLType( nType ), nMaxLength, nColFlags, nMaxLength ); + } + mapPRecordInfo.Insert( pRecordInfo->GetName(), pRecordInfo ); + + return SQL_SUCCESS; +} + + +//----------------------------------------------------------------------------- +// purpose: get a list of triggers from the database. +//----------------------------------------------------------------------------- +SQLRETURN CJobUpdateSchema::YieldingGetTriggers( ESchemaCatalog eSchemaCatalog, int nSchemaID, CUtlVector< CTriggerInfo > &vecTriggerInfo ) +{ + CSQLAccess sqlAccess( eSchemaCatalog ); + + // get some description and the text of the triggers on this database. + // Find the name of the trigger, the name of the table it servers, the type of + // trigger, and its text. Doesn't bring back any disabled or MS-shipped triggers, + // and gets only DML triggers and not DDL triggers. + const char *pchStatement = "SELECT ST.name AS TriggerName, SOT.name AS TableName, ST.is_instead_of_trigger, SC.Text, SC.ColID" + " FROM sys.triggers AS ST" + " JOIN sys.syscomments AS SC ON SC.id = ST.object_id" + " JOIN sys.objects AS SO ON SO.object_id = ST.object_id" + " JOIN sys.objects AS SOT on SOT.object_id = ST.parent_id" + " WHERE ST.type_desc = 'SQL_TRIGGER'" + " AND ST.is_ms_shipped = 0 AND ST.is_disabled = 0" + " AND ST.parent_class = 1" + " AND SO.schema_id = ?" + " ORDER BY TableName, TriggerName, SC.ColID"; + sqlAccess.AddBindParam( nSchemaID ); + RETURN_SQL_ERROR_ON_FALSE( sqlAccess.BYieldingExecute( FILE_AND_LINE, pchStatement ) ); + + // should be one results set + Assert( 1 == sqlAccess.GetResultSetCount() ); + + FOR_EACH_SQL_RESULT( sqlAccess, 0, sqlRecord ) + { + // get the text of the procedure + const char *pchText; + DbgVerify( sqlRecord.BGetStringValue( 3, &pchText ) ); + + // is this instead of? + bool bIsInsteadOf; + DbgVerify( sqlRecord.BGetBoolValue( 2, &bIsInsteadOf ) ); + + // get the table name + const char *pchTableName; + DbgVerify( sqlRecord.BGetStringValue( 1, &pchTableName ) ); + + // and the trigger name + const char *pchTriggerName; + DbgVerify( sqlRecord.BGetStringValue( 0, &pchTriggerName ) ); + + // finally, grab the collation id + int16 nColID; + DbgVerify( sqlRecord.BGetInt16Value( 4, &nColID ) ); + + if ( nColID == 1 ) + { + // the collation ID is one, so we know this is a new one + CTriggerInfo info; + info.m_strText = pchText; + Q_strncpy( info.m_szTriggerName, pchTriggerName, Q_ARRAYSIZE( info.m_szTriggerName ) ); + Q_strncpy( info.m_szTriggerTableName, pchTableName, Q_ARRAYSIZE( info.m_szTriggerTableName ) ); + info.m_eSchemaCatalog = eSchemaCatalog; + vecTriggerInfo.AddToTail( info ); + } + else + { + // the collation ID is not one, so we're concatenating. + Assert( vecTriggerInfo.Count() - 1 >= 0 ); + + // the name could not have changed + Assert( 0 == Q_strcmp( vecTriggerInfo[vecTriggerInfo.Count() - 1].m_szTriggerName, pchTriggerName ) ); + } + } + + return SQL_SUCCESS; +} + + +//----------------------------------------------------------------------------- +// Purpose: retrieves the index information for the specified table, then adds this +// information to the record info. This is a SQL Server-specific implementation +// which gets data describing index features not available through plain ODBC +// queries. +// Input: pSQLConnection - SQL connection to execute on +// pRecordInfo - CRecordInfo to add the index info into +// Output: SQL return code. +//----------------------------------------------------------------------------- +SQLRETURN CJobUpdateSchema::YieldingGetColumnIndexes( ESchemaCatalog eSchemaCatalog, CRecordInfo *pRecordInfo ) +{ + // query the system management views for all of the indexes on this table + static const char *pstrStatement = + "SELECT SI.Name AS INDEX_NAME, SC.Name AS COLUMN_NAME, SI.Type AS [TYPE], IS_INCLUDED_COLUMN, SIC.KEY_Ordinal AS [ORDINAL_POSITION], IS_UNIQUE, IS_PRIMARY_KEY, SI.INDEX_ID" + " FROM sys.indexes SI " + " JOIN sys.index_columns SIC" + " ON SIC.Object_id = SI.Object_Id" + " AND SIC.Index_ID = SI.Index_id" + " JOIN sys.objects SO" + " ON SIC.Object_ID = SO.Object_ID" + " JOIN sys.columns SC" + " ON SC.Object_ID = SO.Object_ID" + " AND SC.column_id = SIC.column_id" + " WHERE SO.Object_ID = ? " + "ORDER BY SIC.Index_id, SIC.Key_Ordinal "; + CSQLAccess sqlAccess( eSchemaCatalog ); + sqlAccess.AddBindParam( pRecordInfo->GetTableID() ); + RETURN_SQL_ERROR_ON_FALSE( sqlAccess.BYieldingExecute( FILE_AND_LINE, pstrStatement ) ); + + // builds a list of columns in a particular index. + CUtlVector<int> vecColumns; + CUtlVector<int> vecIncluded; + int nLastIndexID = -1; + bool bIsClustered = false; + bool bIsIndexUnique = false; + bool bIsPrimaryKey = false; + CFmtStr1024 sIndexName, sColumnName; + + FOR_EACH_SQL_RESULT( sqlAccess, 0, typeRecord ) + { + // Starting a new index? + int nIndexID; + DbgVerify( typeRecord.BGetIntValue( 7, &nIndexID ) ); + if ( nIndexID != nLastIndexID ) + { + // first column! is it our first time through? + if ( vecColumns.Count() > 0 ) + { + // yes. let's add what we had from the previous index + // to the fieldset + FieldSet_t fs( bIsIndexUnique, bIsClustered, vecColumns, sIndexName ); + fs.AddIncludedColumns( vecIncluded ); + int idx = pRecordInfo->AddIndex( fs ); + if ( bIsPrimaryKey ) + { + Assert( bIsIndexUnique ); + Assert( pRecordInfo->GetPKIndex() < 0 ); + pRecordInfo->SetPKIndex( idx ); + } + + // reset the vector + vecColumns.RemoveAll(); + vecIncluded.RemoveAll(); + bIsIndexUnique = false; + bIsClustered = false; + bIsPrimaryKey = false; + } + nLastIndexID = nIndexID; + } + + int nTypeID, nOrdinalPosition; + bool bIsIncluded, bIsColumnUnique; + + DbgVerify( typeRecord.BGetStringValue( 0, &sIndexName ) ); + DbgVerify( typeRecord.BGetStringValue( 1, &sColumnName ) ); + DbgVerify( typeRecord.BGetIntValue( 2, &nTypeID ) ); + DbgVerify( typeRecord.BGetBoolValue( 3, &bIsIncluded ) ); + DbgVerify( typeRecord.BGetIntValue( 4, &nOrdinalPosition ) ); + DbgVerify( typeRecord.BGetBoolValue( 5, &bIsColumnUnique ) ); + DbgVerify( typeRecord.BGetBoolValue( 6, &bIsPrimaryKey ) ); + + RETURN_SQL_ERROR_ON_FALSE( sColumnName.Length() > 0 ); + + int nColumnIndexed = -1; + RETURN_SQL_ERROR_ON_FALSE( pRecordInfo->BFindColumnByName( sColumnName, &nColumnIndexed ) ); + + CColumnInfo & columnInfo = pRecordInfo->GetColumnInfo( nColumnIndexed ); + int nColFlags = 0; + + if ( bIsIncluded ) + { + Assert( nOrdinalPosition == 0 ); + // it's included; no flags + vecIncluded.AddToTail( nColumnIndexed ); + } + else + { + Assert( nOrdinalPosition != 0 ); + if ( bIsPrimaryKey ) + { + // if we're working a primary key, mark those flags + nColFlags = k_nColFlagPrimaryKey | k_nColFlagIndexed | k_nColFlagUnique; + // PKs are always unique + bIsIndexUnique = true; + } + else + { + // if we're working a "regular" index, we need to know the uniqueness of the index ... + + nColFlags = k_nColFlagIndexed; + if ( bIsColumnUnique ) + { + nColFlags |= k_nColFlagUnique; + bIsIndexUnique = true; + } + } + + // clustering type + if ( nTypeID == SQL_INDEX_CLUSTERED ) + { + nColFlags |= k_nColFlagClustered; + bIsClustered = true; + } + columnInfo.SetColFlagBits( nColFlags ); + + // add this column to our list for the index set + vecColumns.AddToTail( nColumnIndexed ); + } + } + + // anything left over? + if ( vecColumns.Count() > 0 ) + { + // yep, add that, too + FieldSet_t fs( bIsIndexUnique, bIsClustered, vecColumns, sIndexName ); + fs.AddIncludedColumns( vecIncluded ); + int idx = pRecordInfo->AddIndex( fs ); + if ( bIsPrimaryKey ) + { + Assert( bIsIndexUnique ); + Assert( pRecordInfo->GetPKIndex() < 0 ); + pRecordInfo->SetPKIndex( idx ); + } + } + + return SQL_SUCCESS; +} + +//----------------------------------------------------------------------------- +// Purpose: Finds the schema info on any FK constraints defined for the table +// Input: pRecordInfo - CRecordInfo to add the index info into (and lookup table name out of) +// Output: SQL return code. +//----------------------------------------------------------------------------- +SQLRETURN CJobUpdateSchema::YieldingGetTableFKConstraints( ESchemaCatalog eSchemaCatalog, CRecordInfo *pRecordInfo ) +{ + // Used below, declared up here because of goto + FKData_t fkData; + + CSQLAccess sqlAccess( eSchemaCatalog ); + const char *pchStatement = "SELECT fk.name AS FKName, current_table.name AS TableName, parent_table.name AS ParentTableName, " + "table_column.name AS ColName, parent_table_column.name AS ParentColName, " + "fk.delete_referential_action_desc AS OnDelete, " + "fk.update_referential_action_desc AS OnUpdate " + "FROM sys.objects AS current_table " + "JOIN sys.foreign_keys AS fk ON fk.parent_object_id=current_table.object_id AND fk.is_ms_shipped=0 " + "JOIN sys.foreign_key_columns AS fk_col ON fk_col.constraint_object_id=fk.object_id " + "JOIN sys.objects AS parent_table ON parent_table.object_id=fk_col.referenced_object_id " + "JOIN sys.columns AS table_column ON table_column.object_id=fk_col.parent_object_id AND table_column.column_id=fk_col.parent_column_id " + "JOIN sys.columns AS parent_table_column ON parent_table_column.object_id=fk_col.referenced_object_id AND parent_table_column.column_id=fk_col.referenced_column_id " + "WHERE current_table.object_id = ? ORDER BY fk.name"; + sqlAccess.AddBindParam( pRecordInfo->GetTableID() ); + RETURN_SQL_ERROR_ON_FALSE( sqlAccess.BYieldingExecute( FILE_AND_LINE, pchStatement ) ); + + FOR_EACH_SQL_RESULT( sqlAccess, 0, sqlRecord ) + { + // get all the data for the FK + const char *pchFKName; + DbgVerify( sqlRecord.BGetStringValue( 0, &pchFKName ) ); + + const char *pchTableName; + DbgVerify( sqlRecord.BGetStringValue( 1, &pchTableName ) ); + + AssertMsg( Q_stricmp( pchTableName, pRecordInfo->GetName() ) == 0, "FOREIGN KEY schema conversion found FK for table not matching search!\n" ); + + const char *pchParentTable; + DbgVerify( sqlRecord.BGetStringValue( 2, &pchParentTable ) ); + + const char *pchColName; + DbgVerify( sqlRecord.BGetStringValue( 3, &pchColName ) ); + + const char *pchParentColName; + DbgVerify( sqlRecord.BGetStringValue( 4, &pchParentColName ) ); + + const char *pchOnDelete; + DbgVerify( sqlRecord.BGetStringValue( 5, &pchOnDelete ) ); + + const char *pchOnUpdate; + DbgVerify( sqlRecord.BGetStringValue( 6, &pchOnUpdate ) ); + + // Is this more data for the FK we are already tracking? If so just append the column data, + // otherwise, assuming some data exists, add the key to the record info + if ( Q_strcmp( fkData.m_rgchName, pchFKName ) == 0 ) + { + int iColRelation = fkData.m_VecColumnRelations.AddToTail(); + FKColumnRelation_t &colRelation = fkData.m_VecColumnRelations[iColRelation]; + Q_strncpy( colRelation.m_rgchCol, pchColName, Q_ARRAYSIZE( colRelation.m_rgchCol ) ); + Q_strncpy( colRelation.m_rgchParentCol, pchParentColName, Q_ARRAYSIZE( colRelation.m_rgchParentCol ) ); + } + else + { + if ( Q_strlen( fkData.m_rgchName ) ) + { + pRecordInfo->AddFK( fkData ); + } + + // Do initial setup of the new key + Q_strncpy( fkData.m_rgchName, pchFKName, Q_ARRAYSIZE( fkData.m_rgchName ) ); + Q_strncpy( fkData.m_rgchParentTableName, pchParentTable, Q_ARRAYSIZE( fkData.m_rgchParentTableName ) ); + fkData.m_eOnDeleteAction = EForeignKeyActionFromName( pchOnDelete ); + fkData.m_eOnUpdateAction = EForeignKeyActionFromName( pchOnUpdate ); + int iColRelation = fkData.m_VecColumnRelations.AddToTail(); + FKColumnRelation_t &colRelation = fkData.m_VecColumnRelations[iColRelation]; + Q_strncpy( colRelation.m_rgchCol, pchColName, Q_ARRAYSIZE( colRelation.m_rgchCol ) ); + Q_strncpy( colRelation.m_rgchParentCol, pchParentColName, Q_ARRAYSIZE( colRelation.m_rgchParentCol ) ); + } + } + + + // Add the last key we were building data for + if ( Q_strlen( fkData.m_rgchName ) ) + { + pRecordInfo->AddFK( fkData ); + } + + return SQL_SUCCESS; +} + + +//----------------------------------------------------------------------------- +// Purpose: Creates a table in the DB to match the recordinfo +// Input: pRecordInfo - CRecordInfo to create a table for +// Output: SQL return code. +//----------------------------------------------------------------------------- +SQLRETURN CJobUpdateSchema::YieldingCreateTable( ESchemaCatalog eSchemaCatalog, CRecordInfo *pRecordInfo ) +{ + CFmtStrMax sCmd; + SQLRETURN nRet = 0; + const char *pchName = pRecordInfo->GetName(); + Assert( pchName && pchName[0] ); + CSQLAccess sqlAccess( eSchemaCatalog ); + + // build the create table command + sCmd.sprintf( "CREATE TABLE %s (", pchName ); + + // add all the columns + for ( int iColumn = 0; iColumn < pRecordInfo->GetNumColumns(); iColumn++ ) + { + char rgchType[k_cSmallBuff]; + const CColumnInfo &columnInfo = pRecordInfo->GetColumnInfo( iColumn ); + char *pchType = SQLTypeFromField( columnInfo, rgchType, Q_ARRAYSIZE( rgchType ) ); + Assert( pchType[0] ); + + // add the name and type of this column + sCmd.AppendFormat( "%s %s", columnInfo.GetName(), pchType ); + + // add any constraints + AppendConstraints( pRecordInfo, &columnInfo, true, sCmd ); + + if ( iColumn < ( pRecordInfo->GetNumColumns() - 1 ) ) + sCmd += ", "; + } + + AppendTableConstraints( pRecordInfo, sCmd ); + sCmd += ")"; + + + // create the table + // metadata operations aren't transactional, so we'll set bAutoTransaction = true + EXIT_ON_BOOLSQL_FAILURE( sqlAccess.BYieldingExecute( FILE_AND_LINE, sCmd ) ); + + // add any indexes + for ( int n = 0; n < pRecordInfo->GetIndexFields( ).Count(); n++ ) + { + const FieldSet_t& refSet = pRecordInfo->GetIndexFields( )[ n ]; + + // already added the PK index, so skip it + if ( n == pRecordInfo->GetPKIndex() ) + continue; + + // call YieldingAddIndex to do the work + nRet = YieldingAddIndex( eSchemaCatalog, pRecordInfo, refSet ); + EXIT_ON_SQL_FAILURE( nRet ); + } + +Exit: + return nRet; +} + +//----------------------------------------------------------------------------- +// Purpose: Adds an index of multiple columns to a table. +// Input: pSQLConnection - connection to use for command +// pRecordInfo - record info describing table +// refFields - description of index to add +// Output: SQL return code. +//----------------------------------------------------------------------------- +SQLRETURN CJobUpdateSchema::YieldingAddIndex( ESchemaCatalog eSchemaCatalog, CRecordInfo *pRecordInfo, const FieldSet_t &refFields ) +{ + CSQLAccess sqlAccess( eSchemaCatalog ); + CUtlString sCmd = GetAddIndexSQL( pRecordInfo, refFields ); + RETURN_SQL_ERROR_ON_FALSE( sqlAccess.BYieldingExecute( FILE_AND_LINE, sCmd ) ); + return SQL_SUCCESS; +} + + +//----------------------------------------------------------------------------- +// Purpose: Depending on the conversion mode, either really perfom the +// conversion, or just log the SQL text so a human being can review +// it and do it later. +//----------------------------------------------------------------------------- + +SQLRETURN CJobUpdateSchema::YieldingProcessUnsafeConversion( ESchemaCatalog eSchemaCatalog, const char *pszSQL, const char *pszComment ) +{ + if ( k_EConversionModeConvertIrreversible != m_eConversionMode ) + { + if ( pszComment ) + { + m_sRecommendedSQL.Append( "-- " ); + m_sRecommendedSQL.Append( pszComment ); + m_sRecommendedSQL.Append( "\n" ); + } + m_sRecommendedSQL.Append( pszSQL ); + m_sRecommendedSQL.Append("\nGO\n \n"); // that space is a kludge, because we are using V_splitlines, which will ignore consecutive seperators + return SQL_SUCCESS; + } + CSQLAccess sqlAccess( eSchemaCatalog ); + RETURN_SQL_ERROR_ON_FALSE( sqlAccess.BYieldingExecute( FILE_AND_LINE, pszSQL ) ); + return SQL_SUCCESS; +} + +//----------------------------------------------------------------------------- +// Purpose: Add a SQL command to cleanup the schema that will actually destroy (supposedly unused) data. +//----------------------------------------------------------------------------- + +void CJobUpdateSchema::AddDataDestroyingConversion( const char *pszSQL, const char *pszComment ) +{ + if ( pszComment ) + { + m_sDataDestroyingCleanupSQL.Append( "-- " ); + m_sDataDestroyingCleanupSQL.Append( pszComment ); + m_sDataDestroyingCleanupSQL.Append( "\n" ); + } + m_sDataDestroyingCleanupSQL.Append( pszSQL ); + m_sDataDestroyingCleanupSQL.Append("\nGO\n \n"); // that space is a kludge, because we are using V_splitlines, which will ignore consecutive seperators +} + +//----------------------------------------------------------------------------- +// Purpose: Adds a column to a table +// Input: pRecordInfo - record info describing table to add a column to +// pColumnInfo - column info describing column to add +// Output: SQL return code. +//----------------------------------------------------------------------------- +SQLRETURN CJobUpdateSchema::YieldingAlterTableAddColumn( ESchemaCatalog eSchemaCatalog, CRecordInfo *pRecordInfo, const CColumnInfo *pColumnInfo ) +{ + CSQLAccess sqlAccess( eSchemaCatalog ); + CFmtStrMax sCmd; + char rgchType[k_cSmallBuff]; + const char *pchTableName = pRecordInfo->GetName(); + Assert( pchTableName ); + const char *pchColumnName = pColumnInfo->GetName(); + Assert( pchColumnName ); + DbgVerify( SQLTypeFromField( *pColumnInfo, rgchType, Q_ARRAYSIZE( rgchType ) )[0] ); + + // build the alter table command + sCmd.sprintf( "ALTER TABLE %s ADD %s %s", pchTableName, pchColumnName, rgchType ); + + // add any constraints + AppendConstraints( pRecordInfo, pColumnInfo, true, sCmd ); + + // !KLUDGE! This is guaranteed to fail if it has "not null" in it. + if ( V_strstr( sCmd, "NOT NULL" ) ) + { + EmitError( SPEW_SQL, "Cannot add column %s to table %s with NOT NULL constraint\n", pchColumnName, pchTableName ); + EmitInfo( SPEW_SQL, SPEW_ALWAYS, LOG_ALWAYS, "Here is the SQL that should be run to add the column:\n" ); + EmitInfo( SPEW_SQL, SPEW_ALWAYS, LOG_ALWAYS, " %s\n", sCmd.String() ); + return SQL_ERROR; + } + + // execute the command. + RETURN_SQL_ERROR_ON_FALSE( sqlAccess.BYieldingExecute( FILE_AND_LINE, sCmd ) ); + return SQL_SUCCESS; +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a constraint to an existing column +// Input: hDBC - SQL connection to execute on +// pRecordInfo - record info describing table +// pColumnInfo - column info describing column to add contraint for +// nColFlagConstraint - constraint to add +// Output: SQL return code. +//----------------------------------------------------------------------------- +SQLRETURN CJobUpdateSchema::YieldingAddConstraint( ESchemaCatalog eSchemaCatalog, CRecordInfo *pRecordInfo, const CColumnInfo *pColumnInfo, int nColFlagConstraint ) +{ + Assert( pRecordInfo ); + Assert( pColumnInfo ); + + CFmtStrMax sCmd; + sCmd.sprintf( "ALTER TABLE App%u.%s ADD", GGCBase()->GetAppID(), pRecordInfo->GetName() ); + + Assert( !pColumnInfo->BIsClustered() ); + AppendConstraint( pRecordInfo->GetName(), pColumnInfo->GetName(), nColFlagConstraint, true, pColumnInfo->BIsClustered(), sCmd, 0 ); + + sCmd.AppendFormat( " (%s)", pColumnInfo->GetName() ); + + return YieldingProcessUnsafeConversion( eSchemaCatalog, sCmd ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Changes type or length of existing column +// Input: hDBC - SQL connection to execute on +// pRecordInfo - record info describing table +// pColumnInfoDesired - column info describing new column type or length +// Output: SQL return code. +//----------------------------------------------------------------------------- +SQLRETURN CJobUpdateSchema::YieldingChangeColumnTypeOrLength( ESchemaCatalog eSchemaCatalog, CRecordInfo *pRecordInfo, const CColumnInfo *pColumnInfoDesired ) +{ + CUtlString sCmd = GetAlterColumnText( pRecordInfo, pColumnInfoDesired ); + return YieldingProcessUnsafeConversion( eSchemaCatalog, sCmd ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Changes constraints/indexes on a column +// Input: hDBC - SQL connection to execute on +// pRecordInfo - record info describing table +// pColumnInfoActual - column info describing existing column properties +// pColumnInfoActual - column info describing desired new column properties +// Output: SQL return code. +//----------------------------------------------------------------------------- +SQLRETURN CJobUpdateSchema::YieldingChangeColumnProperties( ESchemaCatalog eSchemaCatalog, CRecordInfo *pRecordInfo, const CColumnInfo *pColumnInfoActual, + const CColumnInfo *pColumnInfoDesired ) +{ + CSQLAccess sqlAccess( eSchemaCatalog ); + Assert( pRecordInfo ); + Assert( pColumnInfoActual ); + Assert( pColumnInfoDesired ); + + SQLRETURN nRet = SQL_SUCCESS; + + pColumnInfoActual->ValidateColFlags(); + pColumnInfoDesired->ValidateColFlags(); + + Assert( pColumnInfoActual->GetColFlags() != pColumnInfoDesired->GetColFlags() ); + + // all the operations we might have to perform; note we have have to drop one + // thing and add another + bool bAddExplicitIndex = false, bRemoveExplicitIndex = false; + bool bAddUniqueConstraint = false, bRemoveUniqueConstraint = false; + bool bAddPrimaryKeyConstraint = false, bRemovePrimaryKeyConstraint = false; + + // determine which operations need to be performed + if ( !pColumnInfoActual->BIsPrimaryKey() && pColumnInfoDesired->BIsPrimaryKey() ) + { + bAddPrimaryKeyConstraint = true; + } + else if ( pColumnInfoActual->BIsPrimaryKey() && !pColumnInfoDesired->BIsPrimaryKey() ) + { + bRemovePrimaryKeyConstraint = true; + } + + if ( !pColumnInfoActual->BIsExplicitlyUnique() && pColumnInfoDesired->BIsExplicitlyUnique() ) + { + bAddUniqueConstraint = true; + } + else if ( pColumnInfoActual->BIsExplicitlyUnique() && !pColumnInfoDesired->BIsExplicitlyUnique() ) + { + bRemoveUniqueConstraint = true; + } + + if ( !pColumnInfoActual->BIsExplicitlyIndexed() && pColumnInfoDesired->BIsExplicitlyIndexed() ) + { + bAddExplicitIndex = true; + } + else if ( pColumnInfoActual->BIsExplicitlyIndexed() && !pColumnInfoDesired->BIsExplicitlyIndexed() ) + { + bRemoveExplicitIndex = true; + } + + // sanity check + Assert( !( bAddUniqueConstraint && bAddPrimaryKeyConstraint ) ); // primary key constraint adds implicit uniqueness constraint; it's a bug if we decide to do both + Assert( !( bAddUniqueConstraint && bAddExplicitIndex ) ); // uniqueness constraint adds implicit index; it's a bug if we decide to do both + Assert( !( bAddPrimaryKeyConstraint && bAddExplicitIndex ) ); // primary key constraint adds implicit index; it's a bug if we decide to do both + + // The index conversion stuff already handles dropping of unexpected indices + // and primary key constraints. I'm not even sure that this works or is used anymore. + if ( bAddUniqueConstraint ) + { + nRet = YieldingAddConstraint( eSchemaCatalog, pRecordInfo, pColumnInfoActual, k_nColFlagUnique ); + EXIT_ON_SQL_FAILURE( nRet ); + } + +Exit: + return nRet; +} + + + + +//----------------------------------------------------------------------------- +// Purpose: Creates specified trigger +// Input: pSQLConnection - SQL connection to execute on +// refTriggerInfo - trigger to be created +// Output: SQL return code. +//----------------------------------------------------------------------------- +SQLRETURN CJobUpdateSchema::YieldingCreateTrigger( ESchemaCatalog eSchemaCatalog, CTriggerInfo &refTriggerInfo ) +{ + char rgchCmd[ k_cchSQLStatementTextMax]; + CSQLAccess sqlAccess( eSchemaCatalog ); + + // build the create command + Q_snprintf( rgchCmd, Q_ARRAYSIZE( rgchCmd ), + "CREATE TRIGGER [%s] ON [%s] " + "%s " + "AS BEGIN" + "%s\n" + "END", + refTriggerInfo.m_szTriggerName, + refTriggerInfo.m_szTriggerTableName, + refTriggerInfo.GetTriggerTypeString(), + refTriggerInfo.m_strText.Get() ); + + RETURN_SQL_ERROR_ON_FALSE( sqlAccess.BYieldingExecute( FILE_AND_LINE, rgchCmd ) ); + return SQL_SUCCESS; +} + + +//----------------------------------------------------------------------------- +// Purpose: drops specified trigger +// Input: pSQLConnection - SQL connection to execute on +// refTriggerInfo - trigger to be dropped +// Output: SQL return code. +//----------------------------------------------------------------------------- +SQLRETURN CJobUpdateSchema::YieldingDropTrigger( ESchemaCatalog eSchemaCatalog, CTriggerInfo &refTriggerInfo ) +{ + char rgchCmd[ k_cchSQLStatementTextMax]; + CSQLAccess sqlAccess( eSchemaCatalog ); + + // build the create command + Q_snprintf( rgchCmd, Q_ARRAYSIZE( rgchCmd ), + "DROP TRIGGER [%s];", + refTriggerInfo.m_szTriggerName ); + + RETURN_SQL_ERROR_ON_FALSE( sqlAccess.BYieldingExecute( FILE_AND_LINE, rgchCmd ) ); + return SQL_SUCCESS; +} + + + +//----------------------------------------------------------------------------- +// Purpose: Used for SQL/EGCSQLType conversion +//----------------------------------------------------------------------------- +struct SQLTypeMapping_t +{ + const char *m_pchTypeName; + EGCSQLType m_eType; +}; + +static SQLTypeMapping_t g_rSQLTypeMapping[] = +{ + { "tinyint", k_EGCSQLType_int8 }, + { "smallint", k_EGCSQLType_int16 }, + { "int", k_EGCSQLType_int32 }, + { "bigint", k_EGCSQLType_int64 }, + { "real", k_EGCSQLType_float }, + { "float", k_EGCSQLType_double }, + { "text", k_EGCSQLType_String }, + { "ntext", k_EGCSQLType_String }, + { "char", k_EGCSQLType_String }, + { "varchar", k_EGCSQLType_String }, + { "nchar", k_EGCSQLType_String }, + { "nvarchar", k_EGCSQLType_String }, + { "sysname", k_EGCSQLType_String }, + { "varbinary", k_EGCSQLType_Blob }, + { "image", k_EGCSQLType_Image }, +}; +static uint32 g_cSQLTypeMapping = Q_ARRAYSIZE( g_rSQLTypeMapping ); + + +//----------------------------------------------------------------------------- +// Purpose: Returns the EGCSQLType for the specified SQL type name (or Invalid +// for unsupported types.) +//----------------------------------------------------------------------------- +EGCSQLType ETypeFromMSSQLDataType( const char *pchType ) +{ + for( uint32 unMapping = 0; unMapping < g_cSQLTypeMapping; unMapping++ ) + { + if( !Q_stricmp( pchType, g_rSQLTypeMapping[ unMapping ].m_pchTypeName ) ) + return g_rSQLTypeMapping[ unMapping ].m_eType; + } + return k_EGCSQLTypeInvalid; +} + + +//----------------------------------------------------------------------------- +// Purpose: Prepares the type map for use in schema upgrades +//----------------------------------------------------------------------------- +bool CJobUpdateSchema::YieldingBuildTypeMap( ESchemaCatalog eSchemaCatalog ) +{ + CSQLAccess sqlAccess( eSchemaCatalog ); + if( !sqlAccess.BYieldingExecute( FILE_AND_LINE, "select name, system_type_id from sys.types" ) ) + return false; + + FOR_EACH_SQL_RESULT( sqlAccess, 0, typeRecord ) + { + CFmtStr1024 sTypeName; + byte nTypeID; + DbgVerify( typeRecord.BGetStringValue( 0, &sTypeName ) ); + DbgVerify( typeRecord.BGetByteValue( 1, &nTypeID ) ); + + EGCSQLType eType = ETypeFromMSSQLDataType( sTypeName ); + if( eType != k_EGCSQLTypeInvalid ) + m_mapSQLTypeToEType.Insert( nTypeID, eType ); + } + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns the EGCSQLType for the specified type ID out of the database +//----------------------------------------------------------------------------- +EGCSQLType CJobUpdateSchema::GetEGCSQLTypeForMSSQLType( int nType ) +{ + int nIndex = m_mapSQLTypeToEType.Find( nType ); + if( m_mapSQLTypeToEType.IsValidIndex( nIndex ) ) + return m_mapSQLTypeToEType[ nIndex ]; + else + return k_EGCSQLTypeInvalid; +} + +} // namespace GCSDK diff --git a/gcsdk/sqlaccess/sqlaccess.cpp b/gcsdk/sqlaccess/sqlaccess.cpp new file mode 100644 index 0000000..130a446 --- /dev/null +++ b/gcsdk/sqlaccess/sqlaccess.cpp @@ -0,0 +1,953 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Provides access to SQL at a high level +// +//============================================================================= + +#include "stdafx.h" +#include "gcsdk/sqlaccess/sqlaccess.h" +#include "gcsdk/gcsqlquery.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +template< typename LISTENER_FUNC > +static void RunAndClearListenerList( std::vector< LISTENER_FUNC > &vecListeners ) +{ + // Let us not underestimate the ability of random listeners to re-enter everything. + std::vector< LISTENER_FUNC > listenerCopy; + listenerCopy.swap( vecListeners ); + vecListeners.clear(); + + // Why would you consider such a thing + DO_NOT_YIELD_THIS_SCOPE(); + + for ( const auto &listener : listenerCopy ) + { + listener(); + } +} + + +namespace GCSDK +{ +//------------------------------------------------------------------------------------ +// Purpose: Constructor +//------------------------------------------------------------------------------------ +CSQLAccess::CSQLAccess( ESchemaCatalog eSchemaCatalog ) + : m_eSchemaCatalog( eSchemaCatalog) + , m_pCurrentQuery( NULL ) + , m_bInTransaction( false ) +{ + m_pQueryGroup = CGCSQLQueryGroup::Alloc(); +} + + +//------------------------------------------------------------------------------------ +// Purpose: Destructor +//------------------------------------------------------------------------------------ +CSQLAccess::~CSQLAccess( ) +{ + SAFE_RELEASE( m_pQueryGroup ); + Assert( !m_pCurrentQuery ); + SAFE_DELETE( m_pCurrentQuery ); + AssertMsg( !m_bInTransaction, "GCSDK::CSQLAccess object being destroyed with a transaction pending. Use BCommitTransaction or RollbackTransaction to match your BBeginTransaction call." ); +} + + +//------------------------------------------------------------------------------------ +// Purpose: Perform a query +//------------------------------------------------------------------------------------ +bool CSQLAccess::BYieldingExecute( const char *pchName, const char *pchSQLCommand, uint32 *pcRowsAffected, bool bSpewOnError ) +{ + if ( NULL == pchName ) + { + pchName = pchSQLCommand; + } + + bool bStandalone = !BInTransaction(); + if( bStandalone ) + { + BBeginTransaction( pchName ); + } + + CurrentQuery()->SetCommand( pchSQLCommand ); + m_pQueryGroup->AddQuery( m_pCurrentQuery ); + m_pCurrentQuery = NULL; + + bool bSuccess = true; + if( bStandalone ) + { + bSuccess = BCommitTransaction(); + if( bSuccess && pcRowsAffected ) + { + *pcRowsAffected = m_pQueryGroup->GetResults()->GetRowsAffected( 0 ); + } + } + return bSuccess; +} + + +//------------------------------------------------------------------------------------ +// Purpose: Starts a transaction +//------------------------------------------------------------------------------------ +bool CSQLAccess::BBeginTransaction( const char *pchName ) +{ + Assert( !m_bInTransaction ); + if( m_bInTransaction ) + return false; + m_pQueryGroup->Clear(); + m_pQueryGroup->SetName( pchName ); + m_bInTransaction = true; + return true; +} + +//------------------------------------------------------------------------------------ +// Purpose: Returns the string last passed to BBeginTransaction +//------------------------------------------------------------------------------------ +const char *CSQLAccess::PchTransactionName( ) const +{ + return m_pQueryGroup->PchName(); +} + + +//------------------------------------------------------------------------------------ +// Purpose: Commits a transaction to the database +//------------------------------------------------------------------------------------ +bool CSQLAccess::BCommitTransaction( bool bAllowEmpty ) +{ + Assert( BInTransaction() ); + if( !BInTransaction() ) + return false; + + if( !m_pCurrentQuery && !m_pQueryGroup->GetStatementCount() ) + { + if( bAllowEmpty ) + { + // No-op success + m_bInTransaction = false; + RunListeners_Commit(); + return true; + } + else + { + AssertMsg1( false, "BCommitTransaction with empty transaction at %s", m_pQueryGroup->PchName() ); + return false; + } + } + + AssertMsg1( !m_pCurrentQuery, "Unexecuted query present in BCommitTransaction: %s", m_pCurrentQuery->PchCommand() ); + if( m_pCurrentQuery ) + return false; + + m_bInTransaction = false; + + if( !GJobCur().BYieldingRunQuery( m_pQueryGroup, m_eSchemaCatalog ) ) + { + // Notify listeners that the transaction did not succeed + RunListeners_Rollback(); + return false; + } + + // The transaction presumably did make the database, so we do not notify rollback listeners beyond here. + RunListeners_Commit(); + + if( !m_pQueryGroup->GetResults() ) + return false; + + return true; +} + + +//------------------------------------------------------------------------------------ +// Purpose: Rolls back a transaction and clears any queries +//------------------------------------------------------------------------------------ +void CSQLAccess::RollbackTransaction() +{ + bool bWasTransaction = BInTransaction(); + + Assert( bWasTransaction ); + SAFE_DELETE( m_pCurrentQuery ); + m_bInTransaction = false; + + if ( bWasTransaction ) + { + RunListeners_Rollback(); + } + else + { + m_vecCommitListeners.clear(); + m_vecRollbackListeners.clear(); + } +} + +//------------------------------------------------------------------------------------ +// Purpose: Adds a listener to be called synchronously should the transaction successfully commit +//------------------------------------------------------------------------------------ +void CSQLAccess::AddCommitListener( std::function<void (void)> &&listener ) +{ + if ( !BInTransaction() ) + { + AssertMsg( BInTransaction(), "Adding a listener to a non-transaction access, will never fire" ); + return; + } + + m_vecCommitListeners.push_back( std::move( listener ) ); +} + +//------------------------------------------------------------------------------------ +// Purpose: Adds a listener to be called synchronously should the transaction fail or explicitly rollback +//------------------------------------------------------------------------------------ +void CSQLAccess::AddRollbackListener( std::function<void (void)> &&listener ) +{ + if ( !BInTransaction() ) + { + AssertMsg( BInTransaction(), "Adding a listener to a non-transaction access, will never fire" ); + return; + } + + m_vecRollbackListeners.push_back( std::move( listener ) ); +} + +//------------------------------------------------------------------------------------ +// Purpose: Notifies listeners of successful commit. +//------------------------------------------------------------------------------------ +void CSQLAccess::RunListeners_Commit() +{ + RunAndClearListenerList( m_vecCommitListeners ); + // Clear the unused set + m_vecRollbackListeners.clear(); +} + +//------------------------------------------------------------------------------------ +// Purpose: Notifies listeners of a implicitly or explicitly rolled back transactions and clears the listener list. +//------------------------------------------------------------------------------------ +void CSQLAccess::RunListeners_Rollback() +{ + RunAndClearListenerList( m_vecRollbackListeners ); + // Clear the unused set + m_vecCommitListeners.clear(); +} + +//------------------------------------------------------------------------------------ +// Purpose: Perform a query that returns a single string +//------------------------------------------------------------------------------------ +CSQLAccess::EReadSingleResultResult CSQLAccess::BYieldingExecuteSingleResultDataInternal( const char *pchName, const char *pchSQLCommand, EGCSQLType eType, uint8 **ppubData, uint32 *punSize, uint32 *pcRowsAffected, bool bHasDefaultValue ) +{ + AssertMsg( !BInTransaction(), "BYieldingExecuteSingleResultData is not supported in a transaction" ); + if( BInTransaction() ) + return eReadSingle_Error; + + bool bRet = BYieldingExecute( pchName, pchSQLCommand, pcRowsAffected ); + if ( !bRet ) + return eReadSingle_Error; + + if( m_pQueryGroup->GetResults()->GetResultSetCount() != 1 ) + { + AssertMsg1( false, "Expected single result set, found %d", m_pQueryGroup->GetResults()->GetResultSetCount() ); + return eReadSingle_Error; + } + + IGCSQLResultSet *pResultSet = m_pQueryGroup->GetResults()->GetResultSet( 0 ); + + // If we have a default value, getting back zero rows is acceptable. + if( pResultSet->GetRowCount() == 0 && bHasDefaultValue ) + { + return eReadSingle_UseDefault; + } + + // If we either have more than one row or no default value specified, that's an error. + if( pResultSet->GetRowCount() != 1 ) + { + AssertMsg1( false, "Expected single result, found %d", pResultSet->GetRowCount() ); + return eReadSingle_Error; + } + + if( pResultSet->GetColumnCount() != 1 ) + { + AssertMsg1( false, "Expected single column, found %d", pResultSet->GetColumnCount() ); + return eReadSingle_Error; + } + if( pResultSet->GetColumnType( 0 ) != eType ) + { + AssertMsg2( false, "Expected column of type %s, found %s", PchNameFromEGCSQLType( eType ), PchNameFromEGCSQLType( pResultSet->GetColumnType( 0 ) ) ); + return eReadSingle_Error; + } + + return pResultSet->GetData( 0, 0, ppubData, punSize ) + ? eReadSingle_ResultFound + : eReadSingle_Error; +} + + + + +//------------------------------------------------------------------------------------ +// Purpose: Perform a query that returns a single string +//------------------------------------------------------------------------------------ +bool CSQLAccess::BYieldingExecuteString( const char *pchName, const char *pchSQLCommand, CFmtStr1024 *psResult, uint32 *pcRowsAffected ) +{ + uint8 *pubData; + uint32 cubData; + if( CSQLAccess::BYieldingExecuteSingleResultDataInternal( pchName, pchSQLCommand, k_EGCSQLType_String, &pubData, &cubData, pcRowsAffected, false ) != eReadSingle_ResultFound ) + return false; + + *psResult = (char *)pubData; + + return true; +} + +//------------------------------------------------------------------------------------ +// Purpose: Perform a query that returns a single int +//------------------------------------------------------------------------------------ +bool CSQLAccess::BYieldingExecuteScalarInt( const char *pchName, const char *pchSQLCommand, int *pnResult, uint32 *pcRowsAffected ) +{ + return BYieldingExecuteSingleResult<int32, uint32>( pchName, pchSQLCommand, k_EGCSQLType_int32, pnResult, pcRowsAffected ); +} + +bool CSQLAccess::BYieldingExecuteScalarIntWithDefault( const char *pchName, const char *pchSQLCommand, int *pnResult, int iDefaultValue, uint32 *pcRowsAffected ) +{ + return BYieldingExecuteSingleResultWithDefault<int32, uint32>( pchName, pchSQLCommand, k_EGCSQLType_int32, pnResult, iDefaultValue, pcRowsAffected ); +} + +//------------------------------------------------------------------------------------ +// Purpose: Perform a query that returns a single uint32 +//------------------------------------------------------------------------------------ +bool CSQLAccess::BYieldingExecuteScalarUint32( const char *pchName, const char *pchSQLCommand, uint32 *punResult, uint32 *pcRowsAffected ) +{ + return BYieldingExecuteSingleResult<uint32, uint32>( pchName, pchSQLCommand, k_EGCSQLType_int32, punResult, pcRowsAffected ); +} + +bool CSQLAccess::BYieldingExecuteScalarUint32WithDefault( const char *pchName, const char *pchSQLCommand, uint32 *punResult, uint32 unDefaultValue, uint32 *pcRowsAffected ) +{ + return BYieldingExecuteSingleResultWithDefault<uint32, uint32>( pchName, pchSQLCommand, k_EGCSQLType_int32, punResult, unDefaultValue, pcRowsAffected ); +} + +//------------------------------------------------------------------------------------ +// Purpose: A bunch of pass throughs to the query itself +//------------------------------------------------------------------------------------ +void CSQLAccess::AddBindParam( const char *pchValue ) +{ + CurrentQuery()->AddBindParam( pchValue ); +} + +void CSQLAccess::AddBindParam( const int16 nValue ) +{ + CurrentQuery()->AddBindParam( nValue ); +} + +void CSQLAccess::AddBindParam( const uint16 uValue ) +{ + CurrentQuery()->AddBindParam( uValue ); +} + +void CSQLAccess::AddBindParam( const int32 nValue ) +{ + CurrentQuery()->AddBindParam( nValue ); +} + +void CSQLAccess::AddBindParam( const uint32 uValue ) +{ + CurrentQuery()->AddBindParam( uValue ); +} + +void CSQLAccess::AddBindParam( const uint64 ulValue ) +{ + CurrentQuery()->AddBindParam( ulValue ); +} + +void CSQLAccess::AddBindParam( const uint8 *ubValue, const int cubValue ) +{ + CurrentQuery()->AddBindParam( ubValue, cubValue ); +} + +void CSQLAccess::AddBindParam( const float fValue ) +{ + CurrentQuery()->AddBindParam( fValue ); +} + +void CSQLAccess::AddBindParam( const double dValue ) +{ + CurrentQuery()->AddBindParam( dValue ); +} + +void CSQLAccess::AddBindParamRaw( EGCSQLType eType, const byte *pubData, uint32 cubData ) +{ + CurrentQuery()->AddBindParamRaw( eType, pubData, cubData ); +} + +void CSQLAccess::ClearParams() +{ + if( m_pCurrentQuery ) + { + delete m_pCurrentQuery; + m_pCurrentQuery = NULL; + } +} + + +IGCSQLResultSetList *CSQLAccess::GetResults() +{ + return m_pQueryGroup->GetResults(); +} + + +//------------------------------------------------------------------------------------ +// Purpose: Returns the number of result sets +//------------------------------------------------------------------------------------ +uint32 CSQLAccess::GetResultSetCount() +{ + if( m_pQueryGroup->GetResults() ) + return m_pQueryGroup->GetResults()->GetResultSetCount(); + else + return 0; +} + + +//------------------------------------------------------------------------------------ +// Purpose: Returns the number of rows in a result set +//------------------------------------------------------------------------------------ +uint32 CSQLAccess::GetResultSetRowCount( uint32 unResultSet ) +{ + if( m_pQueryGroup->GetResults() && unResultSet < m_pQueryGroup->GetResults()->GetResultSetCount() ) + return m_pQueryGroup->GetResults()->GetResultSet( unResultSet )->GetRowCount(); + else + return 0; +} + + +//------------------------------------------------------------------------------------ +// Purpose: Returns a CSQLRecord object that represents a row in a result set +//------------------------------------------------------------------------------------ +CSQLRecord CSQLAccess::GetResultRecord( uint32 unResultSet, uint32 unRow ) +{ + if( m_pQueryGroup->GetResults() && unResultSet < m_pQueryGroup->GetResults()->GetResultSetCount() ) + { + IGCSQLResultSet *pResultSet = m_pQueryGroup->GetResults()->GetResultSet( unResultSet ); + if( unRow < pResultSet->GetRowCount() ) + return CSQLRecord( unRow, pResultSet ); + } + return CSQLRecord(); // if there was a problem return an empty record +} + +//----------------------------------------------------------------------------- +// Purpose: Inserts a new record into the DS +// Input: pRecordBase - record to insert +// Output: true if successful, false otherwise +//----------------------------------------------------------------------------- +bool CSQLAccess::BYieldingInsertRecord( const CRecordBase *pRecordBase ) +{ + ClearParams(); + + const CRecordInfo *pRecordInfo = pRecordBase->GetPSchema()->GetRecordInfo(); + int cColumns = pRecordInfo->GetNumColumns(); + for ( int nColumn = 0; nColumn < cColumns; nColumn++ ) + { + const CColumnInfo &columnInfo = pRecordInfo->GetColumnInfo( nColumn ); + if ( !columnInfo.BIsInsertable() ) + continue; + + uint8 *pubData; + uint32 cubData; + DbgVerify( pRecordBase->BGetField( nColumn, &pubData, &cubData ) ); + + CurrentQuery()->AddBindParamRaw( columnInfo.GetType(), pubData, cubData ); + } + + uint32 nRows; + const char *pchStatement = pRecordBase->GetPSchema()->GetInsertStatementText(); + + bool bRet = BYieldingExecute( pchStatement, pchStatement, &nRows ); + return ( nRows == 1 || BInTransaction() ) && bRet; +} + + +//----------------------------------------------------------------------------- +// Purpose: Inserts a new record into the DS if such row doesn't exist +// Input: pRecordBase - record to insert +// Output: true if successful, false otherwise +//----------------------------------------------------------------------------- +bool CSQLAccess::BYieldingInsertWhenNotMatchedOnPK( CRecordBase *pRecordBase ) +{ + ClearParams(); + + const CRecordInfo *pRecordInfo = pRecordBase->GetPSchema()->GetRecordInfo(); + int cColumns = pRecordInfo->GetNumColumns(); + for ( int nColumn = 0; nColumn < cColumns; nColumn++ ) + { + const CColumnInfo &columnInfo = pRecordInfo->GetColumnInfo( nColumn ); + if ( !columnInfo.BIsInsertable() ) + { + Assert( columnInfo.BIsInsertable() ); + return false; + } + + uint8 *pubData; + uint32 cubData; + DbgVerify( pRecordBase->BGetField( nColumn, &pubData, &cubData ) ); + + CurrentQuery()->AddBindParamRaw( columnInfo.GetType(), pubData, cubData ); + } + + uint32 nRows; + const char *pchStatement = pRecordBase->GetPSchema()->GetMergeStatementTextOnPKWhenNotMatchedInsert(); + + bool bRet = BYieldingExecute( pchStatement, pchStatement, &nRows ); + return ( nRows == 1 || nRows == 0 || BInTransaction() ) && bRet; +} + +//----------------------------------------------------------------------------- +// Purpose: Inserts a new record into the DS if such row doesn't exist +// updates an existing row if such row is matched by PK +// Input: pRecordBase - record to insert +// Output: true if successful, false otherwise +//----------------------------------------------------------------------------- +bool CSQLAccess::BYieldingInsertOrUpdateOnPK( CRecordBase *pRecordBase ) +{ + ClearParams(); + + const CRecordInfo *pRecordInfo = pRecordBase->GetPSchema()->GetRecordInfo(); + int cColumns = pRecordInfo->GetNumColumns(); + for ( int nColumn = 0; nColumn < cColumns; nColumn++ ) + { + const CColumnInfo &columnInfo = pRecordInfo->GetColumnInfo( nColumn ); + if ( !columnInfo.BIsInsertable() ) + { + Assert( columnInfo.BIsInsertable() ); + return false; + } + + uint8 *pubData; + uint32 cubData; + DbgVerify( pRecordBase->BGetField( nColumn, &pubData, &cubData ) ); + + CurrentQuery()->AddBindParamRaw( columnInfo.GetType(), pubData, cubData ); + } + + uint32 nRows; + const char *pchStatement = pRecordBase->GetPSchema()->GetMergeStatementTextOnPKWhenMatchedUpdateWhenNotMatchedInsert(); + + bool bRet = BYieldingExecute( pchStatement, pchStatement, &nRows ); + return ( nRows == 1 || BInTransaction() ) && bRet; +} + +//----------------------------------------------------------------------------- +// Purpose: Inserts a new record into the DB and reads non-insertable fields back +// into the record. +// Input: pRecordBase - record to insert +// Output: true if successful, false otherwise +//----------------------------------------------------------------------------- +bool CSQLAccess::BYieldingInsertWithIdentity( CRecordBase* pRecordBase ) +{ + AssertMsg( !BInTransaction(), "BYieldingInsertWithIdentity is not supported in a transaction" ); + if( BInTransaction() ) + return false; + ClearParams(); + + TSQLCmdStr sStatement; + CUtlVector<int> vecOutputFields; + CRecordInfo *pRecordInfo = pRecordBase->GetPSchema()->GetRecordInfo(); + BuildInsertAndReadStatementText( &sStatement, &vecOutputFields, pRecordInfo ); + + AssertMsg( vecOutputFields.Count() > 0, "BYieldingInsertAndReadRecord called for a record type with no non-insertable columns" ); + if ( vecOutputFields.Count() == 0 ) + return false; + + int cColumns = pRecordInfo->GetNumColumns(); + for ( int nColumn = 0; nColumn < cColumns; nColumn++ ) + { + const CColumnInfo &columnInfo = pRecordInfo->GetColumnInfo( nColumn ); + if ( !columnInfo.BIsInsertable() ) + { + continue; + } + + uint8 *pubData; + uint32 cubData; + DbgVerify( pRecordBase->BGetField( nColumn, &pubData, &cubData ) ); + + CurrentQuery()->AddBindParamRaw( columnInfo.GetType(), pubData, cubData ); + } + + bool bRet = BYieldingExecute( sStatement, sStatement ); + if( !bRet ) + return false; + + Assert( 1 == GetResultSetCount() ); + if ( 1 != GetResultSetCount() ) + return false; + + IGCSQLResultSet *pResultSet = m_pQueryGroup->GetResults()->GetResultSet( 0 ); + Assert( 1 == pResultSet->GetRowCount() ); + if ( 1 != pResultSet->GetRowCount() ) + return false; + + Assert( (uint32)vecOutputFields.Count() == pResultSet->GetColumnCount() ); + if ( (uint32)vecOutputFields.Count() != pResultSet->GetColumnCount() ) + return false; + + for( uint32 nColumn = 0; nColumn < pResultSet->GetColumnCount(); nColumn++ ) + { + uint8 *pubData; + uint32 cubData; + DbgVerify( pResultSet->GetData( 0, nColumn, &pubData, &cubData ) ); + + int nSchColumn = vecOutputFields[nColumn]; + Assert( pResultSet->GetColumnType( nColumn ) == pRecordInfo->GetColumnInfo( nSchColumn ).GetType() ); + DbgVerify( pRecordBase->BSetField( nSchColumn, pubData, cubData ) ); + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Reads a list of records from the DB according to the specified where +// clause +// Input: pRecordBase - record to read +// readSet - The set of columns to read +// whereSet - The set of columns to query on +// Output: true if successful, false otherwise +//----------------------------------------------------------------------------- +EResult CSQLAccess::YieldingReadRecordWithWhereColumns( CRecordBase *pRecord, const CColumnSet & readSet, const CColumnSet & whereSet, const char* pchOrderClause ) +{ + AssertMsg( !BInTransaction(), "BYieldingReadRecordWithWhereColumns is not supported in a transaction" ); + if( BInTransaction() ) + return k_EResultInvalidState; + + //if there is an order by clause, only take the top one, if there isn't, then validate that we have a single instance + const char* pszTopClause = ( pchOrderClause ) ? "TOP (1)" : "TOP (2)"; + + TSQLCmdStr sStatement; + BuildSelectStatementText( &sStatement, readSet, pszTopClause ); + + // if we actually have some columns for the where clause, + // append a where clause. + if( whereSet.GetColumnCount() ) + { + sStatement.Append( " WHERE " ); + AppendWhereClauseText( &sStatement, whereSet ); + AddRecordParameters( *pRecord, whereSet ); + } + //append the order by if they added one + if( pchOrderClause ) + { + sStatement.Append( " ORDER BY " ); + sStatement.Append( pchOrderClause ); + } + + Assert(!readSet.IsEmpty() ); + if( !BYieldingExecute( sStatement, sStatement ) ) + return k_EResultFail; + + if ( GetResultSetCount() != 1 ) + { + AssertMsg( GetResultSetCount() == 1, "Unexpected number of result sets returned from select statement" ); + return k_EResultFail; + } + + // make sure the types are the same + IGCSQLResultSet *pResultSet = m_pQueryGroup->GetResults()->GetResultSet( 0 ); + if ( pResultSet->GetRowCount() == 0 ) + return k_EResultNoMatch; + + //note that since we only take the top one when there is an order by clause, we don't need to handle that case down here, only if top 2 is selected + if( pResultSet->GetRowCount() != 1 ) + { + // Make sure we aren't failing because there are multiple matching records. + // That is probably a misuse of the API or some unexpected condition. + AssertMsg1( false, "BYieldingReadRecordWithWhereColumns from %s failing because multiple records match WHERE clause", readSet.GetRecordInfo()->GetName() ); + return k_EResultLimitExceeded; + } + FOR_EACH_COLUMN_IN_SET( readSet, nColumnIndex ) + { + EGCSQLType eRecordType = readSet.GetColumnInfo( nColumnIndex ).GetType(); + EGCSQLType eResultType = pResultSet->GetColumnType( nColumnIndex ); + + AssertMsg2( eResultType == eRecordType, "Column %d type mismatch in %s", nColumnIndex, readSet.GetRecordInfo()->GetName() ); + if( eRecordType != eResultType ) + return k_EResultInvalidParam; + } + + CSQLRecord sqlRecord = GetResultRecord( 0, 0 ); + + FOR_EACH_COLUMN_IN_SET( readSet, nColumnIndex ) + { + uint8 *pubData; + uint32 cubData; + + DbgVerify( sqlRecord.BGetColumnData( nColumnIndex, &pubData, (int*)&cubData ) ); + DbgVerify( pRecord->BSetField( readSet.GetColumn( nColumnIndex), pubData, cubData ) ); + } + + return k_EResultOK; +} + +//----------------------------------------------------------------------------- +// Purpose: Updates a record in the DB +// Input: record - data source for columns to match against (whereColumns) and +// columns to assign (updateColumns) +// whereColumns - columns to match against +// updateColumns - columns to update +// Output: true if successful, false otherwise +//----------------------------------------------------------------------------- +bool CSQLAccess::BYieldingUpdateRecord( const CRecordBase & record, const CColumnSet & whereColumns, const CColumnSet & updateColumns, const CSQLOutputParams *pOptionalOutputParams /* = NULL */ ) +{ + return BYieldingUpdateRecords( record, whereColumns, record, updateColumns, pOptionalOutputParams ); +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +bool CSQLAccess::BYieldingUpdateRecords( const CRecordBase & whereRecord, const CColumnSet & whereColumns, const CRecordBase & updateRecord, const CColumnSet & updateColumns, const CSQLOutputParams *pOptionalOutputParams /* = NULL */ ) +{ + ClearParams(); + + Assert( whereColumns.GetRecordInfo() == updateColumns.GetRecordInfo() ); + if ( whereColumns.GetRecordInfo() != updateColumns.GetRecordInfo() ) + return false; + Assert( whereColumns.GetRecordInfo() == whereRecord.GetPSchema()->GetRecordInfo() ); + if ( whereColumns.GetRecordInfo() != whereRecord.GetPSchema()->GetRecordInfo() ) + return false; + Assert( whereColumns.GetRecordInfo() == updateRecord.GetPSchema()->GetRecordInfo() ); + if ( whereColumns.GetRecordInfo() != updateRecord.GetPSchema()->GetRecordInfo() ) + return false; + + AssertMsg( !updateColumns.IsEmpty(), "Someone is calling BYieldingUpdateRecord with no columns to update." ); + if ( updateColumns.IsEmpty() ) + return false; + + // add the columns we're updating as bound params + TSQLCmdStr sStatement; + BuildUpdateStatementText( &sStatement, updateColumns ); + + AddRecordParameters( updateRecord, updateColumns ); + + // did the users specify an OUTPUT block? + if ( pOptionalOutputParams ) + { + TSQLCmdStr sOutput; + BuildOutputClauseText( &sOutput, pOptionalOutputParams->GetColumnSet() ); + sStatement.Append( sOutput ); + + AddRecordParameters( pOptionalOutputParams->GetRecord(), pOptionalOutputParams->GetColumnSet() ); + } + + if ( !whereColumns.IsEmpty() ) + { + sStatement.Append( " WHERE " ); + AppendWhereClauseText( &sStatement, whereColumns ); + + // add the columns we're querying on as bound params + AddRecordParameters( whereRecord, whereColumns ); + } + + return BYieldingExecute( sStatement, sStatement ); +} + +//----------------------------------------------------------------------------- +// Purpose: Deletes this record's row in the table +// Input: record - record to delete +// whereColumns - columns to use when searching for this record +//----------------------------------------------------------------------------- +bool CSQLAccess::BYieldingDeleteRecords( const CRecordBase & record, const CColumnSet & whereColumns ) +{ + Assert( whereColumns.GetRecordInfo() == record.GetPSchema()->GetRecordInfo() ); + if ( whereColumns.GetRecordInfo() != record.GetPSchema()->GetRecordInfo() ) + return false; + + ClearParams(); + AddRecordParameters( record, whereColumns ); + + TSQLCmdStr sStatement; + BuildDeleteStatementText( &sStatement, record.GetPRecordInfo() ); + sStatement.Append( " WHERE " ); + AppendWhereClauseText( &sStatement, whereColumns ); + + uint32 unRowsAffected; + if( !BYieldingExecute( sStatement, sStatement, &unRowsAffected ) ) + return false; + + return unRowsAffected > 0 || BInTransaction(); +} + +//-------------------------------------------------------------------------------------------------------------------------------- +// CSQLUpdateOrInsert +//-------------------------------------------------------------------------------------------------------------------------------- + +CSQLUpdateOrInsert::CSQLUpdateOrInsert( const char* pszName, int nTable, const CColumnSet & whereColumns, const CColumnSet & updateColumns, const char* pszWhereClause, const char* pszUpdateClause ) +{ + const CRecordInfo* pRecordInfo = GSchemaFull().GetSchema( nTable ).GetRecordInfo(); + + //how many columns do we have + const int nNumColumns = pRecordInfo->GetNumColumns(); + + TSQLCmdStr sStatement; + sStatement = "MERGE INTO "; + sStatement.Append( GSchemaFull().GetDefaultSchemaNameForCatalog( pRecordInfo->GetESchemaCatalog() ) ); + sStatement.Append( '.' ); + sStatement.Append( pRecordInfo->GetName() ); + sStatement.Append( " WITH(HOLDLOCK) AS D USING(VALUES(" ); + sStatement.AppendFormat( "%.*s", GetInsertArgStringChars( nNumColumns ), GetInsertArgString() ); + sStatement.Append( "))AS S(" ); + + //add each column that we are adding the values for, along with the parameter from the structure + for( int nCurrColumn = 0; nCurrColumn < nNumColumns; nCurrColumn++ ) + { + const CColumnInfo& colInfo = pRecordInfo->GetColumnInfo( nCurrColumn ); + if( nCurrColumn != 0 ) + sStatement.Append( ',' ); + sStatement.Append( colInfo.GetName() ); + } + + //our where clause + sStatement.Append( ")ON " ); + + if( pszWhereClause ) + { + sStatement.Append( pszWhereClause ); + } + else + { + FOR_EACH_COLUMN_IN_SET( whereColumns, nCurrColumn ) + { + const char* pszColName = pRecordInfo->GetColumnInfo( whereColumns.GetColumn( nCurrColumn ) ).GetName(); + if( nCurrColumn > 0 ) + sStatement.Append( " AND " ); + sStatement.AppendFormat( "D.%s=S.%s", pszColName, pszColName ); + } + } + + //our update clause (if they have provided fields that they want to update) + if( pszUpdateClause || !updateColumns.IsEmpty() ) + { + sStatement.Append( " WHEN MATCHED THEN UPDATE SET " ); + if( pszUpdateClause ) + { + sStatement.Append( pszUpdateClause ); + } + else + { + FOR_EACH_COLUMN_IN_SET( updateColumns, nCurrColumn ) + { + const char* pszColName = pRecordInfo->GetColumnInfo( updateColumns.GetColumn( nCurrColumn ) ).GetName(); + if( nCurrColumn > 0 ) + sStatement.Append( ',' ); + sStatement.AppendFormat( "%s=S.%s", pszColName, pszColName ); + } + } + } + + //our insert clause + sStatement.Append( " WHEN NOT MATCHED THEN INSERT(" ); + bool bFirstColumn = true; + for( int nCurrColumn = 0; nCurrColumn < nNumColumns; nCurrColumn++ ) + { + const CColumnInfo& colInfo = pRecordInfo->GetColumnInfo( nCurrColumn ); + if( !colInfo.BIsInsertable() ) + continue; + + if( !bFirstColumn ) + sStatement.Append( ',' ); + bFirstColumn = false; + sStatement.Append( colInfo.GetName() ); + } + + sStatement.Append( ")VALUES(" ); + bFirstColumn = true; + for( int nCurrColumn = 0; nCurrColumn < nNumColumns; nCurrColumn++ ) + { + const CColumnInfo& colInfo = pRecordInfo->GetColumnInfo( nCurrColumn ); + if( !colInfo.BIsInsertable() ) + continue; + + if( !bFirstColumn ) + sStatement.Append( ',' ); + bFirstColumn = false; + sStatement.AppendFormat( "S.%s", colInfo.GetName() ); + } + sStatement.Append( ");" ); + + //save our results so we can execute it in the future + m_nTable = nTable; + m_sName = pszName; + m_sQuery = sStatement; +} + +bool CSQLUpdateOrInsert::BYieldingExecute( CSQLAccess& sqlAccess, const CRecordBase& record, uint32 *out_punRowsAffected /* = NULL */ ) const +{ + AssertMsg2( record.GetITable() == m_nTable, "Error: Merge was compiled for table %s, but was attempted to be executed against %s", GSchemaFull().GetSchema( m_nTable ).GetRecordInfo()->GetName(), record.GetPRecordInfo()->GetName() ); + + const CRecordInfo* pRecordInfo = record.GetPRecordInfo(); + //how many columns do we have + const int nNumColumns = pRecordInfo->GetNumColumns(); + + sqlAccess.ClearParams(); + for( int nCurrColumn = 0; nCurrColumn < nNumColumns; nCurrColumn++ ) + { + const CColumnInfo& colInfo = pRecordInfo->GetColumnInfo( nCurrColumn ); + uint8 *pubData; + uint32 cubData; + DbgVerify( record.BGetField( nCurrColumn, &pubData, &cubData ) ); + sqlAccess.AddBindParamRaw( colInfo.GetType(), pubData, cubData ); + } + + return sqlAccess.BYieldingExecute( m_sName, m_sQuery, out_punRowsAffected ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds bind parameters to the list based on a set of fields in a record +// Input: record - record to insert +// columnSet - The set of columns to add as params +//----------------------------------------------------------------------------- +void CSQLAccess::AddRecordParameters( const CRecordBase &record, const CColumnSet & columnSet ) +{ + Assert( record.GetPSchema()->GetRecordInfo() == columnSet.GetRecordInfo() ); + if ( record.GetPSchema()->GetRecordInfo() != columnSet.GetRecordInfo() ) + return; + + FOR_EACH_COLUMN_IN_SET( columnSet, nColumnIndex ) + { + const CColumnInfo &columnInfo = columnSet.GetColumnInfo( nColumnIndex ); + uint8 *pubData; + uint32 cubData; + DbgVerify( record.BGetField( columnSet.GetColumn( nColumnIndex ), &pubData, &cubData ) ); + EGCSQLType eType = columnInfo.GetType(); + CurrentQuery()->AddBindParamRaw( eType, pubData, cubData ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: Deletes all records from a table +// Input: iTable - table to wipe +// Output: true if the operation was successful +// Note: PERFORMANCE WARNING: this is slow on big tables, not intended for use +// in production +//----------------------------------------------------------------------------- +bool CSQLAccess::BYieldingWipeTable( int iTable ) +{ + // make a wipe operation + CRecordInfo *pRecordInfo = GSchemaFull().GetSchema( iTable ).GetRecordInfo(); + + CUtlString buf; + buf.Format( "DELETE FROM %s", pRecordInfo->GetName() ); + return BYieldingExecute( buf.String(), buf.String() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns the current query to add stuff to, creating it if there isn't +// already a current query +//----------------------------------------------------------------------------- +CGCSQLQuery *CSQLAccess::CurrentQuery() +{ + if( m_pCurrentQuery ) + return m_pCurrentQuery; + + m_pCurrentQuery = new CGCSQLQuery(); + return m_pCurrentQuery; +} + + +} // namespace GCSDK diff --git a/gcsdk/sqlaccess/sqlrecord.cpp b/gcsdk/sqlaccess/sqlrecord.cpp new file mode 100644 index 0000000..5b285b4 --- /dev/null +++ b/gcsdk/sqlaccess/sqlrecord.cpp @@ -0,0 +1,538 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//============================================================================= + + +#include "stdafx.h" + +//#include "sqlaccess.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CSQLRecord::CSQLRecord( uint32 unRow, IGCSQLResultSet *pResultSet ) +{ + Init( unRow, pResultSet ); +} +CSQLRecord::CSQLRecord() +: m_pResultSet( NULL ), m_unRow( 0 ) +{ +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CSQLRecord::~CSQLRecord() +{ +} + + +//----------------------------------------------------------------------------- +// Purpose: Initializes a blank record +// Input: iTable - table that this record will belong to +//----------------------------------------------------------------------------- +void CSQLRecord::Init( uint32 unRow, IGCSQLResultSet *pResultSet ) +{ + if( unRow >= pResultSet->GetRowCount() ) + { + m_pResultSet = NULL; + m_unRow = 0; + } + else + { + m_pResultSet = pResultSet; + m_unRow = unRow; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets data for a field in this record +// Input: unColumn - field to get +// pubField - pointer to get filled in with pointer to data +// cubField - pointer to get filled in with size of data +// Output: true if successful, false if data not present +//----------------------------------------------------------------------------- +bool CSQLRecord::BGetColumnData( uint32 unColumn, uint8 **ppubField, int *pcubField ) +{ + size_t sz; + bool bRet = BGetColumnData( unColumn, ppubField, &sz ); + *pcubField = static_cast< int >( sz ); + return bRet; +} + +//----------------------------------------------------------------------------- +// Purpose: Gets data for a field in this record +// Input: unColumn - field to get +// pubField - pointer to get filled in with pointer to data +// cubField - pointer to get filled in with size of data +// Output: true if successful, false if data not present +//----------------------------------------------------------------------------- +bool CSQLRecord::BGetColumnData( uint32 unColumn, uint8 **ppubField, size_t *pcubField ) +{ + Assert( ppubField ); + Assert( pcubField ); + *ppubField = NULL; + *pcubField = 0; + + Assert( m_pResultSet ); + + if ( !BValidateColumnIndex( unColumn ) ) + return false; + + *pcubField = 0; + return m_pResultSet->GetData( m_unRow, unColumn, ppubField, (uint32*)pcubField ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets string data for a field in this record +// Input: unColumn - field to get +// ppchVal - pointer to pointer to fill in to string data +// Output: true if successful, false if data not present or not of correct type +//----------------------------------------------------------------------------- +bool CSQLRecord::BGetStringValue( uint32 unColumn, const char **ppchVal ) +{ + Assert( ppchVal ); + *ppchVal = NULL; + + uint8 *pubData = NULL; + int cubData = 0; + Assert( k_EGCSQLType_String == m_pResultSet->GetColumnType( unColumn ) ); + bool bRet = BGetColumnData( unColumn, &pubData, &cubData ); + if ( bRet ) + *ppchVal = (const char *) pubData; + + return bRet; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets string data for a field in this record +// Input: unColumn - field to get +// ppchVal - pointer to pointer to fill in to string data +// Output: true if successful, false if data not present or not of correct type +//----------------------------------------------------------------------------- +bool CSQLRecord::BGetStringValue( uint32 unColumn, CFmtStr1024 *psVal ) +{ + Assert( psVal ); + *psVal = ""; + + uint8 *pubData = NULL; + int cubData = 0; + Assert( k_EGCSQLType_String == m_pResultSet->GetColumnType( unColumn ) ); + bool bRet = BGetColumnData( unColumn, &pubData, &cubData ); + if ( bRet ) + *psVal = (const char *) pubData; + + return bRet; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets int data for a field in this record +// Input: unColumn - field to get +// pnVal - pointer to fill in with data +// Output: true if successful, false if data not present or not of correct type +//----------------------------------------------------------------------------- +bool CSQLRecord::BGetIntValue( uint32 unColumn, int *pnVal ) +{ + Assert( pnVal ); + *pnVal = 0; + + uint8 *pubData = NULL; + int cubData = 0; + bool bRet = BGetColumnData( unColumn, &pubData, &cubData ); + if ( bRet ) + { + switch( m_pResultSet->GetColumnType( unColumn ) ) + { + case k_EGCSQLType_int64: + { + int64 ul = *((int64 *)pubData); + if ( ul >= LONG_MIN && ul <= LONG_MAX ) + { + *pnVal = (int)ul; + return true; + } + else + { + AssertMsg1(false, "GetIntValue tried to catch %lld in an int, which is too small", ul ); + return false; + } + } + break; + + case k_EGCSQLType_int32: + *pnVal = *((int32 *)pubData); + return true; + + case k_EGCSQLType_int16: + *pnVal = *((int16 *)pubData); + return true; + + case k_EGCSQLType_int8: + *pnVal = *((int8 *)pubData); + return true; + + default: + AssertMsg1(false, "GetIntValue tried to catch a %s, which is the wrong type", PchNameFromEGCSQLType( m_pResultSet->GetColumnType( unColumn ) ) ); + return false; + } + } + + return bRet; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets float data for a field in this record +// Input: unColumn - field to get +// pnVal - pointer to fill in with data +// Output: true if successful, false if data not present or not of correct type +//----------------------------------------------------------------------------- +bool CSQLRecord::BGetFloatValue( uint32 unColumn, float *pfVal ) +{ + Assert( pfVal ); + *pfVal = 0.0f; + + uint8 *pubData = NULL; + int cubData = 0; + bool bRet = BGetColumnData( unColumn, &pubData, &cubData ); + if ( bRet ) + { + Assert( k_EGCSQLType_float == m_pResultSet->GetColumnType( unColumn ) ); + AssertMsg2( sizeof( float ) == cubData, "GetValue expected %llu bytes, found %d", (uint64)sizeof( float ), cubData ); + if ( sizeof( float ) != cubData ) + return false; + *pfVal = *( (float *) pubData ); + } + + return bRet; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets double data for a field in this record +// Input: unColumn - field to get +// pnVal - pointer to fill in with data +// Output: true if successful, false if data not present or not of correct type +//----------------------------------------------------------------------------- +bool CSQLRecord::BGetDoubleValue( uint32 unColumn, double *pdVal ) +{ + Assert( pdVal ); + *pdVal = 0.0f; + + uint8 *pubData = NULL; + int cubData = 0; + bool bRet = BGetColumnData( unColumn, &pubData, &cubData ); + if ( bRet ) + { + Assert( k_EGCSQLType_double == m_pResultSet->GetColumnType( unColumn ) ); + AssertMsg2( sizeof( double ) == cubData, "GetValue expected %llu bytes, found %d", (uint64)sizeof( double ), cubData ); + if ( sizeof( double ) != cubData ) + return false; + *pdVal = *( (double *) pubData ); + } + + return bRet; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets int data for a field in this record +// Input: unColumn - field to get +// pVal - pointer to fill in with data +// Output: true if successful, false if data not present or not of correct type +//----------------------------------------------------------------------------- +bool CSQLRecord::BGetByteValue( uint32 unColumn, byte *pVal ) +{ + Assert( pVal ); + *pVal = 0; + + uint8 *pubData = NULL; + int cubData = 0; + bool bRet = BGetColumnData( unColumn, &pubData, &cubData ); + if ( bRet ) + { + Assert( k_EGCSQLType_int8 == m_pResultSet->GetColumnType( unColumn ) ); + AssertMsg1( 1 == cubData, "GetValue expected 1 bytes, found %d", cubData ); + if ( 1 != cubData ) + return false; + *pVal = *( (byte *) pubData ); + } + + return bRet; +} + +//----------------------------------------------------------------------------- +// Purpose: Gets int data for a field in this record +// Input: unColumn - field to get +// pVal - pointer to fill in with data +// Output: true if successful, false if data not present or not of correct type +//----------------------------------------------------------------------------- +bool CSQLRecord::BGetBoolValue( uint32 unColumn, bool *pVal ) +{ + int32 b; + if ( !BGetIntValue( unColumn, &b ) ) + return false; + + // convert to boolean + *pVal = ( b != 0 ); + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets int16 data for a field in this record +// Input: unColumn - field to get +// pnVal - pointer to fill in with data +// Output: true if successful, false if data not present or not of correct type +//----------------------------------------------------------------------------- +bool CSQLRecord::BGetInt16Value( uint32 unColumn, int16 *pnVal ) +{ + Assert( pnVal ); + *pnVal = 0; + + uint8 *pubData = NULL; + int cubData = 0; + bool bRet = BGetColumnData( unColumn, &pubData, &cubData ); + if ( bRet ) + { + Assert( k_EGCSQLType_int16 == m_pResultSet->GetColumnType( unColumn ) ); + AssertMsg1( 2 == cubData, "GetValue expected 2 bytes, found %d", cubData ); + if ( 2 != cubData ) + return false; + *pnVal = *( (int16 *) pubData ); + } + return bRet; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets int64 data for a field in this record +// Input: unColumn - field to get +// puVal - pointer to fill in with data +// Output: true if successful, false if data not present or not of correct type +//----------------------------------------------------------------------------- +bool CSQLRecord::BGetInt64Value( uint32 unColumn, int64 *puVal ) +{ + Assert( puVal ); + *puVal = 0; + + uint8 *pubData = NULL; + int cubData = 0; + bool bRet = BGetColumnData( unColumn, &pubData, &cubData ); + if ( bRet ) + { + Assert( k_EGCSQLType_int64 == m_pResultSet->GetColumnType( unColumn ) ); + AssertMsg1( 8 == cubData, "GetValue expected 8 bytes, found %d", cubData ); + if ( 8 != cubData ) + return false; + *puVal = *( (int64 *) pubData ); + } + + return bRet; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets uint64 data for a field in this record +// Input: unColumn - field to get +// puVal - pointer to fill in with data +// Output: true if successful, false if data not present or not of correct type +//----------------------------------------------------------------------------- +bool CSQLRecord::BGetUint64Value( uint32 unColumn, uint64 *puVal ) +{ + Assert( puVal ); + *puVal = 0; + + uint8 *pubData = NULL; + int cubData = 0; + bool bRet = BGetColumnData( unColumn, &pubData, &cubData ); + if ( bRet ) + { + Assert( k_EGCSQLType_int64 == m_pResultSet->GetColumnType( unColumn ) ); + AssertMsg1( 8 == cubData, "GetValue expected 8 bytes, found %d", cubData ); + if ( 8 != cubData ) + return false; + *puVal = *( (uint64 *) pubData ); + } + + return bRet; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets uint32 data for a field in this record +// Input: unColumn - field to get +// puVal - pointer to fill in with data +// Output: true if successful, false if data not present or not of correct type +//----------------------------------------------------------------------------- +bool CSQLRecord::BGetUint32Value( uint32 unColumn, uint32 *puVal ) +{ + Assert( puVal ); + *puVal = 0; + + uint8 *pubData = NULL; + int cubData = 0; + bool bRet = BGetColumnData( unColumn, &pubData, &cubData ); + if ( bRet ) + { + Assert( k_EGCSQLType_int32 == m_pResultSet->GetColumnType( unColumn ) ); + AssertMsg1( 4 == cubData, "GetValue expected 4 bytes, found %d", cubData ); + if ( 4 != cubData ) + return false; + *puVal = *( (uint32 *) pubData ); + } + + return bRet; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets uint16 data for a field in this record +// Input: unColumn - field to get +// puVal - pointer to fill in with data +// Output: true if successful, false if data not present or not of correct type +//----------------------------------------------------------------------------- +bool CSQLRecord::BGetUint16Value( uint32 unColumn, uint16 *puVal ) +{ + Assert( puVal ); + *puVal = 0; + + uint8 *pubData = NULL; + int cubData = 0; + bool bRet = BGetColumnData( unColumn, &pubData, &cubData ); + if ( bRet ) + { + Assert( k_EGCSQLType_int16 == m_pResultSet->GetColumnType( unColumn ) ); + AssertMsg1( 2 == cubData, "GetValue expected 2 bytes, found %d", cubData ); + if ( 2 != cubData ) + return false; + *puVal = *( (uint16 *) pubData ); + } + + return bRet; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets uint8 data for a field in this record +// Input: unColumn - field to get +// puVal - pointer to fill in with data +// Output: true if successful, false if data not present or not of correct type +//----------------------------------------------------------------------------- +bool CSQLRecord::BGetUint8Value( uint32 unColumn, uint8 *puVal ) +{ + Assert( puVal ); + *puVal = 0; + + uint8 *pubData = NULL; + int cubData = 0; + bool bRet = BGetColumnData( unColumn, &pubData, &cubData ); + if ( bRet ) + { + Assert( k_EGCSQLType_int8 == m_pResultSet->GetColumnType( unColumn ) ); + AssertMsg1( 1 == cubData, "GetValue expected 1 byte, found %d", cubData ); + if ( 1 != cubData ) + return false; + *puVal = *( (uint8 *) pubData ); + } + + return bRet; +} + + +//----------------------------------------------------------------------------- +// Purpose: Validates column index +// Input: unColumn - field to validate +// Output: true if valid, false otherwise +//----------------------------------------------------------------------------- +bool CSQLRecord::BValidateColumnIndex( uint32 unColumn ) +{ + if ( unColumn >= m_pResultSet->GetColumnCount() ) + { + AssertMsg2( false, "CSQLRecord::BValidateColumnIndex: invalid column index %d. # columns: %d", unColumn, + m_pResultSet->GetColumnCount() ); + return false; + } + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Advances the CSQLRecord to the next row +// Output: returns false if this call would advance the record past the last row. +// And makes the record invalid. +//----------------------------------------------------------------------------- +bool CSQLRecord::NextRow() +{ + Assert( m_pResultSet ); + m_unRow++; + + if( m_unRow >= m_pResultSet->GetRowCount() ) + m_pResultSet = NULL; + return IsValid(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Render a field to a buffer +// Input: unColumn - field to render +// cchBuffer - size of render buffer +// pchBuffer - buffer to render into +//----------------------------------------------------------------------------- +void CSQLRecord::RenderField( uint32 unColumn, int cchBuffer, char *pchBuffer ) +{ + Q_strncpy( pchBuffer, "", cchBuffer ); + + uint8 *pubData; + int cubData; + if ( !BGetColumnData( unColumn, &pubData, &cubData ) ) + return; + + // Get the column info and figure out how to interpret the data + ConvertFieldToText( m_pResultSet->GetColumnType( unColumn ), pubData, cubData, pchBuffer, cchBuffer, false ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Copies a CSQLRecord to CRecordBase +//----------------------------------------------------------------------------- +bool CSQLRecord::BWriteToRecord( CRecordBase *pRecord, const CColumnSet & csWriteFields ) +{ + bool bSuccess = true; + FOR_EACH_COLUMN_IN_SET( csWriteFields, unSQLColumn ) + { + uint32 unRecordColumn = csWriteFields.GetColumn( unSQLColumn ); + + uint8 *pubData; + size_t cubData; + if( !BGetColumnData( unSQLColumn, &pubData, &cubData ) ) + { + bSuccess = false; + } + else + { + bSuccess = pRecord->BSetField( unRecordColumn, pubData, cubData ) && bSuccess; + } + + } + return bSuccess; +} + + + +} // namespace GCSDK diff --git a/gcsdk/sqlaccess/sqlutil.cpp b/gcsdk/sqlaccess/sqlutil.cpp new file mode 100644 index 0000000..6744d21 --- /dev/null +++ b/gcsdk/sqlaccess/sqlutil.cpp @@ -0,0 +1,918 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//============================================================================= + + +#include "stdafx.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ +const char *GetInsertArgString() +{ + static char s_str[1024]; + static bool s_bInit = false; + + if ( !s_bInit ) + { + for ( int i = 0; i < 1023; i++ ) + { + s_str[i] = i % 2 == 0 ? '?' : ','; + } + + s_str[1023] = NULL; + s_bInit = true; + } + + return s_str; +} + +uint32 GetInsertArgStringChars( uint32 nNumParams ) +{ + AssertMsg( nNumParams <= GetInsertArgStringMaxParams(), "Error: Requested more characters than are provided by the GetInsertArgString" ); + if( nNumParams == 0 ) + return 0; + + return nNumParams * 2 - 1; +} + +uint32 GetInsertArgStringMaxParams() +{ + return 512; +} + +//----------------------------------------------------------------------------- +// Purpose: Converts array of field data to text for SQL IN clause +// Input: columnInfo - schema of column being converted +// pubData - pointer to array of data to convert +// cubData - size of array of data +// rgchResult - pointer to output buffer +// cubResultLen - size of output buffer +// bForPreparedStatement - Should we prepare the text for a prepared statement or directly place the values? +//----------------------------------------------------------------------------- +void ConvertFieldArrayToInText( const CColumnInfo &columnInfo, byte *pubData, int cubData, char *rgchResult, int cubResultLen, bool bForPreparedStatement ) +{ + int32 cubLength = columnInfo.GetFixedSize(); + Assert( cubData % cubLength == 0 ); + int32 nArrayCount = cubData / cubLength; + + int32 len = 0; + rgchResult[len++] = '('; + for( int i = 0; i < nArrayCount; ++i ) + { + if ( bForPreparedStatement ) + { + if ( i < nArrayCount - 1 ) + { + rgchResult[len++] = '?'; + rgchResult[len++] = ','; + } + else + { + rgchResult[len++] = '?'; + rgchResult[len++] = ')'; + } + } + else + { + switch ( columnInfo.GetType() ) + { + case k_EGCSQLType_int8: + if ( i < nArrayCount - 1 ) + len += Q_snprintf( rgchResult + len, cubResultLen - len, "%d,", *( (byte *) pubData ) ); + else + len += Q_snprintf( rgchResult + len, cubResultLen - len, "%d)", *( (byte *) pubData ) ); + break; + case k_EGCSQLType_int16: + if ( i < nArrayCount - 1 ) + len += Q_snprintf( rgchResult + len, cubResultLen - len, "%d,", *( (short *) pubData ) ); + else + len += Q_snprintf( rgchResult + len, cubResultLen - len, "%d)", *( (short *) pubData ) ); + break; + case k_EGCSQLType_int32: + if ( i < nArrayCount - 1 ) + len += Q_snprintf( rgchResult + len, cubResultLen - len, "%d,", *( (int *) pubData ) ); + else + len += Q_snprintf( rgchResult + len, cubResultLen - len, "%d)", *( (int *) pubData ) ); + break; + case k_EGCSQLType_int64: + if ( i < nArrayCount - 1 ) + len += Q_snprintf( rgchResult + len, cubResultLen - len, "%lld,", *( (int64 *) pubData ) ); + else + len += Q_snprintf( rgchResult + len, cubResultLen - len, "%lld)", *( (int64 *) pubData ) ); + break; + default: + AssertMsg( false, "Unsupported data type for non prepares statement with IN clause\n" ); + rgchResult[0] = 0; + return; + } + } + + if( len >= cubResultLen - 1 ) + { + AssertMsg( false, "Generation of IN clause foverflowed\n" ); + rgchResult[0] = 0; + return; + } + pubData += cubLength; + } + + rgchResult[len] = 0; + return; +} + + +//----------------------------------------------------------------------------- +// Purpose: Converts field data to text equivalent for SQL statement +// Input: eFieldType - The type of the field to convert to text +// pubRecord - pointer to record data to convert +// cubRecord - size of record data +// rgchField - pointer to output buffer +// cchField - size of output buffer +//----------------------------------------------------------------------------- +void ConvertFieldToText( EGCSQLType eFieldType, uint8 *pubRecord, int cubRecord, char *rgchField, int cchField, bool bQuoteString ) +{ + char rgchTmp[k_cMedBuff]; + + switch ( eFieldType ) + { + case k_EGCSQLType_int8: + Q_snprintf( rgchField, cchField, "%d", *( (byte *) pubRecord ) ); + break; + case k_EGCSQLType_int16: + Q_snprintf( rgchField, cchField, "%d", *( (short *) pubRecord ) ); + break; + case k_EGCSQLType_int32: + Q_snprintf( rgchField, cchField, "%d", *( (int *) pubRecord ) ); + break; + case k_EGCSQLType_int64: + Q_snprintf( rgchField, cchField, "%lld", *( (int64 *) pubRecord ) ); + break; + case k_EGCSQLType_float: + Q_snprintf( rgchField, cchField, "%f", *((float*) pubRecord) ); + break; + case k_EGCSQLType_double: + Q_snprintf( rgchField, cchField, "%f", *((double*) pubRecord) ); + break; + case k_EGCSQLType_String: + if ( pubRecord && *pubRecord ) + { + Assert( cubRecord + 1 < Q_ARRAYSIZE( rgchTmp ) ); + + Q_memcpy( rgchTmp, (char *) pubRecord, cubRecord ); + rgchTmp[cubRecord] = 0; + + if ( bQuoteString ) + { + EscapeStringValue( rgchTmp, Q_ARRAYSIZE( rgchTmp ) ); + Q_snprintf( rgchField, cchField, "'%s'", rgchTmp ); + } + else + { + Q_strncpy( rgchField, rgchTmp, cchField ); + } + } + else + { + if ( bQuoteString ) + { + Q_strncpy( rgchField, "''", cchField ); + } + else + { + Q_strncpy( rgchField, "", cchField ); + } + } + break; + case k_EGCSQLType_Blob: + case k_EGCSQLType_Image: + Q_strncpy( rgchField, "0x", cchField ); + Q_binarytohex( pubRecord, cubRecord, rgchField + 2, cchField - 2 ); + break; + default: + Assert( false ); + break; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns the text SQL type for a given field +// Input: field - field to determine type for +// pchBuf - pointer to output buffer +// cchBuf - size of output buffer +// Output: returns pchBuf for convenience of one-line usage +//----------------------------------------------------------------------------- +char *SQLTypeFromField( const CColumnInfo &colInfo, char *pchBuf, int cchBuf ) +{ + EGCSQLType eType = colInfo.GetType(); + *pchBuf = 0; + switch ( eType ) + { + case k_EGCSQLType_int8: + Q_strncpy( pchBuf, "TINYINT", cchBuf ); + break; + case k_EGCSQLType_int16: + Q_strncpy( pchBuf, "SMALLINT", cchBuf ); + break; + case k_EGCSQLType_int32: + Q_strncpy( pchBuf, "INT", cchBuf ); + break; + case k_EGCSQLType_int64: + Q_strncpy( pchBuf, "BIGINT", cchBuf ); + break; + case k_EGCSQLType_float: + Q_strncpy( pchBuf, "REAL", cchBuf ); + break; + case k_EGCSQLType_double: + Q_strncpy( pchBuf, "FLOAT", cchBuf ); + break; + case k_EGCSQLType_String: + Q_snprintf( pchBuf, cchBuf, "VARCHAR(%d)", colInfo.GetMaxSize() ); + break; + case k_EGCSQLType_Blob: + Q_snprintf( pchBuf, cchBuf, "VARBINARY(%d)", colInfo.GetMaxSize() ); + break; + case k_EGCSQLType_Image: + Q_strncpy( pchBuf, "IMAGE", cchBuf ); + break; + default: + Assert( false ); + break; + } + + return pchBuf; +} + + + +//----------------------------------------------------------------------------- +// Purpose: Escapes any single quotes to a string value to double single quotes +// Input: rgchField - text to escape +// cchField - size of text buffer +// Notes: The text will be escaped and expanded in place in the buffer. +// In the worst case, the text may expand by 2x. (If the field is all +// single quotes.) So, you must pass in a buffer which is at least +// twice as long as the text length so we can guarantee to be able to +// escape the string. +//----------------------------------------------------------------------------- +void EscapeStringValue( char *rgchField, int cchField ) +{ + // TODO - what else do we need to escape? %() ... + char *pubCur = rgchField; + int nLen = 0; + int cSingleQuotes = 0; + + // This function gets called on every text field we write but most text fields + // don't need to be escaped, so try to be as fast as possible in the normal case. + + // first, walk through the string and count the string length and number of single quotes + while ( *pubCur ) + { + if ( '\'' == *pubCur ) + cSingleQuotes++; + nLen ++; + pubCur++; + } + + // if no single quotes, nothing to do + if ( !cSingleQuotes ) + return; + + // caller must pass in a buffer that's long enough for expansion + Assert( nLen + cSingleQuotes + 1 <= cchField ); + if ( !( nLen + cSingleQuotes + 1 <= cchField ) ) + return; + + // We know exactly how many characters the string will expand by (the # of single quotes). Walk backward + // and copy the characters into the right places. This touches each character only once. + pubCur = rgchField + nLen + cSingleQuotes; + *pubCur = 0; + pubCur--; + while ( pubCur > rgchField && cSingleQuotes > 0 ) + { + // read pointer is offset from write pointer by # of remaining single quotes + char *pubRead = pubCur - cSingleQuotes; + Assert( pubRead >= rgchField ); + // copy each character + *pubCur = *pubRead; + if ( '\'' == *pubRead ) + { + // if the character is a single quote, back up one more and insert another single quote to escape it + pubCur --; + *pubCur = '\''; + // decrement # of single quotes remaining + cSingleQuotes --; + Assert( cSingleQuotes >= 0 ); + } + pubCur--; + } +} +//----------------------------------------------------------------------------- +// Purpose: Adds constraint information to a SQL command to add or remove constraint +// Input: pchTableName - name of table +// pchColumnName - name of column +// nColFlagConstraint - flag with which constraint to +// bForAdd - whether constraint is being added or removed +// pchCmd - buffer to append SQL command to +// cchCmd - size of buffer +//----------------------------------------------------------------------------- +void AppendConstraint( const char *pchTableName, const char *pchColumnName, int nColFlagConstraint, bool bForAdd, + bool bClustered, CFmtStrMax & sCmd, int nFillFactor ) +{ + Assert( pchTableName && pchTableName[0] ); + Assert( pchColumnName && pchColumnName[0] ); + + switch ( nColFlagConstraint ) + { + case k_nColFlagPrimaryKey: + sCmd.AppendFormat( " CONSTRAINT %s_%s_PrimaryKey", pchTableName, pchColumnName); + if ( bForAdd ) + { + sCmd += " PRIMARY KEY "; + if ( bClustered ) + { + sCmd.AppendFormat( " CLUSTERED WITH (FILLFACTOR = %d) ", nFillFactor ); + } + else + { + sCmd += "NONCLUSTERED"; + } + } + break; + case k_nColFlagUnique: + /* do nothing - the uniqueness will be handled by creation of an index */ + break; + case k_nColFlagAutoIncrement: + sCmd += " IDENTITY"; + break; + default: + AssertMsg( false, "CSQLThread::AppendContraint: invalid constraint type" ); + break; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds constraint information to a SQL command to add or remove constraint +// Input: pRecordInfo - record info describing table +// pColumnInfo - record info describing column +// bForAdd - whether constraint is being added or removed +// pchCmd - buffer to append SQL command to +// cchCmd - size of buffer +//----------------------------------------------------------------------------- +void AppendConstraints( const CRecordInfo *pRecordInfo, const CColumnInfo *pColumnInfo, bool bForAdd, CFmtStrMax & sCmd ) +{ + Assert( pRecordInfo != NULL ); + Assert( pColumnInfo != NULL ); + + if ( pColumnInfo->BIsPrimaryKey() ) + { + // any column in a PK can't be NULL. + if ( bForAdd ) + { + sCmd += " NOT NULL"; + } + + // only add primary key constraint here if it is a single-column PK + if ( pRecordInfo->GetPrimaryKeyType() == k_EPrimaryKeyTypeSingle ) + { + // get the fields on the primary key + const CUtlVector< FieldSet_t > &refFields = pRecordInfo->GetIndexFields( ); + int nFillFactor = refFields.Element( pRecordInfo->GetPKIndex() ).GetFillFactor(); + AppendConstraint( pRecordInfo->GetName(), pColumnInfo->GetName(), k_nColFlagPrimaryKey, bForAdd, pColumnInfo->BIsClustered(), sCmd, nFillFactor ); + } + } + else if ( pColumnInfo->BIsUnique() ) + { + AppendConstraint( pRecordInfo->GetName(), pColumnInfo->GetName(), k_nColFlagUnique, bForAdd, pColumnInfo->BIsClustered(), sCmd, 0 ); + } + + if ( pColumnInfo->BIsAutoIncrement() ) + { + AppendConstraint( pRecordInfo->GetName(), pColumnInfo->GetName(), k_nColFlagAutoIncrement, bForAdd, pColumnInfo->BIsClustered(), sCmd, 0 ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: Generates the "CONSTRAINT ..." text for the table primary key +//----------------------------------------------------------------------------- +void BuildTablePKConstraintText( TSQLCmdStr *psStatement, CRecordInfo *pRecordInfo ) +{ + const FieldSet_t& vecFields = pRecordInfo->GetPKFields( ); + + psStatement->sprintf( "CONSTRAINT %s_PrimaryKey PRIMARY KEY %s ( ", + pRecordInfo->GetName(), + vecFields.IsClustered() ? "CLUSTERED" : "NONCLUSTERED" ); + + for ( int nField = 0; nField < vecFields.GetCount(); nField++ ) + { + // what field is the next column in our index? + int nThisField = vecFields.GetField( nField ); + const CColumnInfo& columnInfo = pRecordInfo->GetColumnInfo(nThisField); + + if (nField != 0) + { + *psStatement += ", "; + } + *psStatement += columnInfo.GetName(); + } + + // close our list + *psStatement += ") "; + + if ( vecFields.GetFillFactor() != 0 ) + { + // non-default fill factor, so specify it + psStatement->AppendFormat( " WITH FILLFACTOR = %d ", + vecFields.GetFillFactor() ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: Adds constraint information to a SQL command to add or remove table-level constraints +// Input: pRecordInfo - record info describing table +// pchCmd - buffer to append SQL command to +// cchCmd - size of buffer +//----------------------------------------------------------------------------- + +void AppendTableConstraints( CRecordInfo *pRecordInfo, CFmtStrMax & sCmd ) +{ + // the only supported table constraint is for PKs or FKs + if ( pRecordInfo->GetPrimaryKeyType() == k_EPrimaryKeyTypeMulti ) + { + TSQLCmdStr tmp; + BuildTablePKConstraintText( &tmp, pRecordInfo ); + sCmd += ", "; + sCmd += tmp; + } + + // Look for FKs required on this table + // the only supported table constraint is for PKs or FKs + int cFKs = pRecordInfo->GetFKCount(); + for( int i=0; i < cFKs; ++i ) + { + FKData_t &fkData = pRecordInfo->GetFKData( i ); + + CFmtStr sColumns, sParentColumns; + FOR_EACH_VEC( fkData.m_VecColumnRelations, nCol ) + { + FKColumnRelation_t &colRelation = fkData.m_VecColumnRelations[nCol]; + if ( nCol > 0) + { + sColumns += ","; + sParentColumns += ","; + } + sColumns += colRelation.m_rgchCol; + sParentColumns += colRelation.m_rgchParentCol; + } + + TSQLCmdStr sTmp; + sTmp.sprintf( ", CONSTRAINT %s FOREIGN KEY (%s) REFERENCES %s(%s) ON DELETE %s ON UPDATE %s", + fkData.m_rgchName, sColumns.Access(), fkData.m_rgchParentTableName, sParentColumns.Access(), + PchNameFromEForeignKeyAction( fkData.m_eOnDeleteAction ), PchNameFromEForeignKeyAction( fkData.m_eOnUpdateAction ) ); + + // add to the command + sCmd += sTmp; + } +} + + + + +//----------------------------------------------------------------------------- +// Purpose: Builds a SQL INSERT statement +// Input: psStatement - The string to put the statement into +// pRecordInfo - record info describing table inserting into +//----------------------------------------------------------------------------- +void BuildInsertStatementText( TSQLCmdStr *psStatement, const CRecordInfo *pRecordInfo ) +{ + psStatement->sprintf("INSERT INTO %s.%s (", GSchemaFull().GetDefaultSchemaNameForCatalog( pRecordInfo->GetESchemaCatalog() ), pRecordInfo->GetName() ); + + // build a string of the field names + int cColumns = pRecordInfo->GetNumColumns(); + int nInsertable = 0; + bool bAddedBefore = false; + for ( int iColumn = 0; iColumn < cColumns; iColumn++ ) + { + const CColumnInfo &columnInfo = pRecordInfo->GetColumnInfo( iColumn ); + if ( !columnInfo.BIsInsertable() ) + continue; + + nInsertable++; + + if ( bAddedBefore ) + psStatement->Append( ',' ); + bAddedBefore = true; + psStatement->Append( columnInfo.GetName() ); + } + + psStatement->AppendFormat( ") VALUES (%.*s)", GetInsertArgStringChars( nInsertable ), GetInsertArgString() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Builds a SQL INSERT statement +// IMPORTANT NOTE - This Insert statement will use the Microsoft SQL Server +// specific clause 'OUTPUT Inserted.ColumnName' +// The result of that will be that the SQL statement will return to us +// the columns that could not be specified by the Insert. +// At the time of writing, that is primarily AutoIncrement columns, +// however in theory we should be able to recover any computed column +// from SQL server, with the caveats specified at : +// http://msdn.microsoft.com/en-us/library/ms177564.aspx +// +// Input: psStatement - The output statement string +// pRecordInfo - record info describing table inserting into +//----------------------------------------------------------------------------- + +void BuildInsertAndReadStatementText( TSQLCmdStr *psStatement, CUtlVector<int> *pvecOutputFields, const CRecordInfo *pRecordInfo ) +{ + psStatement->sprintf("INSERT INTO %s.%s (", GSchemaFull().GetDefaultSchemaNameForCatalog( pRecordInfo->GetESchemaCatalog() ), pRecordInfo->GetName() ); + + // build a string of the field names + int nInsertable = 0; + int cColumns = pRecordInfo->GetNumColumns(); + bool bAddedBefore = false; + for ( int iColumn = 0; iColumn < cColumns; iColumn++ ) + { + const CColumnInfo &columnInfo = pRecordInfo->GetColumnInfo( iColumn ); + if ( !columnInfo.BIsInsertable() ) + continue; + + nInsertable++; + + if ( bAddedBefore ) + psStatement->Append( ',' ); + bAddedBefore = true; + psStatement->Append( columnInfo.GetName() ); + } + + bAddedBefore = false ; + int nOutputColumn = 0; + for( int iColumn = 0; iColumn < cColumns; iColumn++ ) + { + const CColumnInfo &columnInfo = pRecordInfo->GetColumnInfo( iColumn ) ; + + // + // If we can't Insert it - we want SQL Server to tell us what value was stored + // in the column !! + // + if( !columnInfo.BIsInsertable() ) + { + if( bAddedBefore ) + psStatement->Append( ", INSERTED." ); + else + psStatement->Append( ") OUTPUT INSERTED." ); + bAddedBefore = true ; + psStatement->Append( columnInfo.GetName() ); + pvecOutputFields->AddToTail( iColumn ); + nOutputColumn++; + } + } + + // add field values to SQL statement + psStatement->AppendFormat( " VALUES (%.*s)", GetInsertArgStringChars( nInsertable ), GetInsertArgString() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Builds a SQL MERGE statement update or insert using in-flight values table +// Input: psStatement - The string to put the statement into +// pRecordInfo - record info describing table inserting into +//----------------------------------------------------------------------------- +void BuildMergeStatementTextOnPKWhenMatchedUpdateWhenNotMatchedInsert( TSQLCmdStr *psStatement, const CRecordInfo *pRecordInfo ) +{ + psStatement->sprintf( "MERGE INTO %s.%s WITH( HOLDLOCK, ROWLOCK ) T USING ( VALUES (%.*s) ) AS S(", + GSchemaFull().GetDefaultSchemaNameForCatalog( pRecordInfo->GetESchemaCatalog() ), pRecordInfo->GetName(), + GetInsertArgStringChars( pRecordInfo->GetNumColumns() ), GetInsertArgString() ); + + { + int cColumns = pRecordInfo->GetNumColumns(); + for ( int iColumn = 0; iColumn < cColumns; iColumn++ ) + { + const CColumnInfo &columnInfo = pRecordInfo->GetColumnInfo( iColumn ); + if ( iColumn ) + psStatement->Append( ',' ); + psStatement->Append( columnInfo.GetName() ); + } + } + + psStatement->Append( ") ON " ); + + // build a string of the PK columns + const FieldSet_t &fsPK = pRecordInfo->GetIndexFields()[pRecordInfo->GetPKIndex()]; + { + int cColumns = fsPK.GetCount(); + for ( int iColumn = 0; iColumn < cColumns; iColumn++ ) + { + const CColumnInfo &columnInfo = pRecordInfo->GetColumnInfo( fsPK.GetField( iColumn ) ); + if ( iColumn ) + psStatement->Append( " AND " ); + psStatement->Append( "T." ); + psStatement->Append( columnInfo.GetName() ); + psStatement->Append( "=S." ); + psStatement->Append( columnInfo.GetName() ); + } + } + + psStatement->Append( " WHEN MATCHED THEN UPDATE SET " ); + + // build the update string + { + int cColumns = pRecordInfo->GetNumColumns(); + bool bAddedBefore = false; + for ( int iColumn = 0; iColumn < cColumns; iColumn++ ) + { + bool bThisColumnIsPartOfPK = false; + for ( int ipkCheck = 0; ipkCheck < fsPK.GetCount(); ++ipkCheck ) + { + if ( iColumn == fsPK.GetField( ipkCheck ) ) + { + bThisColumnIsPartOfPK = true; + break; + } + } + if ( bThisColumnIsPartOfPK ) + continue; + + const CColumnInfo &columnInfo = pRecordInfo->GetColumnInfo( iColumn ); + if ( bAddedBefore ) + psStatement->Append( ',' ); + bAddedBefore = true; + psStatement->Append( columnInfo.GetName() ); + psStatement->Append( "=S." ); + psStatement->Append( columnInfo.GetName() ); + } + } + + psStatement->Append( " WHEN NOT MATCHED BY TARGET THEN INSERT (" ); + + // build a string of the field names + { + int cColumns = pRecordInfo->GetNumColumns(); + bool bAddedBefore = false; + for ( int iColumn = 0; iColumn < cColumns; iColumn++ ) + { + const CColumnInfo &columnInfo = pRecordInfo->GetColumnInfo( iColumn ); + if ( !columnInfo.BIsInsertable() ) + continue; + + if ( bAddedBefore ) + psStatement->Append( ',' ); + bAddedBefore = true; + psStatement->Append( columnInfo.GetName() ); + } + } + + psStatement->Append( ") VALUES (" ); + { + int cColumns = pRecordInfo->GetNumColumns(); + bool bAddedBefore = false; + for ( int iColumn = 0; iColumn < cColumns; iColumn++ ) + { + const CColumnInfo &columnInfo = pRecordInfo->GetColumnInfo( iColumn ); + if ( !columnInfo.BIsInsertable() ) + continue; + + if ( bAddedBefore ) + psStatement->Append( ',' ); + bAddedBefore = true; + psStatement->Append( "S." ); + psStatement->Append( columnInfo.GetName() ); + } + } + psStatement->Append( ");" ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Builds a SQL MERGE statement using CTE_MergeParams as supplied table holding rows +// Input: psStatement - The string to put the statement into +// pRecordInfo - record info describing table inserting into +//----------------------------------------------------------------------------- +void BuildMergeStatementTextOnPKWhenNotMatchedInsert( TSQLCmdStr *psStatement, const CRecordInfo *pRecordInfo ) +{ + psStatement->sprintf( "MERGE INTO %s.%s WITH( HOLDLOCK, ROWLOCK ) T USING ( VALUES (%.*s) ) AS S(", + GSchemaFull().GetDefaultSchemaNameForCatalog( pRecordInfo->GetESchemaCatalog() ), pRecordInfo->GetName(), + GetInsertArgStringChars( pRecordInfo->GetNumColumns() ), GetInsertArgString() ); + + { + int cColumns = pRecordInfo->GetNumColumns(); + for ( int iColumn = 0; iColumn < cColumns; iColumn++ ) + { + const CColumnInfo &columnInfo = pRecordInfo->GetColumnInfo( iColumn ); + if ( iColumn ) + psStatement->Append( ',' ); + psStatement->Append( columnInfo.GetName() ); + } + } + + psStatement->Append( ") ON " ); + + // build a string of the PK columns + const FieldSet_t &fsPK = pRecordInfo->GetIndexFields()[pRecordInfo->GetPKIndex()]; + { + int cColumns = fsPK.GetCount(); + for ( int iColumn = 0; iColumn < cColumns; iColumn++ ) + { + const CColumnInfo &columnInfo = pRecordInfo->GetColumnInfo( fsPK.GetField( iColumn ) ); + if ( iColumn ) + psStatement->Append( " AND " ); + psStatement->Append( "T." ); + psStatement->Append( columnInfo.GetName() ); + psStatement->Append( "=S." ); + psStatement->Append( columnInfo.GetName() ); + } + } + + psStatement->Append( " WHEN NOT MATCHED BY TARGET THEN INSERT (" ); + + // build a string of the field names + { + int cColumns = pRecordInfo->GetNumColumns(); + bool bAddedBefore = false; + for ( int iColumn = 0; iColumn < cColumns; iColumn++ ) + { + const CColumnInfo &columnInfo = pRecordInfo->GetColumnInfo( iColumn ); + if ( !columnInfo.BIsInsertable() ) + continue; + + if ( bAddedBefore ) + psStatement->Append( ',' ); + bAddedBefore = true; + psStatement->Append( columnInfo.GetName() ); + } + } + + psStatement->Append( ") VALUES (" ); + { + int cColumns = pRecordInfo->GetNumColumns(); + bool bAddedBefore = false; + for ( int iColumn = 0; iColumn < cColumns; iColumn++ ) + { + const CColumnInfo &columnInfo = pRecordInfo->GetColumnInfo( iColumn ); + if ( !columnInfo.BIsInsertable() ) + continue; + + if ( bAddedBefore ) + psStatement->Append( ',' ); + bAddedBefore = true; + psStatement->Append( "S." ); + psStatement->Append( columnInfo.GetName() ); + } + } + psStatement->Append( ");" ); +} + +void BuildSelectStatementText( TSQLCmdStr *psStatement, const CColumnSet & selectSet, const char *pchTopClause ) +{ + *psStatement = "SELECT "; + + if( pchTopClause ) + { + psStatement->Append( pchTopClause ); + psStatement->Append( ' ' ); + } + + // build a string of the field names + bool bAddedBefore = false; + FOR_EACH_COLUMN_IN_SET( selectSet, nColumnIndex ) + { + const CColumnInfo &columnInfo = selectSet.GetColumnInfo( nColumnIndex ); + if ( bAddedBefore ) + psStatement->Append( ',' ); + bAddedBefore = true; + psStatement->Append( columnInfo.GetName() ); + } + + psStatement->Append( " FROM "); + psStatement->Append( GSchemaFull().GetDefaultSchemaNameForCatalog( selectSet.GetRecordInfo()->GetESchemaCatalog() ) ); + psStatement->Append( '.' ); + psStatement->Append( selectSet.GetRecordInfo()->GetName() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Builds a SQL UPDATE statement +// Input: pRecordInfo - record info describing table inserting into +// bForPreparedStatement - if true, inserts values as '?' for later +// binding. If false, values are inserted in text. +// pchStatement - pointer to buffer to build statement in +// cchStatement - size of buffer +// pSQLRecord - pointer to record with data to update +// iColumnMatch - column to use for WHERE condition +// pvMatch - data value to use for WHERE condition +// cubMatch - size of pvMatch data +// rgiColumnUpdate - array of column #'s to update +// ciColumnUpdate - count of column #'s to update +//----------------------------------------------------------------------------- +void BuildUpdateStatementText( TSQLCmdStr *psStatement, const CColumnSet & updateColumns ) +{ + // build the UPDATE statement + psStatement->sprintf( "UPDATE %s.%s SET ", GSchemaFull().GetDefaultSchemaNameForCatalog( updateColumns.GetRecordInfo()->GetESchemaCatalog() ), updateColumns.GetRecordInfo()->GetName() ); + + // add each field we're updating to the UPDATE statement + FOR_EACH_COLUMN_IN_SET( updateColumns, nColumnIndex ) + { + const CColumnInfo &columnInfo = updateColumns.GetColumnInfo( nColumnIndex ); + + if( nColumnIndex > 0 ) + psStatement->Append( ',' ); + psStatement->Append( columnInfo.GetName() ); + psStatement->Append( "=?" ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Builds a SQL UPDATE statement +//----------------------------------------------------------------------------- +void BuildDeleteStatementText( TSQLCmdStr *psStatement, const CRecordInfo *pRecordInfo ) +{ + psStatement->sprintf( "DELETE FROM %s.%s", GSchemaFull().GetDefaultSchemaNameForCatalog( pRecordInfo->GetESchemaCatalog() ), pRecordInfo->GetName() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Builds a where clause for the provided fields +//----------------------------------------------------------------------------- +void AppendWhereClauseText( TSQLCmdStr *psClause, const CColumnSet & columnSet ) +{ + // add each field we're updating to the UPDATE statement + FOR_EACH_COLUMN_IN_SET( columnSet, nColumnIndex ) + { + const CColumnInfo &columnInfo = columnSet.GetColumnInfo( nColumnIndex ); + + if( nColumnIndex > 0 ) + psClause->Append( " AND "); + psClause->Append( columnInfo.GetName() ); + psClause->Append( "=?" ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: Builds an OUTPUT [fields] INTO [table] for the provided fields/data +//----------------------------------------------------------------------------- +void BuildOutputClauseText( TSQLCmdStr *psClause, const CColumnSet & columnSet ) +{ + *psClause = " OUTPUT "; + + FOR_EACH_COLUMN_IN_SET( columnSet, nColumnIndex ) + { + const CColumnInfo &columnInfo = columnSet.GetColumnInfo( nColumnIndex ); + + if( nColumnIndex > 0 ) + psClause->Append( ", "); + + psClause->Append( " ? AS " ); + psClause->Append( columnInfo.GetName() ); + } + + psClause->Append( " INTO " ); + psClause->Append( columnSet.GetRecordInfo()->GetName() ); +} + +////----------------------------------------------------------------------------- +//// Purpose: our own special "upsert" into a column with a uniqueness constraint +////----------------------------------------------------------------------------- +//EResult UpdateOrInsertUnique( CSQLAccess &sqlAccess, int iTable, int iField, CRecordBase *pRecordBase, int iIndexID ) +//{ +// // attempt an update - if it fails due to duplicate primary key, they can't use this +// // url (it's taken) - if it succeeds but affects 0 rows, they didn't have a vanity url +// // and we need to do an insert (which could again fail due to primary key constraints) +// int cRecordsUpdated = 0; +// bool bRet = sqlAccess.BYieldingUpdateFieldFromRecordWithIndex( iTable, &cRecordsUpdated, iField, pRecordBase, iIndexID ); +// if ( !bRet ) +// { +// // ODBC is the suck - give me Spring JDBC templates, please. +// if ( sqlAccess.GetLastError()->IsDuplicateInsertAttempt() ) +// { +// return k_EResultDuplicateName; +// } +// return k_EResultFail; +// } +// else if ( 0 == cRecordsUpdated ) +// { +// // the user didn't have an entry, so insert one. +// bRet = sqlAccess.BYieldingInsertRecord( iTable, pRecordBase ); +// if ( !bRet ) +// { +// // ODBC is the suck - give me Spring JDBC templates, please. +// if ( sqlAccess.GetLastError()->IsDuplicateInsertAttempt() ) +// { +// return k_EResultDuplicateName; +// } +// return k_EResultFail; +// } +// } +// return k_EResultOK; +//} +// + +} // namespace GCSDK diff --git a/gcsdk/stackstring.h b/gcsdk/stackstring.h new file mode 100644 index 0000000..1d6e474 --- /dev/null +++ b/gcsdk/stackstring.h @@ -0,0 +1,107 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +// +// A string class that keeps all its memory on the stack and performs no +// memory allocation +//=============================================================================// + +#ifndef STACKSTRING_H +#define STACKSTRING_H + +#ifdef _WIN32 +#pragma once +#endif + + +template< size_t cubSize > +class CStackString +{ +public: + class CStackString() { Reset(); } + class CStackString( const char *pchRHS ) { Set( pchRHS ); } + + void Reset() { m_rchBuffer[0] = 0; m_pchEnd = m_rchBuffer; } + void Set( const char *pchValue ); + const char *Get() const { return m_rchBuffer; } + static size_t GetCubSize() { return cubSize; } + size_t Length() const { return m_pchEnd - &m_rchBuffer[0]; } + + CStackString & operator=( const char *pchValue ) { Set( pchValue ); return *this; } + CStackString & operator+=( const char *pchValue ) { Append( pchValue ); return *this; } + operator const char*() const { return m_rchBuffer; } + + void Format( PRINTF_FORMAT_STRING const char *pchFormat, ... ); + void AppendFormat( PRINTF_FORMAT_STRING const char *pchFormat, ... ); + void AppendFormatV( const char *pchFormat, va_list marker ); + void Append( const char *pchValue ); + + void SetIndent( uint32 unCount, char chIndent = '\t' ); +private: + char m_rchBuffer[ cubSize ]; + char *m_pchEnd; +}; + +typedef CStackString<256> CStackString256; +typedef CStackString<512> CStackString512; +typedef CStackString<1024> CStackString1024; +typedef CStackString<8192> CStackStringMax; + +template< size_t cubSize > +void CStackString<cubSize>::Set( const char *pchValue ) +{ + V_strcpy_safe( m_rchBuffer, pchValue ); + m_pchEnd = &m_rchBuffer[Q_strlen(m_rchBuffer)]; +} + +template< size_t cubSize > +void CStackString<cubSize>::AppendFormatV( const char *pchFormat, va_list marker ) +{ + size_t cubPrinted = Q_vsnprintf( m_pchEnd, GetCubSize() - Length(), pchFormat, marker ); + m_pchEnd += cubPrinted; +} + + +template< size_t cubSize > +void CStackString<cubSize>::Format( PRINTF_FORMAT_STRING const char *pchFormat, ... ) +{ + va_list marker; + va_start( marker, pchFormat ); + size_t cubPrinted = Q_vsnprintf( m_rchBuffer, GetCubSize(), pchFormat, marker ); + m_pchEnd = m_rchBuffer + cubPrinted; + va_end( marker ); +} + + +template< size_t cubSize > +void CStackString<cubSize>::AppendFormat( PRINTF_FORMAT_STRING const char *pchFormat, ... ) +{ + va_list marker; + va_start( marker, pchFormat ); + AppendFormatV( pchFormat, marker ); + va_end( marker ); +} + +template< size_t cubSize > +void CStackString<cubSize>::Append( const char *pchValue ) +{ + Q_strcat( m_rchBuffer, pchValue, GetCubSize() ); + m_pchEnd = m_rchBuffer + Q_strlen(m_rchBuffer); +} + +template< size_t cubSize > +void CStackString<cubSize>::SetIndent( uint32 unCount, char chIndent ) +{ + Reset(); + for ( uint32 x = 0; x < unCount && x < (GetCubSize()-1); x++ ) + { + *m_pchEnd = chIndent; + m_pchEnd++; + } + *m_pchEnd = '\0'; +} + + +#endif diff --git a/gcsdk/stdafx.cpp b/gcsdk/stdafx.cpp new file mode 100644 index 0000000..02b21e4 --- /dev/null +++ b/gcsdk/stdafx.cpp @@ -0,0 +1,9 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// stdafx.cpp : source file that includes just the standard includes +// stdafx.pch will be the pre-compiled header +// stdafx.obj will contain the pre-compiled type information + +#include "stdafx.h" + +// TODO: reference any additional headers you need in STDAFX.H +// and not in this file diff --git a/gcsdk/stdafx.h b/gcsdk/stdafx.h new file mode 100644 index 0000000..ab651db --- /dev/null +++ b/gcsdk/stdafx.h @@ -0,0 +1,14 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// stdafx.h : include file for standard system include files, +// or project specific include files that are used frequently, but +// are changed infrequently +// + +#include "gcsdk/gcsdk_auto.h" + +#include "tier1/kvpacker.h" + +using GCSDK::SPEW_SYSTEM_MISC; + +#include "tier0/memdbgon.h" + diff --git a/gcsdk/steamextra/clientenums.h b/gcsdk/steamextra/clientenums.h new file mode 100644 index 0000000..a2888d8 --- /dev/null +++ b/gcsdk/steamextra/clientenums.h @@ -0,0 +1,611 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//============================================================================= + +#ifndef CLIENTENUMS_H +#define CLIENTENUMS_H +#ifdef _WIN32 +#pragma once +#endif + + +enum ELogonState +{ + k_ELogonStateNotLoggedOn = 0, + k_ELogonStateLoggingOn = 1, + k_ELogonStateLoggingOff = 2, + k_ELogonStateLoggedOn = 3 +}; + + +// Enums for all personal questions supported by the system. +enum EPersonalQuestion +{ + // Never ever change these after initial release. + k_EPSMsgNameOfSchool = 0, // Question: What is the name of your school? + k_EPSMsgFavoriteTeam = 1, // Question: What is your favorite team? + k_EPSMsgMothersName = 2, // Question: What is your mother's maiden name? + k_EPSMsgNameOfPet = 3, // Question: What is the name of your pet? + k_EPSMsgChildhoodHero = 4, // Question: Who was your childhood hero? + k_EPSMsgCityBornIn = 5, // Question: What city were you born in? + + k_EPSMaxPersonalQuestion +}; + + +// account flags (stored in DB) +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASEd +enum EAccountFlags +{ + m_EAccountFlagNormalUser = 0, // Standard user level (yes, this is meant to be zero) + k_EAccountFlagPersonaNameSet = ( 1 << 0 ), // true if the user has set the persona name they really want, instead of the auto-generated one + k_EAccountFlagUnbannable = ( 1 << 1 ), // whatever happens, this account can't be banned + k_EAccountFlagPasswordSet = ( 1 << 2 ), // we've set the password at least once on this account + k_EAccountFlagSupport = ( 1 << 3 ), // Enables use of web support tool + k_EAccountFlagAdmin = ( 1 << 4 ), // The name says it all, can do everything + k_EAccountFlagSupervisor = ( 1 << 5 ), // support supervisory role + k_EAccountFlagAppEditor = ( 1 << 6 ), // Can edit app info + k_EAccountFlagHWIDSet = ( 1 << 7 ), // Set HWID once + k_EAccountFlagPersonalQASet = ( 1 << 8 ), // user has personal Question & anser set + k_EAccountFlagVacBeta = ( 1 << 9 ), // user participates in VAC beta tests + k_EAccountFlagDebug = ( 1 << 10 ), // user is in debug mode, eg VAC doesn't kick etc + k_EAccountFlagDisabled = ( 1 << 11 ), // account is disabled. + k_EAccountFlagLimitedUser = ( 1 << 12 ), // account is limited user account because it doesnt own anything + k_EAccountFlagLimitedUserForce = ( 1 << 13 ), // account is limited user account because we forced it to be + k_EAccountFlagEmailValidated = ( 1 << 14 ), // user has verified email address via WG + k_EAccountFlagMarketingTreatment = ( 1 << 15), // account is flagged as being in a treatment for marketing/sales experiments +}; + +// profile state (stored in DB) +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +enum ECommunityProfileState +{ + k_ECommunityProfileNotCreated = 0, // user hasn't setup community account yet + k_ECommunityProfileActive = 1, // user joined community, site is public + k_ECommunityProfilePrivate = 2, // user joined community, site is private + k_ECommunityProfileLocked = 3, // user got locked, content can't be changed but is still accessible + k_ECommunityProfileDisabled = 4, // user got disabled, site not accessible anymore +}; + + +// profile privacy option setting (stored in DB) +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +enum ECommunityPrivacyState +{ + k_ECommunityPrivacyInvalid = 0, + k_ECommunityPrivacyPrivate = 1, // ain't nobody can see it + k_ECommunityPrivacyFriendsOnly = 2, // only your friends can see it + k_ECommunityPrivacyPublic = 3, // anybody could see it +}; + +enum ECommunityVisibilityState +{ + k_ECommunityVisibilityPrivate = 1, // private, requester see only public fields + k_ECommunityVisibilityFriendsOnly = 2, // friends only, requester sees only public fields + k_ECommunityVisibilityOpen = 3, // it is visible to requester; they are owner or friend or public + k_ECommunityVisibilitySupportPrivate = 4, // was private, but it's a support account asking + k_ECommunityVisibilitySupportFriendsOnly = 5,// was friends only, but it's a support account asking +}; + +enum ECommentPermission +{ + k_ECommentPermissionFriendsOnly = 0, // only friends can leave a comment + k_ECommentPermissionAnyone = 1, // anybody can leave a comment + k_ECommentPermissionSelfOnly = 2, // only the account owner can leave a comment +}; + +// Payment methods for purchases - BIT FLAGS so can be used to indicate +// acceptable payment methods for packages +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +enum EPaymentMethod +{ + k_EPaymentMethodNone = 0x000, // user got the license for free + k_EPaymentMethodActivationCode = 0x001, // user paid by entering unused CD-Key or other activation code + k_EPaymentMethodCreditCard = 0x002, // user paid with credit card + k_EPaymentMethodPayPal = 0x004, // user paid with via paypal + k_EPaymentMethodGuestPass = 0x008, // user paid by redeeming a guest pass + k_EPaymentMethodHardwarePromo = 0x010, // user presented machine credentials + k_EPaymentMethodClickAndBuy = 0x020, // ClickandBuy + k_EPaymentMethodAutoGrant = 0x040, // server side purchased package, things like German specific TF2 free weekend + k_EPaymentMethodWallet = 0x080, // user paid with wallet + k_EPaymentMethodOEMTicket = 0x100, // user paid by redeeming a OEM license ticket + k_EPaymentMethodSplit = 0x200, // user paid with wallet AND a provider +}; + +// Sources for WalletTxn records +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +enum EWalletSource +{ + k_EWalletSourceInvalid = 0, + k_EWalletSourcePurchase = 1, // Created from a purchase, refund, chargeback, or reverse chargeback (PurchaseRefGID -> TransID) + k_EWalletSourceGuestPass = 2, // Created from a guest pass (PurchaseRefGID -> GuestPassID) + k_EWalletSourceConversion = 3, // Created from a wallet conversion (PurchaseRefGID -> GID shared between debit & credit records) + k_EWalletSourceRebate = 4, // Created from a rebate (PurchaseRefGID -> TransID) +}; + +// License types +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +enum ELicenseType +{ + k_ENoLicense = 0, // for shipped goods + k_ESinglePurchase = 1, // single purchase + k_ESinglePurchaseLimitedUse = 2, // single purchase w/ expiration + k_ERecurringCharge = 3, // recurring subscription + k_ERecurringChargeLimitedUse = 4, // recurring subscription w/ limited minutes per period + k_ERecurringChargeLimitedUseWithOverages = 5, // like above but w/ soft limit and overage charges +}; + +// Flags for licenses - BITS +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +enum ELicenseFlags +{ + k_ELicenseFlagNone = 0x00, // just a place holder + k_ELicenseFlagRenew = 0x01, // Renew this license next period + k_ELicenseFlagRenewalFailed = 0x02, // Auto-renew failed + k_ELicenseFlagPending = 0x04, // Purchase or renewal is pending + k_ELicenseFlagExpired = 0x08, // Set if no longer active (whatever the reason) + k_ELicenseFlagCancelledByUser = 0x10, // Cancelled by the user + k_ELicenseFlagCancelledByAdmin = 0x20, // Cancelled by customer support + k_ELicenseFlagLowViolenceContent = 0x40,// license is for low violence content +}; + +// Status of a package +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +enum EPackageStatus +{ + k_EPackageAvailable = 0, // Available for purchase and use + k_EPackagePreorder = 1, // Available for purchase, as a pre-order + k_EPackageUnavailable = 2, // Not available for new purchases, may still be owned + k_EPackageInvalid = 3, // Either an unknown package or a deleted one that nobody should own +}; + +// Purchase status +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +enum EPurchaseStatus +{ + k_EPurchasePending = 0, // purchase is pending, valid but pending subscription + k_EPurchaseSucceeded = 1, // purchase successful, valid subscription + k_EPurchaseFailed = 2, // purchase failed, no subscription + k_EPurchaseRefunded = 3, // we refunded the purchase and removed subscription + k_EPurchaseInit = 4, // user started purchase + k_EPurchaseChargedback = 5, // the user issued a chargeback, we removed subscription + k_EPurchaseRevoked = 6, // we revoked the purchase and removed subscription. Usually stolen CD-Keys + k_EPurchaseInDispute = 7, // the purchase is being disputed by the user, preliminary to a chargeback +}; + +// LineItemTypes +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASEd +enum ELineItemType +{ + k_ELineItemTypeInvalid = 0, // Unknown - load all purchase line items + k_ELineItemTypeMicroTxn = ( 1 << 0 ), // Transaction has data in MicroTxnLineItem table + k_ELineItemTypeWallet = ( 1 << 1 ), // Transaction has data in WalletLineItem table + k_ELineItemTypePkg = ( 1 << 2 ), // Transaction has data in PurchaseLineItem table +}; + +// Enum for the types of news push items you can get +enum ENewsUpdateType +{ + k_EAppNews = 0, // news about a particular app + k_ESteamAds = 1, // Marketing messages + k_ESteamNews = 2, // EJ's corner and the like + k_ECDDBUpdate = 3, // backend has a new CDDB for you to load + k_EClientUpdate = 4, // new version of the steam client is available +}; + +// Detailed purchase result codes for the client +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +enum EPurchaseResultDetail +{ + k_EPurchaseResultNoDetail = 0, + k_EPurchaseResultAVSFailure = 1, + k_EPurchaseResultInsufficientFunds = 2, + k_EPurchaseResultContactSupport = 3, + k_EPurchaseResultTimeout = 4, + + k_EPurchaseResultInvalidPackage = 5, + k_EPurchaseResultInvalidPaymentMethod = 6, + k_EPurchaseResultInvalidData = 7, + k_EPurchaseResultOthersInProgress = 8, + k_EPurchaseResultAlreadyPurchased = 9, + k_EPurchaseResultWrongPrice = 10, + k_EPurchaseResultFraudCheckFailed = 11, + k_EPurchaseResultCancelledByUser = 12, + k_EPurchaseResultRestrictedCountry = 13, + k_EPurchaseResultBadActivationCode = 14, // this code gives no receipt + k_EPurchaseResultDuplicateActivationCode = 15, + + k_EPurchaseResultUseOtherPaymentMethod = 16, // User should try a different payment method + k_EPurchaseResultUseOtherFundingSource = 17, // Select a different funding source (paypal) + k_EPurchaseResultInvalidShippingAddress = 18, // Shipping address is invalid (paypal) + k_EPurchaseResultRegionNotSupported = 19, // This region is not supported with this payment type + + k_EPurchaseResultAcctIsBlocked = 20, // Acct has been blocked by provider - user should contact provider to resolve + k_EPurchaseResultAcctNotVerified = 21, // Provider indicated account needs to be verified for transaction to complete + + k_EPurchaseResultInvalidAccount = 22, // Provider indicated the account is invalid or no longer usable + k_EPurchaseResultStoreBillingCountryMismatch = 23, // store country code & billing country code do not match + k_EPurchaseResultDoesNotOwnRequiredApp = 24, // user does not own one of the apps required for purchase + k_EPurchaseResultCanceledByNewTransaction = 25, // user made a new transaction which canceled an old, pending transaction + k_EPurchaseResultForceCanceledPending = 26, // A pending transaction was force canceled (no response from provider) + k_EPurchaseResultFailCurrencyTransProvider = 27, // selected transaction provider does not support calculated currency + k_EPurchaseResultFailedCyberCafe = 28, // cybercafe account tried to purchase or use an activation code + + k_EPurchaseResultNeedsPreApproval = 29, // Transaction needs approval from support + k_EPurchaseResultPreApprovalDenied = 30, // Transaction was denied by support + k_EPurchaseResultWalletCurrencyMismatch = 31, // Currency of purchase does not match currency of wallet +}; + +// Type of system IM. The client can use this to do special UI handling in specific circumstances +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +enum ESystemIMType +{ + k_ESystemIMRawText = 0, + k_ESystemIMInvalidCard = 1, + k_ESystemIMRecurringPurchaseFailed = 2, + k_ESystemIMCardWillExpire = 3, + k_ESystemIMSubscriptionExpired = 4, + k_ESystemIMGuestPassReceived = 5, // User has received a guest pass from a friend + k_ESystemIMGuestPassGranted = 6, // System has granted a user a guest pass to give out + k_ESystemIMGiftRevoked = 7, // We revoked a gift due to chargeback, etc + + // + k_ESystemIMTypeMax +}; + +// Ways an external cd key can be munged onto a users PC +enum ELegacyKeyRegistrationMethod +{ + eLegacyKeyRegistrationMethodNone = 0, // doesn't support legacy cd keys + eLegacyKeyRegistrationMethodRegistry, // just place it into the registry + eLegacyKeyRegistrationMethodDisk, // put it in a file on disk +}; + +// Support events, generated by system or support input +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +enum ESupportEvent +{ + // support activity + eSupportNote = 0, // a generic support note + eSupportLogin = 1, // support account logged in out, data: IP:Port + eSupportLogoff = 2, // support account logged out, text: IP:Port + eSupportTicketCreated = 3, // a support ticket was created + eSupportTicketClosed = 4, // problem was solved, ticket closed + + // account changes + eSupportEnableAccount = 5, // support enabled account, text: reason + eSupportDisableAccount = 6, // support disabled account, text: reason + eSupportChangeAccountPassword = 7, // support or user changed password, data: DONT include old password + eSupportChangeAccountEmail = 8, // support or user changed email, data: old email + eSupportChangeAccountName = 9, // support or user changed name, data: old name + eSupportChangeAccountPlayer = 10, // support or user changed player name, data: old name + + eSupportPurchaseChargedback = 11, // a charge back was issued + eSupportPurchaseRefunded = 12, // a refund was issued + eSupportPurchaseForcedCompletion = 13, // support forced a pending purchase to complete + + // license handling + eSupportLicenseAdded = 14, // support added a license, text: reason + eSupportLicenseCanceled = 15, // support or user cancel a license + eSupportLicenseChanged = 16, // support removed a license, text: reason + eSupportBannedGame = 17, // support banned game for an account + eSupportUnbannedGame = 18, // support unbanned game for an account + + // purchase activity + eSupportRunPurchase = 19, + eSupportChangedCreditCard = 20, // support updated a credit card + eSupportSetNoFraudCheckFlag = 21, // support disabled fraud check, data: reason + + // banning + eSupportBannedCreditCard = 22, // support banned a credit card + eSupportBannedIP = 23, // support banned an IP + eSupportBannedCDKey = 24, // support banned an CDKey + eSupportBannedCountry = 25, // support banned a country + eSupportBannedPayPalAccount = 26, + + eSupportPurchaseCanceled = 27, // support forced a pending purchase to cancel + + eSupportChangeAvatar = 28, + eSupportChangeProfileURL = 29, + + eSupportRegisterCDKey = 30, // support added a CD key to this account + eSupportGrantGuestPass = 31, // support granted a guest pass to this account + eSupportResubmitTransaction = 32, // support resubmitted a transaction + + eSupportResetContent = 33, // reset user content based on abuse reports + eSupportLockProfile = 34, // temp block a user from modifying community content + eSupportSetCommunityState = 35, // perm lock a user profile, can't be modified + eSupportDeleteAbuseReports = 36, // deleted abuse reports for the SteamID + eSupportSetAccountFlags = 37, // changed account flags + + eSupportChargebackStatusUpdate = 38, // support updated the pending chargeback status + + eSupportRefundForcedCompletion = 39, // support forced a pending refund to complete + eSupportRefundCanceled = 40, // support forced a pending refund to cancel + + eSupportRevokeActivationKey = 41, // support revoked and unlocked activation key + eSupportReverseChargeback = 42, // a charge back was reversed + eSupportRejectedActivationKey = 43, // activation key was rejected (invalid or already used) + + eSupportPurchaseError = 44, // Purchase error + eSupportAudited = 45, + + // items + eSupportBanItems = 46, // support banned a user's items + eSupportRestoreBannedItems = 47, // support restored banned items + eSupportRestoreDeletedItems = 48, // support restored deleted items + + // comments + eSupportDeleteComments = 49, // cleared comments on this account (not written by this account) + + eSupportDeleteItems = 50, // support deleted a user's items + eSupportDeleteCachedCard = 51, // support deleted a user's cached credit card + + eSupportConvertedWallet = 52, // user's wallet was converted + + eSupportTxnApproved = 53, // PreApproval granted + eSupportTxnDenied = 54, // PreApproval denied + + eSupportGCAction = 55, // used for all support actions from the GC +}; + + +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +enum ESupportTicket +{ + // all kinds of problems tickets that have to be handled by support + k_ETicketUnknown = 0, // an unknown problem. yay. + k_ETicketManual = 1, // a problem manually entered by support. + k_ETicketFraudRedFlag = 2, // fraud detection marked this account + k_ETicketPurchaseError = 3, // a purchase error happened + k_ETicketChargeback = 4, // a chargeback needs attention +}; + +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +enum ESupportTicketState +{ + // all kinds of problems tickets that have to be handled by support + k_ETicketStateUnknown = 0, // support issue state is unknown + k_ETicketStateUnassigned = 1, // support issue is 'open' but not assigned yet + k_ETicketStateInProcess = 2, // support issue is assigned to an support actor + k_ETicketStateResolved = 3, // support issue is fixed and closed + k_ETicketStateUnresolved = 4, // support issue couldn't be fixed. closed anyway + k_ETicketStateAutoClosed = 5, // System closed the ticket automatically +}; + + +//----------------------------------------------------------------------------- +// types of content that can be reported as abused +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +//----------------------------------------------------------------------------- +enum ECommunityContentType +{ + k_EContentUnspecified = 0, + k_EContentAll = 1, // reset all community content + k_EContentAvatarImage = 2, // clear avatar image + k_EContentProfileText = 3, // reset profile text + k_EContentWebLinks = 4, // delete web links + k_EContentAnnouncement = 5, + k_EContentEventText = 6, + k_EContentCustomCSS = 7, + k_EContentProfileURL = 8, // delete community URL ID + k_EContentComments = 9, // just comments this guy has written +}; + + +//----------------------------------------------------------------------------- +// types of reasons why a violation report was issued +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +//----------------------------------------------------------------------------- +enum EAbuseReportType +{ + k_EAbuseUnspecified = 0, + k_EAbuseInappropriate = 1, // just not ok to post + k_EAbuseProhibited = 2, // prohibited by EULA or general law + k_EAbuseSpamming = 3, // excessive spamming + k_EAbuseAdvertisement = 4, // unwanted advertisement + k_EAbuseExploit = 5, // content data attempts to exploit code issue + k_EAbuseSpoofing = 6, // user/group is impersonating an official contact + k_EAbuseLanguage = 7, // bad language + k_EAbuseAdultContent = 8, // any kind of adult material, references etc + k_EAbuseHarassment = 9, // harassment, discrimination, racism etc +}; + +//----------------------------------------------------------------------------- +// actions for a user within a clan for logging in the ClanHistory table +//----------------------------------------------------------------------------- + +enum EClanAction +{ + k_EJoined = 1, // joined the clan + k_ELeft = 2, // left the clan + k_EPromoted = 3, // promoted to officer + k_EDemoted = 4, // demoted from officer + k_EKicked = 5, // kicked off the clan + k_ECreated = 6, // clan was created + k_EInvited = 7, // invited someone + k_EEventCreated = 8, // clan event created + k_EEventUpdated = 9, // clan event updated + k_EEventDeleted = 10, // clan event deleted + k_EPermissionsChanged = 11, // clan permissions were changed + k_EAnnouncementCreated = 12, // clan announcement created + k_EAnnouncementUpdated = 13, // clan announcement updated + k_EAnnouncementDeleted = 14, // clan announcement deleted + k_EPOTWChanged = 15, // changed the POTW + k_ELinksChanged = 16, // links changed + k_EDetailsChanged = 17, // details changed + k_ESupportResetContent = 18, // support reset some or all of the clan content + k_ESupportLockedGroup = 19, // support locked this clan, it can't be modified anymore + k_ESupportUnlockedGroup = 20, // support unlocked this clan + k_ESupportChangedOwner = 21, // support transfered ownership + k_EMadePublic = 22, // made from private into public + k_EMadePrivate = 23, // made from public into private + k_ESupportDisabledGroup = 24, // support disabled this group, nobody can see it anymore + k_EKickedChat = 25, // kicked from chat + k_EBannedChat = 26, // banned from chat + k_EUnBannedChat = 27, // un-banned from chat + + k_EHighestValidAction // keep me updated, please! +}; + + +//----------------------------------------------------------------------------- +// types of events for use in the Clan Event Type table +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +//----------------------------------------------------------------------------- +enum EClanEventType +{ + k_EOtherEvent = 1, + k_EGameEvent = 2, + k_EPartyEvent = 3, + k_EMeetingEvent = 4, + k_ESpecialCauseEvent = 5, + k_EMusicAndArtsEvent = 6, + k_ESportsEvent = 7, + k_ETripEvent = 8, + k_EChatEvent = 9, + k_EGameReleaseEvent = 10, +}; + +//----------------------------------------------------------------------------- +// types of marketing messages displayed to users +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +//----------------------------------------------------------------------------- +enum EMarketingMessageType +{ + k_EMarketingMessageNowAvailable = 1, + k_EMarketingMessageWeekendDeal = 2, + k_EMarketingMessagePrePurchase = 3, + k_EMarketingMessagePlayNow = 4, + k_EMarketingMessagePreloadNow = 5, + k_EMarketingMessageGeneral = 6, + k_EMarketingMessageDemoQuit = 7, + k_EMarketingMessageGifting = 8, + k_EMarketingMessageEJsKorner = 9, +}; + +//----------------------------------------------------------------------------- +// types of associations a marketing message may have +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +//----------------------------------------------------------------------------- +enum EMarketingMessageAssociationType +{ + k_EMarketingMessageNoAssociation = 0, + k_EMarketingMessageAppAssociation = 1, + k_EMarketingMessageSubscriptionAssociation = 2, + k_EMarketingMessagePublisherAssociation = 3, + k_EMarketingMessageGenreAssociation = 4, +}; + +//----------------------------------------------------------------------------- +// Marketing message visibility +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +//----------------------------------------------------------------------------- +enum EMarketingMessageVisibility +{ + k_EMarketingMessageVisibleBeta = 1, + k_EMarketingMessageVisiblePublic = 2, +}; + +//----------------------------------------------------------------------------- +// Structures used in multiple messages +//----------------------------------------------------------------------------- +// Purchase message constants +// WARNING: Do not change these if an instance of this record may exist in a database!!! +// BUGBUG derrick - Since these also define schema, they should be moved into steamschema.h +const int k_cchCCNumMax = 16 + 1; +const int k_cchHolderNameMax = 100 + 1; +const int k_cchExpYearMax = 4 + 1; +const int k_cchExpMonthMax = 2 + 1; +const int k_cchCVV2Max = 4 + 1; +const int k_cchAddressMax = 128 + 1; +const int k_cchAddress2Max = k_cchAddressMax; +const int k_cchCityMax = 50 + 1; +const int k_cchPostcodeMax = 16 + 1; +const int k_cchStateMax = 32 + 1; +const int k_cchPhoneMax = 20 + 1; +const int k_cchEmailMax = 100 + 1; +const int k_cchCountryCodeMax = 2 + 1; +const int k_cchPayPalCheckoutTokenMax = 20 + 1; +const int k_cchStateCodeMax = 3 + 1; +const int k_cchCurrencyCodeMax = 3 + 1; + +const int k_cubMaxDfsURL = 128; // Max size for URL descriptors on DFS + + +// License information +struct LicenseInfo_t +{ + PackageId_t m_unPackageID; + RTime32 m_RTime32Created; + RTime32 m_RTime32NextProcess; + int32 m_nMinuteLimit; + int32 m_nMinutesUsed; + EPaymentMethod m_ePaymentMethod; + uint32 m_nFlags; + char m_rgchPurchaseCountryCode[k_cchCountryCodeMax]; + int32 m_nTerritoryCode; +}; + +// Supported Currency Codes +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +enum ECurrencyCode +{ + k_ECurrencyCodeInvalid = 0, + k_ECurrencyCodeUSD = 1, + k_ECurrencyCodeGBP = 2, + k_ECurrencyCodeEUR = 3, + + k_ECurrencyCodeMax = 4 +}; + +enum ETaxType +{ + k_ETaxTypeInvalid = 0, + k_ETaxTypeUSState = 1, + k_ETaxTypeVAT = 2 +}; + +// client stat list +// needs to be kept in the same order, since it's part of the protocol +enum EClientStat +{ + k_EClientStatP2PConnectionsUDP = 0, + k_EClientStatP2PConnectionsRelay = 1, + k_EClientStatP2PGameConnections = 2, + k_EClientStatP2PVoiceConnections = 3, + + k_EClientStatBytesDownloaded = 4, + + k_EClientStatMax, // must be last, used as array's of data +}; + +enum EP2PState +{ + k_EP2PStateNotConnected, + k_EP2PStateUDP, + k_EP2PStateRelay, +}; + +// User response for authentication request +enum EMicroTxnAuthResponse +{ + k_EMicroTxnAuthResponseInvalid = 0, // Invalid value + k_EMicroTxnAuthResponseAuthorize = 1, // user accepted microtransaction + k_EMicroTxnAuthResponseDeny = 2, // user denied microtransaction + k_EMicroTxnAuthResponseAutoDeny = 3, // client automatically denied microtransaction (user wasn't in game, etc.) +}; + +// Result of authorization request, returned to client +enum EMicroTxnAuthResult +{ + k_EMicroTxnAuthResultInvalid = 0, // Invalid value + k_EMicroTxnAuthResultOK = 1, // Successfully authorized + k_EMicroTxnAuthResultFail = 2, // An error occurred + k_EMicroTxnAuthResultInsufficientFunds = 3, // User has insufficient funds to complete microtransaction +}; + +#endif diff --git a/gcsdk/steamextra/gamecoordinator/igamecoordinator.h b/gcsdk/steamextra/gamecoordinator/igamecoordinator.h new file mode 100644 index 0000000..80eb7a4 --- /dev/null +++ b/gcsdk/steamextra/gamecoordinator/igamecoordinator.h @@ -0,0 +1,33 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +//============================================================================= + +#ifndef IGAMECOORDINATOR_H +#define IGAMECOORDINATOR_H +#ifdef _WIN32 +#pragma once +#endif + +typedef uint32 AppId_t; +class CSteamID; +class IGameCoordinatorHost; +class IGCSQLResultSetList; + +class IGameCoordinator +{ +public: + virtual bool BInit( AppId_t unAppID, const char *pchAppPath, IGameCoordinatorHost *pHost ) = 0; + virtual bool BMainLoopOncePerFrame( uint64 ulLimitMicroseconds ) = 0; + virtual bool BMainLoopUntilFrameCompletion( uint64 ulLimitMicroseconds ) = 0; + virtual void Shutdown() = 0; + virtual void Uninit() = 0; + virtual void MessageFromClient( const CSteamID & senderID, uint32 unMsgType, void *pubData, uint32 cubData ) = 0; + virtual void Validate( CValidator &validator, const char *pchName ) = 0; + virtual void SQLResults( GID_t gidContextID ) = 0; +}; + +#define GAMECOORDINATOR_INTERFACE_VERSION "GAMECOORDINATOR003" + +#endif // IGAMECOORDINATOR_H diff --git a/gcsdk/steamextra/gamecoordinator/igamecoordinatorhost.h b/gcsdk/steamextra/gamecoordinator/igamecoordinatorhost.h new file mode 100644 index 0000000..d94f306 --- /dev/null +++ b/gcsdk/steamextra/gamecoordinator/igamecoordinatorhost.h @@ -0,0 +1,31 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Provides an interface that the server hosting a GC must implement +// +// $NoKeywords: $ +//============================================================================= + +#ifndef IGAMECOORDINATORHOST_H +#define IGAMECOORDINATORHOST_H +#ifdef _WIN32 +#pragma once +#endif + +class CSteamID; +class IGCSQLQuery; + +class IGameCoordinatorHost +{ +public: + virtual bool BSendMessageToClient( AppId_t unAppID, const CSteamID & steamIDTarget, uint32 unMsgType, const void *pubData, uint32 cubData ) = 0; + virtual GID_t GenerateGID() = 0; + virtual void EmitMessage( const char *pchGroupName, SpewType_t spewType, int iSpewLevel, int iLevelLog, const char *pchMsg ) = 0; + virtual void SQLQuery( GID_t gidContextID, IGCSQLQuery *pQuery, int eSchemaCatalog ) = 0; + virtual void StartupComplete( bool bSuccess ) = 0; + virtual void ShutdownComplete() = 0; + virtual EUniverse GetUniverse() = 0; +}; + +#define GAMECOORDINATORHOST_INTERFACE_VERSION "GAMECOORDINATORHOST002" + +#endif // IGAMECOORDINATORHOST_H diff --git a/gcsdk/steamextra/gamecoordinator/igcsqlquery.h b/gcsdk/steamextra/gamecoordinator/igcsqlquery.h new file mode 100644 index 0000000..cdfd3ad --- /dev/null +++ b/gcsdk/steamextra/gamecoordinator/igcsqlquery.h @@ -0,0 +1,67 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//============================================================================= + +#ifndef IGCSQLQUERY_H +#define IGCSQLQUERY_H +#ifdef _WIN32 +#pragma once +#endif + + +// the type of the parameter +enum EGCSQLType +{ + k_EGCSQLTypeInvalid = -1, + + // Variable length types + k_EGCSQLType_Blob, + k_EGCSQLType_String, + + // fixed length types + k_EGCSQLType_int8, // also uint8 + k_EGCSQLType_int16, // also uint16 + k_EGCSQLType_int32, // also uint32 + k_EGCSQLType_int64, // also uint64 + k_EGCSQLType_float, + k_EGCSQLType_double, + k_EGCSQLType_Binary, // raw binary data of fixed size (i.e. a C struct). + k_EGCSQLType_Image, + k_EGCSQLType_bool, +}; + +class IGCSQLResultSetList; + +class IGCSQLQuery +{ +protected: + // call Destroy() instead of deleting this object directly + virtual ~IGCSQLQuery() {} + +public: + // returns the number of statements in the transaction + // represented by this query object + virtual uint32 GetStatementCount() = 0; + + // returns a string that represents where in the GC this + // query came from. Usually this is FILE_AND_LINE. + virtual const char *PchName() = 0; + + // get the null-terminated query string itself + virtual const char *PchCommand( uint32 unStatement ) = 0; + + // gets the parameter data + virtual uint32 CnParams( uint32 unStatement ) = 0; + virtual EGCSQLType EParamType( uint32 unStatement, uint32 uIndex ) = 0; + virtual byte *PubParam( uint32 unStatement, uint32 uIndex ) = 0; + virtual uint32 CubParam( uint32 unStatement, uint32 uIndex ) = 0; + + // reports the result + virtual void SetResults( IGCSQLResultSetList *pResults ) = 0; +}; + + +#endif // IGCSQLQUERY_H diff --git a/gcsdk/steamextra/gamecoordinator/igcsqlresultsetlist.h b/gcsdk/steamextra/gamecoordinator/igcsqlresultsetlist.h new file mode 100644 index 0000000..ced12f2 --- /dev/null +++ b/gcsdk/steamextra/gamecoordinator/igcsqlresultsetlist.h @@ -0,0 +1,62 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//============================================================================= + +#ifndef IGCSQLRESULTSETLIST_H +#define IGCSQLRESULTSETLIST_H +#ifdef _WIN32 +#pragma once +#endif + +#include "igcsqlquery.h" + +enum EGCSQLError +{ + k_EGCSQLErrorNone = 0, + k_EGCSQLErrorUnknown, + k_EGCSQLErrorBacklog, + k_EGCSQLErrorBadQueryParameters, + k_EGCSQLErrorConnectionError, + k_EGCSQLErrorDataTruncated, + k_EGCSQLErrorDeadlockLoser, + k_EGCSQLErrorDuplicateKey, + k_EGCSQLErrorGenericError, + k_EGCSQLErrorNoResultSet, + k_EGCSQLErrorSyntaxError, + k_EGCSQLErrorTableOrViewNotFound, + k_EGCSQLErrorTimeout, + k_EGCSQLErrorConstraintViolation, + k_EGCSQLErrorNumericValueOutOfRange, + k_EGCSQLErrorRollbackFailed, + k_EGCSQLErrorColumnNotFound, +}; + + +class IGCSQLResultSet +{ +public: + virtual uint32 GetColumnCount() = 0; + virtual EGCSQLType GetColumnType( uint32 nColumn ) = 0; + virtual const char *GetColumnName( uint32 nColumn ) = 0; + + virtual uint32 GetRowCount() = 0; + virtual bool GetData( uint32 unRow, uint32 unColumn, uint8 **ppData, uint32 *punSize ) = 0; +}; + + +class IGCSQLResultSetList +{ +public: + virtual EGCSQLError GetError() = 0; + virtual uint32 GetResultSetCount() = 0; + virtual IGCSQLResultSet *GetResultSet( uint32 nResultSetIndex ) = 0; + virtual uint32 GetRowsAffected( uint32 unWhichStatement ) = 0; + virtual void Destroy() = 0; + virtual const char *GetErrorText() = 0; +}; + + +#endif // IGCSQLRESULTSETLIST_H diff --git a/gcsdk/steamextra/misc.cpp b/gcsdk/steamextra/misc.cpp new file mode 100644 index 0000000..ed3facb --- /dev/null +++ b/gcsdk/steamextra/misc.cpp @@ -0,0 +1,187 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Miscellaneous code +// +//============================================================================= + +#include "stdafx.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + + +//----------------------------------------------------------------------------- +// Purpose: Tells us whether an account name looks like a VTT account name +// (used as an exception for IP-based rate limiting) +//----------------------------------------------------------------------------- +bool IsVTTAccountName( const char *szAccountName ) +{ + const static char *k_szCafe = "valvecafepc"; + + if ( 0 == Q_strncmp( szAccountName, k_szCafe, Q_strlen( k_szCafe ) ) ) + return true; + return false; +} + + +//----------------------------------------------------------------------------- +// Random number generation +// Use this system for all random number generation. It's very fast, and is +// integrated with our automated tests so that we can perform reproducibly "random" +// test cases. +//----------------------------------------------------------------------------- +uint32 g_unRandCur = 0; +int g_iunRandMask = 0; + +// k_rgunMask +// Set of masks used by our quick and dirty random number generator. +const uint32 k_rgunMask[17] = +{ + 0x1739a3b0, + 0xb8907fe1, + 0x8290d3b7, + 0x72839cd0, + 0x242df096, + 0x3829750b, + 0x38de7a77, + 0x72f0924c, + 0x44783927, + 0x01925372, + 0x20902714, + 0x27585920, + 0x27890632, + 0x82910476, + 0x72906721, + 0x28798904, + 0x78592700, +}; + + +//----------------------------------------------------------------------------- +// Purpose: Quickly generates a vaguely random number. +// Output: A vaguely random number. +//----------------------------------------------------------------------------- +uint32 UNRandFast() +{ + g_iunRandMask++; + g_unRandCur += 637429601; // Just add a large prime number (we'll wrap frequently) + return ( g_unRandCur ^ k_rgunMask[ g_iunRandMask % 17 ] ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Quickly generates a vaguely random character. +// Output: A vaguely random char in the range [32,126]. +//----------------------------------------------------------------------------- +char CHRandFast() +{ + return ( UNRandFast() % 95 ) + 32; +} + + +//----------------------------------------------------------------------------- +// Purpose: Sets the random number seed (note that we actually break this down +// into two parts: g_unRandCur and g_iunRandMask). +// Input: ulRandSeed: Value to use as our seed +//----------------------------------------------------------------------------- +void SetRandSeed( uint64 ulRandSeed ) +{ + g_unRandCur = ulRandSeed >> 32; + g_iunRandMask = ulRandSeed & 0xffffffff; +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns the current random number seed (actually a composite of +// g_unRandCur and g_iunRandMask) +// Output: Our current 64 bit random number seed. +//----------------------------------------------------------------------------- +uint64 GetRandSeed() +{ + return ( ( ((uint64)g_unRandCur) << 32 ) + g_iunRandMask ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Quickly fill a memory block with random bytes +//----------------------------------------------------------------------------- +void RandMem(void *dest, int count) +{ + unsigned char *pDest = (unsigned char *)dest; + + while ( count >= 4 ) + { + *(uint32*)(pDest) = UNRandFast(); + pDest+=4; + count-=4; + } + + while ( count > 0 ) + { + *pDest = UNRandFast(); + pDest++; + count--; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Calculates the percentage of numerator/demoninator, or 0 if +// denominator is 0. +//----------------------------------------------------------------------------- +float SafeCalcPct( uint64 ulNumerator, uint64 ulDenominator ) +{ + if ( 0 == ulDenominator ) + return 0; + return ( 100.0f * (float) ulNumerator / (float) ulDenominator ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Common code to reject an operation due to a time backlog. +// Does a gradual fade of 0% rejections at the specified threshold +// up to 100% at the limit. The gradual fade reduces system oscillations +// that could occur if you abruptly stop allowing all operations. +// Input: nBacklogCur - the current backlog (in arbitrary units) +// nBacklogThreshold - the threshold backlog at which to begin rejecting +// nBacklogLimit - hard limit at which to reject 100% of operations +// iItem - a monotonically increasing counter of items submitted. Used +// to determine which operations are allowed if there is a partial +// rejection rate. +//----------------------------------------------------------------------------- +bool BRejectDueToBacklog( int nBacklogCur, int nBacklogThreshold, int nBacklogLimit, int iItem ) +{ + bool bRefuse = false; + + if ( nBacklogCur >= nBacklogLimit ) + { + // if we're over the hard backlog limit, refuse all operations + bRefuse = true; + } + else if ( nBacklogCur >= nBacklogThreshold ) + { + // if we're near the hard backlog limit, start refusing an increasing % of operations, + // so we don't snap abruptly in and out of accepting operations and potentially cause oscillations + + // ramp from refusing 0% of operations at the backlog threshold up to 100% at the backlog hard limit + + // calculate refuse % to nearest 10%, to make it easy to mod the item # and get a good distribution + float nRefusePctDecile = 10 * (float) ( nBacklogCur - nBacklogThreshold ) / + (float) ( nBacklogLimit - nBacklogThreshold ); + Assert( nRefusePctDecile >= 0.0 ); + Assert( nRefusePctDecile <= 10.0 ); + + // compare the operations submitted count mod 10 to the refusal percent decile to decide if we should + // accept or refuse this particular operation + if ( ( iItem % 10 ) < nRefusePctDecile ) + bRefuse = true; + } + + return bRefuse; +} + + +//----------------------------------------------------------------------------- +// Purpose: Defines the head of the CDumpMemFnReg linked list +//----------------------------------------------------------------------------- +CDumpMemFnReg *CDumpMemFnReg::sm_Head = NULL;
\ No newline at end of file diff --git a/gcsdk/steamextra/misc.h b/gcsdk/steamextra/misc.h new file mode 100644 index 0000000..d1ced65 --- /dev/null +++ b/gcsdk/steamextra/misc.h @@ -0,0 +1,121 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Miscellaneous platform-specific code +// +//============================================================================= + + +#ifndef MISC_H +#define MISC_H + +// Random number utilities +uint32 UNRandFast(); +char CHRandFast(); +void SetRandSeed( uint64 ulRandSeed ); +uint64 GetRandSeed(); +void RandMem(void *dest, int count); + +bool IsVTTAccountName( const char *szAccountName ); + +float SafeCalcPct( uint64 ulNumerator, uint64 ulDenominator ); + +bool BRejectDueToBacklog( int nBacklogCur, int nBacklogThreshold, int nBacklogLimit, int iItem ); + +#define SAFE_CLOSE_HANDLE( x ) if ( INVALID_HANDLE_VALUE != ( x ) ) { CloseHandle( x ); ( x ) = INVALID_HANDLE_VALUE; } +#define SAFE_DELETE( x ) if ( NULL != ( x ) ) { delete ( x ); ( x ) = NULL; } +#define SAFE_CLOSE_HCONNECTION( x ) if ( 0 != ( x ) ) { CNet::BClose( ( x ) ); ( x ) = 0; } +#define SAFE_RELEASE( x ) if ( NULL != ( x ) ) { ( x )->Release(); x = NULL; } + +#define DECLARE_STEAM_CLASS_SIMPLE( className, baseClassName ) \ + typedef baseClassName BaseClass; \ + typedef className ThisClass; \ + +#define DECLARE_STEAM_CLASS_NOBASE( className ) \ + typedef className ThisClass; \ + +// Type for our memory output debugging. +class IDisplayMemPoolStats +{ +public: + virtual void Display( const char *pszClassName, uint64 ulClassSize, uint32 unPoolInstanceCount, uint64 ulPoolMemoryUsage, uint32 unPoolPeakInstanceCount, uint64 ulPoolPeakMemoryUsage ) = 0; +}; + +// A class for registering functions that will print memory usage information +typedef void (*DumpMemFn_t)( IDisplayMemPoolStats * ); +class CDumpMemFnReg +{ +public: + static CDumpMemFnReg *sm_Head; + + CDumpMemFnReg( DumpMemFn_t fn ) + : m_fn( fn ), + m_pNext( sm_Head ) + { + sm_Head = this; + } + + DumpMemFn_t m_fn; + CDumpMemFnReg *m_pNext; +}; + +// Helper macros for creating and using CClassMemoryPool on our frequently allocated objects +#define DECLARE_CLASS_MEMPOOL( className ) \ +private: \ + static CUtlMemoryPool sm_classMemPool; \ +public: \ + static void* operator new ( size_t nSize ); \ + static void* operator new ( size_t nSize, int nBlockUse, const char *pFileName, int nLine ); \ + static void operator delete ( void *pMem ); \ + static void DumpMemStats( IDisplayMemPoolStats *pDisplayer ); \ + + +#define IMPLEMENT_CLASS_MEMPOOL( className, initSize, growMode ) \ +CUtlMemoryPool className::sm_classMemPool( sizeof( className ), ( initSize ), ( growMode ), MEM_ALLOC_CLASSNAME( className ) ); \ + \ +void* className::operator new ( size_t nSize ) \ +{ \ + if ( nSize != sizeof( className ) ) \ + { \ + EmitError( SPEW_CONSOLE, #className"::operator new() called on wrong size! Expected %llu, Got %llu\n", (uint64)sizeof( className ), (uint64)nSize ); \ + return NULL; \ + } \ + \ + return sm_classMemPool.Alloc(); \ +} \ + \ +void* className::operator new ( size_t nSize, int nBlockUse, const char *pFileName, int nLine ) \ +{ \ + if ( nSize != sizeof( className ) ) \ + { \ + EmitError( SPEW_CONSOLE, #className"::operator new() called on wrong size! Expected %llu, Got %llu\n", (uint64)sizeof( className ), (uint64)nSize ); \ + return NULL; \ + } \ + \ + return sm_classMemPool.Alloc(); \ +} \ + \ +void className::operator delete ( void *pMem ) \ +{ \ + sm_classMemPool.Free( (className *)pMem ); \ +} \ + \ +void className::DumpMemStats( IDisplayMemPoolStats *pDisplayer ) \ +{ \ + Assert( pDisplayer ); \ + pDisplayer->Display \ + ( \ + #className, \ + sizeof( className ), \ + sm_classMemPool.Count(), \ + sm_classMemPool.Count() * sizeof( className ), \ + ( ( sm_classMemPool.PeakCount() + ( initSize ) ) / ( initSize ) ) * ( initSize ) , \ + ( ( sm_classMemPool.PeakCount() + ( initSize ) ) / ( initSize ) ) * ( initSize ) * sizeof( className ) \ + ); \ +} \ + \ +static CDumpMemFnReg s_##className##RegDumpMemory( &className::DumpMemStats ); + +// useful macro for rendering an IP +#define iptod(x) ((x)>>24&0xff), ((x)>>16&0xff), ((x)>>8&0xff), ((x)&0xff) + +#endif // MISC_H diff --git a/gcsdk/steamextra/rtime.cpp b/gcsdk/steamextra/rtime.cpp new file mode 100644 index 0000000..14081ce --- /dev/null +++ b/gcsdk/steamextra/rtime.cpp @@ -0,0 +1,1216 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Encapsulates real world (wall clock) time +// +//============================================================================= + +#include "stdafx.h" +#ifdef POSIX +#include <sys/time.h> +#else +#include "winlite.h" +#endif +#include "rtime.h" +#include <time.h> + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +#if defined( WIN32 ) || defined( _PS3 ) +// This strptime implementation is taken from the Goolge Site Map Generator project: + +// Copyright 2009 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Implement strptime under windows +static const char* kWeekFull[] = { + "Sunday", "Monday", "Tuesday", "Wednesday", + "Thursday", "Friday", "Saturday" +}; + +static const char* kWeekAbbr[] = { + "Sun", "Mon", "Tue", "Wed", + "Thu", "Fri", "Sat" +}; + +static const char* kMonthFull[] = { + "January", "February", "March", "April", "May", "June", + "July", "August", "September", "October", "November", "December" +}; + +static const char* kMonthAbbr[] = { + "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" +}; + +static const char* _parse_num(const char* s, int low, int high, int* value) { + const char* p = s; + for (*value = 0; *p != NULL && V_isdigit(*p); ++p) { + *value = (*value) * 10 + static_cast<int>(*p) - static_cast<int>('0'); + } + + if (p == s || *value < low || *value > high) return NULL; + return p; +} + +static char* _strptime(const char *s, const char *format, struct tm *tm) { + while (*format != NULL && *s != NULL) { + if (*format != '%') { + if (*s != *format) return NULL; + + ++format; + ++s; + continue; + } + + ++format; + int len = 0; + switch (*format) { + // weekday name. + case 'a': + case 'A': + tm->tm_wday = -1; + for (int i = 0; i < 7; ++i) { + len = static_cast<int>(strlen(kWeekAbbr[i])); + if (V_strnicmp(kWeekAbbr[i], s, len) == 0) { + tm->tm_wday = i; + break; + } + + len = static_cast<int>(strlen(kWeekFull[i])); + if ( V_strnicmp(kWeekFull[i], s, len) == 0) { + tm->tm_wday = i; + break; + } + } + if (tm->tm_wday == -1) return NULL; + s += len; + break; + + // month name. + case 'b': + case 'B': + case 'h': + tm->tm_mon = -1; + for (int i = 0; i < 12; ++i) { + len = static_cast<int>(strlen(kMonthAbbr[i])); + if ( V_strnicmp(kMonthAbbr[i], s, len) == 0) { + tm->tm_mon = i; + break; + } + + len = static_cast<int>(strlen(kMonthFull[i])); + if ( V_strnicmp(kMonthFull[i], s, len) == 0) { + tm->tm_mon = i; + break; + } + } + if (tm->tm_mon == -1) return NULL; + s += len; + break; + + // month [1, 12]. + case 'm': + s = _parse_num(s, 1, 12, &tm->tm_mon); + if (s == NULL) return NULL; + --tm->tm_mon; + break; + + // day [1, 31]. + case 'd': + case 'e': + s = _parse_num(s, 1, 31, &tm->tm_mday); + if (s == NULL) return NULL; + break; + + // hour [0, 23]. + case 'H': + s = _parse_num(s, 0, 23, &tm->tm_hour); + if (s == NULL) return NULL; + break; + + // minute [0, 59] + case 'M': + s = _parse_num(s, 0, 59, &tm->tm_min); + if (s == NULL) return NULL; + break; + + // seconds [0, 60]. 60 is for leap year. + case 'S': + s = _parse_num(s, 0, 60, &tm->tm_sec); + if (s == NULL) return NULL; + break; + + // year [1900, 9999]. + case 'Y': + s = _parse_num(s, 1900, 9999, &tm->tm_year); + if (s == NULL) return NULL; + tm->tm_year -= 1900; + break; + + // year [0, 99]. + case 'y': + s = _parse_num(s, 0, 99, &tm->tm_year); + if (s == NULL) return NULL; + if (tm->tm_year <= 68) { + tm->tm_year += 100; + } + break; + // arbitray whitespace. + case 't': + case 'n': + while (V_isspace(*s)) ++s; + break; + + // '%'. + case '%': + if (*s != '%') return NULL; + ++s; + break; + + // All the other format are not supported. + default: + AssertMsg( false, "Invalid format string to strptime!" ); + return NULL; + } + ++format; + } + + if (*format != NULL) { + return NULL; + } else { + return const_cast<char*>(s); + } +} + +char* strptime(const char *buf, const char *fmt, struct tm *tm) { + return _strptime(buf, fmt, tm); +} +#endif // WIN32 + + +// Our cached copy of the current time +RTime32 CRTime::sm_nTimeLastSystemTimeUpdate = 0; // initialize to large negative value to trigger immediate FileTimeCur update +char CRTime::sm_rgchLocalTimeCur[16]=""; +char CRTime::sm_rgchLocalDateCur[16]=""; +RTime32 CRTime::sm_nTimeCur = 0; + + + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CRTime::CRTime() +{ + if ( sm_nTimeCur == 0 ) + { + sm_nTimeCur = time(NULL); + } + + m_nStartTime = sm_nTimeCur; + m_bGMT = false; +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns the amount of time that's passed between our time and the +// current time. +// Output: Time that's passed between our time and the current time +//----------------------------------------------------------------------------- +int CRTime::CSecsPassed() const +{ + return( sm_nTimeCur - m_nStartTime ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Updates our current time value. We only +// update the time once per frame-- the rest of the time, we just +// access a cached copy of the time. +// NOTE: This should only be called once per frame. +//----------------------------------------------------------------------------- +void CRTime::UpdateRealTime() +{ + // BUGBUG Alfred: update this less often than once per frame? + RTime32 nTimePrev = sm_nTimeCur; + sm_nTimeCur = time(NULL); + + if ( sm_nTimeCur < nTimePrev ) + { + // time can go backwards sometimes if clock sync adjusts system time; warn when this happens + EmitInfo( SPEW_SYSTEM_MISC, SPEW_ALWAYS, LOG_ALWAYS, "Warning: system time went backward by %d seconds\n", ( nTimePrev - sm_nTimeCur ) ); + } + + // update our time from file time once per second + if ( sm_nTimeCur - sm_nTimeLastSystemTimeUpdate >= 1 ) + { +#ifdef _WIN32 + // get the local time, generate time & date strings and cache the strings, as we will need these + // frequently for logs. + SYSTEMTIME systemTimeLocal; + GetLocalTime( &systemTimeLocal ); + GetTimeFormat( LOCALE_USER_DEFAULT, 0, &systemTimeLocal, "HH:mm:ss", sm_rgchLocalTimeCur, Q_ARRAYSIZE( sm_rgchLocalTimeCur ) ); + GetDateFormat( LOCALE_USER_DEFAULT, 0, &systemTimeLocal, "MM/dd/yy", sm_rgchLocalDateCur, Q_ARRAYSIZE( sm_rgchLocalDateCur ) ); +#elif defined(POSIX) + time_t now; + time( &now ); + struct tm tmStruct; + struct tm *localTime = Plat_gmtime( &now, &tmStruct ); + strftime( sm_rgchLocalTimeCur, Q_ARRAYSIZE( sm_rgchLocalTimeCur ), "%H:%M:%S", localTime ); + strftime( sm_rgchLocalDateCur, Q_ARRAYSIZE( sm_rgchLocalDateCur ), "%m/%d/%y", localTime ); +#else +#error "Implement me" +#endif + sm_nTimeLastSystemTimeUpdate = sm_nTimeCur; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Sets the system clock on this box to specified world time +// Input: rTime32Current - world time to set +//----------------------------------------------------------------------------- +void CRTime::SetSystemClock( RTime32 rTime32Current ) +{ +#ifdef _WIN32 + FILETIME fileTime; + SYSTEMTIME systemTime = {0}; + // convert from seconds since 1/1/1970 to filetime (100 nanoseconds since 1/1/1601) with this magic formula courtesy of MSDN + uint64 ulTmp = ( ( (uint64) rTime32Current ) * 10 * k_nMillion ) + 116444736000000000; + fileTime.dwLowDateTime = (DWORD) ulTmp; + fileTime.dwHighDateTime = ulTmp >> 32; + + // convert from filetime to system time (note this also does time zone conversion to UTC) + BOOL bRet = FileTimeToSystemTime( &fileTime, &systemTime ); + Assert( bRet ); // should never fail + if ( !bRet ) // but if it does, don't set system clock to garbage + return; + + // set system time in UTC + bRet = SetSystemTime( &systemTime ); + Assert( bRet ); + + // update our cached time + sm_nTimeCur = rTime32Current; +#else + Assert( !"Not implemented" ); +#endif // _WIN32 +} + + +//----------------------------------------------------------------------------- +// Purpose: Renders the time +// Output : ptr to time string +//----------------------------------------------------------------------------- +const char* CRTime::Render( char (&buf)[k_RTimeRenderBufferSize] ) const +{ + return Render( m_nStartTime, buf ); +} + +//----------------------------------------------------------------------------- +// Purpose: Renders the time - static function +// Input : rTime32 - time to render +// Output : ptr to time string +//----------------------------------------------------------------------------- +const char* CRTime::Render( const RTime32 rTime32, char (&buf)[k_RTimeRenderBufferSize] ) +{ + if ( !buf ) + { + Assert( buf ); + return nullptr; + } + + // The return value string contains exactly 26 characters and has the form: Wed Jan 02 02:03:55 1980\n\0 + time_t tTime = rTime32; + char pchTime[32]; + if ( !Plat_ctime( &tTime, pchTime, Q_ARRAYSIZE( pchTime ) ) ) + return 0; + + // Remove '\n' + Assert( Q_strlen( pchTime ) == 25 ); + pchTime[ 24 ] = '\0'; + + if ( rTime32 == k_RTime32Infinite ) + Q_strncpy( buf, "Infinite time value", k_RTimeRenderBufferSize ); + else if ( rTime32 == k_RTime32Nil ) + Q_strncpy( buf, "Nil time value", k_RTimeRenderBufferSize ); + else if ( rTime32 < k_RTime32MinValid ) + Q_strncpy( buf, "Invalid time value", k_RTimeRenderBufferSize ); + else + Q_strncpy( buf, pchTime, k_RTimeRenderBufferSize ); + + return buf; +} + +//----------------------------------------------------------------------------- +// Purpose: Get the calendar year (absolute) for the current time +//----------------------------------------------------------------------------- +int CRTime::GetYear() const +{ + time_t timeCur = m_nStartTime; + struct tm tmStruct; + struct tm *ptmCur = m_bGMT ? Plat_gmtime( &timeCur, &tmStruct ) : Plat_localtime( &timeCur, &tmStruct ); + return ptmCur->tm_year + 1900; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the calendar month (0-11) for the current time +//----------------------------------------------------------------------------- +int CRTime::GetMonth() const +{ + time_t timeCur = m_nStartTime; + struct tm tmStruct; + struct tm *ptmCur = m_bGMT ? Plat_gmtime( &timeCur, &tmStruct ) : Plat_localtime( &timeCur, &tmStruct ); + return ptmCur->tm_mon; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the day of the calendar year (0-365) for the current time +//----------------------------------------------------------------------------- +int CRTime::GetDayOfYear() const +{ + time_t timeCur = m_nStartTime; + struct tm tmStruct; + struct tm *ptmCur = m_bGMT ? Plat_gmtime( &timeCur, &tmStruct ) : Plat_localtime( &timeCur, &tmStruct ); + return ptmCur->tm_yday; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the day of the month (1-31) for the current time +//----------------------------------------------------------------------------- +int CRTime::GetDayOfMonth() const +{ + time_t timeCur = m_nStartTime; + struct tm tmStruct; + struct tm *ptmCur = m_bGMT ? Plat_gmtime( &timeCur, &tmStruct ) : Plat_localtime( &timeCur, &tmStruct ); + return ptmCur->tm_mday; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the day of the week (0-6, 0=Sunday) for the current time +//----------------------------------------------------------------------------- +int CRTime::GetDayOfWeek() const +{ + time_t timeCur = m_nStartTime; + struct tm tmStruct; + struct tm *ptmCur = m_bGMT ? Plat_gmtime( &timeCur, &tmStruct ) : Plat_localtime( &timeCur, &tmStruct ); + return ptmCur->tm_wday; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the current hour (0-23) +//----------------------------------------------------------------------------- +int CRTime::GetHour( ) const +{ + time_t timeCur = m_nStartTime; + struct tm tmStruct; + struct tm *ptmCur = m_bGMT ? Plat_gmtime( &timeCur, &tmStruct ) : Plat_localtime( &timeCur, &tmStruct ); + return ptmCur->tm_hour; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the current minute value (0-59) +//----------------------------------------------------------------------------- +int CRTime::GetMinute( ) const +{ + time_t timeCur = m_nStartTime; + struct tm tmStruct; + struct tm *ptmCur = m_bGMT ? Plat_gmtime( &timeCur, &tmStruct ) : Plat_localtime( &timeCur, &tmStruct ); + return ptmCur->tm_min; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the current second value (0-59) +//----------------------------------------------------------------------------- +int CRTime::GetSecond() const +{ + time_t timeCur = m_nStartTime; + struct tm tmStruct; + struct tm *ptmCur = m_bGMT ? Plat_gmtime( &timeCur, &tmStruct ) : Plat_localtime( &timeCur, &tmStruct ); + return ptmCur->tm_sec; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the ISO week number +//----------------------------------------------------------------------------- +int CRTime::GetISOWeekOfYear() const +{ + int nDay = GetDayOfYear() - ( 1 + GetDayOfWeek() ); + int nISOWeek = nDay / 7; + return nISOWeek; +} + + +//----------------------------------------------------------------------------- +// Purpose: let me know if this is a leap year or not +//----------------------------------------------------------------------------- +/* static */ bool CRTime::BIsLeapYear( int nYear ) +{ + // every for years, unless it is a century; or if it is every 4th century + if ( ( nYear % 4 == 0 && nYear % 100 != 0) || nYear % 400 == 0) + return true; /* leap */ + else + return false; /* no leap */ +} + + +//----------------------------------------------------------------------------- +// Purpose: Calculate and return a time value corresponding to given sting +// Using a format string to convert +// Input: pchFmt - Format string that describes how to parse the value +// YY or YYYY is year, MM month, DD day of the month, +// hh mm ss is hour minute second. +// Z0000 is a time-zone offset, eg -0700. +// Everything except YY is optional (will be considered 0 if not given) +// pchValue - String containing the value to covert +// Output: RTime32 +//----------------------------------------------------------------------------- +// STATIC +RTime32 CRTime::RTime32FromFmtString( const char *pchFmt, const char* pchValue ) +{ + struct tm tm; + + char rgchNum[8]; + char rgchValue[64]; + + Q_memset( &tm, 0x0, sizeof( tm ) ); + Q_strncpy( rgchValue, pchValue, sizeof( rgchValue) ); + + int cchFmt = Q_strlen( pchFmt ); + int cchValue = Q_strlen( rgchValue ); + if ( cchFmt != cchValue || cchFmt < 4 ) + { + Assert( false ); + return k_RTime32Nil; + } + + const char *pchYYYY = Q_strstr( pchFmt, "YYYY" ); + const char *pchYY = Q_strstr( pchFmt, "YY" ); + const char *pchMM = Q_strstr( pchFmt, "MM" ); + const char *pchMnt = Q_strstr( pchFmt, "Mnt" ); + const char *pchDD = Q_strstr( pchFmt, "DD" ); + const char *pchThh = Q_strstr( pchFmt, "hh" ); + const char *pchTmm = Q_strstr( pchFmt, "mm" ); + const char *pchTss = Q_strstr( pchFmt, "ss" ); + const char *pchTzone = Q_strstr( pchFmt, "Z0000" ); + + if ( pchYYYY ) + { + pchYYYY = rgchValue + ( pchYYYY - pchFmt ); + Q_strncpy( rgchNum, pchYYYY, 5 ); + tm.tm_year = strtol( rgchNum, 0, 10 ) - 1900; + } + else if ( pchYY ) + { + pchYY = rgchValue + ( pchYY - pchFmt ); + Q_strncpy( rgchNum, pchYY, 3 ); + tm.tm_year = strtol( rgchNum, 0, 10 ) + 100; + } + else + return k_RTime32Nil; // must have a year + + if ( pchMM ) + { + pchMM = rgchValue + ( pchMM - pchFmt ); + Q_strncpy( rgchNum, pchMM, 3 ); + tm.tm_mon = strtol( rgchNum, 0, 10 ) - 1; + } + if ( pchMnt ) + { + static const char *rgszMonthNames[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; + pchMnt = rgchValue + ( pchMnt - pchFmt ); + int i; + for ( i = 0; i < 12; i++ ) + { + if ( !V_strnicmp( rgszMonthNames[i], pchMnt, 3 ) ) + break; + } + if ( i < 12 ) + tm.tm_mon = i; + } + if ( pchDD ) + { + pchDD = rgchValue + (pchDD - pchFmt ); + Q_strncpy( rgchNum, pchDD, 3 ); + tm.tm_mday = strtol( rgchNum, 0, 10 ); + } + if ( pchThh ) + { + pchThh = rgchValue + ( pchThh - pchFmt ); + Q_strncpy( rgchNum, pchThh, 3 ); + tm.tm_hour = strtol( rgchNum, 0, 10 ); + } + if ( pchTmm ) + { + pchTmm = rgchValue + ( pchTmm - pchFmt ); + Q_strncpy( rgchNum, pchTmm, 3 ); + tm.tm_min = strtol( rgchNum, 0, 10 ); + } + if ( pchTss ) + { + pchTss = rgchValue + (pchTss - pchFmt ); + Q_strncpy( rgchNum, pchTss, 3 ); + tm.tm_sec = strtol( rgchNum, 0, 10 ); + } + if ( pchTzone ) + { + long nOffset = 0; + pchTzone = rgchValue + (pchTzone - pchFmt); + Q_strncpy( rgchNum, pchTzone, 6 ); + nOffset = strtol( rgchNum, 0, 10 ); + tm.tm_hour -= nOffset / 100; // to go from -0700 to UTC, need to ADD seven + + // is this a sub-hour timezone? eg +0545 Kathmandu + int nMinutesOffset = nOffset % 100; + if ( nMinutesOffset ) + tm.tm_min -= nMinutesOffset; + + // OK, so this is somewhat lame: mktime assumes our tm units are in LOCAL time. + // However, we have just created a UTC time by using the supplied timezone offset. + // The rational thing to do here would be to call mkgmtime() instead of mktime(), + // but that function isn't available in unix-land. + // SO, instead we will MANUALLY convert this tm back to local time + +#if ( defined( _MSC_VER ) && _MSC_VER >= 1900 ) + #define timezone _timezone + #define daylight _daylight +#endif + + // subtract timezone, which is in SECONDS. timezone is (UTC - local), so local = UTC - timezone + tm.tm_sec -= timezone; + // timezone does NOT account for DST, so if we are in DST, we need to ADD an hour. + // This is because the value of timezone we subtracted was one hour TOO LARGE + tm.tm_hour += daylight ? 1 : 0; + + } + + // We don't know if DST is in effect, let the CRT + // figure it out + tm.tm_isdst = -1; + + return mktime( &tm ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Calculate and return a time value corresponding to given sting which is +// expected to be in one of the common HTTP date formats. +//----------------------------------------------------------------------------- +// STATIC +RTime32 CRTime::RTime32FromHTTPDateString( const char* pchValue ) +{ + // First format here is RFC 822/1123 format + struct tm tm; + if ( strptime( pchValue, "%a, %e %b %Y %H:%M:%S", &tm ) ) + { + return Plat_timegm( &tm ); + } + + // If that failed, try RFC 850/1036 format + if ( strptime( pchValue, "%a, %e-%b-%y %H:%M:%S", &tm ) ) + { + return Plat_timegm( &tm ); + } + + // If that also failed, give up + return k_RTime32Nil; +} + + +//----------------------------------------------------------------------------- +// Purpose: Parse time from string RFC3339 format (assumes UTC) +//----------------------------------------------------------------------------- +// STATIC +RTime32 CRTime::RTime32FromRFC3339UTCString( const char* pchValue ) +{ + + // UTC only from RFC 3339. Should be 2005-05-15T17:11:51Z + struct tm tm; + if ( strptime( pchValue, "%Y-%m-%dT%H:%M:%SZ", &tm ) ) + { + return Plat_timegm( &tm ); + } + + // If that also failed, give up + return k_RTime32Nil; + +} + + +//----------------------------------------------------------------------------- +// Purpose: Output time in RFC3339 format (assumes UTC) +//----------------------------------------------------------------------------- +// STATIC +const char* CRTime::RTime32ToRFC3339UTCString( const RTime32 rTime32, char (&buf)[k_RTimeRenderBufferSize] ) +{ + if ( !buf ) + { + Assert( buf ); + return nullptr; + } + + // Store the result in a temporary buffer, so that you can use several in a single printf. + time_t tTime = rTime32; + struct tm tmStruct; + struct tm *ptm = Plat_gmtime( &tTime, &tmStruct ); + + if ( rTime32 == k_RTime32Nil || !ptm ) + return "NIL"; + + if ( rTime32 == k_RTime32Infinite ) + return "Infinite time value"; + + if ( rTime32 < k_RTime32MinValid || !ptm ) + return "Invalid time value"; + + Q_snprintf( buf, k_RTimeRenderBufferSize, "%04u-%02u-%02uT%02u:%02u:%02uZ", ptm->tm_year+1900, ptm->tm_mon+1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec ); + return buf; +} + + +//----------------------------------------------------------------------------- +// Purpose: Calculate and return a time value corresponding to given sting +// "YYYY-MM-DD hh:mm:ss" (copied from sqlhelpers.cpp) +//----------------------------------------------------------------------------- +// STATIC +RTime32 CRTime::RTime32FromString( const char* pszValue ) +{ + struct tm tm; + + char num[5]; + char szValue[64]; + + Q_memset( &tm, 0x0, sizeof( tm ) ); + Q_strncpy( szValue, pszValue, sizeof( szValue) ); + + const char *str= szValue; + + num[0] =*str++; num[1] =*str++; num[2] =*str++; num[3] =*str++; num[4] = 0; + tm.tm_year = strtol( num, 0, 10 ) - 1900; + if (*str == '-') str++; + num[0] = *str++; num[1] = *str++; num[2] = 0; + tm.tm_mon = strtol( num, 0, 10 ) - 1; + if (*str == '-') str++; + num[0] = *str++; num[1] = *str++; num[2] = 0; + tm.tm_mday = strtol( num, 0, 10 ); + + if ( *str != 0 ) + { + // skip an optional space or T between date and time + if ( *str == ' ' || *str == 'T' ) + str++; + + // time is given too + num[0] = *str++; num[1] = *str++; num[2] = 0; + tm.tm_hour = strtol( num, 0, 10 ); + if (*str == ':') str++; + num[0] = *str++; num[1] = *str++; num[2] = 0; + tm.tm_min = strtol( num, 0, 10 ); + if (*str == ':') str++; + num[0] = *str++; num[1] = *str++; num[2] = 0; + tm.tm_sec = strtol( num, 0, 10 ); + } + tm.tm_isdst = -1; + + return mktime( &tm ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns a static string "YYYY-MM-DD hh:mm:ss" for given RTime32 +// Input: rTime32 - +// bNoPunct - No dashes, colons or spaces will be in the output string +// bOnlyDate - Only output the date +// Output: const char * -- only usable till the next yield +//----------------------------------------------------------------------------- +// STATIC +const char* CRTime::RTime32ToString( const RTime32 rTime32, char (&buf)[k_RTimeRenderBufferSize], bool bNoPunct /*=false*/, bool bOnlyDate /*= false*/ ) +{ + if ( !buf ) + { + return nullptr; + } + + // Store the result in a temporary buffer, so that you can use several in a single printf. + time_t tTime = rTime32; + struct tm tmStruct; + struct tm *ptm = Plat_localtime( &tTime, &tmStruct ); + + const char *pchOnlyDateFmt = bNoPunct ? "%04u%02u%02u" : "%04u-%02u-%02u"; + const char *pchDateTimeFmt = bNoPunct ? "%04u%02u%02u%02u%02u%02u" : "%04u-%02u-%02u %02u:%02u:%02u"; + if ( rTime32 == k_RTime32Nil || !ptm ) + return "NIL"; + + if ( rTime32 == k_RTime32Infinite ) + return "Infinite time value"; + + if ( rTime32 < k_RTime32MinValid || !ptm ) + return "Invalid time value"; + + if ( bOnlyDate ) + { + Q_snprintf( buf, k_RTimeRenderBufferSize, pchOnlyDateFmt, + ptm->tm_year+1900, ptm->tm_mon+1, ptm->tm_mday ); + } + else + { + Q_snprintf( buf, k_RTimeRenderBufferSize, pchDateTimeFmt, + ptm->tm_year+1900, ptm->tm_mon+1, ptm->tm_mday, + ptm->tm_hour, ptm->tm_min, ptm->tm_sec ); + } + + return buf; +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns a static string like "Aug 21" for given RTime32 +// Input: rTime32 - +// +// Output: const char * -- only usable till the next yield +//----------------------------------------------------------------------------- +// STATIC +const char* CRTime::RTime32ToDayString( const RTime32 rTime32, char (&buf)[k_RTimeRenderBufferSize], bool bGMT ) +{ + if ( !buf ) + { + return nullptr; + } + + // Store the result in a temporary buffer, so that you can use several in a single printf. + time_t tTime = rTime32; + struct tm tmStruct; + struct tm *ptm = bGMT ? Plat_gmtime( &tTime, &tmStruct ) : Plat_localtime( &tTime, &tmStruct ); + + DbgVerify( strftime( buf, k_RTimeRenderBufferSize, "%b %d", ptm ) ); + return buf; +} + + + +//----------------------------------------------------------------------------- +// Purpose: Calculate and return a time value corresponding to the beginning of +// the day represented by rtime32 +//----------------------------------------------------------------------------- +// STATIC +RTime32 CRTime::RTime32BeginningOfDay( const RTime32 rtime32 ) +{ + time_t timeCur = rtime32; + struct tm tmStruct; + struct tm *ptmCur = Plat_localtime( &timeCur, &tmStruct ); + if ( !ptmCur ) + return k_RTime32Nil; + + // midnight + ptmCur->tm_hour = 0; + ptmCur->tm_min = 0; + ptmCur->tm_sec = 0; + + // Let it compute DST + ptmCur->tm_isdst = -1; + + return mktime( ptmCur ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Calculate and return a time value corresponding to the beginning of +// the next day after rtime32 +//----------------------------------------------------------------------------- +// STATIC +RTime32 CRTime::RTime32BeginningOfNextDay( const RTime32 rtime32 ) +{ + time_t timeCur = rtime32; + struct tm tmStruct; + struct tm *ptmCur = Plat_localtime( &timeCur, &tmStruct ); + if ( !ptmCur ) + return k_RTime32Nil; + + // It will move to the next month etc if need be + ptmCur->tm_mday++; + + // midnight + ptmCur->tm_hour = 0; + ptmCur->tm_min = 0; + ptmCur->tm_sec = 0; + + // Let it compute DST + ptmCur->tm_isdst = -1; + + return mktime( ptmCur ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Calculate and return a time value corresponding to the first day of +// the month indicated by rtime32 +//----------------------------------------------------------------------------- +// STATIC +RTime32 CRTime::RTime32FirstDayOfMonth( const RTime32 rtime32 ) +{ + time_t timeCur = rtime32; + struct tm tmStruct; + struct tm *ptmCur = Plat_localtime( &timeCur, &tmStruct ); + if ( !ptmCur ) + return k_RTime32Nil; + + // first day of month + ptmCur->tm_mday = 1; + + // midnight + ptmCur->tm_hour = 0; + ptmCur->tm_min = 0; + ptmCur->tm_sec = 0; + + // Let it compute DST + ptmCur->tm_isdst = -1; + + return mktime( ptmCur ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Calculate and return a time value corresponding to the last day of +// the month indicated by rtime32 +//----------------------------------------------------------------------------- +// STATIC +RTime32 CRTime::RTime32LastDayOfMonth( const RTime32 rtime32 ) +{ + time_t timeCur = rtime32; + struct tm tmStruct; + struct tm *ptmCur = Plat_localtime( &timeCur, &tmStruct ); + if ( !ptmCur ) + return k_RTime32Nil; + + // Zeroth day of month N becomes last day of month (N-1) + ptmCur->tm_mon++; + ptmCur->tm_mday = 0; + + // midnight + ptmCur->tm_hour = 0; + ptmCur->tm_min = 0; + ptmCur->tm_sec = 0; + + // Let it compute DST + ptmCur->tm_isdst = -1; + + return mktime( ptmCur ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Calculate and return a time value corresponding to the first day of +// the month after the one indicated by rtime32 +//----------------------------------------------------------------------------- +// STATIC +RTime32 CRTime::RTime32FirstDayOfNextMonth( const RTime32 rtime32 ) +{ + time_t timeCur = rtime32; + struct tm tmStruct; + struct tm *ptmCur = Plat_localtime( &timeCur, &tmStruct ); + if ( !ptmCur ) + return k_RTime32Nil; + + ptmCur->tm_mon++; + ptmCur->tm_mday = 1; + + // midnight + ptmCur->tm_hour = 0; + ptmCur->tm_min = 0; + ptmCur->tm_sec = 0; + + // Let it compute DST + ptmCur->tm_isdst = -1; + + return mktime( ptmCur ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Calculate and return a time value corresponding to the last day of +// the month after the one indicated by rtime32 +//----------------------------------------------------------------------------- +// STATIC +RTime32 CRTime::RTime32LastDayOfNextMonth( const RTime32 rtime32 ) +{ + time_t timeCur = rtime32; + struct tm tmStruct; + struct tm *ptmCur = Plat_localtime( &timeCur, &tmStruct ); + if ( !ptmCur ) + return k_RTime32Nil; + + // use zeroth-day trick - skip 2 months then back a day + ptmCur->tm_mon += 2; + ptmCur->tm_mday = 0; + + // midnight + ptmCur->tm_hour = 0; + ptmCur->tm_min = 0; + ptmCur->tm_sec = 0; + + // Let it compute DST + ptmCur->tm_isdst = -1; + + return mktime( ptmCur ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Calculate and return a time value corresponding to the Nth day of +// the month. If that month only has K days, K < N, it will return +// the Kth day. The input should be reasonable (don't ask for the -5th +// day of the month). +// +// Input: rtime32 - Time representing some time in the month interested in +// nDay - The day of that month you want the return to be set to +// +// Return: Time value equal to midnight on that day. +//----------------------------------------------------------------------------- +// STATIC +RTime32 CRTime::RTime32NthDayOfMonth( const RTime32 rtime32, int nDay ) +{ + Assert( nDay > 0 ); + Assert( nDay < 32 ); + + time_t timeCur = rtime32; + struct tm tmStruct; + struct tm *ptmCur = Plat_localtime( &timeCur, &tmStruct ); + if ( !ptmCur ) + return k_RTime32Nil; + + int nCurMonth = ptmCur->tm_mon; + + ptmCur->tm_mday = nDay; + + // midnight + ptmCur->tm_hour = 0; + ptmCur->tm_min = 0; + ptmCur->tm_sec = 0; + + // Let it compute DST + ptmCur->tm_isdst = -1; + + // This call will modify ptmCur in-place + time_t timeThen = mktime( ptmCur ); + + // See if the month changed + if ( ptmCur->tm_mon != nCurMonth ) + { + // use zeroth-day trick to just get the last day of this month + ptmCur->tm_mday = 0; + // Let it compute DST + ptmCur->tm_isdst = -1; + timeThen = mktime( ptmCur ); + } + + return timeThen; +} + + +//----------------------------------------------------------------------------- +// Purpose: Add X months to the current date, and return the Nth day of that +// month. +// +// Input: nNthDayOfMonth - Day of the target month to return a date for +// rtime32StartDate - Time value to add X months to +// nMonthsToAdd - X +// +// Return: Time value equal to midnight on that day. +//----------------------------------------------------------------------------- +// STATIC +RTime32 CRTime::RTime32MonthAddChooseDay( int nNthDayOfMonth, RTime32 rtime32StartDate, int nMonthsToAdd ) +{ + // Get the first day of start month + RTime32 rtime32FirstDayOfStartMonth = CRTime( rtime32StartDate ).GetFirstDayOfMonth(); + + // Add X months to that - guaranteed to be correct month + RTime32 rtime32FirstDayOfTargetMonth = CRTime::RTime32DateAdd( rtime32FirstDayOfStartMonth, nMonthsToAdd, k_ETimeUnitMonth ); + + // Then get the Nth day of that month + RTime32 rtime32Target = CRTime::RTime32NthDayOfMonth( rtime32FirstDayOfTargetMonth, nNthDayOfMonth ); + + return rtime32Target; +} + + +//----------------------------------------------------------------------------- +// Purpose: Add or subtract N units of time from the current value. +// Units may be days, weeks, seconds, etc +// Input: rtime32 - Reference time +// nAmount - Number of units to add (neg for subtract) +// eTimeFlagAmountType - Indicates what units are on nAmount +// +// Return: The newly calculated offset time (the input is unmodified) +//----------------------------------------------------------------------------- +// STATIC +RTime32 CRTime::RTime32DateAdd( const RTime32 rtime32, int nAmount, ETimeUnit eTimeAmountType ) +{ + time_t timeCur = rtime32; + struct tm tmStruct; + struct tm *ptmCur = Plat_localtime( &timeCur, &tmStruct ); + if ( !ptmCur ) + return k_RTime32Nil; + + // mktime() is smart enough to take day-of-month values that are out of range and adjust + // everything to make sense. So you can go back 3 weeks by just subtracting 21 from tm_mday. + switch ( eTimeAmountType ) + { + default: + AssertMsg( false, "Bad flag in RTime32DateAdd" ); + break; + case k_ETimeUnitForever: + return k_RTime32Infinite; + case k_ETimeUnitYear: + ptmCur->tm_year += nAmount; + break; + case k_ETimeUnitMonth: + ptmCur->tm_mon += nAmount; + break; + case k_ETimeUnitWeek: + ptmCur->tm_mday += 7 * nAmount; + break; + case k_ETimeUnitDay: + ptmCur->tm_mday += nAmount; + break; + case k_ETimeUnitHour: + ptmCur->tm_hour += nAmount; + break; + case k_ETimeUnitMinute: + ptmCur->tm_min += nAmount; + break; + case k_ETimeUnitSecond: + ptmCur->tm_sec += nAmount; + break; + } + + // Let it compute DST + ptmCur->tm_isdst = -1; + + return mktime( ptmCur ); +} + +//----------------------------------------------------------------------------- +// Purpose: Compare two times and evaluate what calendar boundaries have +// been crossed (eg day, month, hour) between the two times. +// +// Note: in general, the crossing of a large boundary will be accompanied +// by the crossing of all smaller boundaries. The exception is Week: +// the Week boundary is from Saturday to Sunday, and it is possible to +// go over a Month or Year boundary without beginning a new week. +// +// So, the return value is the largest time boundary that was crossed. +// However, the pbWeekChanged value will be set to indicate if the week +// changed in cases where the return value is Month or Year. +// +// Input: unTime1 - First time value +// unTime2 - Second time value +// pbWeekChanged - Indicates if the Week changed +// +// Return: Largest time boundary crossed +//----------------------------------------------------------------------------- +// STATIC +ETimeUnit CRTime::FindTimeBoundaryCrossings( RTime32 unTime1, RTime32 unTime2, bool *pbWeekChanged ) +{ + time_t time1 = unTime1; + time_t time2 = unTime2; + + // have to cache the first one locally, because it's a global object + struct tm tmStruct1; + struct tm *ptmTime1 = Plat_localtime( &time1, &tmStruct1 ); + if ( !ptmTime1 ) + return k_ETimeUnitForever; + struct tm _tmTime1 = *ptmTime1; + ptmTime1 = &_tmTime1; + struct tm tmStruct2; + struct tm *ptmTime2 = Plat_localtime( &time2, &tmStruct2 ); + if ( !ptmTime2 ) + return k_ETimeUnitForever; + + // Need a little extra logic to find week boundaries + // Find this out first, because it may or may not be true even if a + // month / year boundary was crossed. + *pbWeekChanged = false; + + // If the difference is more than 6 days, we crossed a week boundary + if ( ( ( unTime1 > unTime2 ) && ( ( unTime1 - unTime2 ) > k_cSecondsPerWeek ) ) + || ( ( unTime2 > unTime1 ) && ( ( unTime2 - unTime1 ) > k_cSecondsPerWeek ) ) ) + { + *pbWeekChanged = true; + } + else if ( ptmTime1->tm_yday != ptmTime2->tm_yday ) + { + // Otherwise, we have to look at wday - if the later time + // has a lower or equal wday value, then we crossed a week boundary + if ( unTime2 > unTime1 ) + { + if ( ptmTime2->tm_wday <= ptmTime1->tm_wday ) + *pbWeekChanged = true; + } + else + { + if ( ptmTime1->tm_wday <= ptmTime2->tm_wday ) + *pbWeekChanged = true; + } + } + + // Evaluate larger boundaries first. As soon as we detect + // that we've crossed a boundary, we consider all smaller boundaries + // crossed too. + + // Year + if ( ptmTime1->tm_year != ptmTime2->tm_year ) + return k_ETimeUnitYear; + + // Month + if ( ptmTime1->tm_mon != ptmTime2->tm_mon ) + return k_ETimeUnitMonth; + + // If the week changed, return that now + if ( *pbWeekChanged ) + return k_ETimeUnitWeek; + + // Day + if ( ptmTime1->tm_yday != ptmTime2->tm_yday ) + return k_ETimeUnitDay; + + // Hour + if ( ptmTime1->tm_hour != ptmTime2->tm_hour ) + return k_ETimeUnitHour; + + // If DST changed, make sure that we know an hour boundary was crossed + // (overlap from the "fall back" case may otherwise trick us) + if ( ptmTime1->tm_isdst != ptmTime2->tm_isdst ) + return k_ETimeUnitHour; + + // Minute + if ( ptmTime1->tm_min != ptmTime2->tm_min ) + return k_ETimeUnitMinute; + + // Second + if ( ptmTime1->tm_sec != ptmTime2->tm_sec ) + return k_ETimeUnitSecond; + + return k_ETimeUnitNone; + +} + diff --git a/gcsdk/steamextra/rtime.h b/gcsdk/steamextra/rtime.h new file mode 100644 index 0000000..579cdca --- /dev/null +++ b/gcsdk/steamextra/rtime.h @@ -0,0 +1,187 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Encapsulates real world (wall clock) time +// +//============================================================================= +#ifndef RTIME_H +#define RTIME_H +#ifdef _WIN32 +#pragma once +#endif + + +#ifdef WIN32 +char* strptime(const char *s, const char *format, struct tm *tm); +#endif + +#include <time.h> +#include <ctype.h> +#include <string.h> + + + +class CSTime; + +// Invalid time values +const RTime32 k_RTime32Nil = 0; +// time values between Nil and MinValid are available for special constants +const RTime32 k_RTime32MinValid = 10; +// infinite time value +const RTime32 k_RTime32Infinite = 0x7FFFFFFF; //01-18-2038 + +// Render buffer size +const size_t k_RTimeRenderBufferSize = 25; + +// Flags for component fields. Longer durations must be > shorter ones +// WARNING: DO NOT RENUMBER EXISTING VALUES - STORED IN DATABASE +enum ETimeUnit +{ + k_ETimeUnitNone = 0, + k_ETimeUnitSecond = 1, + k_ETimeUnitMinute = 2, + k_ETimeUnitHour = 3, + k_ETimeUnitDay = 4, + k_ETimeUnitWeek = 5, + k_ETimeUnitMonth = 6, + k_ETimeUnitYear = 7, + k_ETimeUnitForever +}; + +// CRTime +// This is our primary real time structure. +// It offers 1 second resolution beginning on January 1, 1970 (i.e unix time) +// This represents wall clock time +class CRTime +{ +public: + // default constructor initializes to the current time + CRTime(); + CRTime( RTime32 nTime ) : m_nStartTime( nTime ), m_bGMT( false ) {} + + void SetToCurrentTime() { m_nStartTime = sm_nTimeCur; } + void SetFromCurrentTime( int dSecOffset ) { m_nStartTime = sm_nTimeCur + dSecOffset; } + + // Amount of seconds that have passed between this CRTime being set and the current wall clock time. + int CSecsPassed() const; + + // Time accessors + RTime32 GetRTime32() const { return m_nStartTime; } + void SetRTime32( RTime32 rTime32 ) { m_nStartTime = rTime32; } + + // Access our cached current time value + static void UpdateRealTime(); + static void SetSystemClock( RTime32 nCurrentTime ); + static RTime32 RTime32TimeCur() { return sm_nTimeCur; } + + // Render + const char *Render( char (&buf)[k_RTimeRenderBufferSize] ) const; + static const char *Render( const RTime32 rTime32, char (&buf)[k_RTimeRenderBufferSize] ); + + // Return a representation of the current system time + static char *PchTimeCur() { return sm_rgchLocalTimeCur; } + // Return a representation of the current system date + static char *PchDateCur() { return sm_rgchLocalDateCur; } + + // comparisons against other CRTime objects + bool operator==( const CRTime &val ) const { return val.m_nStartTime == m_nStartTime; } + bool operator<( const CRTime &val ) const { return m_nStartTime < val.m_nStartTime; } + bool operator<=( const CRTime &val ) const { return m_nStartTime <= val.m_nStartTime; } + bool operator>( const CRTime &val ) const { return m_nStartTime > val.m_nStartTime; } + bool operator>=( const CRTime &val ) const { return m_nStartTime >= val.m_nStartTime; } + + // comparisons against RTime32 numbers (to avoid implicit construct/destruct of a temporary CRTime) + bool operator==( const RTime32 &val ) const { return m_nStartTime == val; } + bool operator<( const RTime32 &val ) const { return m_nStartTime < val; } + bool operator<=( const RTime32 &val ) const { return m_nStartTime <= val; } + bool operator>( const RTime32 &val ) const { return m_nStartTime > val; } + bool operator>=( const RTime32 &val ) const { return m_nStartTime >= val; } + + const CRTime& operator=( const RTime32 &val ) { m_nStartTime = val; return *this; } + const CRTime& operator=( const CRTime &val ) { m_nStartTime = val.m_nStartTime; return *this; } + const CRTime operator+( const int &nVal ) { return m_nStartTime + nVal; } + // Add exactly X seconds to this time + const CRTime& operator+=( const int &nVal ) { m_nStartTime += nVal; return *this; } + + // Component Details + int GetYear() const; + int GetMonth() const; // returns 0..11 + int GetDayOfYear() const; + int GetDayOfMonth() const; + int GetDayOfWeek() const; + int GetHour() const; + int GetMinute() const; + int GetSecond() const; + int GetISOWeekOfYear() const; + + // Handy references to nearby time boundaries + static RTime32 RTime32BeginningOfDay( const RTime32 ); // at 00:00:00 + static RTime32 RTime32BeginningOfNextDay( const RTime32 ); // at 00:00:00 + static RTime32 RTime32FirstDayOfMonth( const RTime32 ); // at 00:00:00 + static RTime32 RTime32LastDayOfMonth( const RTime32 ); // at 00:00:00 + static RTime32 RTime32FirstDayOfNextMonth( const RTime32 ); // at 00:00:00 + static RTime32 RTime32LastDayOfNextMonth( const RTime32 ); // at 00:00:00 + + static bool BIsLeapYear( int nYear ); + bool BIsLeapYear() const { return CRTime::BIsLeapYear( GetYear() ); } + + // Parse time using a format string with strptime format + static RTime32 RTime32FromFmtString( const char *pchFmt, const char* pchValue ); + + // Parse time from string in standard HTTP date format + static RTime32 RTime32FromHTTPDateString( const char* pchValue ); + + // Parse time from string RFC3339 format (assumes UTC) + static RTime32 RTime32FromRFC3339UTCString( const char* pchValue ); + static const char* RTime32ToRFC3339UTCString( const RTime32 rTime32, char (&buf)[k_RTimeRenderBufferSize] ); + + // parse time from string "YYYY-MM-DD hh:mm:ss" or just "YYYY-MM-DD", the ' ',':','-' are optional + static RTime32 RTime32FromString( const char* pszValue ); + + // turns RTime32 in a string like "YYYY-MM-DD hh:mm:ss" + static const char* RTime32ToString( const RTime32 rTime32, char (&buf)[k_RTimeRenderBufferSize], bool bNoPunct = false, bool bOnlyDate = false ); + + // turns RTime32 into a string like "Aug 21" + static const char* RTime32ToDayString( const RTime32 rTime32, char (&buf)[k_RTimeRenderBufferSize], bool bGMT = false ); + + // If the month only has K days, K < N, returns Kth day + static RTime32 RTime32NthDayOfMonth( const RTime32, int nDay ); // at 00:00:00 + + // Add X months but return the Nth day of that month. If the month only has K days, K < N, returns Kth day. + static RTime32 RTime32MonthAddChooseDay( int nNthDayOfMonth, RTime32 rtime32StartDate, int nMonthsToAdd ); + + RTime32 GetBeginningOfDay() const { return RTime32BeginningOfDay( m_nStartTime ); } + RTime32 GetBeginningOfNextDay() const { return RTime32BeginningOfNextDay( m_nStartTime ); } + RTime32 GetFirstDayOfMonth() const { return RTime32FirstDayOfMonth( m_nStartTime ); } + RTime32 GetLastDayOfMonth() const { return RTime32LastDayOfMonth( m_nStartTime ); } + RTime32 GetFirstDayOfNextMonth() const { return RTime32FirstDayOfNextMonth( m_nStartTime ); } + RTime32 GetLastDayOfNextMonth() const { return RTime32LastDayOfNextMonth( m_nStartTime ); } + RTime32 GetNthDayOfMonth( int nDay ) const { return RTime32NthDayOfMonth( m_nStartTime, nDay ); } + RTime32 MonthAddChooseDay( int nNthDayOfMonth, int nMonthsToAdd ) const { return RTime32MonthAddChooseDay( nNthDayOfMonth, m_nStartTime, nMonthsToAdd ); } + + // Add or subtract N days, weeks, minutes, etc from the current time + static RTime32 RTime32DateAdd( const RTime32, int nAmount, ETimeUnit eTimeAmountType ); + RTime32 DateAdd( int nAmount, ETimeUnit eTimeAmountType ) const { return RTime32DateAdd( m_nStartTime, nAmount, eTimeAmountType ); } + + // Compare two times, and return what the largest time boundary crossed between the two was. + // Week boundaries do not line up with Month or Year boundaries, so you must rely on pbWeekChanged for them! + static ETimeUnit FindTimeBoundaryCrossings( RTime32 unTime1, RTime32 unTime2, bool *pbWeekChanged ); + + void SetToGMT( bool bUseGMT ) { m_bGMT = bUseGMT;} + bool BIsGMT() const { return m_bGMT; } + +private: + // prevent assignment or copy construction from the server time type + const CRTime& operator=( const CSTime &val ) { return *this; } + CRTime( const CSTime& ) {} + + RTime32 m_nStartTime; // the time store by this instance (wall clock, in seconds) + static RTime32 sm_nTimeCur; // current system wide wall clock time + + static char sm_rgchLocalTimeCur[16]; // string version of time for logging + static char sm_rgchLocalDateCur[16]; // string version of date for logging + static RTime32 sm_nTimeLastSystemTimeUpdate; // last time we updated above two logging strings + + bool m_bGMT; +}; + +#endif // RTIME_H diff --git a/gcsdk/steamextra/steam/isteamgamecoordinator.h b/gcsdk/steamextra/steam/isteamgamecoordinator.h new file mode 100644 index 0000000..63f3c1b --- /dev/null +++ b/gcsdk/steamextra/steam/isteamgamecoordinator.h @@ -0,0 +1,64 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: interface to the game coordinator for this application +// +//============================================================================= + +#ifndef ISTEAMGAMECOORDINATOR +#define ISTEAMGAMECOORDINATOR +#ifdef _WIN32 +#pragma once +#endif + +#include "steam/steamtypes.h" +#include "steam/steamclientpublic.h" +#include "steam/isteamclient.h" + +// list of possible return values from the ISteamGameCoordinator API +enum EGCResults +{ + k_EGCResultOK = 0, + k_EGCResultNoMessage = 1, // There is no message in the queue + k_EGCResultBufferTooSmall = 2, // The buffer is too small for the requested message + k_EGCResultNotLoggedOn = 3, // The client is not logged onto Steam + k_EGCResultInvalidMessage = 4, // Something was wrong with the message being sent with SendMessage +}; + + +//----------------------------------------------------------------------------- +// Purpose: Functions for sending and receiving messages from the Game Coordinator +// for this application +//----------------------------------------------------------------------------- +class ISteamGameCoordinator +{ +public: + + // sends a message to the Game Coordinator + virtual EGCResults SendMessage( uint32 unMsgType, const void *pubData, uint32 cubData ) = 0; + + // returns true if there is a message waiting from the game coordinator + virtual bool IsMessageAvailable( uint32 *pcubMsgSize ) = 0; + + // fills the provided buffer with the first message in the queue and returns k_EGCResultOK or + // returns k_EGCResultNoMessage if there is no message waiting. pcubMsgSize is filled with the message size. + // If the provided buffer is not large enough to fit the entire message, k_EGCResultBufferTooSmall is returned + // and the message remains at the head of the queue. + virtual EGCResults RetrieveMessage( uint32 *punMsgType, void *pubDest, uint32 cubDest, uint32 *pcubMsgSize ) = 0; + +}; +#define STEAMGAMECOORDINATOR_INTERFACE_VERSION "SteamGameCoordinator001" + +// callback notification - A new message is available for reading from the message queue +struct GCMessageAvailable_t +{ + enum { k_iCallback = k_iSteamGameCoordinatorCallbacks + 1 }; + uint32 m_nMessageSize; +}; + +// callback notification - A message failed to make it to the GC. It may be down temporarily +struct GCMessageFailed_t +{ + enum { k_iCallback = k_iSteamGameCoordinatorCallbacks + 2 }; +}; + +#endif // ISTEAMGAMECOORDINATOR diff --git a/gcsdk/steamextra/steamid.cpp b/gcsdk/steamextra/steamid.cpp new file mode 100644 index 0000000..6a4eb01 --- /dev/null +++ b/gcsdk/steamextra/steamid.cpp @@ -0,0 +1,729 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//============================================================================= + +#ifdef TF +#include "steamcommon.h" +#define INCLUDED_STEAM2_USERID_STRUCTS +#include "steam/steamclientpublic.h" +#endif + +#include "stdafx.h" +#ifdef HL1 +#include "steamcommon.h" +#include "steam/steamclientpublic.h" +#endif + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +#ifndef UINT64_MAX +#define UINT64_MAX ((uint64)-1) +#endif + +static const char *DecimalToUint64( const char *pchStr, uint64 unLimit, + uint64 *punVal ) +{ + const char *pchStart = pchStr; + uint64 unVal = 0; + + while ( *pchStr >= '0' && *pchStr <= '9' ) + { + uint64 unNext = unVal * 10; + + if ( unNext < unVal ) + { + // 64-bit overflow. + return NULL; + } + + unVal = unNext + (uint64)( *pchStr - '0' ); + if ( unVal > unLimit ) + { + // Limit overflow. + return NULL; + } + + pchStr++; + } + if ( pchStr == pchStart ) + { + // No number at all. + return NULL; + } + + *punVal = unVal; + return pchStr; +} + +//----------------------------------------------------------------------------- +// Purpose: Constructor +// Input : pchSteamID - text representation of a Steam ID +//----------------------------------------------------------------------------- +CSteamID::CSteamID( const char *pchSteamID, EUniverse eDefaultUniverse /* = k_EUniverseInvalid */ ) +{ + SetFromString( pchSteamID, eDefaultUniverse ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Initializes a steam ID from a string +// Input : pchSteamID - text representation of a Steam ID +//----------------------------------------------------------------------------- +void CSteamID::SetFromString( const char *pchSteamID, EUniverse eDefaultUniverse ) +{ + uint nAccountID = 0; + uint nInstance = 1; + EUniverse eUniverse = eDefaultUniverse; + EAccountType eAccountType = k_EAccountTypeIndividual; +#ifdef DBGFLAG_ASSERT + // TF Merge -- Assert is debug-only and we have unused variable warnings on :-/ + const char *pchSteamIDString = pchSteamID; +#endif + CSteamID StrictID; + + StrictID.SetFromStringStrict( pchSteamID, eDefaultUniverse ); + + if ( *pchSteamID == '[' ) + pchSteamID++; + + // BUGBUG Rich use the Q_ functions + if (*pchSteamID == 'A') + { + // This is test only + pchSteamID++; // skip the A + eAccountType = k_EAccountTypeAnonGameServer; + if (*pchSteamID == '-' || *pchSteamID == ':') + pchSteamID++; // skip the optional - or : + + if ( strchr( pchSteamID, '(' ) ) + sscanf( strchr( pchSteamID, '(' ), "(%u)", &nInstance ); + const char *pchColon = strchr( pchSteamID, ':' ); + if ( pchColon && *pchColon != 0 && strchr( pchColon+1, ':' )) + { + sscanf( pchSteamID, "%u:%u:%u", (uint*)&eUniverse, &nAccountID, &nInstance ); + } + else if ( pchColon ) + { + sscanf( pchSteamID, "%u:%u", (uint*)&eUniverse, &nAccountID ); + } + else + { + sscanf( pchSteamID, "%u", &nAccountID ); + } + + if ( nAccountID == 0 ) + { + // i dont care what number you entered + CreateBlankAnonLogon(eUniverse); + } + else + { + InstancedSet( nAccountID, nInstance, eUniverse, eAccountType ); + } + // Catch cases where we're allowing sloppy input that we + // might not want to allow. + AssertMsg1( this->operator==( StrictID ), "Steam ID does not pass strict parsing: '%s'", pchSteamIDString ); + return; + } + else if (*pchSteamID == 'G') + { + pchSteamID++; // skip the G + eAccountType = k_EAccountTypeGameServer; + if (*pchSteamID == '-' || *pchSteamID == ':') + pchSteamID++; // skip the optional - or : + } + else if (*pchSteamID == 'C') + { + pchSteamID++; // skip the C + eAccountType = k_EAccountTypeContentServer; + if (*pchSteamID == '-' || *pchSteamID == ':') + pchSteamID++; // skip the optional - or : + } + else if (*pchSteamID == 'g') + { + pchSteamID++; // skip the g + eAccountType = k_EAccountTypeClan; + nInstance = 0; + if (*pchSteamID == '-' || *pchSteamID == ':') + pchSteamID++; // skip the optional - or : + } + else if (*pchSteamID == 'c') + { + pchSteamID++; // skip the c + eAccountType = k_EAccountTypeChat; + nInstance = k_EChatInstanceFlagClan; + if (*pchSteamID == '-' || *pchSteamID == ':') + pchSteamID++; // skip the optional - or : + } + else if (*pchSteamID == 'L') + { + pchSteamID++; // skip the c + eAccountType = k_EAccountTypeChat; + nInstance = k_EChatInstanceFlagLobby; + if (*pchSteamID == '-' || *pchSteamID == ':') + pchSteamID++; // skip the optional - or : + } + else if (*pchSteamID == 'T') + { + pchSteamID++; // skip the T + eAccountType = k_EAccountTypeChat; + nInstance = 0; // Anon chat + if (*pchSteamID == '-' || *pchSteamID == ':') + pchSteamID++; // skip the optional - or : + } + else if (*pchSteamID == 'U') + { + pchSteamID++; // skip the U + eAccountType = k_EAccountTypeIndividual; + nInstance = 1; + if (*pchSteamID == '-' || *pchSteamID == ':') + pchSteamID++; // skip the optional - or : + } + else if (*pchSteamID == 'i') + { + pchSteamID++; // skip the i + eAccountType = k_EAccountTypeInvalid; + nInstance = 1; + if (*pchSteamID == '-' || *pchSteamID == ':') + pchSteamID++; // skip the optional - or : + } + + if ( strchr( pchSteamID, ':' ) ) + { + if (*pchSteamID == '[') + pchSteamID++; // skip the optional [ + sscanf( pchSteamID, "%u:%u", (uint*)&eUniverse, &nAccountID ); + if ( eUniverse == k_EUniverseInvalid ) + eUniverse = eDefaultUniverse; + } + else + { + uint64 unVal64 = 0; + + sscanf( pchSteamID, "%llu", &unVal64 ); + if ( unVal64 > UINT_MAX ) + { + // Assume a full 64-bit Steam ID. + SetFromUint64( unVal64 ); + // Catch cases where we're allowing sloppy input that we + // might not want to allow. + AssertMsg1( this->operator==( StrictID ), "Steam ID does not pass strict parsing: '%s'", pchSteamIDString ); + return; + } + else + { + nAccountID = (uint)unVal64; + } + } + + Assert( (eUniverse > k_EUniverseInvalid) && (eUniverse < k_EUniverseMax) ); + + InstancedSet( nAccountID, nInstance, eUniverse, eAccountType ); + + // Catch cases where we're allowing sloppy input that we + // might not want to allow. + AssertMsg1( this->operator==( StrictID ), "Steam ID does not pass strict parsing: '%s'", pchSteamIDString ); +} + +// SetFromString allows many partially-correct strings, constraining how +// we might be able to change things in the future. +// SetFromStringStrict requires the exact string forms that we support +// and is preferred when the caller knows it's safe to be strict. +// Returns whether the string parsed correctly. The ID may +// still be invalid even if the string parsed correctly. +// If the string didn't parse correctly the ID will always be invalid. +bool CSteamID::SetFromStringStrict( const char *pchSteamID, EUniverse eDefaultUniverse ) +{ + uint nAccountID = 0; + uint nInstance = 1; + uint unMaxVal = 2; + EUniverse eUniverse = eDefaultUniverse; + EAccountType eAccountType = k_EAccountTypeIndividual; + char chPrefix; + bool bBracket = false; + bool bValid = true; + uint64 unVal[3]; + const char *pchEnd; + + // Start invalid. + Clear(); + + if ( !pchSteamID ) + { + return false; + } + + if ( *pchSteamID == '[' ) + { + pchSteamID++; + bBracket = true; + } + + chPrefix = *pchSteamID; + switch( chPrefix ) + { + case 'A': + // This is test only + eAccountType = k_EAccountTypeAnonGameServer; + unMaxVal = 3; + break; + + case 'G': + eAccountType = k_EAccountTypeGameServer; + break; + + case 'C': + eAccountType = k_EAccountTypeContentServer; + break; + + case 'g': + eAccountType = k_EAccountTypeClan; + nInstance = 0; + break; + + case 'c': + eAccountType = k_EAccountTypeChat; + nInstance = k_EChatInstanceFlagClan; + break; + + case 'L': + eAccountType = k_EAccountTypeChat; + nInstance = k_EChatInstanceFlagLobby; + break; + + case 'T': + eAccountType = k_EAccountTypeChat; + nInstance = 0; // Anon chat + break; + + case 'U': + eAccountType = k_EAccountTypeIndividual; + nInstance = 1; + break; + + case 'i': + eAccountType = k_EAccountTypeInvalid; + nInstance = 1; + break; + + default: + // We're reserving other leading characters so + // this should only be the plain-digits case. + if (chPrefix < '0' || chPrefix > '9') + { + bValid = false; + } + chPrefix = 0; + break; + } + + if ( chPrefix ) + { + pchSteamID++; // skip the prefix + if (*pchSteamID == '-' || *pchSteamID == ':') + pchSteamID++; // skip the optional - or : + } + + uint unIdx = 0; + + for (;;) + { + pchEnd = DecimalToUint64( pchSteamID, UINT64_MAX, &unVal[unIdx] ); + if ( !pchEnd ) + { + bValid = false; + break; + } + + unIdx++; + + // For 'A' we can have a trailing instance, which must + // be the end of the string. + if ( *pchEnd == '(' && + chPrefix == 'A' ) + { + if ( unIdx > 2 ) + { + // Two instance IDs provided. + bValid = false; + } + + pchEnd = DecimalToUint64( pchEnd + 1, k_unSteamAccountInstanceMask, &unVal[2] ); + if ( !pchEnd || + *pchEnd != ')' ) + { + bValid = false; + break; + } + else + { + nInstance = (uint)unVal[2]; + + pchEnd++; + if ( *pchEnd == ':' ) + { + // Not expecting more values. + bValid = false; + break; + } + } + } + + if ( *pchEnd != ':' ) + { + if ( bBracket ) + { + if ( *pchEnd != ']' || + *(pchEnd + 1) != 0 ) + { + bValid = false; + } + } + else if ( *pchEnd != 0 ) + { + bValid = false; + } + + break; + } + + if ( unIdx >= unMaxVal ) + { + bValid = false; + break; + } + + pchSteamID = pchEnd + 1; + } + + if ( unIdx > 2 ) + { + if ( unVal[2] <= k_unSteamAccountInstanceMask ) + { + nInstance = (uint)unVal[2]; + } + else + { + bValid = false; + } + } + if ( unIdx > 1 ) + { + if ( unVal[0] >= k_EUniverseInvalid && + unVal[0] < k_EUniverseMax ) + { + eUniverse = (EUniverse)unVal[0]; + if ( eUniverse == k_EUniverseInvalid ) + eUniverse = eDefaultUniverse; + } + else + { + bValid = false; + } + + if ( unVal[1] <= k_unSteamAccountIDMask ) + { + nAccountID = (uint)unVal[1]; + } + else + { + bValid = false; + } + } + else if ( unIdx > 0 ) + { + if ( unVal[0] <= k_unSteamAccountIDMask ) + { + nAccountID = (uint)unVal[0]; + } + else if ( !chPrefix ) + { + if ( bValid ) + { + SetFromUint64( unVal[0] ); + } + return bValid; + } + else + { + bValid = false; + } + } + else + { + bValid = false; + } + + if ( bValid ) + { + if ( chPrefix == 'A' ) + { + if ( nAccountID == 0 ) + { + // i dont care what number you entered + CreateBlankAnonLogon(eUniverse); + return bValid; + } + } + + InstancedSet( nAccountID, nInstance, eUniverse, eAccountType ); + } + + return bValid; +} + + +#if defined( INCLUDED_STEAM2_USERID_STRUCTS ) +//----------------------------------------------------------------------------- +// Purpose: Initializes a steam ID from a Steam2 ID string +// Input: pchSteam2ID - Steam2 ID (as a string #:#:#) to convert +// eUniverse - universe this ID belongs to +// Output: true if successful, false otherwise +//----------------------------------------------------------------------------- +bool CSteamID::SetFromSteam2String( const char *pchSteam2ID, EUniverse eUniverse ) +{ + Assert( pchSteam2ID ); + + // Convert the Steam2 ID string to a Steam2 ID structure + TSteamGlobalUserID steam2ID; + steam2ID.m_SteamInstanceID = 0; + steam2ID.m_SteamLocalUserID.Split.High32bits = 0; + steam2ID.m_SteamLocalUserID.Split.Low32bits = 0; + + const char *pchTSteam2ID = pchSteam2ID; + + // Customer support is fond of entering steam IDs in the following form: STEAM_n:x:y + const char *pchOptionalLeadString = "STEAM_"; + if ( V_strnicmp( pchSteam2ID, pchOptionalLeadString, V_strlen( pchOptionalLeadString ) ) == 0 ) + pchTSteam2ID = pchSteam2ID + V_strlen( pchOptionalLeadString ); + + char cExtraCharCheck = 0; + + int cFieldConverted = sscanf( pchTSteam2ID, "%hu:%u:%u%c", &steam2ID.m_SteamInstanceID, + &steam2ID.m_SteamLocalUserID.Split.High32bits, &steam2ID.m_SteamLocalUserID.Split.Low32bits, &cExtraCharCheck ); + + // Validate the conversion ... a special case is steam2 instance ID 1 which is reserved for special DoD handling + if ( cExtraCharCheck != 0 || cFieldConverted == EOF || cFieldConverted < 2 || ( cFieldConverted < 3 && steam2ID.m_SteamInstanceID != 1 ) ) + return false; + + // Now convert to steam ID from the Steam2 ID structure + SetFromSteam2( &steam2ID, eUniverse ); + return true; +} +#endif + +//----------------------------------------------------------------------------- +// Purpose: Renders the steam ID to a buffer. NOTE: for convenience of calling +// code, this code returns a pointer to a static buffer and is NOT thread-safe. +// Output: buffer with rendered Steam ID +//----------------------------------------------------------------------------- +const char * CSteamID::Render() const +{ + // longest length of returned string is k_cBufLen + // [A:%u:%u:%u] + // %u == 10 * 3 + 6 == 36, plus terminator == 37 + const int k_cBufLen = 37; + + const int k_cBufs = 4; // # of static bufs to use (so people can compose output with multiple calls to Render() ) + static char rgchBuf[k_cBufs][k_cBufLen]; + static int nBuf = 0; + char * pchBuf = rgchBuf[nBuf]; // get pointer to current static buf + nBuf ++; // use next buffer for next call to this method + nBuf %= k_cBufs; + + if ( k_EAccountTypeAnonGameServer == m_steamid.m_comp.m_EAccountType ) + { + V_snprintf( pchBuf, k_cBufLen, "[A:%u:%u:%u]", m_steamid.m_comp.m_EUniverse, m_steamid.m_comp.m_unAccountID, m_steamid.m_comp.m_unAccountInstance ); + } + else if ( k_EAccountTypeGameServer == m_steamid.m_comp.m_EAccountType ) + { + V_snprintf( pchBuf, k_cBufLen, "[G:%u:%u]", m_steamid.m_comp.m_EUniverse, m_steamid.m_comp.m_unAccountID ); + } + else if ( k_EAccountTypeMultiseat == m_steamid.m_comp.m_EAccountType ) + { + V_snprintf( pchBuf, k_cBufLen, "[M:%u:%u:%u]", m_steamid.m_comp.m_EUniverse, m_steamid.m_comp.m_unAccountID, m_steamid.m_comp.m_unAccountInstance ); + } + else if ( k_EAccountTypePending == m_steamid.m_comp.m_EAccountType ) + { + V_snprintf( pchBuf, k_cBufLen, "[P:%u:%u]", m_steamid.m_comp.m_EUniverse, m_steamid.m_comp.m_unAccountID ); + } + else if ( k_EAccountTypeContentServer == m_steamid.m_comp.m_EAccountType ) + { + V_snprintf( pchBuf, k_cBufLen, "[C:%u:%u]", m_steamid.m_comp.m_EUniverse, m_steamid.m_comp.m_unAccountID ); + } + else if ( k_EAccountTypeClan == m_steamid.m_comp.m_EAccountType ) + { + // 'g' for "group" + V_snprintf( pchBuf, k_cBufLen, "[g:%u:%u]", m_steamid.m_comp.m_EUniverse, m_steamid.m_comp.m_unAccountID ); + } + else if ( k_EAccountTypeChat == m_steamid.m_comp.m_EAccountType ) + { + if ( m_steamid.m_comp.m_unAccountInstance & k_EChatInstanceFlagClan ) + { + V_snprintf( pchBuf, k_cBufLen, "[c:%u:%u]", m_steamid.m_comp.m_EUniverse, m_steamid.m_comp.m_unAccountID ); + } + else if ( m_steamid.m_comp.m_unAccountInstance & k_EChatInstanceFlagLobby ) + { + V_snprintf( pchBuf, k_cBufLen, "[L:%u:%u]", m_steamid.m_comp.m_EUniverse, m_steamid.m_comp.m_unAccountID ); + } + else // Anon chat + { + V_snprintf( pchBuf, k_cBufLen, "[T:%u:%u]", m_steamid.m_comp.m_EUniverse, m_steamid.m_comp.m_unAccountID ); + } + } + else if ( k_EAccountTypeInvalid == m_steamid.m_comp.m_EAccountType ) + { + V_snprintf( pchBuf, k_cBufLen, "[I:%u:%u]", m_steamid.m_comp.m_EUniverse, m_steamid.m_comp.m_unAccountID ); + } + else if ( k_EAccountTypeIndividual == m_steamid.m_comp.m_EAccountType ) + { + if ( m_steamid.m_comp.m_unAccountInstance != k_unSteamUserDesktopInstance ) + V_snprintf( pchBuf, k_cBufLen, "[U:%u:%u:%u]", m_steamid.m_comp.m_EUniverse, m_steamid.m_comp.m_unAccountID, m_steamid.m_comp.m_unAccountInstance ); + else + V_snprintf( pchBuf, k_cBufLen, "[U:%u:%u]", m_steamid.m_comp.m_EUniverse, m_steamid.m_comp.m_unAccountID ); + } + else if ( k_EAccountTypeAnonUser == m_steamid.m_comp.m_EAccountType ) + { + V_snprintf( pchBuf, k_cBufLen, "[a:%u:%u]", m_steamid.m_comp.m_EUniverse, m_steamid.m_comp.m_unAccountID ); + } + else + { + V_snprintf( pchBuf, k_cBufLen, "[i:%u:%u]", m_steamid.m_comp.m_EUniverse, m_steamid.m_comp.m_unAccountID ); + } + return pchBuf; +} + + +//----------------------------------------------------------------------------- +// Purpose: Renders the passed-in steam ID to a buffer. NOTE: for convenience of calling +// code, this code returns a pointer to a static buffer and is NOT thread-safe. +// Input: 64-bit representation of Steam ID to render +// Output: buffer with rendered Steam ID +//----------------------------------------------------------------------------- +const char * CSteamID::Render( uint64 ulSteamID ) +{ + CSteamID steamID( ulSteamID ); + return steamID.Render(); +} + + +//----------------------------------------------------------------------------- +// Purpose: some steamIDs are for internal use only +// This is really debug code, but we run with asserts on in retail, so ... +//----------------------------------------------------------------------------- +bool CSteamID::BValidExternalSteamID() const +{ + if ( m_steamid.m_comp.m_EAccountType == k_EAccountTypePending ) + return false; + if ( m_steamid.m_comp.m_EAccountType != k_EAccountTypeAnonGameServer && m_steamid.m_comp.m_EAccountType != k_EAccountTypeContentServer && m_steamid.m_comp.m_EAccountType != k_EAccountTypeAnonUser ) + { + if ( m_steamid.m_comp.m_unAccountID == 0 && m_steamid.m_comp.m_unAccountInstance == 0 ) + return false; + } + return true; +} + +#ifdef STEAM +//----------------------------------------------------------------------------- +// Purpose: Returns the matching chat steamID, with the default instance of 0 +// Input: SteamID, either a Clan or a Chat type +// Output: SteamID with account type changed to chat, and the Clan flag set. +// If account type was not chat to start with, instance will be set to 0 +//----------------------------------------------------------------------------- +CSteamID ChatIDFromSteamID( const CSteamID &steamID ) +{ + if ( steamID.GetEAccountType() == k_EAccountTypeChat ) + return steamID; + + return ChatIDFromClanID( steamID ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns the matching chat steamID, with the default instance of 0 +// Input: SteamID, either a Clan type or a Chat type w/ the Clan flag set +// Output: SteamID with account type changed to clan. +// If account type was not clan to start with, instance will be set to 0 +//----------------------------------------------------------------------------- +CSteamID ClanIDFromSteamID( const CSteamID &steamID ) +{ + if ( steamID.GetEAccountType() == k_EAccountTypeClan ) + return steamID; + + return ClanIDFromChatID( steamID ); +} + + +// Asserts steamID type before conversion +CSteamID ChatIDFromClanID( const CSteamID &steamIDClan ) +{ + Assert( steamIDClan.GetEAccountType() == k_EAccountTypeClan ); + + return CSteamID( steamIDClan.GetAccountID(), k_EChatInstanceFlagClan, steamIDClan.GetEUniverse(), k_EAccountTypeChat ); +} + + +// Asserts steamID type before conversion +CSteamID ClanIDFromChatID( const CSteamID &steamIDChat ) +{ + Assert( steamIDChat.GetEAccountType() == k_EAccountTypeChat ); + Assert( k_EChatInstanceFlagClan & steamIDChat.GetUnAccountInstance() ); + + return CSteamID( steamIDChat.GetAccountID(), 0, steamIDChat.GetEUniverse(), k_EAccountTypeClan ); +} + + +//----------------------------------------------------------------------------- +// Purpose: CGameID "hidden" functions +// move these somewhere else maybe +//----------------------------------------------------------------------------- +CGameID::CGameID( const char *pchGameID ) +{ + m_ulGameID = 0; + + sscanf( pchGameID, "%llu", &m_ulGameID ); + + switch ( m_gameID.m_nType ) + { + default: + AssertMsg( false, "Unknown GameID type" ); + m_ulGameID = 0; + break; + case k_EGameIDTypeApp: + case k_EGameIDTypeGameMod: + case k_EGameIDTypeShortcut: + case k_EGameIDTypeP2P: + break; + } +} + + +// renders this Game ID to string +const char * CGameID::Render() const +{ + // longest buffer is log10(2**64) == 20 + 1 == 21 + const int k_cBufLen = 21; + + const int k_cBufs = 4; // # of static bufs to use (so people can compose output with multiple calls to Render() ) + static char rgchBuf[k_cBufs][k_cBufLen]; + static int nBuf = 0; + char * pchBuf = rgchBuf[nBuf]; // get pointer to current static buf + nBuf ++; // use next buffer for next call to this method + nBuf %= k_cBufs; + + V_snprintf( pchBuf, k_cBufLen, "%llu", m_ulGameID ); + + return pchBuf; +} + +// static method to render a uint64 representation of a Game ID to a string +const char * CGameID::Render( uint64 ulGameID ) +{ + CGameID nGameID( ulGameID ); + return nGameID.Render(); +} +#endif diff --git a/gcsdk/steamextra/tier0/t0constants.h b/gcsdk/steamextra/tier0/t0constants.h new file mode 100644 index 0000000..6c1524d --- /dev/null +++ b/gcsdk/steamextra/tier0/t0constants.h @@ -0,0 +1,35 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: declares a variety of constants +// +// $NoKeywords: $ +//============================================================================= + +#ifndef T0CONSTANTS_H +#define T0CONSTANTS_H +#ifdef _WIN32 +#pragma once +#endif + +//----------------------------------------------------------------------------- +// numeric constants to avoid typos with wrong number of zeros +//----------------------------------------------------------------------------- +const int64 k_nMillion = 1000000; +const int64 k_nThousand = 1000; +const int64 k_nKiloByte = 1024; +const int64 k_nMegabyte = k_nKiloByte * k_nKiloByte; + +//----------------------------------------------------------------------------- +// Timing constants +//----------------------------------------------------------------------------- + +const unsigned int k_nSecondsPerHour = 60*60; +const unsigned int k_nSecondsPerDay = k_nSecondsPerHour * 24; + +const int k_cSecondsPerMinute = 60; +const int k_cSecondsPerHour = k_cSecondsPerMinute * 60; +const int k_cSecondsPerDay = k_cSecondsPerHour * 24; +const int k_cSecondsPerWeek = k_cSecondsPerDay * 7; +const int k_cSecondsPerYear = k_cSecondsPerDay * 365; + +#endif // T0CONSTANTS_H
\ No newline at end of file diff --git a/gcsdk/steamextra/tier1/hashglobals.cpp b/gcsdk/steamextra/tier1/hashglobals.cpp new file mode 100644 index 0000000..775e1da --- /dev/null +++ b/gcsdk/steamextra/tier1/hashglobals.cpp @@ -0,0 +1,35 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//============================================================================= + + +#include "stdafx.h" +#include "pearsonshash.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +// This is a table of the values 0-255 in pseudo random order for use in our pearsons hash +// variant implemented below +const unsigned char g_CTHashRandomValues[256] = +{ + 131, 184, 146, 42, 124, 142, 226, 76, 8, 135, 215, 116, 228, 49, 144, 231, + 238, 25, 156, 125, 128, 87, 223, 38, 98, 122, 105, 4, 35, 180, 188, 160, + 179, 59, 218, 0, 192, 207, 209, 150, 227, 143, 140, 161, 73, 84, 111, 162, + 239, 74, 210, 195, 15, 225, 104, 221, 245, 12, 72, 47, 109, 187, 40, 178, + 208, 56, 190, 193, 126, 95, 33, 45, 177, 170, 186, 123, 202, 149, 60, 194, + 168, 102, 71, 148, 46, 121, 52, 119, 196, 247, 127, 145, 75, 79, 61, 254, + 9, 44, 23, 211, 18, 175, 251, 130, 203, 108, 85, 167, 29, 250, 138, 182, + 101, 213, 159, 92, 36, 10, 86, 32, 176, 80, 17, 134, 181, 114, 64, 165, + 89, 68, 6, 14, 205, 137, 117, 7, 39, 132, 26, 19, 214, 99, 166, 163, + 69, 174, 157, 100, 201, 118, 2, 28, 235, 236, 139, 244, 70, 20, 155, 82, + 51, 154, 115, 94, 93, 83, 136, 27, 198, 43, 50, 243, 183, 153, 53, 206, + 77, 55, 57, 3, 220, 147, 253, 110, 37, 246, 97, 13, 120, 103, 91, 169, + 58, 11, 133, 22, 152, 189, 222, 151, 141, 88, 224, 1, 48, 191, 249, 173, + 106, 113, 252, 172, 232, 66, 219, 96, 237, 21, 233, 62, 242, 54, 230, 65, + 78, 248, 16, 41, 31, 200, 90, 112, 255, 171, 164, 24, 199, 81, 212, 197, + 185, 67, 5, 234, 30, 129, 216, 63, 204, 158, 217, 229, 107, 240, 241, 34, +}; diff --git a/gcsdk/steamextra/tier1/murmurhash3.cpp b/gcsdk/steamextra/tier1/murmurhash3.cpp new file mode 100644 index 0000000..72fd8aa --- /dev/null +++ b/gcsdk/steamextra/tier1/murmurhash3.cpp @@ -0,0 +1,74 @@ +//======= Copyright � Valve Corporation, All rights reserved. ================= +// +// Public domain MurmurHash3 by Austin Appleby is a very solid general-purpose +// hash with a 32-bit output. References: +// http://code.google.com/p/smhasher/ (home of MurmurHash3) +// https://sites.google.com/site/murmurhash/avalanche +// http://www.strchr.com/hash_functions +// +//============================================================================= + +#include <stdafx.h> +#include "murmurhash3.h" + +//----------------------------------------------------------------------------- + +uint32 MurmurHash3_32( const void * key, size_t len, uint32 seed, bool bCaselessStringVariant ) +{ + const uint8 * data = (const uint8*)key; + const ptrdiff_t nblocks = len / 4; + uint32 uSourceBitwiseAndMask = 0xDFDFDFDF | ((uint32)bCaselessStringVariant - 1); + + uint32 h1 = seed; + + //---------- + // body + + const uint32 * blocks = (const uint32 *)(data + nblocks*4); + + for(ptrdiff_t i = -nblocks; i; i++) + { + uint32 k1 = LittleDWord(blocks[i]); + k1 &= uSourceBitwiseAndMask; + + k1 *= 0xcc9e2d51; + k1 = (k1 << 15) | (k1 >> 17); + k1 *= 0x1b873593; + + h1 ^= k1; + h1 = (h1 << 13) | (h1 >> 19); + h1 = h1*5+0xe6546b64; + } + + //---------- + // tail + + const uint8 * tail = (const uint8*)(data + nblocks*4); + + uint32 k1 = 0; + + switch(len & 3) + { + case 3: k1 ^= tail[2] << 16; + case 2: k1 ^= tail[1] << 8; + case 1: k1 ^= tail[0]; + k1 &= uSourceBitwiseAndMask; + k1 *= 0xcc9e2d51; + k1 = (k1 << 15) | (k1 >> 17); + k1 *= 0x1b873593; + h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; + + h1 ^= h1 >> 16; + h1 *= 0x85ebca6b; + h1 ^= h1 >> 13; + h1 *= 0xc2b2ae35; + h1 ^= h1 >> 16; + + return h1; +} diff --git a/gcsdk/steamextra/tier1/murmurhash3.h b/gcsdk/steamextra/tier1/murmurhash3.h new file mode 100644 index 0000000..8f477cd --- /dev/null +++ b/gcsdk/steamextra/tier1/murmurhash3.h @@ -0,0 +1,100 @@ +//======= Copyright � Valve Corporation, All rights reserved. ================= +// +// Public domain MurmurHash3 by Austin Appleby is a very solid general-purpose +// hash with a 32-bit output. References: +// http://code.google.com/p/smhasher/ (home of MurmurHash3) +// https://sites.google.com/site/murmurhash/avalanche +// http://www.strchr.com/hash_functions +// +//============================================================================= + +#ifndef MURMURHASH3_H +#define MURMURHASH3_H + +#if defined(_WIN32) +#pragma once +#endif + +uint32 MurmurHash3_32( const void *key, size_t len, uint32 seed, bool bCaselessStringVariant = false ); + +inline uint32 MurmurHash3String( const char *pszKey, size_t len ) +{ + return MurmurHash3_32( pszKey, len, 1047 /*anything will do for a seed*/, false ); +} + +inline uint32 MurmurHash3StringCaseless( const char *pszKey, size_t len ) +{ + return MurmurHash3_32( pszKey, len, 1047 /*anything will do for a seed*/, true ); +} + +inline uint32 MurmurHash3String( const char *pszKey ) +{ + return MurmurHash3String( pszKey, strlen( pszKey ) ); +} + +inline uint32 MurmurHash3StringCaseless( const char *pszKey ) +{ + return MurmurHash3StringCaseless( pszKey, strlen( pszKey ) ); +} + +template <typename T> +inline uint32 MurmurHash3Item( const T &item ) +{ + return MurmurHash3_32( &item, sizeof(item), 1047 ); +} + +inline uint32 MurmurHash3Int( uint32 h ) +{ + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + return h; +} + + +template <> +inline uint32 MurmurHash3Item( const uint32 &item ) +{ + return MurmurHash3Int( item ); +} + +template <> +inline uint32 MurmurHash3Item( const int32 &item ) +{ + return MurmurHash3Int( item ); +} + + +template<typename T> +struct MurmurHash3Functor +{ + typedef uint32 TargetType; + TargetType operator()(const T &key) const + { + return MurmurHash3Item( key ); + } +}; + +template<> +struct MurmurHash3Functor<char *> +{ + typedef uint32 TargetType; + TargetType operator()(const char *key) const + { + return MurmurHash3String( key ); + } +}; + +template<> +struct MurmurHash3Functor<const char *> +{ + typedef uint32 TargetType; + TargetType operator()(const char *key) const + { + return MurmurHash3String( key ); + } +}; + +#endif // MURMURHASH3_H diff --git a/gcsdk/steamextra/tier1/pearsonshash.h b/gcsdk/steamextra/tier1/pearsonshash.h new file mode 100644 index 0000000..29da4a5 --- /dev/null +++ b/gcsdk/steamextra/tier1/pearsonshash.h @@ -0,0 +1,268 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: PearsonsHash.h +// +// This file contains implementation definitions of 'Pearson's' Hash' +// The generic implementation is a template, plus a couple of specializations +// for commonly used types. +// +// Take a look at http://en.wikipedia.org/wiki/Pearson_hashing for more info. +// +//============================================================================= + +#ifndef _PEARSONSHASH_H_ +#define _PEARSONSHASH_H_ + +#if defined(_WIN32) || defined(_WIN64) +#pragma once +#endif + +extern const unsigned char g_CTHashRandomValues[256] ; + +template<typename T> +struct PearsonsHashFunctor +{ + typedef uint32 TargetType ; + TargetType operator()(const T& unKey) const + { + // This is a pearsons hash variant that returns a maximum of 32 bits + size_t size = sizeof(T); + const uint8 * k = (const uint8 *) &unKey; + uint32 byte_one = 0, byte_two = 0, byte_three = 0, byte_four = 0, n; + while (size) + { + --size; + n = *k++; + byte_one = g_CTHashRandomValues[byte_one ^ n]; + + if (size) + { + --size; + n = *k++; + byte_two = g_CTHashRandomValues[byte_two ^ n]; + } + else + break; + + if (size) + { + --size; + n = *k++; + byte_three = g_CTHashRandomValues[byte_three ^ n]; + } + else + break; + + if (size) + { + --size; + n = *k++; + byte_four = g_CTHashRandomValues[byte_four ^ n]; + } + else + break; + } + + TargetType idx = ( byte_four << 24 ) | ( byte_three << 16 ) | ( byte_two << 8 ) | byte_one; + return ( idx ); + } +}; + +// +// We use this specialization for pointer types - it allows somebody +// to define a specialization for some complicated type, and then use a +// pointer to that type as a key, and have that automatically go to the +// specialization for the complicated type ! +// +template<typename T> +struct PearsonsHashFunctor<T*> +{ + typedef uint32 TargetType ; + TargetType operator()(const T* key) const + { + PearsonsHashFunctor<T> functor ; + return functor(*key) ; + } +}; + +// +// This functor specializes for unsigned 32 bit integers, a commonly used type in Steam. +// It should return the exact same result as the unspecialized version on Intel Architecture machines. +// +template<> +struct PearsonsHashFunctor<uint32> +{ + typedef uint32 TargetType ; + TargetType operator()(const uint32 unKey) const + { + uint32 byte_one = g_CTHashRandomValues[(unKey>>0) & 0xff]; + uint32 byte_two = g_CTHashRandomValues[(unKey>>8) & 0xff]; + uint32 byte_three = g_CTHashRandomValues[(unKey>>16) & 0xff]; + uint32 byte_four = g_CTHashRandomValues[(unKey>>24)&0xff]; + return ( byte_four << 24 ) | ( byte_three << 16 ) | ( byte_two << 8 ) | byte_one; + } +}; + +// +// This functor specializes for unsigned 64 bit integers, another commonly used type in Steam. +// It should return the exact same result as the unspecialized version on Intel Architecture machines. +// +template<> +struct PearsonsHashFunctor<uint64> +{ + typedef uint32 TargetType ; + TargetType operator()(const uint64 unKey) const + { + // + // Note that we pull apart the 64 bits in Intel's endian order. + // + uint32 n; + // + // On Intel Machines, to make this return the exact same result as the generic version + // we have to go from least significant byte to most significant byte ! + // + n = static_cast<uint32>((unKey >> (0)) & 0xff) ; + uint32 byte_one = g_CTHashRandomValues[n]; + n = static_cast<uint32>((unKey >> (8)) & 0xff) ; + uint32 byte_two = g_CTHashRandomValues[n]; + n = static_cast<uint32>((unKey >> (16)) & 0xff) ; + uint32 byte_three = g_CTHashRandomValues[n]; + n = static_cast<uint32>((unKey >> (24)) & 0xff) ; + uint32 byte_four = g_CTHashRandomValues[n]; + n = static_cast<uint32>((unKey >> (32)) & 0xff) ; + byte_one = g_CTHashRandomValues[n ^ byte_one]; + n = static_cast<uint32>((unKey >> (8+32)) & 0xff) ; + byte_two = g_CTHashRandomValues[n ^ byte_two]; + n = static_cast<uint32>((unKey >> (16+32)) & 0xff) ; + byte_three = g_CTHashRandomValues[n ^ byte_three]; + n = static_cast<uint32>((unKey >> (24+32)) & 0xff) ; + byte_four = g_CTHashRandomValues[n ^ byte_four]; + return ( byte_four << 24 ) | ( byte_three << 16 ) | ( byte_two << 8 ) | byte_one; + } +}; + +// +// This functor specializes for C standard NULL terminated strings ! +// It is setup so that if you had a char array containing a NULL terminated string +// and correctly sized, ie char rgch[16] = { "123456789012345" } and a +// null terminated string i.e. char *sz = "123456789012345" these will return identical +// results, and both include the NULL terminator in the hash calculation. +// +template<> +struct PearsonsHashFunctor<char*> +{ + typedef uint32 TargetType ; + TargetType operator()(const char* szKey) const + { + const uint8 * k = (const uint8 *) szKey ; + uint32 byte_one = 0, byte_two = 0, byte_three = 0, byte_four = 0, n; + do + { + n = *k++; + byte_one = g_CTHashRandomValues[byte_one ^ n]; + if (n=='\0') + break; + + n = *k++; + byte_two = g_CTHashRandomValues[byte_two ^ n]; + if (n=='\0') + break; + + n = *k++; + byte_three = g_CTHashRandomValues[byte_three ^ n]; + if (n=='\0') + break; + + n = *k++; + byte_four = g_CTHashRandomValues[byte_four ^ n]; + } while(n!='\0') ; + + TargetType idx = ( byte_four << 24 ) | ( byte_three << 16 ) | ( byte_two << 8 ) | byte_one; + return ( idx ); + } +}; + +// +// This functor compares two objects of a particular type and returns a result +// that follows the strcmp/memcmp. +// If the type doesn't provide an operator== or operator< then you can provide +// a type specific specialization to override this defualt functor ! +// +template<typename T> +struct ComparisonFunctor +{ + int operator()(const T &lhs, const T &rhs) const + { + if( lhs == rhs ) + return 0 ; + else if( lhs < rhs ) + return -1 ; + else + return 1 ; + } + +}; + +// +// I expect people to build Comparison specializations for full types, +// not pointer types - this Specialization should allow the C++ compiler +// to bind to a specialization for a particular type. +// We do not want to compare pointers for equality, and a memcmp() may +// not be good for a complicated type ! +// +template<typename T> +struct ComparisonFunctor<T*> +{ + int operator()(const T *lhs, const T *rhs) const + { + ComparisonFunctor<T> functor ; + return functor(*lhs, *rhs) ; + } +}; + +template<> +struct ComparisonFunctor<int> +{ + int operator()(const int lhs, const int rhs) const + { + return lhs-rhs ; + } +}; + +template<> +struct ComparisonFunctor<unsigned int> +{ + int operator()(const unsigned int lhs, const unsigned int rhs) const + { + return static_cast<int>(lhs) - static_cast<int>(rhs) ; + } +}; + + +template<> +struct ComparisonFunctor<uint64> +{ + int operator()(const uint64 lhs, const uint64 rhs) const + { + if( lhs < rhs ) + return -1 ; + else if( lhs == rhs ) + return 0 ; + else + return 1 ; + } +}; + +template<> +struct ComparisonFunctor<char*> +{ + int operator()(const char * lhs, const char* rhs) const + { + return Q_strcmp(lhs, rhs) ; + } +}; + + +#endif // _PEARSONSHASH_H_ + + diff --git a/gcsdk/steamextra/tier1/tsmempool.cpp b/gcsdk/steamextra/tier1/tsmempool.cpp new file mode 100644 index 0000000..71b0f5b --- /dev/null +++ b/gcsdk/steamextra/tier1/tsmempool.cpp @@ -0,0 +1,255 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// The copyright to the contents herein is the property of Valve, L.L.C. +// The contents may be used and/or copied only with the written permission of +// Valve, L.L.C., or in accordance with the terms and conditions stipulated in +// the agreement/contract under which the contents have been supplied. +// +// Purpose: +//============================================================================= + + +//#include "pch_vstdlib.h" + +#include "stdafx.h" +#include "tier0/tslist.h" +#include "tier0/t0constants.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +static const uint k_cubBytesAllocatedToConsiderFreeingMemory = 5 * k_nMegabyte; +static const int k_cBlocksAllocatedToConsiderFreeingMemory = 10; + +typedef TSLNodeBase_t FreeListItem_t; + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CThreadSafeMemoryPool::CThreadSafeMemoryPool( int blockSize, int numElements, int growMode ) +{ + m_ptslistFreeBlocks = new CTSListBase; + + // round up to the nearest 8-byte boundary + if ( blockSize % TSLIST_NODE_ALIGNMENT != 0 ) + { + blockSize += TSLIST_NODE_ALIGNMENT - (blockSize % TSLIST_NODE_ALIGNMENT); + } + Assert( blockSize % TSLIST_NODE_ALIGNMENT == 0 ); + Assert( blockSize > sizeof(FreeListItem_t) ); + m_nGrowMode = growMode; + m_cubBlockSize = blockSize; + m_nGrowSize = numElements; + m_cubAllocated = 0; +} + + +//----------------------------------------------------------------------------- +// Purpose: Frees the memory contained in the mempool +//----------------------------------------------------------------------------- +CThreadSafeMemoryPool::~CThreadSafeMemoryPool() +{ + AUTO_LOCK_SPIN_WRITE( m_threadRWLock ); + FOR_EACH_VEC( m_vecBlockSets, i ) + { + _aligned_free( m_vecBlockSets[i].m_pubBlockSet ); + } + + delete m_ptslistFreeBlocks; +} + + +//----------------------------------------------------------------------------- +// Purpose: Frees everything +//----------------------------------------------------------------------------- +void CThreadSafeMemoryPool::Clear() +{ + AUTO_LOCK_SPIN_WRITE( m_threadRWLock ); + ClearNoLock(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Frees everything +//----------------------------------------------------------------------------- +void CThreadSafeMemoryPool::ClearNoLock() +{ + FOR_EACH_VEC( m_vecBlockSets, i ) + { + _aligned_free( m_vecBlockSets[i].m_pubBlockSet ); + } + m_ptslistFreeBlocks->Detach(); + m_cubAllocated = 0; + m_cBlocksInUse = 0; + m_vecBlockSets.RemoveAll(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Allocates a single block of memory from the pool. +//----------------------------------------------------------------------------- +void *CThreadSafeMemoryPool::Alloc() +{ + return Alloc( m_cubBlockSize ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Allocates a single block of memory from the pool. +//----------------------------------------------------------------------------- +void *CThreadSafeMemoryPool::Alloc( unsigned int amount ) +{ + // loop attempting to get memory + // there appears to be a case where memory corruption can get this into an infinite loop + // normally 1 or 2 attempts are necessary to get a block, so if we hit 1000 we know something is wrong + int cAttempts = 1000; + while ( --cAttempts ) + { + // pull first from the free list + m_threadRWLock.LockForRead(); + FreeListItem_t *pFreeListItem = m_ptslistFreeBlocks->Pop(); + if ( pFreeListItem ) + { + m_threadRWLock.UnlockRead(); + m_cBlocksInUse++; + return (void *)pFreeListItem; + } + m_threadRWLock.UnlockRead(); + + // no free items, add a new block + + // switch from a read lock to a write lock + AUTO_LOCK_SPIN_WRITE( m_threadRWLock ); + + // another thread may have allocated memory; try the free list again if so + if ( m_ptslistFreeBlocks->Count() > 0 ) + continue; + + size_t cubBlob = m_nGrowSize * m_cubBlockSize; + if ( m_nGrowMode == GROW_FAST ) + { + cubBlob *= (m_vecBlockSets.Count() + 1); + } + + // don't grow if we're told not to + if ( m_nGrowMode == GROW_NONE && m_vecBlockSets.Count() == 1 ) + return NULL; + + // allocate, but we can fail + byte *pBlobBase = (byte *)MemAlloc_AllocAligned( cubBlob, TSLIST_NODE_ALIGNMENT /*, (m_nGrowMode == GROW_TIL_YOU_CANT)*/ ); + if ( !pBlobBase ) + return NULL; + + byte *pBlobEnd = pBlobBase + cubBlob; + // add all the items to the pool + for ( byte *pBlob = pBlobBase; pBlob < pBlobEnd; pBlob += m_cubBlockSize ) + { + m_ptslistFreeBlocks->Push( (FreeListItem_t *)pBlob ); + } + + m_cubAllocated += cubBlob; + BlockSet_t blockSet = { pBlobBase, cubBlob }; + m_vecBlockSets.AddToTail( blockSet ); + } + + return NULL; +} + + +//----------------------------------------------------------------------------- +// Purpose: Frees a block of memory +//----------------------------------------------------------------------------- +void CThreadSafeMemoryPool::Free( void *pMem ) +{ + Free( pMem, m_cubBlockSize ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Frees a block of memory +//----------------------------------------------------------------------------- +void CThreadSafeMemoryPool::Free( void *pMem, int cubAlloc ) +{ + m_threadRWLock.LockForRead(); + + // push the item back onto the free list + m_ptslistFreeBlocks->Push( (FreeListItem_t *)pMem ); + m_cBlocksInUse--; + + m_threadRWLock.UnlockRead(); + + // if we're completely free, and have too much memory allocated, free some + if ( m_cBlocksInUse == 0 + && m_cubAllocated >= k_cubBytesAllocatedToConsiderFreeingMemory + && m_vecBlockSets.Count() >= k_cBlocksAllocatedToConsiderFreeingMemory ) + { + AUTO_LOCK_SPIN_WRITE( m_threadRWLock ); + + // check again nothing is in use + if ( m_cBlocksInUse == 0 ) + { + // free all the blocks + ClearNoLock(); + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: display +//----------------------------------------------------------------------------- +void CThreadSafeMemoryPool::PrintStats() +{ + AUTO_LOCK_SPIN_WRITE( m_threadRWLock ); + int cBlocksInUse = m_cBlocksInUse; + Msg( "Block size: %-11s Alloc'd: %8d Num blobs: %5d (%s)\n", Q_pretifymem( m_cubBlockSize, 2, true ), + cBlocksInUse, m_vecBlockSets.Count(), Q_pretifymem( m_cubAllocated, 2, true ) ); +} + + +//----------------------------------------------------------------------------- +// Purpose: data accessor +//----------------------------------------------------------------------------- +size_t CThreadSafeMemoryPool::CubTotalSize() +{ + return m_cubAllocated; +} + + +//----------------------------------------------------------------------------- +// Purpose: data accessor +//----------------------------------------------------------------------------- +size_t CThreadSafeMemoryPool::CubSizeInUse() +{ + return m_cBlocksInUse * m_cubBlockSize; +} + + +//----------------------------------------------------------------------------- +// Purpose: data accessor +//----------------------------------------------------------------------------- +int CThreadSafeMemoryPool::Count() +{ + return m_cBlocksInUse; +} + + +#ifdef DBGFLAG_VALIDATE +//----------------------------------------------------------------------------- +// Purpose: Run a global validation pass on all of our data structures and memory +// allocations. +// Input: validator - Our global validator object +// pchName - Our name (typically a member var in our container) +//----------------------------------------------------------------------------- +void CThreadSafeMemoryPool::Validate( CValidator &validator, const char *pchName ) +{ + AUTO_LOCK_SPIN_WRITE( m_threadRWLock ); + VALIDATE_SCOPE(); + FOR_EACH_VEC( m_vecBlockSets, i ) + { + validator.ClaimMemory( MemAlloc_Unalign( m_vecBlockSets[i].m_pubBlockSet ) ); + } + ValidateObj( m_vecBlockSets ); + validator.ClaimMemory( MemAlloc_Unalign( m_ptslistFreeBlocks ) ); +} +#endif // DBGFLAG_VALIDATE diff --git a/gcsdk/steamextra/tier1/tsmempool.h b/gcsdk/steamextra/tier1/tsmempool.h new file mode 100644 index 0000000..402681c --- /dev/null +++ b/gcsdk/steamextra/tier1/tsmempool.h @@ -0,0 +1,170 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// The copyright to the contents herein is the property of Valve, L.L.C. +// The contents may be used and/or copied only with the written permission of +// Valve, L.L.C., or in accordance with the terms and conditions stipulated in +// the agreement/contract under which the contents have been supplied. +// +// Purpose: +// +// $Workfile: $ +// $Date: $ +// +//----------------------------------------------------------------------------- +// $Log: $ +// +// $NoKeywords: $ +//============================================================================= + +#ifndef TSMEMPOOL_H +#define TSMEMPOOL_H + +#ifdef _WIN32 +#pragma once +#endif + +#undef new + +//----------------------------------------------------------------------------- +// Purpose: Optimized pool memory allocator +//----------------------------------------------------------------------------- +class CThreadSafeMemoryPool +{ +public: + enum + { + GROW_NONE=0, // Don't allow new blobs. + GROW_FAST=1, // New blob size is numElements * (i+1) (ie: the blocks it allocates get larger and larger each time it allocates one). + GROW_SLOW=2, // New blob size is numElements. + GROW_TIL_YOU_CANT=3 // GROW_SLOW til alloc fails - then STOP and dont assert! + }; + + CThreadSafeMemoryPool( int blockSize, int numElements, int growMode = GROW_FAST ); + ~CThreadSafeMemoryPool(); + + void *Alloc(); // Allocate the element size you specified in the constructor. + void *Alloc( unsigned int cubAlloc ); + void Free( void *pMem ); + void Free( void *pMem, int cubAlloc ); + + // Frees everything + void Clear(); + + // display + void PrintStats(); + size_t CubTotalSize(); + size_t CubSizeInUse(); + int Count(); + + static void * operator new( size_t size ) + { + CThreadSafeMemoryPool *pNode = (CThreadSafeMemoryPool *)MemAlloc_AllocAligned( size, 8, __FILE__, __LINE__ +#ifdef STEAM + , true // new operator +#endif + ); + return pNode; + } + + static void * operator new( size_t size, int nBlockUse, const char *pFileName, int nLine ) + { + CThreadSafeMemoryPool *pNode = (CThreadSafeMemoryPool *)MemAlloc_AllocAligned( size, 8, pFileName, nLine +#ifdef STEAM + , true // new operator +#endif + ); + return pNode; + } + + static void operator delete( void *p) + { + MemAlloc_FreeAligned( p +#ifdef STEAM + , true // new operator +#endif + ); + } + + static void operator delete( void *p, int nBlockUse, const char *pFileName, int nLine ) + { + MemAlloc_FreeAligned( p +#ifdef STEAM + , true // new operator +#endif + ); + } + +#ifdef DBGFLAG_VALIDATE + void Validate( CValidator &validator, const char *pchName ); // Validate our internal structures +#endif // DBGFLAG_VALIDATE + +private: + // These ain't gonna work + static void * operator new[] ( size_t size ); + static void operator delete [] ( void *p); + + // CThreadSpinRWLock needs 8 byte alignment to work but we new() CThreadSafeMemoryPool + // so simply place it at the start of the class to make sure it fits on the 8-byte boundary + CThreadSpinRWLock m_threadRWLock; + + int m_nGrowMode; + int m_cubBlockSize; + int m_nGrowSize; + + void ClearNoLock(); + + CInterlockedInt m_cBlocksInUse; + size_t m_cubAllocated; + + struct BlockSet_t + { + byte *m_pubBlockSet; + size_t m_cubAllocated; + }; + CUtlVector<BlockSet_t> m_vecBlockSets; + + class CTSListBase *m_ptslistFreeBlocks; +}; + + +//----------------------------------------------------------------------------- +// Wrapper macro to make an allocator that returns particular typed allocations +// and construction and destruction of objects. +//----------------------------------------------------------------------------- +template< class T > +class CThreadSafeClassMemoryPool : public CThreadSafeMemoryPool +{ +public: + CThreadSafeClassMemoryPool(int numElements, int growMode = GROW_FAST) : + CThreadSafeMemoryPool( sizeof(T), numElements, growMode ) {} + + T* Alloc(); + void Free( T *pMem ); +}; + + +template< class T > +T* CThreadSafeClassMemoryPool<T>::Alloc() +{ + T *pRet = (T*)CThreadSafeMemoryPool::Alloc(); + if ( pRet ) + { + Construct( pRet ); + } + return pRet; +} + + +template< class T > +void CThreadSafeClassMemoryPool<T>::Free(T *pMem) +{ + if ( pMem ) + { + Destruct( pMem ); + } + + CThreadSafeMemoryPool::Free( pMem ); +} + + +#endif // TSMEMPOOL_H diff --git a/gcsdk/steamextra/tier1/tsmultimempool.cpp b/gcsdk/steamextra/tier1/tsmultimempool.cpp new file mode 100644 index 0000000..86a8b50 --- /dev/null +++ b/gcsdk/steamextra/tier1/tsmultimempool.cpp @@ -0,0 +1,383 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +// +//=============================================================================// + +#include <stdafx.h> +#include "tier0/t0constants.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +static const int k_cubMemBlockPrefixSize = sizeof(uint32); + +#define ALLOCSIZE_TO_LOOKUP( cubAlloc ) ( (cubAlloc - 1) >> 5 ) +#define LOOKUP_TO_ALLOCSIZE( iLookup ) ( (iLookup << 5) + 1 ) + + +//----------------------------------------------------------------------------- +// Purpose: constructor, the sizes in pMemPoolConfig must be in ascending order +//----------------------------------------------------------------------------- +CThreadSafeMultiMemoryPool::CThreadSafeMultiMemoryPool( const MemPoolConfig_t *pMemPoolConfig, int cnMemPoolConfig, int nGrowMode /*= GROW_FAST*/ ) +{ + m_cubReallocedTotal = 0; + m_MapRawAllocation.SetLessFunc( DefLessFunc( void * ) ); + + for ( int iMemPoolConfig = 0; iMemPoolConfig < cnMemPoolConfig; iMemPoolConfig++ ) + { + MemPoolRecord_t memPoolRecord; + // verify that the mem pool sizes are in ascending order + Assert( iMemPoolConfig == 0 || ( iMemPoolConfig > 0 && pMemPoolConfig[ iMemPoolConfig - 1 ].m_cubBlockSize < pMemPoolConfig[ iMemPoolConfig].m_cubBlockSize ) ); + AssertMsg( pMemPoolConfig[ iMemPoolConfig].m_cubBlockSize % 32 == 0, "Mempools sizes must be multiples of 32" ); + // add an int to the block size so we can note the alloc size + memPoolRecord.m_pMemPool = new CThreadSafeMemoryPool( pMemPoolConfig[ iMemPoolConfig ].m_cubBlockSize + k_cubMemBlockPrefixSize, + pMemPoolConfig[ iMemPoolConfig ].m_cubDefaultPoolSize, nGrowMode ); + Assert( memPoolRecord.m_pMemPool ); + memPoolRecord.m_nBlockSize = pMemPoolConfig[ iMemPoolConfig ].m_cubBlockSize; + m_VecMemPool.AddToTail( memPoolRecord ); + + // update the largest blocksize + m_nBlockSizeMax = MAX( m_nBlockSizeMax, memPoolRecord.m_nBlockSize ); + } + + // build the lookup table + int nLookupMax = m_nBlockSizeMax >> 5; + m_VecMemPoolLookup.AddMultipleToTail( nLookupMax ); + for ( int i = 0; i < nLookupMax; i++ ) + { + uint32 cubAllocSize = LOOKUP_TO_ALLOCSIZE( i ); + for ( int iMemPool = 0; iMemPool < m_VecMemPool.Count(); iMemPool++ ) + { + if ( m_VecMemPool[iMemPool].m_nBlockSize >= cubAllocSize ) + { + m_VecMemPoolLookup[i] = &m_VecMemPool[iMemPool]; + break; + } + } + } + +#if defined(_DEBUG) + // validate the lookup table + for ( int i = 1; i < (int)m_nBlockSizeMax; i++ ) + { + for ( int iMemPool = 0; iMemPool < m_VecMemPool.Count(); iMemPool++ ) + { + if ( (int)m_VecMemPool[iMemPool].m_nBlockSize >= i ) + { + AssertMsg( m_VecMemPoolLookup[ALLOCSIZE_TO_LOOKUP(i)] == &m_VecMemPool[iMemPool], "Invalid mempool block size, can't generate lookup table" ); + break; + } + } + } +#endif // _DEBUG +} + + +//----------------------------------------------------------------------------- +// Purpose: destructor +//----------------------------------------------------------------------------- +CThreadSafeMultiMemoryPool::~CThreadSafeMultiMemoryPool() +{ + AUTO_LOCK( m_mutexRawAllocations ); + + for ( int iMemPool = 0; iMemPool < m_VecMemPool.Count(); iMemPool ++ ) + { + delete m_VecMemPool[iMemPool].m_pMemPool; + } + + FOR_EACH_MAP_FAST( m_MapRawAllocation, iRawAllocation ) + { + FreePv( m_MapRawAllocation[iRawAllocation].m_pvMem ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Allocates a block of memory at of least nAllocSize bytes +// Input : nAllocSize - number of bytes to alloc +// Output : pointer to memory alloc'd, NULL on error +//----------------------------------------------------------------------------- +void *CThreadSafeMultiMemoryPool::Alloc( uint32 cubAllocSize ) +{ + if ( cubAllocSize == 0 ) + return NULL; + + if ( cubAllocSize <= m_nBlockSizeMax ) + { + MemPoolRecord_t *pMemPoolRecord = m_VecMemPoolLookup[ALLOCSIZE_TO_LOOKUP( cubAllocSize )]; + void *pvMem = pMemPoolRecord->m_pMemPool->Alloc( cubAllocSize + k_cubMemBlockPrefixSize ); + *(uint32 *)pvMem = cubAllocSize; + return ( (char *)pvMem + k_cubMemBlockPrefixSize ); + } + + + // can't fit in our mem pools, alloc it in our one off buffer + RawAllocation_t rawAllocation; + rawAllocation.m_nBlockSize = cubAllocSize; + rawAllocation.m_pvMem = PvAlloc( cubAllocSize + k_cubMemBlockPrefixSize ); + if ( !rawAllocation.m_pvMem ) + { + return NULL; + } + *(uint32 *)rawAllocation.m_pvMem = rawAllocation.m_nBlockSize; + AUTO_LOCK( m_mutexRawAllocations ); + m_MapRawAllocation.Insert( rawAllocation.m_pvMem, rawAllocation ); + return ( (char *)rawAllocation.m_pvMem + k_cubMemBlockPrefixSize ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Free a previously alloc'd block +// Input : pMem - memory to free +//----------------------------------------------------------------------------- +void CThreadSafeMultiMemoryPool::Free( void *pvMem ) +{ + if ( !pvMem ) + return; + + uint32 cubAllocSize = *( (uint32 *)pvMem - 1 ); + + if ( cubAllocSize <= m_nBlockSizeMax ) + { + MemPoolRecord_t *pMemPoolRecord = m_VecMemPoolLookup[ALLOCSIZE_TO_LOOKUP( cubAllocSize )]; + pMemPoolRecord->m_pMemPool->Free( (char *)pvMem - k_cubMemBlockPrefixSize, cubAllocSize + k_cubMemBlockPrefixSize ); + return; + } + + AUTO_LOCK( m_mutexRawAllocations ); + + // must have been alloc'd from the raw heap, find it in map + void *pvAllocedMem = (char *)pvMem - k_cubMemBlockPrefixSize; + int iRawAllocation = m_MapRawAllocation.Find( pvAllocedMem ); + if ( m_MapRawAllocation.InvalidIndex() == iRawAllocation ) + { + AssertMsg3( false, "CThreadSafeMultiMemoryPool::Free: raw allocation %p (original alloc: %p, %d bytes) not found in allocation map", + pvMem, pvAllocedMem, cubAllocSize ); + return; + + } + + FreePv( m_MapRawAllocation[iRawAllocation].m_pvMem ); + m_MapRawAllocation.RemoveAt( iRawAllocation); +} + + +//----------------------------------------------------------------------------- +// Purpose: Return the size alloc'd for this block +// Input : pMem - memory to report +// Output : size in bytes of this memory +//----------------------------------------------------------------------------- +int CThreadSafeMultiMemoryPool::CubAllocSize(void *pvMem) +{ + if ( !pvMem ) + { + return -1; + } + + return *(((uint32 *)pvMem) -1); +} + + +//----------------------------------------------------------------------------- +// Purpose: Frees all previously alloc'd memory +//----------------------------------------------------------------------------- +void CThreadSafeMultiMemoryPool::Clear() +{ + AUTO_LOCK( m_mutexRawAllocations ); + + for ( int iMemPool = 0; iMemPool < m_VecMemPool.Count(); iMemPool++ ) + { + m_VecMemPool[iMemPool].m_pMemPool->Clear(); + } + + FOR_EACH_MAP_FAST( m_MapRawAllocation, iRawAllocation ) + { + FreePv( m_MapRawAllocation[iRawAllocation].m_pvMem ); + } + m_MapRawAllocation.RemoveAll(); +} + + +//----------------------------------------------------------------------------- +// Purpose: print to the console info about our storage +//----------------------------------------------------------------------------- +void CThreadSafeMultiMemoryPool::PrintStats() +{ + for ( int iMemPool= 0; iMemPool < m_VecMemPool.Count(); iMemPool++ ) + { + m_VecMemPool[iMemPool].m_pMemPool->PrintStats(); + } + int cubRawBytesAllocd = 0; + AUTO_LOCK( m_mutexRawAllocations ); + FOR_EACH_MAP_FAST( m_MapRawAllocation, iRawAllocation ) + { + cubRawBytesAllocd += m_MapRawAllocation[iRawAllocation].m_nBlockSize; + } + Msg( "Raw bytes alloc'd: %s\n", Q_pretifymem( cubRawBytesAllocd, 2, true ) ); + Msg( "Cumulative bytes re-alloced: %s\n", Q_pretifymem( m_cubReallocedTotal, 2, true ) ); +} + + +//----------------------------------------------------------------------------- +// Purpose: return the total mem alloced by this pool in MB +//----------------------------------------------------------------------------- +int CThreadSafeMultiMemoryPool::CMBPoolSize() const +{ + uint64 cubRawBytesAllocd = 0; + for ( int iMemPool= 0; iMemPool < m_VecMemPool.Count(); iMemPool++ ) + { + cubRawBytesAllocd += ( m_VecMemPool[iMemPool].m_pMemPool->CubTotalSize() ); + } + AUTO_LOCK( m_mutexRawAllocations ); + FOR_EACH_MAP_FAST( m_MapRawAllocation, iRawAllocation ) + { + cubRawBytesAllocd += m_MapRawAllocation[iRawAllocation].m_nBlockSize; + } + + return ( cubRawBytesAllocd / k_nMegabyte ); +} + +//----------------------------------------------------------------------------- +// Purpose: return the total mem alloced by this pool in MB +//----------------------------------------------------------------------------- +int CThreadSafeMultiMemoryPool::CMBPoolSizeInUse() const +{ + uint64 cubRawBytesAllocd = 0; + for ( int iMemPool= 0; iMemPool < m_VecMemPool.Count(); iMemPool++ ) + { + cubRawBytesAllocd += ( m_VecMemPool[iMemPool].m_pMemPool->CubSizeInUse() ); + } + AUTO_LOCK( m_mutexRawAllocations ); + FOR_EACH_MAP_FAST( m_MapRawAllocation, iRawAllocation ) + { + cubRawBytesAllocd += m_MapRawAllocation[iRawAllocation].m_nBlockSize; + } + + return ( cubRawBytesAllocd / k_nMegabyte ); +} + + +//----------------------------------------------------------------------------- +// Purpose: return number of mempool blocks alloc'd +//----------------------------------------------------------------------------- +int CThreadSafeMultiMemoryPool::Count() +{ + int cCount = 0; + for ( int iMemPool = 0; iMemPool < m_VecMemPool.Count(); iMemPool++ ) + { + cCount += m_VecMemPool[iMemPool].m_pMemPool->Count(); + } + return cCount; +} + + +//----------------------------------------------------------------------------- +// Purpose: reallocate an existing block of memory to a new size (and copy the data +// Input: pvMem - a pointer to the existing memory +// cubAlloc - number of bytes to alloc +// Output: returns a pointer to the memory allocated (NULL on error) +//----------------------------------------------------------------------------- +void *CThreadSafeMultiMemoryPool::ReAlloc( void *pvMem, uint32 cubAlloc ) +{ + uint32 cubOldAlloc = CubAllocSize(pvMem); + if ( pvMem && cubAlloc <= cubOldAlloc ) + return pvMem; + + if ( cubOldAlloc > m_nBlockSizeMax ) + { + AUTO_LOCK( m_mutexRawAllocations ); + // okay, must have been alloc'd from the raw heap, search for it + void *pvAllocedMem = (char *)pvMem - k_cubMemBlockPrefixSize; + int iRawAllocation = m_MapRawAllocation.Find( pvAllocedMem ); + if ( m_MapRawAllocation.InvalidIndex() == iRawAllocation ) + { + AssertMsg3( false, "CThreadSafeMultiMemoryPool::ReAlloc: raw allocation %p (original alloc: %p, %d bytes) not found in allocation map", + pvMem, pvAllocedMem, cubOldAlloc ); + return NULL; + } + + // realloc the memory + void *pvNewMem = PvRealloc( pvAllocedMem, cubAlloc + k_cubMemBlockPrefixSize ); + if ( !pvNewMem ) + { + m_MapRawAllocation.RemoveAt( iRawAllocation ); + return NULL; + } + + // update our tracking + *(uint32 *)pvNewMem = cubAlloc; + if ( pvAllocedMem == pvNewMem ) + { + // if pointer is the same, use the same map entry with the same key (the pointer given to caller) + m_MapRawAllocation[iRawAllocation].m_pvMem = pvNewMem; + m_MapRawAllocation[iRawAllocation].m_nBlockSize = cubAlloc; + } + else + { + // if pointer changed, need to remove the old entry and re-insert with new key + m_MapRawAllocation.RemoveAt( iRawAllocation ); + RawAllocation_t rawAllocation; + rawAllocation.m_pvMem = pvNewMem; + rawAllocation.m_nBlockSize = cubAlloc; + m_MapRawAllocation.Insert( rawAllocation.m_pvMem, rawAllocation ); + } + return ( (char *)pvNewMem + k_cubMemBlockPrefixSize ); + } + else + { + // see if we can stay in the same block + MemPoolRecord_t *pMemPoolRecord = m_VecMemPoolLookup[ALLOCSIZE_TO_LOOKUP( cubOldAlloc )]; + if ( cubAlloc <= pMemPoolRecord->m_nBlockSize ) + { + // re-assign the size + *((uint32 *)pvMem - 1) = cubAlloc; + return pvMem; + } + + void *pvNewMem = Alloc( cubAlloc ); + if ( !pvNewMem ) + { + return NULL; + } + m_cubReallocedTotal += cubOldAlloc; + Q_memcpy( pvNewMem, pvMem, cubOldAlloc ); + Free( pvMem ); // now free the old memory buffer we had + return pvNewMem; + } +} + + +#ifdef DBGFLAG_VALIDATE +//----------------------------------------------------------------------------- +// Purpose: Ensure that all of our internal structures are consistent, and +// account for all memory that we've allocated. +// Input: validator - Our global validator object +// pchName - Our name (typically a member var in our container) +//----------------------------------------------------------------------------- +void CThreadSafeMultiMemoryPool::Validate( CValidator &validator, const char *pchName ) +{ + validator.Push( "CThreadSafeMultiMemoryPool", this, pchName ); + + ValidateObj( m_VecMemPool ); + for( int iMemPool = 0; iMemPool < m_VecMemPool.Count(); iMemPool++ ) + { + validator.ClaimMemory_Aligned( m_VecMemPool[iMemPool].m_pMemPool ); + m_VecMemPool[iMemPool].m_pMemPool->Validate( validator, "m_VecMemPool[iMemPool].m_pMemPool" ); + } + + AUTO_LOCK( m_mutexRawAllocations ); + ValidateObj( m_MapRawAllocation ); + FOR_EACH_MAP_FAST( m_MapRawAllocation, iRawAllocation ) + { + validator.ClaimMemory( m_MapRawAllocation[iRawAllocation].m_pvMem ); + } + + ValidateObj( m_VecMemPoolLookup ); + + validator.Pop(); +} +#endif // DBGFLAG_VALIDATE + diff --git a/gcsdk/steamextra/tier1/tsmultimempool.h b/gcsdk/steamextra/tier1/tsmultimempool.h new file mode 100644 index 0000000..860e9df --- /dev/null +++ b/gcsdk/steamextra/tier1/tsmultimempool.h @@ -0,0 +1,97 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// The copyright to the contents herein is the property of Valve, L.L.C. +// The contents may be used and/or copied only with the written permission of +// Valve, L.L.C., or in accordance with the terms and conditions stipulated in +// the agreement/contract under which the contents have been supplied. +// +// Purpose: +// +// $Workfile: $ +// $Date: $ +// +//----------------------------------------------------------------------------- +// $Log: $ +// +// $NoKeywords: $ +//============================================================================= + +#ifndef TSMULTIMEMPOOL_H +#define TSMULTIMEMPOOL_H + +#ifdef _WIN32 +#pragma once +#endif + +#include "tier1/utlmap.h" +#include "tier1/mempool.h" +#include "tier1/tsmempool.h" + +//----------------------------------------------------------------------------- +// Purpose: A container of a range of mem pool sizes (for network buffers for example) +// and a raw alloc capability (for sizes greater than any contained mem pool). +//----------------------------------------------------------------------------- +class CThreadSafeMultiMemoryPool +{ +public: + struct MemPoolConfig_t + { + uint32 m_cubBlockSize; + uint32 m_cubDefaultPoolSize; + }; + + CThreadSafeMultiMemoryPool( const MemPoolConfig_t *pnBlock, int cnMemPoolConfig, int nGrowMode = UTLMEMORYPOOL_GROW_FAST ); + ~CThreadSafeMultiMemoryPool(); + + // Allocate a block of at least nAllocSize bytes + void* Alloc( uint32 cubAlloc ); + // Free a previously alloc'd block + void Free(void *pvMem); + // ReAllocate a previously allocated block to a new size + void* ReAlloc( void *pvMem, uint32 cubAlloc ); + + // Frees everything + void Clear(); + + // alloc size for this bit of alloc'd memory + int CubAllocSize( void *pvMem ); + // prints details about our contained memory + void PrintStats(); + + // total number of alloc'd elements + int Count(); + + // Return the total size in MB allocated for this pool + int CMBPoolSize() const; + // Return the amount of memory in use + int CMBPoolSizeInUse() const; + +#ifdef DBGFLAG_VALIDATE + void Validate( CValidator &validator, const char *pchName ); // Validate our internal structures +#endif // DBGFLAG_VALIDATE + +private: + struct MemPoolRecord_t + { + CThreadSafeMemoryPool *m_pMemPool; + uint32 m_nBlockSize; + }; + + CUtlVector<MemPoolRecord_t> m_VecMemPool; // stores our list of mem pools + + uint32 m_nBlockSizeMax; + CUtlVector<MemPoolRecord_t *> m_VecMemPoolLookup; // quick lookup table of mempools + + struct RawAllocation_t + { + void *m_pvMem; + uint32 m_nBlockSize; + }; + CUtlMap<void *,RawAllocation_t,int> m_MapRawAllocation; // stores our list of raw alloc'd mem + CThreadFastMutex m_mutexRawAllocations; + + uint32 m_cubReallocedTotal; +}; + + +#endif // TSMULTIMEMPOOL_H diff --git a/gcsdk/steamextra/tier1/utlhashmaplarge.h b/gcsdk/steamextra/tier1/utlhashmaplarge.h new file mode 100644 index 0000000..5525930 --- /dev/null +++ b/gcsdk/steamextra/tier1/utlhashmaplarge.h @@ -0,0 +1,693 @@ +//========= Copyright Valve Corporation, All rights reserved. =================// +// +// Purpose: index-based hash map container well suited for large and growing +// datasets. It uses less memory than other hash maps and incrementally +// rehashes to reduce reallocation spikes. +// +//=============================================================================// + +#ifndef UTLHASHMAPLARGE_H +#define UTLHASHMAPLARGE_H + +#ifdef _WIN32 +#pragma once +#endif + +#include "tier0/dbg.h" +#include "bitvec.h" +#include "tier1/murmurhash3.h" + +// fast mod for power of 2 numbers +namespace basetypes +{ +template <class T> +inline bool IsPowerOf2(T n) +{ + return n > 0 && (n & (n-1)) == 0; +} + +template <class T1, class T2> +inline T2 ModPowerOf2(T1 a, T2 b) +{ + return T2(a) & (b-1); +} +} + +// default comparison operator +template <typename T> +class CDefEquals +{ +public: + CDefEquals() {} + CDefEquals( int i ) {} + inline bool operator()( const T &lhs, const T &rhs ) const { return ( lhs == rhs ); } + inline bool operator!() const { return false; } +}; + + +// Specialization to compare pointers +template <typename T> +class CDefEquals<T*> +{ +public: + CDefEquals() {} + CDefEquals( int i ) {} + inline bool operator()( const T *lhs, const T *rhs ) const + { + if ( lhs == rhs ) + return true; + else if ( NULL == lhs || NULL == rhs ) + return false; + else + return ( *lhs == *rhs ); + } + inline bool operator!() const { return false; } +}; + + +// Hash specialization for CUtlStrings +template<> +struct MurmurHash3Functor<CUtlString> +{ + typedef uint32 TargetType ; + TargetType operator()(const CUtlString &strKey) const + { + return MurmurHash3Functor<const char*>()( strKey.String() ); + } +}; + +//hash 3 function for a general case sensitive string compares +struct MurmurHash3ConstCharPtr +{ + typedef uint32 TargetType ; + TargetType operator()( const char* pszKey ) const { return MurmurHash3Functor<const char*>()( pszKey ); } +}; +struct CaseSensitiveStrEquals +{ + bool operator()( const char* pszLhs, const char* pszRhs ) const { return strcmp( pszLhs, pszRhs ) == 0; } +}; + +//----------------------------------------------------------------------------- +// +// Purpose: An associative container. Pretty much identical to CUtlMap without the ability to walk in-order +// This container is well suited for large and growing datasets. It uses less +// memory than other hash maps and incrementally rehashes to reduce reallocation spikes. +// However, it is slower (by about 20%) than CUtlHashTable +// +//----------------------------------------------------------------------------- +template <typename K, typename T, typename L = CDefEquals<K>, typename H = MurmurHash3Functor<K> > +class CUtlHashMapLarge +{ +public: + // This enum exists so that FOR_EACH_MAP and FOR_EACH_MAP_FAST cannot accidentally + // be used on a type that is not a CUtlMap. If the code compiles then all is well. + // The check for IsUtlMap being true should be free. + // Using an enum rather than a static const bool ensures that this trick works even + // with optimizations disabled on gcc. + enum CompileTimeCheck + { + IsUtlMap = 1 + }; + + typedef K KeyType_t; + typedef T ElemType_t; + typedef int IndexType_t; + typedef L EqualityFunc_t; + typedef H HashFunc_t; + + CUtlHashMapLarge() + { + m_cElements = 0; + m_nMaxElement = 0; + m_nMinRehashedBucket = InvalidIndex(); + m_nMaxRehashedBucket = InvalidIndex(); + m_iNodeFreeListHead = InvalidIndex(); + } + + CUtlHashMapLarge( int cElementsExpected ) + { + m_cElements = 0; + m_nMaxElement = 0; + m_nMinRehashedBucket = InvalidIndex(); + m_nMaxRehashedBucket = InvalidIndex(); + m_iNodeFreeListHead = InvalidIndex(); + EnsureCapacity( cElementsExpected ); + } + + ~CUtlHashMapLarge() + { + RemoveAll(); + } + + // gets particular elements + ElemType_t & Element( IndexType_t i ) { return m_memNodes.Element( i ).m_elem; } + const ElemType_t & Element( IndexType_t i ) const { return m_memNodes.Element( i ).m_elem; } + ElemType_t & operator[]( IndexType_t i ) { return m_memNodes.Element( i ).m_elem; } + const ElemType_t & operator[]( IndexType_t i ) const { return m_memNodes.Element( i ).m_elem; } + KeyType_t & Key( IndexType_t i ) { return m_memNodes.Element( i ).m_key; } + const KeyType_t & Key( IndexType_t i ) const { return m_memNodes.Element( i ).m_key; } + + // Num elements + IndexType_t Count() const { return m_cElements; } + + // Max "size" of the vector + IndexType_t MaxElement() const { return m_nMaxElement; } + + // Checks if a node is valid and in the map + bool IsValidIndex( IndexType_t i ) const { return i >= 0 && i < m_nMaxElement && !IsFreeNodeID( m_memNodes[i].m_iNextNode ); } + + // Invalid index + static IndexType_t InvalidIndex() { return -1; } + + // Insert method + IndexType_t Insert( const KeyType_t &key, const ElemType_t &insert ) { return InsertInternal( key, insert, eInsert_UpdateExisting ); } + IndexType_t Insert( const KeyType_t &key ) { return InsertInternal( key, ElemType_t(), eInsert_UpdateExisting ); } + IndexType_t InsertWithDupes( const KeyType_t &key, const ElemType_t &insert ) { return InsertInternal( key, insert, eInsert_CreateDupes ); } + IndexType_t FindOrInsert( const KeyType_t &key, const ElemType_t &insert ) { return InsertInternal( key, insert, eInsert_LeaveExisting ); } + IndexType_t InsertOrReplace( const KeyType_t &key, const ElemType_t &insert ) { return InsertInternal( key, insert, eInsert_UpdateExisting ); } + + + // Finds an element + IndexType_t Find( const KeyType_t &key ) const; + + // has an element + bool HasElement( const KeyType_t &key ) const + { + return Find( key ) != InvalidIndex(); + } + + void EnsureCapacity( int num ); + + void RemoveAt( IndexType_t i ); + bool Remove( const KeyType_t &key ) + { + int iMap = Find( key ); + if ( iMap != InvalidIndex() ) + { + RemoveAt( iMap ); + return true; + } + return false; + } + void RemoveAll(); + void Purge(); + void PurgeAndDeleteElements(); + + void Swap( CUtlHashMapLarge<K,T,L,H> &rhs ) + { + m_vecHashBuckets.Swap( rhs.m_vecHashBuckets ); + V_swap( m_bitsMigratedBuckets, rhs.m_bitsMigratedBuckets ); + m_memNodes.Swap( rhs.m_memNodes ); + V_swap( m_iNodeFreeListHead, rhs.m_iNodeFreeListHead ); + V_swap( m_cElements, rhs.m_cElements ); + V_swap( m_nMaxElement, rhs.m_nMaxElement ); + V_swap( m_nMinRehashedBucket, rhs.m_nMinRehashedBucket ); + V_swap( m_nMaxRehashedBucket, rhs.m_nMaxRehashedBucket ); + V_swap( m_EqualityFunc, rhs.m_EqualityFunc ); + V_swap( m_HashFunc, rhs.m_HashFunc ); + } + +private: + enum EInsertPolicy { eInsert_UpdateExisting, eInsert_LeaveExisting, eInsert_CreateDupes }; + IndexType_t InsertInternal( const KeyType_t &key, const ElemType_t &insert, EInsertPolicy ePolicy ); + + inline IndexType_t FreeNodeIDToIndex( IndexType_t i ) const { return (0-i)-3; } + inline IndexType_t FreeNodeIndexToID( IndexType_t i ) const { return (-3)-i; } + inline bool IsFreeNodeID( IndexType_t i ) const { return i < InvalidIndex(); } + + int FindInBucket( int iBucket, const KeyType_t &key ) const; + int AllocNode(); + void RehashNodesInBucket( int iBucket ); + void LinkNodeIntoBucket( int iBucket, int iNewNode ); + void UnlinkNodeFromBucket( int iBucket, int iNewNode ); + bool RemoveNodeFromBucket( int iBucket, int iNodeToRemove ); + void IncrementalRehash(); + + struct HashBucket_t + { + IndexType_t m_iNode; + }; + CUtlVector<HashBucket_t> m_vecHashBuckets; + + CLargeVarBitVec m_bitsMigratedBuckets; + + struct Node_t + { + KeyType_t m_key; + ElemType_t m_elem; + int m_iNextNode; + }; + CUtlMemory<Node_t> m_memNodes; + IndexType_t m_iNodeFreeListHead; + + IndexType_t m_cElements; + IndexType_t m_nMaxElement; + IndexType_t m_nMinRehashedBucket, m_nMaxRehashedBucket; + EqualityFunc_t m_EqualityFunc; + HashFunc_t m_HashFunc; +}; + + +//----------------------------------------------------------------------------- +// Purpose: inserts an item into the map +//----------------------------------------------------------------------------- +template <typename K, typename T, typename L, typename H> +inline int CUtlHashMapLarge<K,T,L,H>::InsertInternal( const KeyType_t &key, const ElemType_t &insert, EInsertPolicy ePolicy ) +{ + // make sure we have room in the hash table + if ( m_cElements >= m_vecHashBuckets.Count() ) + EnsureCapacity( MAX( 16, m_vecHashBuckets.Count() * 2 ) ); + if ( m_cElements >= m_memNodes.Count() ) + m_memNodes.Grow( m_memNodes.Count() * 2 ); + + // rehash incrementally + IncrementalRehash(); + + // hash the item + uint32 hash = m_HashFunc( key ); + + // migrate data forward, if necessary + int cBucketsToModAgainst = m_vecHashBuckets.Count() >> 1; + int iBucket = basetypes::ModPowerOf2(hash, cBucketsToModAgainst); + while ( iBucket >= m_nMinRehashedBucket + && !m_bitsMigratedBuckets.Get( iBucket ) ) + { + RehashNodesInBucket( iBucket ); + cBucketsToModAgainst >>= 1; + iBucket = basetypes::ModPowerOf2(hash, cBucketsToModAgainst); + } + + // prevent duplicates if necessary + if ( ( ePolicy != eInsert_CreateDupes ) && m_cElements ) + { + // look in the bucket to see if we have a conflict + int iBucket2 = basetypes::ModPowerOf2( hash, m_vecHashBuckets.Count() ); + IndexType_t iNode = FindInBucket( iBucket2, key ); + if ( iNode != InvalidIndex() ) + { + // a duplicate - update in place (matching CUtlMap) + if( ePolicy == eInsert_UpdateExisting ) + { + m_memNodes[iNode].m_elem = insert; + } + return iNode; + } + } + + // make an item + int iNewNode = AllocNode(); + m_memNodes[iNewNode].m_iNextNode = InvalidIndex(); + CopyConstruct( &m_memNodes[iNewNode].m_key, key ); + CopyConstruct( &m_memNodes[iNewNode].m_elem, insert ); + + iBucket = basetypes::ModPowerOf2( hash, m_vecHashBuckets.Count() ); + + // link ourselves in + // ::OutputDebugStr( CFmtStr( "insert %d into bucket %d\n", key, iBucket ).Access() ); + LinkNodeIntoBucket( iBucket, iNewNode ); + + // return the new node + return iNewNode; +} + +//----------------------------------------------------------------------------- +// Purpose: grows the map to fit the specified amount +//----------------------------------------------------------------------------- +template <typename K, typename T, typename L, typename H> +inline void CUtlHashMapLarge<K,T,L,H>::EnsureCapacity( int amount ) +{ + m_memNodes.EnsureCapacity( amount ); + // ::OutputDebugStr( CFmtStr( "grown m_memNodes from %d to %d\n", m_cElements, m_memNodes.Count() ).Access() ); + + if ( amount <= m_vecHashBuckets.Count() ) + return; + int cBucketsNeeded = MAX( 16, m_vecHashBuckets.Count() ); + while ( cBucketsNeeded < amount ) + cBucketsNeeded *= 2; + + // ::OutputDebugStr( CFmtStr( "grown m_vecHashBuckets from %d to %d\n", m_vecHashBuckets.Count(), cBucketsNeeded ).Access() ); + + // grow the hash buckets + int grow = cBucketsNeeded - m_vecHashBuckets.Count(); + int iFirst = m_vecHashBuckets.AddMultipleToTail( grow ); + // clear all the new data to invalid bits + memset( &m_vecHashBuckets[iFirst], 0xFFFFFFFF, grow*sizeof(m_vecHashBuckets[iFirst]) ); + Assert( basetypes::IsPowerOf2( m_vecHashBuckets.Count() ) ); + + // we'll have to rehash, all the buckets that existed before growth + m_nMinRehashedBucket = 0; + m_nMaxRehashedBucket = iFirst; + if ( m_cElements > 0 ) + { + // remove all the current bits + m_bitsMigratedBuckets.Resize( 0 ); + // re-add new bits; these will all be reset to 0 + m_bitsMigratedBuckets.Resize( m_vecHashBuckets.Count() ); + } + else + { + // no elements - no rehashing + m_nMinRehashedBucket = m_vecHashBuckets.Count(); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: gets a new node, from the free list if possible +//----------------------------------------------------------------------------- +template <typename K, typename T, typename L, typename H> +inline int CUtlHashMapLarge<K,T,L,H>::AllocNode() +{ + // if we're out of free elements, get the max + if ( m_cElements == m_nMaxElement ) + { + m_cElements++; + return m_nMaxElement++; + } + + // pull from the free list + Assert( m_iNodeFreeListHead != InvalidIndex() ); + int iNewNode = m_iNodeFreeListHead; + m_iNodeFreeListHead = FreeNodeIDToIndex( m_memNodes[iNewNode].m_iNextNode ); + m_cElements++; + return iNewNode; +} + + +//----------------------------------------------------------------------------- +// Purpose: takes a bucket of nodes and re-hashes them into a more optimal bucket +//----------------------------------------------------------------------------- +template <typename K, typename T, typename L, typename H> +inline void CUtlHashMapLarge<K,T,L,H>::RehashNodesInBucket( int iBucketSrc ) +{ + // mark us as migrated + m_bitsMigratedBuckets.Set( iBucketSrc ); + + // walk the list of items, re-hashing them + IndexType_t iNode = m_vecHashBuckets[iBucketSrc].m_iNode; + while ( iNode != InvalidIndex() ) + { + IndexType_t iNodeNext = m_memNodes[iNode].m_iNextNode; + Assert( iNodeNext != iNode ); + + // work out where the node should go + const KeyType_t &key = m_memNodes[iNode].m_key; + uint32 hash = m_HashFunc( key ); + int iBucketDest = basetypes::ModPowerOf2( hash, m_vecHashBuckets.Count() ); + + // if the hash bucket has changed, move it + if ( iBucketDest != iBucketSrc ) + { + // ::OutputDebugStr( CFmtStr( "moved key %d from bucket %d to %d\n", key, iBucketSrc, iBucketDest ).Access() ); + + // remove from this bucket list + UnlinkNodeFromBucket( iBucketSrc, iNode ); + + // link into new bucket list + LinkNodeIntoBucket( iBucketDest, iNode ); + } + iNode = iNodeNext; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: searches for an item by key, returning the index handle +//----------------------------------------------------------------------------- +template <typename K, typename T, typename L, typename H> +inline int CUtlHashMapLarge<K,T,L,H>::Find( const KeyType_t &key ) const +{ + if ( m_cElements == 0 ) + return InvalidIndex(); + + // hash the item + uint32 hash = m_HashFunc( key ); + + // find the bucket + int cBucketsToModAgainst = m_vecHashBuckets.Count(); + int iBucket = basetypes::ModPowerOf2( hash, cBucketsToModAgainst ); + + // look in the bucket for the item + int iNode = FindInBucket( iBucket, key ); + if ( iNode != InvalidIndex() ) + return iNode; + + // not found? we may have to look in older buckets + cBucketsToModAgainst >>= 1; + while ( cBucketsToModAgainst >= m_nMinRehashedBucket ) + { + iBucket = basetypes::ModPowerOf2( hash, cBucketsToModAgainst ); + + if ( !m_bitsMigratedBuckets.Get( iBucket ) ) + { + int iNode2 = FindInBucket( iBucket, key ); + if ( iNode2 != InvalidIndex() ) + return iNode2; + } + + cBucketsToModAgainst >>= 1; + } + + return InvalidIndex(); +} + + +//----------------------------------------------------------------------------- +// Purpose: searches for an item by key, returning the index handle +//----------------------------------------------------------------------------- +template <typename K, typename T, typename L, typename H> +inline int CUtlHashMapLarge<K,T,L,H>::FindInBucket( int iBucket, const KeyType_t &key ) const +{ + if ( m_vecHashBuckets[iBucket].m_iNode != InvalidIndex() ) + { + IndexType_t iNode = m_vecHashBuckets[iBucket].m_iNode; + Assert( iNode < m_nMaxElement ); + while ( iNode != InvalidIndex() ) + { + // equality check + if ( m_EqualityFunc( key, m_memNodes[iNode].m_key ) ) + return iNode; + + iNode = m_memNodes[iNode].m_iNextNode; + } + } + + return InvalidIndex(); +} + + +//----------------------------------------------------------------------------- +// Purpose: links a node into a bucket +//----------------------------------------------------------------------------- +template <typename K, typename T, typename L, typename H> +void CUtlHashMapLarge<K,T,L,H>::LinkNodeIntoBucket( int iBucket, int iNewNode ) +{ + // add into the start of the bucket's list + m_memNodes[iNewNode].m_iNextNode = m_vecHashBuckets[iBucket].m_iNode; + m_vecHashBuckets[iBucket].m_iNode = iNewNode; +} + + +//----------------------------------------------------------------------------- +// Purpose: unlinks a node from the bucket +//----------------------------------------------------------------------------- +template <typename K, typename T, typename L, typename H> +void CUtlHashMapLarge<K,T,L,H>::UnlinkNodeFromBucket( int iBucket, int iNodeToUnlink ) +{ + int iNodeNext = m_memNodes[iNodeToUnlink].m_iNextNode; + + // if it's the first node, just update the bucket to point to the new place + int iNode = m_vecHashBuckets[iBucket].m_iNode; + if ( iNode == iNodeToUnlink ) + { + m_vecHashBuckets[iBucket].m_iNode = iNodeNext; + return; + } + + // walk the list to find where + while ( iNode != InvalidIndex() ) + { + if ( m_memNodes[iNode].m_iNextNode == iNodeToUnlink ) + { + m_memNodes[iNode].m_iNextNode = iNodeNext; + return; + } + iNode = m_memNodes[iNode].m_iNextNode; + } + + // should always be valid to unlink + Assert( false ); +} + + +//----------------------------------------------------------------------------- +// Purpose: removes a single item from the map +//----------------------------------------------------------------------------- +template <typename K, typename T, typename L, typename H> +inline void CUtlHashMapLarge<K,T,L,H>::RemoveAt( IndexType_t i ) +{ + if ( !IsValidIndex( i ) ) + { + Assert( false ); + return; + } + + // unfortunately, we have to re-hash to find which bucket we're in + uint32 hash = m_HashFunc( m_memNodes[i].m_key ); + int cBucketsToModAgainst = m_vecHashBuckets.Count(); + int iBucket = basetypes::ModPowerOf2( hash, cBucketsToModAgainst ); + if ( RemoveNodeFromBucket( iBucket, i ) ) + return; + + // wasn't found; look in older buckets + cBucketsToModAgainst >>= 1; + while ( cBucketsToModAgainst >= m_nMinRehashedBucket ) + { + iBucket = basetypes::ModPowerOf2( hash, cBucketsToModAgainst ); + + if ( !m_bitsMigratedBuckets.Get( iBucket ) ) + { + if ( RemoveNodeFromBucket( iBucket, i ) ) + return; + } + + cBucketsToModAgainst >>= 1; + } + + // never found, container is busted + Assert( false ); +} + + +//----------------------------------------------------------------------------- +// Purpose: removes a node from the bucket, return true if it was found +//----------------------------------------------------------------------------- +template <typename K, typename T, typename L, typename H> +inline bool CUtlHashMapLarge<K,T,L,H>::RemoveNodeFromBucket( IndexType_t iBucket, int iNodeToRemove ) +{ + IndexType_t iNode = m_vecHashBuckets[iBucket].m_iNode; + while ( iNode != InvalidIndex() ) + { + if ( iNodeToRemove == iNode ) + { + // found it, remove + UnlinkNodeFromBucket( iBucket, iNodeToRemove ); + Destruct( &m_memNodes[iNode].m_key ); + Destruct( &m_memNodes[iNode].m_elem ); + + // link into free list + m_memNodes[iNode].m_iNextNode = FreeNodeIndexToID( m_iNodeFreeListHead ); + m_iNodeFreeListHead = iNode; + m_cElements--; + if ( m_cElements == 0 ) + { + m_nMinRehashedBucket = m_vecHashBuckets.Count(); + } + return true; + } + + iNode = m_memNodes[iNode].m_iNextNode; + } + + return false; +} + + +//----------------------------------------------------------------------------- +// Purpose: removes all items from the hash map +//----------------------------------------------------------------------------- +template <typename K, typename T, typename L, typename H> +inline void CUtlHashMapLarge<K,T,L,H>::RemoveAll() +{ + FOR_EACH_MAP_FAST( *this, i ) + { + Destruct( &m_memNodes[i].m_key ); + Destruct( &m_memNodes[i].m_elem ); + } + + m_cElements = 0; + m_nMaxElement = 0; + m_iNodeFreeListHead = InvalidIndex(); + m_nMinRehashedBucket = m_vecHashBuckets.Count(); + m_nMaxRehashedBucket = InvalidIndex(); + m_bitsMigratedBuckets.Resize( 0 ); + memset( m_vecHashBuckets.Base(), 0xFF, m_vecHashBuckets.Count() * sizeof(HashBucket_t) ); +} + + +//----------------------------------------------------------------------------- +// Purpose: removes all items from the hash map and releases memory +//----------------------------------------------------------------------------- +template <typename K, typename T, typename L, typename H> +inline void CUtlHashMapLarge<K,T,L,H>::Purge() +{ + FOR_EACH_MAP_FAST( *this, i ) + { + Destruct( &m_memNodes[i].m_key ); + Destruct( &m_memNodes[i].m_elem ); + } + + m_cElements = 0; + m_nMaxElement = 0; + m_iNodeFreeListHead = InvalidIndex(); + m_nMinRehashedBucket = InvalidIndex(); + m_nMaxRehashedBucket = InvalidIndex(); + + m_bitsMigratedBuckets.Resize( 0 ); + m_memNodes.Purge(); + m_vecHashBuckets.Purge(); +} + + +//----------------------------------------------------------------------------- +// Purpose: removes and deletes all items from the hash map and releases memory +//----------------------------------------------------------------------------- +template <typename K, typename T, typename L, typename H> +inline void CUtlHashMapLarge<K,T,L,H>::PurgeAndDeleteElements() +{ + FOR_EACH_MAP_FAST( *this, i ) + { + delete this->Element( i ); + } + + Purge(); +} + + +//----------------------------------------------------------------------------- +// Purpose: rehashes buckets +//----------------------------------------------------------------------------- +template <typename K, typename T, typename L, typename H> +inline void CUtlHashMapLarge<K,T,L,H>::IncrementalRehash() +{ + if ( m_nMinRehashedBucket < m_nMaxRehashedBucket ) + { + while ( m_nMinRehashedBucket < m_nMaxRehashedBucket ) + { + // see if the bucket needs rehashing + if ( m_vecHashBuckets[m_nMinRehashedBucket].m_iNode != InvalidIndex() + && !m_bitsMigratedBuckets.Get(m_nMinRehashedBucket) ) + { + // rehash this bucket + RehashNodesInBucket( m_nMinRehashedBucket ); + // only actively do one - don't want to do it too fast since we may be on a rapid growth path + ++m_nMinRehashedBucket; + break; + } + + // nothing to rehash in that bucket - increment and look again + ++m_nMinRehashedBucket; + } + + if ( m_nMinRehashedBucket >= m_nMaxRehashedBucket ) + { + // we're done; don't need any bits anymore + m_nMinRehashedBucket = m_vecHashBuckets.Count(); + m_nMaxRehashedBucket = InvalidIndex(); + m_bitsMigratedBuckets.Resize( 0 ); + } + } +} + + +#endif // UTLHASHMAPLARGE_H diff --git a/gcsdk/steammessages.proto b/gcsdk/steammessages.proto new file mode 100644 index 0000000..5eda1a4 --- /dev/null +++ b/gcsdk/steammessages.proto @@ -0,0 +1,860 @@ +//====== Copyright 1996-2010, Valve Corporation, All rights reserved. ======= +// +// Purpose: The file defines our Google Protocol Buffers which are used in over +// the wire messages between servers as well as between clients and servers. +// +//============================================================================= + +// We care more about speed than code size +option optimize_for = SPEED; + +// We don't use the service generation functionality +option cc_generic_services = false; + + +// +// STYLE NOTES: +// +// Use CamelCase CMsgMyMessageName style names for messages. +// +// Use lowercase _ delimited names like my_steam_id for field names, this is non-standard for Steam, +// but plays nice with the Google formatted code generation. +// +// Try not to use required fields ever. Only do so if you are really really sure you'll never want them removed. +// Optional should be preffered as it will make versioning easier and cleaner in the future if someone refactors +// your message and wants to remove or rename fields. +// +// Use fixed64 for JobId_t, GID_t, or SteamID. This is appropriate for any field that is normally +// going to be larger than 2^56. Otherwise use int64 for 64 bit values that are frequently smaller +// than 2^56 as it will safe space on the wire in those cases. +// +// Similar to fixed64, use fixed32 for RTime32 or other 32 bit values that are frequently larger than +// 2^28. It will safe space in those cases, otherwise use int32 which will safe space for smaller values. +// An exception to this rule for RTime32 is if the value will frequently be zero rather than set to an actual +// time. +// + +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + optional bool key_field = 60000 [ default = false ]; +} + + +extend google.protobuf.MessageOptions{ + // Allows us to customize the pooling for different messages + optional int32 msgpool_soft_limit = 60000 [default=32]; + optional int32 msgpool_hard_limit = 60001 [default=384]; +} + +enum GCProtoBufMsgSrc +{ + GCProtoBufMsgSrc_Unspecified = 0; + GCProtoBufMsgSrc_FromSystem = 1; + GCProtoBufMsgSrc_FromSteamID = 2; + GCProtoBufMsgSrc_FromGC = 3; + GCProtoBufMsgSrc_ReplySystem = 4; +}; + +// +// Message header, every protcol buffer based message starts with this. +// +message CMsgProtoBufHeader +{ + option (msgpool_soft_limit) = 256; + option (msgpool_hard_limit) = 1024; + + // All fields here are optional. + + // Client message header fields + optional fixed64 client_steam_id = 1; // SteamID of the client sending this, typically set in all client originated messages. + optional int32 client_session_id = 2; // SessionID of the client on the CM + + // Source appId for inter-gc messages + optional uint32 source_app_id = 3; // appId of source GC message sender + + // Job routing (may be set on client or inter-server messages) + optional fixed64 job_id_source = 10 [ default = 0xFFFFFFFFFFFFFFFF ]; // JobID that sent this message + optional fixed64 job_id_target = 11 [ default = 0xFFFFFFFFFFFFFFFF ]; // The target job which is expected to be waiting on this message + optional string target_job_name = 12; // the type of job to start when this message is received + + optional int32 eresult = 13 [default = 2]; // For response jobs, the corresponding eresult + optional string error_message = 14; // Optionally an error message in case of failure. Mostly used for debugging purpose. + + // Where did this message originally enter the system? From a client, from another GC, etc + optional GCProtoBufMsgSrc gc_msg_src = 200; + // If this came from another GC, what is the GC that it came from + optional uint32 gc_dir_index_source = 201; +} + + +// +// Used to serialize CWebAPIKey objects. +// +message CMsgWebAPIKey +{ + optional uint32 status = 1 [ default = 0xFF ]; + optional uint32 account_id = 2 [ default = 0 ]; + optional uint32 publisher_group_id = 3 [ default = 0 ]; + optional uint32 key_id = 4; + optional string domain = 5; +} + + +// +// An HTTP request message +// +message CMsgHttpRequest +{ + message RequestHeader + { + optional string name = 1; + optional string value = 2; + } + + message QueryParam + { + optional string name = 1; + optional bytes value = 2; + } + + optional uint32 request_method = 1; + optional string hostname = 2; + optional string url = 3; + repeated RequestHeader headers = 4; + repeated QueryParam get_params = 5; + repeated QueryParam post_params = 6; + optional bytes body = 7; + optional uint32 absolute_timeout = 8; +} + + +// +// A web API request +// +message CMsgWebAPIRequest +{ + optional string UNUSED_job_name = 1; // no longer used + optional string interface_name = 2; + optional string method_name = 3; + optional uint32 version = 4; + optional CMsgWebAPIKey api_key = 5; + optional CMsgHttpRequest request = 6; + optional uint32 routing_app_id = 7; +} + + +// +// An HTTP response +// +message CMsgHttpResponse +{ + message ResponseHeader + { + optional string name = 1; + optional string value = 2; + } + + optional uint32 status_code = 1; + repeated ResponseHeader headers = 2; + optional bytes body = 3; +} + +// +// Message struct for k_EMsgAMFindAccounts +// +message CMsgAMFindAccounts +{ + optional uint32 search_type = 1; + optional string search_string = 2; +} + + +// +// Message struct for k_EMsgAMFindAccountsResponse +// +message CMsgAMFindAccountsResponse +{ + repeated fixed64 steam_id = 1; +} + +// +// k_EMsgNotifyWatchdog +// +message CMsgNotifyWatchdog +{ + optional uint32 source = 1; // Alert source + optional uint32 alert_type = 2; // type of alert + optional uint32 alert_destination = 3; // destination for alert + optional bool critical = 4; // Is the alert critical + optional uint32 time = 5; // world time that alert occurred + optional uint32 appid = 6; // app to forward the alert to for alerts with alert_type set to AppID + optional string text = 7; // Alert text +} + +// +// k_EGCMsgGetLicenses +// +message CMsgAMGetLicenses +{ + optional fixed64 steamid = 1; // the steam ID to fetch licenses for +} + + +// +// Used by CMsgAMGetLicensesResponse +// +message CMsgPackageLicense +{ + optional uint32 package_id = 1; // ID of the package this license is for + optional uint32 time_created = 2; // RTime32 when the license was granted + optional uint32 owner_id = 3; // the original owner if this license. if this is different from given steamid, it's a borrowed package +} + +// +// k_EMsgAMGetLicensesResponse +// +message CMsgAMGetLicensesResponse +{ + repeated CMsgPackageLicense license = 1; // the list of licenses the user owns + optional uint32 result = 2; // result code, k_EResultOK on success + +} + + +// +// k_EMsgAMGetUserGameStats +// +message CMsgAMGetUserGameStats +{ + optional fixed64 steam_id = 1; // ID of user + optional fixed64 game_id = 2; // Game ID of stats to get + repeated uint32 stats = 3; +} + + +// +// k_EMsgAMGetUserGameStatsResponse +// +message CMsgAMGetUserGameStatsResponse +{ + optional fixed64 steam_id = 1; // ID of user + optional fixed64 game_id = 2; // Game ID + optional int32 eresult = 3 [default = 2]; // EResult with result of query. (Fields following are only valid if this is EResultOK.) + + message Stats + { + optional uint32 stat_id = 1; + optional uint32 stat_value = 2; // There are 4 of these, really only 8 bits each. Yay for compression! + } + + message Achievement_Blocks + { + optional uint32 achievement_id = 1; + optional uint32 achievement_bit_id = 2; + optional fixed32 unlock_time = 3; // There are only 32 of these, matching the achievment bitfields, we check on the receiver that + } + + repeated Stats stats = 4; + + repeated Achievement_Blocks achievement_blocks = 5; + +} + + +// k_EMsgAdminGCGetCommandList +message CMsgGCGetCommandList +{ + optional uint32 app_id = 1; + optional string command_prefix = 2; // prefix of the command to filter by +}; + +// k_EMsgAdminGCGetCommandListResponse +message CMsgGCGetCommandListResponse +{ + repeated string command_name = 1; // a list of command names +}; + +// +// k_EGCMsgMemCachedGet +// +message CGCMsgMemCachedGet +{ + repeated string keys = 1; +} + +// +// k_EGCMsgMemCachedGetResponse +// +message CGCMsgMemCachedGetResponse +{ + message ValueTag + { + optional bool found = 1; + optional bytes value = 2; + } + + repeated ValueTag values = 1; +} + +// +// k_EGCMsgMemCachedSet +// +message CGCMsgMemCachedSet +{ + message KeyPair + { + optional string name = 1; + optional bytes value = 2; + } + + repeated KeyPair keys = 1; +} + +// +// k_EGCMsgMemCachedDelete +// +message CGCMsgMemCachedDelete +{ + repeated string keys = 1; +} + +// +// k_EGCMsgMemCachedStats +// +message CGCMsgMemCachedStats +{ + // Nothing, yet. +} + +// +// k_EGCMsgMemCachedStatsResponse +// +message CGCMsgMemCachedStatsResponse +{ + optional uint64 curr_connections = 1; + optional uint64 cmd_get = 2; + optional uint64 cmd_set = 3; + optional uint64 cmd_flush = 4; + optional uint64 get_hits = 5; + optional uint64 get_misses = 6; + optional uint64 delete_hits = 7; + optional uint64 delete_misses = 8; + optional uint64 bytes_read = 9; + optional uint64 bytes_written = 10; + optional uint64 limit_maxbytes = 11; + optional uint64 curr_items = 12; + optional uint64 evictions = 13; + optional uint64 bytes = 14; +} + +// +// k_EGCMsgSQLStats +// +message CGCMsgSQLStats +{ + optional uint32 schema_catalog = 1; +} + +// +// k_EGCMsgSQLStatsResponse +// +message CGCMsgSQLStatsResponse +{ + optional uint32 threads = 1; + optional uint32 threads_connected = 2; + optional uint32 threads_active = 3; + optional uint32 operations_submitted = 4; + optional uint32 prepared_statements_executed = 5; + optional uint32 non_prepared_statements_executed = 6; + optional uint32 deadlock_retries = 7; + optional uint32 operations_timed_out_in_queue = 8; + optional uint32 errors = 9; +} + +// k_EMsgAMAddFreeLicense +message CMsgAMAddFreeLicense +{ + optional fixed64 steamid = 1; // SteamID of account + optional uint32 ip_public = 2; // IP of client (zero if not a client-initiated message) + optional uint32 packageid = 3; // ID for package to purchase. Should be k_uPackageIdInvalid if shopping cart gid set + optional string store_country_code = 4; // country code to use for purchase +}; + +// k_EMsgAMAddFreeLicenseResponse +message CMsgAMAddFreeLicenseResponse +{ + optional int32 eresult = 1 [default = 2]; // EResult with result of Purchase. + optional int32 purchase_result_detail = 2; // Detailed result information + optional fixed64 transid = 3; // ID of the created transaction +}; + + +// +// k_EGCMsgGetIPLocation +// +message CGCMsgGetIPLocation +{ + repeated fixed32 ips = 1; +} + +// +// k_EGCMsgGetIPLocationResponse +// +message CIPLocationInfo +{ + optional uint32 ip = 1; + optional float latitude = 2; + optional float longitude = 3; + optional string country = 4; + optional string state = 5; + optional string city = 6; +} + +message CGCMsgGetIPLocationResponse +{ + repeated CIPLocationInfo infos = 1; +} + + +// +// k_EGCMsgSystemStatsSchema +// +message CGCMsgSystemStatsSchema +{ + optional uint32 gc_app_id = 1; + optional bytes schema_kv = 2; +} + +// +// k_EGCMsgGetSystemStats +// +message CGCMsgGetSystemStats +{ +} + +// +// k_EGCMsgGetSystemStatsResponse +// +message CGCMsgGetSystemStatsResponse +{ + optional uint32 gc_app_id = 1; + optional bytes stats_kv = 2; + // statically included in GCHost's stats + optional uint32 active_jobs = 3; + optional uint32 yielding_jobs = 4; + optional uint32 user_sessions = 5; + optional uint32 game_server_sessions = 6; + optional uint32 socaches = 7; + optional uint32 socaches_to_unload = 8; + optional uint32 socaches_loading = 9; + optional uint32 writeback_queue = 10; + optional uint32 steamid_locks = 11; + optional uint32 logon_queue = 12; + optional uint32 logon_jobs = 13; +} + +// k_EGCMsgSendEmail +message CMsgAMSendEmail +{ + message ReplacementToken + { + optional string token_name = 1; + optional string token_value = 2; + } + message PersonaNameReplacementToken + { + optional fixed64 steamid = 1; + optional string token_name = 2; + } + optional fixed64 steamid = 1; + optional uint32 email_msg_type = 2; + optional uint32 email_format = 3; + repeated PersonaNameReplacementToken persona_name_tokens = 5; + + optional uint32 source_gc = 6; + repeated ReplacementToken tokens = 7; +} + +// k_EGCMsgSendEmailResponse +message CMsgAMSendEmailResponse +{ + optional uint32 eresult = 1 [default = 2]; +} + +// k_EGCMsgGetEmailTemplate +message CMsgGCGetEmailTemplate +{ + optional uint32 app_id = 1; + optional uint32 email_msg_type = 2; + optional int32 email_lang = 3; + optional int32 email_format = 4; +} + +// k_EGCMsgGetEmailTemplateResponse +message CMsgGCGetEmailTemplateResponse +{ + optional uint32 eresult = 1 [default = 2]; + optional bool template_exists = 2; + optional string template = 3; +} + +// k_EMsgAMGrantGuestPasses2 +message CMsgAMGrantGuestPasses2 +{ + optional fixed64 steam_id = 1; + optional uint32 package_id = 2; + optional int32 passes_to_grant = 3; + optional int32 days_to_expiration = 4; + optional int32 action = 5; +} + +// k_EMsgAMGrantGuestPasses2Response +message CMsgAMGrantGuestPasses2Response +{ + optional int32 eresult = 1 [default = 2]; + optional int32 passes_granted = 2 [default = 0]; +} + +// k_EGCMsgGetAccountDetails +message CGCSystemMsg_GetAccountDetails +{ + option (msgpool_soft_limit) = 128; + option (msgpool_hard_limit) = 512; + + optional fixed64 steamid = 1; // User to get details for + optional uint32 appid = 2; // appid of the source GC +} + +message CGCSystemMsg_GetAccountDetails_Response +{ + option (msgpool_soft_limit) = 128; + option (msgpool_hard_limit) = 512; + + optional uint32 eresult_deprecated = 1 [ default = 2 ]; // Result of the request + optional string account_name = 2; // Login name for the user + optional string persona_name = 3; // Diplay name for the user + optional bool is_profile_public = 4; // Is the user's profile public + optional bool is_inventory_public = 5; // Is the user's inventory public + //optional bool is_trusted = 6; // Is the user trusted + optional bool is_vac_banned = 7; // Is the user vac banned + optional bool is_cyber_cafe = 8; // Is the user a cybe cafe + optional bool is_school_account = 9; // Is the user a school account + optional bool is_limited = 10; // Is the user limited + optional bool is_subscribed = 11; // Is the user subscribed to this app + optional uint32 package = 12; // The package the user owns the app through + optional bool is_free_trial_account = 13; // Is the user playing the game for free + optional uint32 free_trial_expiration = 14; // If the user is playing for free, when does it expire? + optional bool is_low_violence = 15; // Is the user restricted to low-violence for this app + optional bool is_account_locked_down = 16; // Is the user's account locked + optional bool is_community_banned = 17; // Is the user banned from performing community actions + optional bool is_trade_banned = 18; // Is the user banned from trading items to other users on Steam + optional uint32 trade_ban_expiration = 19; // The time at which the user is unbanned from trading + optional uint32 accountid = 20; // The account ID of the user we're responding about + optional uint32 suspension_end_time = 21; // If suspended (ban that ends), the date/time the suspension ends + optional string currency = 22; // The currency associated with this account + optional uint32 steam_level = 23; // The Steam level of the user + optional uint32 friend_count = 24; // Number of friends + optional uint32 account_creation_time = 25; // Time when the account was created + optional bool is_steamguard_enabled = 27; // Is SteamGuard enabled + optional bool is_phone_verified = 28; // Has a verified phone number + optional bool is_two_factor_auth_enabled = 29; // Has SteamGuard two factor auth + optional uint32 two_factor_enabled_time = 30; // Time two factor was added + optional uint32 phone_verification_time = 31; // Time phone was verified + optional uint64 phone_id = 33; // Phone identifier + optional bool is_phone_identifying = 34; // Phone is useful for identity +} + +// +// k_EGCMsgCheckClanMembership +// +message CMsgGCCheckClanMembership +{ + optional fixed64 steamid = 1; // [ (description) = "User whose group memberships we want" ]; + optional uint32 clanid = 2; // [ (description) = "Clan to check membership against" ]; + +} + +message CMsgGCCheckClanMembership_Response +{ + optional bool ismember = 1; // [ (description) = "Whether the user in question is a member of the group in question" ]; +} + +// +// k_EGCMsgGetPersonaNames +// +message CMsgGCGetPersonaNames +{ + repeated fixed64 steamids = 1; // Users whose persona names we want +} + +message CMsgGCGetPersonaNames_Response +{ + message PersonaName + { + optional fixed64 steamid = 1; // User we could get a name for + optional string persona_name = 2; // Display name for that user + } + + repeated PersonaName succeeded_lookups = 1; // Users we could get names for + repeated fixed64 failed_lookup_steamids = 2; // Users we failed to get a names for +} + + +// +// k_EGCMsgCheckFriendship +// +message CMsgGCCheckFriendship +{ + optional fixed64 steamid_left = 1; // User whose friends list we'll load + optional fixed64 steamid_right = 2; // User to look for in the list we load +} + +message CMsgGCCheckFriendship_Response +{ + optional bool success = 1; // Whether the API calls all succeeded + optional bool found_friendship = 2; // Denotes whether the users are friends (false on API failure) +} + + +// +// k_EGCMsgMasterSetDirectory +// +message CMsgGCMsgMasterSetDirectory +{ + message SubGC + { + optional uint32 dir_index = 1; // The index in the GC directory indicating what role this GC serves + optional string name = 2; // A string to give the GC a name for asserts/logs/connection, etc + optional string box = 3; // The box that this GC is expected to be associated with + optional string command_line = 4; // Additional command line parameters to provide for this GC instance + optional string gc_binary = 5; // The binary that should be launched for this GC. This can be left blank to launch the default binary + } + + optional uint32 master_dir_index = 1; // The index of the master GC so that it knows how to setup the routing tables to include the master and sub GCs + repeated SubGC dir = 2; // A listing of the various sub GCs that the GCH should create +} + +message CMsgGCMsgMasterSetDirectory_Response +{ + optional int32 eresult = 1 [default = 2]; // Could the GC start the processes? It doesn't mean they will all get initialized and each sub GC still needs to ack, but catches failure earlier +} + +// +// k_EGCMsgWebAPIJobRequestForwardResponse +// +message CMsgGCMsgWebAPIJobRequestForwardResponse +{ + optional uint32 dir_index = 1; // The directory index of the GC which has been delegated to handle this particular WebAPI request +} + +// +// k_EGCMsgGetPurchaseTrustStatus +// +message CGCSystemMsg_GetPurchaseTrust_Request +{ + optional fixed64 steamid = 1; // User to get details for +} + +message CGCSystemMsg_GetPurchaseTrust_Response +{ + optional bool has_prior_purchase_history = 1; // The user has prior purchase history with no recent gaps in purchase activity + optional bool has_no_recent_password_resets = 2; // The user hasn't had their password reset recently + optional bool is_wallet_cash_trusted = 3; // False if the user has recently used a new payment method to fund his or her wallet + optional uint32 time_all_trusted = 4; // The time that the user will be trusted in all of the given fields if he or she were to complete a microtransaction right now +} + +// k_EMsgGCHAccountVacStatusChange +message CMsgGCHAccountVacStatusChange +{ + optional fixed64 steam_id = 1; + optional uint32 app_id = 2; + optional uint32 rtime_vacban_starts = 3; + optional bool is_banned_now = 4; + optional bool is_banned_future = 5; +} + +// k_EMsgGCHAccountTradeBanStatusChange +message CMsgGCHAccountTradeBanStatusChange +{ + optional fixed64 steamid = 1; + optional uint32 appid = 2; + optional bool is_banned = 3; + optional uint32 time_banned_until = 4; +} + +// k_EMsgGCHAccountLockStatusChange +message CMsgGCHAccountLockStatusChange +{ + optional fixed64 steamid = 1; + optional uint32 appid = 2; + optional bool is_locked = 3; +} + +// k_EMsgGCHVacVerificationChange +message CMsgGCHVacVerificationChange +{ + optional fixed64 steamid = 1; + optional uint32 appid = 2; + optional bool is_verified = 3; +} + +// k_EMsgGCHAccountPhoneNumberChange +message CMsgGCHAccountPhoneNumberChange +{ + optional fixed64 steamid = 1; + optional uint32 appid = 2; + optional uint64 phone_id = 3; + optional bool is_verified = 4; + optional bool is_identifying = 5; +} + +// k_EMsgGCHAccountTwoFactorChange +message CMsgGCHAccountTwoFactorChange +{ + optional fixed64 steamid = 1; + optional uint32 appid = 2; + optional bool twofactor_enabled = 3; +} + +// +// k_EGCMsgGetPartnerAccountLink +// +message CMsgGCGetPartnerAccountLink +{ + optional fixed64 steamid = 1; // User whose partner account link details we want to get +} + +message CMsgGCGetPartnerAccountLink_Response +{ + optional uint32 pwid = 1; // Perfect World ID (not specified if not linked) + optional uint32 nexonid = 2; // Nexon ID (not specified if not linked) +} + + +// +// k_EGCMsgMasterSetWebAPIRouting and k_EGCMsgMasterSetClientMsgRouting +// +message CMsgGCRoutingInfo +{ + enum RoutingMethod + { + RANDOM = 0; // random instead of round-robin so that we don't need to track state per routing pool + DISCARD = 1; + CLIENT_STEAMID = 2; + PROTOBUF_FIELD_UINT64 = 3; + WEBAPI_PARAM_UINT64 = 4; + } + + repeated uint32 dir_index = 1; // One or more directory indices which are potential targets for this route + optional RoutingMethod method = 2 [ default = RANDOM ]; // Method by which the route choses its target from multiple dir_index values + optional RoutingMethod fallback = 3 [ default = DISCARD ]; // Fallback method to use when default method is not applicable (eg, field can't be parsed) + optional uint32 protobuf_field = 4; // For PROTOBUF_FIELD_UINT64, the protobuf field number to decode as a uint64 for routing + optional string webapi_param = 5; // For WEBAPI_PARAM_UINT64 method, the case-insensitive name of the webapi parameter +} + +message CMsgGCMsgMasterSetWebAPIRouting +{ + message Entry + { + optional string interface_name = 1; + optional string method_name = 2; + optional CMsgGCRoutingInfo routing = 3; + } + repeated Entry entries = 1; +} + +message CMsgGCMsgMasterSetClientMsgRouting +{ + message Entry + { + optional uint32 msg_type = 1; // Client message ID to be routed; top bit is ignored for historical reasons + optional CMsgGCRoutingInfo routing = 2; + } + repeated Entry entries = 1; +} + +message CMsgGCMsgMasterSetWebAPIRouting_Response +{ + optional int32 eresult = 1 [ default = 2 ]; // Success or failure code from the GCH when processing k_EGCMsgMasterSetWebAPIRouting +} + +message CMsgGCMsgMasterSetClientMsgRouting_Response +{ + optional int32 eresult = 1 [ default = 2 ]; // Success or failure code from the GCH when processing k_EGCMsgMasterSetClientMsgRouting +} + + +// k_EGCMsgSetOptions +message CMsgGCMsgSetOptions +{ + enum Option + { + // Notifications (aka "data streams" - unsoliticed messages from Steam) - default disabled, specify to opt-in + NOTIFY_USER_SESSIONS = 0; + NOTIFY_SERVER_SESSIONS = 1; + NOTIFY_ACHIEVEMENTS = 2; + NOTIFY_VAC_ACTION = 3; + + // todo: other options? should start options higher up, like 20+, to save room for streams? + } + repeated Option options = 1; + + + // The client_msg_ranges field indicates which client messages, if any, this GC should receive copies of + message MessageRange + { + required uint32 low = 1; + required uint32 high = 2; + } + repeated MessageRange client_msg_ranges = 2; + + + // The GCSQL version field is only read by the GC.EXE host to determine support for enums, etc; the GCH doesn't care + enum GCSQLVersion + { + GCSQL_VERSION_BASELINE = 1; // baseline + GCSQL_VERSION_BOOLTYPE = 2; // added explicit support for bool types (instead of int8) + } + optional GCSQLVersion gcsql_version = 3; +} + + +// k_EMsgGCHUpdateSession +message CMsgGCHUpdateSession +{ + optional fixed64 steam_id = 1; + optional uint32 app_id = 2; + optional bool online = 3; + optional fixed64 server_steam_id = 4; // For a game client, the steam_id of the current server + optional uint32 server_addr = 5; // The IP address of the current server (or self for server state) + optional uint32 server_port = 6; // The IP port of the server (or self for server state) + optional uint32 os_type = 7; + optional uint32 client_addr = 8; // For a game client, the public IP address of the client + + message ExtraField + { + optional string name = 1; + optional string value = 2; + } + repeated ExtraField extra_fields = 9; +} + +// +// k_EGCMsgVSReportedSuspiciousActivity +// +message CMsgNotificationOfSuspiciousActivity +{ + optional fixed64 steamid = 1; + optional uint32 appid = 2; + + message MultipleGameInstances + { + optional uint32 app_instance_count = 1; + repeated fixed64 other_steamids = 2; + } + optional MultipleGameInstances multiple_instances = 3; +} + +// Do not remove this comment due to a bug on the Mac OS X protobuf compiler + diff --git a/gcsdk/steammessages_include.vpc b/gcsdk/steammessages_include.vpc new file mode 100644 index 0000000..035f5ac --- /dev/null +++ b/gcsdk/steammessages_include.vpc @@ -0,0 +1,24 @@ +$MacroRequired GENERATED_PROTO_DIR + +$Project +{ + $Folder "Protobuf Files" + { + $File "$SRCDIR\gcsdk\steammessages.proto" + $Folder "Generated Files" + { + $DynamicFile "$GENERATED_PROTO_DIR\steammessages.pb.h" + $DynamicFile "$GENERATED_PROTO_DIR\steammessages.pb.cc" [!$OSXALL] + { + $Configuration + { + $Compiler + { + $Create/UsePrecompiledHeader "Not Using Precompiled Headers" + } + } + } + } + } + +} diff --git a/gcsdk/webapi_response.cpp b/gcsdk/webapi_response.cpp new file mode 100644 index 0000000..82f1105 --- /dev/null +++ b/gcsdk/webapi_response.cpp @@ -0,0 +1,2562 @@ +//========= Copyright � 1996-2010, Valve LLC, All rights reserved. ============ +// +// Purpose: Implementation for CWebAPIResponse objects +// +//============================================================================= + +#include "stdafx.h" +#include "thirdparty/JSON_parser/JSON_parser.h" + +using namespace GCSDK; + +#include "tier0/memdbgoff.h" + +// !FIXME! DOTAMERGE +//IMPLEMENT_CLASS_MEMPOOL_MT( CWebAPIValues, 1000, UTLMEMORYPOOL_GROW_SLOW ); + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +//----------------------------------------------------------------------------- +// Purpose: Helper for emitting properly escaped json string values +//----------------------------------------------------------------------------- +void EmitJSONString( CUtlBuffer &outputBuffer, const char *pchValue ) +{ + outputBuffer.PutChar( '"' ); + + if ( pchValue ) + { + int i = 0; + while( pchValue[i] ) + { + switch ( pchValue[i] ) + { + case '"': + outputBuffer.Put( "\\\"", 2 ); + break; + case '\\': + outputBuffer.Put( "\\\\", 2 ); + break; + case '\n': + outputBuffer.Put( "\\n", 2 ); + break; + case '\r': + outputBuffer.Put( "\\r", 2 ); + break; + case '\t': + outputBuffer.Put( "\\t", 2 ); + break; + default: + if ( (uint8) pchValue[i] < 32 ) + { + outputBuffer.Put( "\\u00", 4 ); + outputBuffer.PutChar( ( pchValue[i] & 16 ) ? '1' : '0' ); + outputBuffer.PutChar( "0123456789abcdef"[ pchValue[i] & 0xF ] ); + } + else + { + outputBuffer.PutChar( pchValue[i] ); + } + } + ++i; + } + } + + outputBuffer.PutChar( '"' ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Helper for emitting properly escaped XML string values, we always use UTF8, +// so we only really need to encode & ' " < > +//----------------------------------------------------------------------------- +void EmitXMLString( CUtlBuffer &outputBuffer, const char *pchValue ) +{ + if ( pchValue ) + { + int i = 0; + while( pchValue[i] ) + { + switch ( pchValue[i] ) + { + case '&': + outputBuffer.Put( "&", 5 ); + break; + case '\'': + outputBuffer.Put( "'", 6 ); + break; + case '"': + outputBuffer.Put( """, 6 ); + break; + case '<': + outputBuffer.Put( "<", 4 ); + break; + case '>': + outputBuffer.Put( ">", 4 ); + break; + default: + outputBuffer.PutChar( pchValue[i] ); + } + ++i; + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Helper for emitting properly escaped VDF string values, we always use UTF8, +// and we escape only " and \ +//----------------------------------------------------------------------------- +void EmitVDFString( CUtlBuffer &outputBuffer, const char *pchValue ) +{ + outputBuffer.PutChar( '"' ); + + if ( pchValue ) + { + int i = 0; + while( pchValue[i] ) + { + switch ( pchValue[i] ) + { + case '\\': + outputBuffer.Put( "\\\\", 2 ); + break; + case '"': + outputBuffer.Put( "\\\"", 2 ); + break; + default: + outputBuffer.PutChar( pchValue[i] ); + } + ++i; + } + } + + outputBuffer.PutChar( '"' ); +} + +namespace GCSDK +{ + +enum { k_LineBreakEveryNGroups = 18 }; // line break every 18 groups of 4 characters (every 72 characters) + +uint32 Base64EncodeMaxOutput( const uint32 cubData, const char *pszLineBreak ) +{ + // terminating null + 4 chars per 3-byte group + line break after every 18 groups (72 output chars) + final line break + uint32 nGroups = (cubData+2)/3; + uint32 cchRequired = 1 + nGroups*4 + ( pszLineBreak ? Q_strlen(pszLineBreak)*(1+(nGroups-1)/k_LineBreakEveryNGroups) : 0 ); + return cchRequired; +} + +bool Base64Encode( const uint8 *pubData, uint32 cubData, char *pchEncodedData, uint32 *pcchEncodedData, const char *pszLineBreak ) +{ + if ( pchEncodedData == NULL ) + { + AssertMsg( *pcchEncodedData == 0, "NULL output buffer with non-zero size passed to Base64Encode" ); + *pcchEncodedData = Base64EncodeMaxOutput( cubData, pszLineBreak ); + return true; + } + + const uint8 *pubDataEnd = pubData + cubData; + char *pchEncodedDataStart = pchEncodedData; + uint32 unLineBreakLen = pszLineBreak ? Q_strlen( pszLineBreak ) : 0; + int nNextLineBreak = unLineBreakLen ? k_LineBreakEveryNGroups : INT_MAX; + + const char * const pszBase64Chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + uint32 cchEncodedData = *pcchEncodedData; + if ( cchEncodedData == 0 ) + goto out_of_space; + + --cchEncodedData; // pre-decrement for the terminating null so we don't forget about it + + // input 3 x 8-bit, output 4 x 6-bit + while ( pubDataEnd - pubData >= 3 ) + { + if ( cchEncodedData < 4 + unLineBreakLen ) + goto out_of_space; + + if ( nNextLineBreak == 0 ) + { + memcpy( pchEncodedData, pszLineBreak, unLineBreakLen ); + pchEncodedData += unLineBreakLen; + cchEncodedData -= unLineBreakLen; + nNextLineBreak = k_LineBreakEveryNGroups; + } + + uint32 un24BitsData; + un24BitsData = (uint32) pubData[0] << 16; + un24BitsData |= (uint32) pubData[1] << 8; + un24BitsData |= (uint32) pubData[2]; + pubData += 3; + + pchEncodedData[0] = pszBase64Chars[ (un24BitsData >> 18) & 63 ]; + pchEncodedData[1] = pszBase64Chars[ (un24BitsData >> 12) & 63 ]; + pchEncodedData[2] = pszBase64Chars[ (un24BitsData >> 6) & 63 ]; + pchEncodedData[3] = pszBase64Chars[ (un24BitsData ) & 63 ]; + pchEncodedData += 4; + cchEncodedData -= 4; + --nNextLineBreak; + } + + // Clean up remaining 1 or 2 bytes of input, pad output with '=' + if ( pubData != pubDataEnd ) + { + if ( cchEncodedData < 4 + unLineBreakLen ) + goto out_of_space; + + if ( nNextLineBreak == 0 ) + { + memcpy( pchEncodedData, pszLineBreak, unLineBreakLen ); + pchEncodedData += unLineBreakLen; + cchEncodedData -= unLineBreakLen; + } + + uint32 un24BitsData; + un24BitsData = (uint32) pubData[0] << 16; + if ( pubData+1 != pubDataEnd ) + { + un24BitsData |= (uint32) pubData[1] << 8; + } + pchEncodedData[0] = pszBase64Chars[ (un24BitsData >> 18) & 63 ]; + pchEncodedData[1] = pszBase64Chars[ (un24BitsData >> 12) & 63 ]; + pchEncodedData[2] = pubData+1 != pubDataEnd ? pszBase64Chars[ (un24BitsData >> 6) & 63 ] : '='; + pchEncodedData[3] = '='; + pchEncodedData += 4; + cchEncodedData -= 4; + } + + if ( unLineBreakLen ) + { + if ( cchEncodedData < unLineBreakLen ) + goto out_of_space; + memcpy( pchEncodedData, pszLineBreak, unLineBreakLen ); + pchEncodedData += unLineBreakLen; + cchEncodedData -= unLineBreakLen; + } + + *pchEncodedData = 0; + *pcchEncodedData = pchEncodedData - pchEncodedDataStart; + return true; + +out_of_space: + *pchEncodedData = 0; + *pcchEncodedData = Base64EncodeMaxOutput( cubData, pszLineBreak ); + AssertMsg( false, "CCrypto::Base64Encode: insufficient output buffer (up to n*4/3+5 bytes required, plus linebreaks)" ); + return false; +} + +bool Base64Decode( const char *pchData, uint32 cchDataMax, uint8 *pubDecodedData, uint32 *pcubDecodedData, bool bIgnoreInvalidCharacters ) +{ + uint32 cubDecodedData = *pcubDecodedData; + uint32 cubDecodedDataOrig = cubDecodedData; + + if ( pubDecodedData == NULL ) + { + AssertMsg( *pcubDecodedData == 0, "NULL output buffer with non-zero size passed to Base64Decode" ); + cubDecodedDataOrig = cubDecodedData = ~0u; + } + + // valid base64 character range: '+' (0x2B) to 'z' (0x7A) + // table entries are 0-63, -1 for invalid entries, -2 for '=' + static const char rgchInvBase64[] = { + 62, -1, -1, -1, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + -1, -1, -1, -2, -1, -1, -1, 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, -1, -1, -1, -1, -1, -1, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, + 47, 48, 49, 50, 51 + }; + COMPILE_TIME_ASSERT( Q_ARRAYSIZE(rgchInvBase64) == 0x7A - 0x2B + 1 ); + + uint32 un24BitsWithSentinel = 1; + while ( cchDataMax-- > 0 ) + { + char c = *pchData++; + + if ( (uint8)(c - 0x2B) >= Q_ARRAYSIZE( rgchInvBase64 ) ) + { + if ( c == '\0' ) + break; + + if ( !bIgnoreInvalidCharacters && !( c == '\r' || c == '\n' || c == '\t' || c == ' ' ) ) + goto decode_failed; + else + continue; + } + + c = rgchInvBase64[(uint8)(c - 0x2B)]; + if ( c < 0 ) + { + if ( c == -2 ) // -2 -> terminating '=' + break; + + if ( !bIgnoreInvalidCharacters ) + goto decode_failed; + else + continue; + } + + un24BitsWithSentinel <<= 6; + un24BitsWithSentinel |= c; + if ( un24BitsWithSentinel & (1<<24) ) + { + if ( cubDecodedData < 3 ) // out of space? go to final write logic + break; + if ( pubDecodedData ) + { + pubDecodedData[0] = (uint8)( un24BitsWithSentinel >> 16 ); + pubDecodedData[1] = (uint8)( un24BitsWithSentinel >> 8); + pubDecodedData[2] = (uint8)( un24BitsWithSentinel ); + pubDecodedData += 3; + } + cubDecodedData -= 3; + un24BitsWithSentinel = 1; + } + } + + // If un24BitsWithSentinel contains data, output the remaining full bytes + if ( un24BitsWithSentinel >= (1<<6) ) + { + // Possibilities are 3, 2, 1, or 0 full output bytes. + int nWriteBytes = 3; + while ( un24BitsWithSentinel < (1<<24) ) + { + nWriteBytes--; + un24BitsWithSentinel <<= 6; + } + + // Write completed bytes to output + while ( nWriteBytes-- > 0 ) + { + if ( cubDecodedData == 0 ) + { + AssertMsg( false, "CCrypto::Base64Decode: insufficient output buffer (up to n*3/4+2 bytes required)" ); + goto decode_failed; + } + if ( pubDecodedData ) + { + *pubDecodedData++ = (uint8)(un24BitsWithSentinel >> 16); + } + --cubDecodedData; + un24BitsWithSentinel <<= 8; + } + } + + *pcubDecodedData = cubDecodedDataOrig - cubDecodedData; + return true; + +decode_failed: + *pcubDecodedData = cubDecodedDataOrig - cubDecodedData; + return false; +} + +} + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CWebAPIResponse::CWebAPIResponse() +{ + m_pValues = NULL; + m_unExpirationSeconds = 0; + m_rtLastModified = 0; + m_bExtendedArrays = false; + m_bJSONAnonymousRootNode = false; + m_eStatusCode = k_EHTTPStatusCode500InternalServerError; +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CWebAPIResponse::~CWebAPIResponse() +{ + if ( m_pValues) + delete m_pValues; + m_pValues = NULL; +} + + +//----------------------------------------------------------------------------- +// Purpose: Outputs formatted data to buffer +//----------------------------------------------------------------------------- +bool CWebAPIResponse::BEmitFormattedOutput( EWebAPIOutputFormat eFormat, CUtlBuffer &outputBuffer, size_t unMaxResultSize ) +{ + VPROF_BUDGET( "CWebAPIResponse::BEmitFormattedOutput", VPROF_BUDGETGROUP_STEAM ); + outputBuffer.Clear(); + + switch( eFormat ) + { + case k_EWebAPIOutputFormat_JSON: + return BEmitJSON( outputBuffer, unMaxResultSize ); + case k_EWebAPIOutputFormat_XML: + return BEmitXML( outputBuffer, unMaxResultSize ); + case k_EWebAPIOutputFormat_VDF: + return BEmitVDF( outputBuffer, unMaxResultSize ); + case k_EWebAPIOutputFormat_ParameterEncoding: + return BEmitParameterEncoding( outputBuffer ); + default: + return false; + } +} + +//----------------------------------------------------------------------------- +// Purpose: Emits JSON formatted representation of response +//----------------------------------------------------------------------------- +bool CWebAPIResponse::BEmitJSON( CUtlBuffer &outputBuffer, size_t unMaxResultSize ) +{ + outputBuffer.PutChar( '{' ); + outputBuffer.PutChar( '\n' ); + + CWebAPIValues *pValues = m_pValues; + + //if we have an anonymous root, get the first child instead of the root itself + if ( m_bJSONAnonymousRootNode && m_pValues ) + { + pValues = m_pValues->GetFirstChild(); + } + + if ( pValues ) + { + if( !CWebAPIValues::BEmitJSONRecursive( pValues, outputBuffer, 1, unMaxResultSize, m_bExtendedArrays ) ) + return false; + } + + outputBuffer.PutChar( '\n' ); + outputBuffer.PutChar( '}' ); + if ( !outputBuffer.IsValid() ) + return false; + + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Emits KeyValues .vdf style formatted representation of response +//----------------------------------------------------------------------------- +bool CWebAPIResponse::BEmitVDF( CUtlBuffer &outputBuffer, size_t unMaxResultSize ) +{ + if ( m_pValues ) + if( !CWebAPIValues::BEmitVDFRecursive( m_pValues, outputBuffer, 0, 0, unMaxResultSize, m_bExtendedArrays ) ) + return false; + + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Emits XML formatted representation of response +//----------------------------------------------------------------------------- +bool CWebAPIResponse::BEmitXML( CUtlBuffer &outputBuffer, size_t unMaxResultSize ) +{ + const char *pchProlog = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"; + outputBuffer.Put( pchProlog, Q_strlen( pchProlog ) ); + + outputBuffer.Put( "<!DOCTYPE ", Q_strlen( "<!DOCTYPE " ) ); + if ( m_pValues && m_pValues->GetName() ) + EmitXMLString( outputBuffer, m_pValues->GetName() ); + outputBuffer.PutChar('>'); + outputBuffer.PutChar('\n'); + + if ( m_pValues ) + if( !CWebAPIValues::BEmitXMLRecursive( m_pValues, outputBuffer, 0, unMaxResultSize ) ) + return false; + + if ( !outputBuffer.IsValid() ) + return false; + + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Emits Parameter Encoding formatted representation of response +//----------------------------------------------------------------------------- +bool CWebAPIResponse::BEmitParameterEncoding( CUtlBuffer &outputBuffer ) +{ + if ( !m_pValues ) + return true; + + CWebAPIValues *pValue = m_pValues->GetFirstChild(); + while ( pValue != NULL ) + { + outputBuffer.Put( pValue->GetName(), Q_strlen( pValue->GetName() ) ); + outputBuffer.Put( "=", 1 ); + + CUtlString sValue; + switch ( pValue->GetType() ) + { + case k_EWebAPIValueType_Object: + Assert( false ); + return false; // no cursive values + case k_EWebAPIValueType_NumericArray: + Assert( false ); + return false; // no arrays + case k_EWebAPIValueType_BinaryBlob: + Assert( false ); + return false; // no binary + + case k_EWebAPIValueType_Int32: + sValue = CNumStr( pValue->GetInt32Value() ); + break; + case k_EWebAPIValueType_Int64: + sValue = CNumStr( pValue->GetInt64Value() ); + break; + case k_EWebAPIValueType_UInt32: + sValue = CNumStr( pValue->GetUInt32Value() ); + break; + case k_EWebAPIValueType_UInt64: + sValue = CNumStr( pValue->GetUInt64Value() ); + break; + case k_EWebAPIValueType_Double: + sValue = CNumStr( pValue->GetDoubleValue() ); + break; + case k_EWebAPIValueType_String: + pValue->GetStringValue( sValue ); + break; + case k_EWebAPIValueType_Bool: + sValue = CNumStr( pValue->GetBoolValue() ); + break; + } + outputBuffer.Put( sValue, sValue.Length() ); + + pValue = pValue->GetNextChild(); + if ( pValue ) + outputBuffer.Put( "&", 1 ); + } + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Resets the response to be empty +//----------------------------------------------------------------------------- +void CWebAPIResponse::Clear() +{ + if ( m_pValues ) + delete m_pValues; + + m_pValues = NULL; +} + +//----------------------------------------------------------------------------- +// Purpose: Access the root value element in the response +//----------------------------------------------------------------------------- +CWebAPIValues *CWebAPIResponse::CreateRootValue( const char *pchName ) +{ + if ( m_pValues ) + { + AssertMsg( false, "CWwebAPIResponse::CreateRootValue called while root already existed." ); + Clear(); + } + + m_pValues = new CWebAPIValues( pchName ); + return m_pValues; +} + + +//---------------------------------------------------------------------------- +// Purpose: Gets a singleton buffer pool for webapi values +//---------------------------------------------------------------------------- +#ifdef GC +static GCConVar webapi_values_max_pool_size_mb( "webapi_values_max_pool_size_mb", "10", "Maximum size in bytes of the WebAPIValues buffer pool" ); +static GCConVar webapi_values_init_buffer_size( "webapi_values_init_buffer_size", "65536", "Initial buffer size for buffers in the WebAPIValues buffer pool" ); +/*static*/ CBufferPoolMT &CWebAPIValues::GetBufferPool() +{ + static CBufferPoolMT s_bufferPool( "WebAPIValues", webapi_values_max_pool_size_mb, webapi_values_init_buffer_size ); + return s_bufferPool; +} +#endif + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CWebAPIValues::CWebAPIValues( CWebAPIValues *pParent, const char *pchName, EWebAPIValueType eValueType, const char *pchArrayElementNames ) +{ + InitInternal( pParent, -1, eValueType, pchArrayElementNames ); + SetName( pchName ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CWebAPIValues::CWebAPIValues( const char *pchName ) +{ + InitInternal( NULL, -1, k_EWebAPIValueType_Object, NULL ); + SetName( pchName ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CWebAPIValues::CWebAPIValues( const char *pchName, const char *pchArrayElementNames ) +{ + InitInternal( NULL, -1, k_EWebAPIValueType_NumericArray, pchArrayElementNames ); + SetName( pchName ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CWebAPIValues::CWebAPIValues( CWebAPIValues *pParent, int nNamePos, EWebAPIValueType eValueType, const char *pchArrayElementNames ) +{ + InitInternal( pParent, nNamePos, eValueType, pchArrayElementNames ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +void CWebAPIValues::InitInternal( CWebAPIValues *pParent, int nNamePos, EWebAPIValueType eValueType, const char *pchArrayElementNames ) +{ + if ( NULL == pParent ) + { +#ifdef GC + m_pStringBuffer = GetBufferPool().GetBuffer(); +#else + m_pStringBuffer = new CUtlBuffer; +#endif + } + else + { + m_pStringBuffer = pParent->m_pStringBuffer; + } + + m_nNamePos = nNamePos; + + m_eValueType = eValueType; + if ( m_eValueType == k_EWebAPIValueType_NumericArray ) + { + Assert( pchArrayElementNames ); + m_nArrayChildElementNamePos = m_pStringBuffer->TellPut(); + m_pStringBuffer->PutString( pchArrayElementNames ); + } + + m_pFirstChild = NULL; + m_pLastChild = NULL; + m_pNextPeer = NULL; + m_pParent = pParent; +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CWebAPIValues::~CWebAPIValues() +{ + ClearValue(); + + CWebAPIValues *pChild = m_pFirstChild; + while( pChild ) + { + CWebAPIValues *pDelete = pChild; + pChild = pChild->m_pNextPeer; + delete pDelete; + } + m_pFirstChild = NULL; + m_pNextPeer = NULL; + + if ( NULL == m_pParent ) + { +#ifdef GC + GetBufferPool().ReturnBuffer( m_pStringBuffer ); +#else + delete m_pStringBuffer; +#endif + } + + // This two ptrs are just for optimized traversal at runtime, deleting just + // our first child and next peer will lead to the full tree being deleted correctly. + m_pLastChild = NULL; + m_pParent = NULL; +} + + +//----------------------------------------------------------------------------- +// Purpose: Sets the name of the values node +//----------------------------------------------------------------------------- +void CWebAPIValues::SetName( const char * pchName ) +{ + if ( pchName == NULL ) + { + AssertMsg( false, "CWebAPIValues constructed with NULL name, breaks some output serialization. Shouldn't do this." ); + m_nNamePos = -1; + } + else + { + // Shouldn't use ', ", &, <, or > in names since they can't output well in XML. : is no good since it implies namespacing in XML. + // Assert about it so we don't end up with responses that are badly formed in XML output. + int unLen = 0; + while ( pchName[unLen] != 0 ) + { + if ( pchName[unLen] == '\'' || pchName[unLen] == '"' || pchName[unLen] == '&' + || pchName[unLen] == '>' || pchName[unLen] == '>' || pchName[unLen] == ':' ) + { + AssertMsg( false, "Shouldn't use any of '\"&<>: in CWebAPIValues node names, you used %s", pchName ); + break; + } + ++unLen; + } + + m_nNamePos = m_pStringBuffer->TellPut(); + m_pStringBuffer->PutString( pchName ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: Assert that we don't have any child nodes, this is used when setting a +// native type value. We don't support having both our own value and children. You +// are either an array of more values, or you are a value yourself. +//----------------------------------------------------------------------------- +void CWebAPIValues::AssertNoChildren() +{ + AssertMsg( m_pFirstChild == NULL, "CWebAPIValues has child nodes, but you are trying to set a direct value for it. Can't have both children and your own value." ); +} + +//----------------------------------------------------------------------------- +// Purpose: Clears any existing value, freeing memory if needed +//----------------------------------------------------------------------------- +void CWebAPIValues::ClearValue() +{ + m_eValueType = k_EWebAPIValueType_Object; +} + + +//----------------------------------------------------------------------------- +// Purpose: Setter +//----------------------------------------------------------------------------- +void CWebAPIValues::SetStringValue( const char *pchValue ) +{ + ClearValue(); + AssertNoChildren(); + m_eValueType = k_EWebAPIValueType_String; + if ( pchValue == NULL ) + { + m_nStrValuePos = -1; + } + else + { + m_nStrValuePos = m_pStringBuffer->TellPut(); + m_pStringBuffer->PutString( pchValue ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Setter +//----------------------------------------------------------------------------- +void CWebAPIValues::SetInt32Value( int32 nValue ) +{ + ClearValue(); + AssertNoChildren(); + m_eValueType = k_EWebAPIValueType_Int32; + m_nValue = nValue; +} + + +//----------------------------------------------------------------------------- +// Purpose: Setter +//----------------------------------------------------------------------------- +void CWebAPIValues::SetUInt32Value( uint32 unValue ) +{ + ClearValue(); + AssertNoChildren(); + m_eValueType = k_EWebAPIValueType_UInt32; + m_unValue = unValue; +} + + +//----------------------------------------------------------------------------- +// Purpose: Setter +//----------------------------------------------------------------------------- +void CWebAPIValues::SetInt64Value ( int64 lValue ) +{ + ClearValue(); + AssertNoChildren(); + m_eValueType = k_EWebAPIValueType_Int64; + m_lValue = lValue; +} + + +//----------------------------------------------------------------------------- +// Purpose: Setter +//----------------------------------------------------------------------------- +void CWebAPIValues::SetUInt64Value( uint64 ulValue ) +{ + ClearValue(); + AssertNoChildren(); + m_eValueType = k_EWebAPIValueType_UInt64; + m_ulValue = ulValue; +} + + +//----------------------------------------------------------------------------- +// Purpose: Setter +//----------------------------------------------------------------------------- +void CWebAPIValues::SetDoubleValue( double flValue ) +{ + ClearValue(); + AssertNoChildren(); + m_eValueType = k_EWebAPIValueType_Double; + m_flValue = flValue; +} + + +//----------------------------------------------------------------------------- +// Purpose: Setter +//----------------------------------------------------------------------------- +void CWebAPIValues::SetBoolValue( bool bValue ) +{ + ClearValue(); + AssertNoChildren(); + m_eValueType = k_EWebAPIValueType_Bool; + m_bValue = bValue; +} + + +//----------------------------------------------------------------------------- +// Purpose: Setter +//----------------------------------------------------------------------------- +void CWebAPIValues::SetNullValue() +{ + ClearValue(); + AssertNoChildren(); + m_eValueType = k_EWebAPIValueType_Null; +} + + +//----------------------------------------------------------------------------- +// Purpose: Setter +//----------------------------------------------------------------------------- +void CWebAPIValues::SetBinaryValue( const uint8 *pValue, uint32 unBytes ) +{ + ClearValue(); + AssertNoChildren(); + m_eValueType = k_EWebAPIValueType_BinaryBlob; + if ( pValue == NULL || unBytes < 1 ) + { + m_BinaryValue.m_nDataPos = 0; + m_BinaryValue.m_unBytes = 0; + } + else + { + m_BinaryValue.m_unBytes = unBytes; + m_BinaryValue.m_nDataPos = m_pStringBuffer->TellPut(); + m_pStringBuffer->Put( pValue, unBytes ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the type currently held by the node +//----------------------------------------------------------------------------- +EWebAPIValueType CWebAPIValues::GetType() const +{ + return m_eValueType; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get int32 value +//----------------------------------------------------------------------------- +int32 CWebAPIValues::GetInt32Value() const +{ + // we can read uint64 values this way too + switch ( m_eValueType ) + { + case k_EWebAPIValueType_Int32: + return m_nValue; + case k_EWebAPIValueType_UInt32: // because we can't tell the type of an int when we parse this node might have different type + case k_EWebAPIValueType_UInt64: + case k_EWebAPIValueType_String: + { + uint32 uiVal = GetUInt32Value(); + AssertMsg( uiVal < INT_MAX, "GetInt32Value called on node with %u, which is too big to fit in an Int32", uiVal ); + return (int32)uiVal; + } + } + + AssertMsg( false, "Shouldn't call CWebAPIValues::GetInt32Value unless value type is int32, uint32, uint64, or string. %d does not.", m_eValueType ); + return 0; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get uint32 value +//----------------------------------------------------------------------------- +uint32 CWebAPIValues::GetUInt32Value() const +{ + // we can read uint64 values this way too + switch ( m_eValueType ) + { + case k_EWebAPIValueType_UInt32: + return m_unValue; + case k_EWebAPIValueType_UInt64: + case k_EWebAPIValueType_String: + { + uint64 uiVal = GetUInt64Value(); + AssertMsg( uiVal <= UINT_MAX, "GetUInt32Value called on node with %llu, which is too big to fit in an UInt32", uiVal ); + return (uint32)uiVal; + } + } + + AssertMsg( false, "Shouldn't call CWebAPIValues::GetUInt32Value unless value type is uint32, uint64, or string. %d does not", m_eValueType ); + return 0; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get int64 value +//----------------------------------------------------------------------------- +int64 CWebAPIValues::GetInt64Value() const +{ + // we can read int32 values this way too + switch ( m_eValueType ) + { + case k_EWebAPIValueType_Int32: + return GetInt32Value(); + case k_EWebAPIValueType_Int64: + return m_lValue; + case k_EWebAPIValueType_String: + if ( m_nStrValuePos < 0 ) + { + return 0ull; + } + else + { +#if defined(_PS3) || defined(POSIX) + return strtoll( (const char *)m_pStringBuffer->Base() + m_nStrValuePos, NULL, 10); +#else + return _strtoi64( (const char *)m_pStringBuffer->Base() + m_nStrValuePos, NULL, 10); +#endif + } + default: + AssertMsg1( false, "Shouldn't call CWebAPIValues::GetInt64Value unless value type matches. %d does not", m_eValueType ); + return 0; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Get uint64 value +//----------------------------------------------------------------------------- +uint64 CWebAPIValues::GetUInt64Value() const +{ + // we can read uint32 values this way too + switch ( m_eValueType ) + { + case k_EWebAPIValueType_UInt32: + return GetUInt32Value(); + case k_EWebAPIValueType_UInt64: + return m_ulValue; + case k_EWebAPIValueType_String: + if ( m_nStrValuePos < 0 ) + { + return 0ull; + } + else + { +#if defined(_PS3) || defined(POSIX) + return strtoull( (const char *)m_pStringBuffer->Base() + m_nStrValuePos, NULL, 10); +#else + return _strtoui64( (const char *)m_pStringBuffer->Base() + m_nStrValuePos, NULL, 10); +#endif + } + default: + AssertMsg1( false, "Shouldn't call CWebAPIValues::GetUInt64Value unless value type matches. %d does not", m_eValueType ); + return 0; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Get double value +//----------------------------------------------------------------------------- +double CWebAPIValues::GetDoubleValue() const +{ + switch ( m_eValueType ) + { + case k_EWebAPIValueType_Int32: + return (double)m_nValue; + case k_EWebAPIValueType_UInt32: + return (double)m_unValue; + case k_EWebAPIValueType_Int64: + return (double)m_lValue; + case k_EWebAPIValueType_UInt64: + return (double)m_ulValue; + case k_EWebAPIValueType_Double: + return m_flValue; + default: + AssertMsg1( false, "Shouldn't call CWebAPIValues::GetDoubleValue unless value type matches. %d does not", m_eValueType ); + return 0.0f; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Get bool value +//----------------------------------------------------------------------------- +bool CWebAPIValues::GetBoolValue() const +{ + if ( m_eValueType != k_EWebAPIValueType_Bool ) + { + AssertMsg( false, "Shouldn't call CWebAPIValues::GetBoolValue unless value type matches" ); + return false; + } + + return m_bValue; +} + + + +//----------------------------------------------------------------------------- +// Purpose: Get string value +//----------------------------------------------------------------------------- +void CWebAPIValues::GetStringValue( CUtlString &stringOut ) const +{ + switch ( m_eValueType ) + { + case k_EWebAPIValueType_String: + if ( m_nStrValuePos < 0 ) + { + stringOut.Clear(); + } + else + { + stringOut = (const char *)m_pStringBuffer->Base() + m_nStrValuePos; + } + + return; + case k_EWebAPIValueType_Int32: + stringOut = CNumStr( m_nValue ).String(); + return; + case k_EWebAPIValueType_Int64: + stringOut = CNumStr( m_lValue ).String(); + return; + case k_EWebAPIValueType_UInt32: + stringOut = CNumStr( m_unValue ).String(); + return; + case k_EWebAPIValueType_UInt64: + stringOut = CNumStr( m_ulValue ).String(); + return; + case k_EWebAPIValueType_Double: + stringOut = CNumStr( m_flValue ).String(); + return; + case k_EWebAPIValueType_Bool: + stringOut = m_bValue ? "true" : "false"; + return; + default: + AssertMsg1( false, "CWebAPIValues::GetStringValue(), unable to convert data type %d to string", m_eValueType ); + stringOut = ""; + return; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Get binary blob value +//----------------------------------------------------------------------------- +void CWebAPIValues::GetBinaryValue( CUtlBuffer &bufferOut ) const +{ + if ( m_eValueType != k_EWebAPIValueType_BinaryBlob ) + { + AssertMsg( false, "Shouldn't call CWebAPIValues::GetBinaryValue unless value type matches" ); + bufferOut.Clear(); + return; + } + + bufferOut.Clear(); + bufferOut.EnsureCapacity( m_BinaryValue.m_unBytes ); + if ( m_BinaryValue.m_unBytes ) + bufferOut.Put( (byte*)m_pStringBuffer->Base() + m_BinaryValue.m_nDataPos, m_BinaryValue.m_unBytes ); + + return; +} + + +//----------------------------------------------------------------------------- +// Purpose: Create a child array of this node. Note that array nodes can only +// have un-named children, in XML the pchArrayElementNames value will be used +// as the element name for each of the children of the array, in JSON it will simply +// be a numerically indexed [] array. +//----------------------------------------------------------------------------- +CWebAPIValues * CWebAPIValues::CreateChildArray( const char *pchName, const char *pchArrayElementNames ) +{ + return CreateChildInternal( pchName, k_EWebAPIValueType_NumericArray, pchArrayElementNames ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Create a child of this node. Note, it's possible to create multiple, +// children with the same name, but you really don't want to. We'll assert about it +// in debug builds to detect, but not in release. If you do create duplicates you'll +// have broken JSON output. +//----------------------------------------------------------------------------- +CWebAPIValues * CWebAPIValues::CreateChildObject( const char *pchName ) +{ + return CreateChildInternal( pchName, k_EWebAPIValueType_Object ); +} + +//----------------------------------------------------------------------------- +// Purpose: Return an existing child object - otherwise create one and return that. +//----------------------------------------------------------------------------- +CWebAPIValues *CWebAPIValues::FindOrCreateChildObject( const char *pchName ) +{ + CWebAPIValues *pChild = FindChild( pchName ); + if ( pChild ) + { + return pChild; + } + + return CreateChildObject( pchName ); +} + +//----------------------------------------------------------------------------- +// Purpose: Add a child object to the array, this should only be called on objects that are of the array type +//----------------------------------------------------------------------------- +CWebAPIValues * CWebAPIValues::AddChildObjectToArray() +{ + if ( m_eValueType != k_EWebAPIValueType_NumericArray ) + { + AssertMsg( m_eValueType == k_EWebAPIValueType_NumericArray, "Can't add array elements to CWebAPIVAlues unless type is of numeric array." ); + return NULL; + } + + // Use child element array name as name of all children of arrays + return CreateChildInternal( NULL, k_EWebAPIValueType_Object ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Add a child array to the array, this should only be called on objects that are of the array type +//----------------------------------------------------------------------------- +CWebAPIValues * CWebAPIValues::AddChildArrayToArray( const char * pchArrayElementNames ) +{ + if ( m_eValueType != k_EWebAPIValueType_NumericArray ) + { + AssertMsg( m_eValueType == k_EWebAPIValueType_NumericArray, "Can't add array elements to CWebAPIVAlues unless type is of numeric array." ); + return NULL; + } + + // Use child element array name as name of all children of arrays + return CreateChildInternal( NULL, k_EWebAPIValueType_NumericArray, pchArrayElementNames ); +} + +//----------------------------------------------------------------------------- +// Purpose: Internal helper for creating children +//----------------------------------------------------------------------------- +CWebAPIValues * CWebAPIValues::CreateChildInternal( const char *pchName, EWebAPIValueType eValueType, const char *pchArrayElementNames ) +{ + // Shouldn't create children if you have a direct value. You are either an object or array of children, + // or a native value type. Not both. + + AssertMsg( m_eValueType == k_EWebAPIValueType_Object || m_eValueType == k_EWebAPIValueType_NumericArray, "You are trying to create a child node of a CWebAPIValues object, but it has a direct value already. Can't have children and a value." ); + if ( m_eValueType != k_EWebAPIValueType_Object && m_eValueType != k_EWebAPIValueType_NumericArray ) + ClearValue(); + + // Shouldn't create named children if you are a numeric array + CWebAPIValues *pNewNode; + if ( m_eValueType == k_EWebAPIValueType_NumericArray ) + { + if ( pchName ) + { + AssertMsg( false, "Can't create named child of CWebAPIValues object of type NumericArray. Should call AddArrayElement instead of CreateChild*." ); + } + // Force name to match what all items in the array should use + pNewNode = new CWebAPIValues( this, m_nArrayChildElementNamePos, eValueType, pchArrayElementNames ); + } + else + { + pNewNode = new CWebAPIValues( this, pchName, eValueType, pchArrayElementNames ); + } + + if ( eValueType == k_EWebAPIValueType_NumericArray ) + { + Assert( pchArrayElementNames ); + } + + if ( !m_pFirstChild ) + { + m_pLastChild = m_pFirstChild = pNewNode; + return m_pFirstChild; + } + else + { + + CWebAPIValues *pCurLastChild = m_pLastChild; + +#ifdef _DEBUG + // In debug, traverse all children so we can check for duplicate names, which will break JSON output! + pCurLastChild = m_pFirstChild; + if ( m_eValueType != k_EWebAPIValueType_NumericArray ) + { + if ( Q_stricmp( pCurLastChild->GetName(), pchName ) == 0 ) + { + AssertMsg( false, "Trying to create CWebAPIValues child with name %s that conflicts with existing child. Breaks JSON output!", pchName ); + } + } + + while ( pCurLastChild->m_pNextPeer ) + { + pCurLastChild = pCurLastChild->m_pNextPeer; + if ( m_eValueType != k_EWebAPIValueType_NumericArray ) + { + if ( Q_stricmp( pCurLastChild->GetName(), pchName ) == 0 ) + { + AssertMsg( false, "Trying to create CWebAPIValues child with name %s that conflicts with existing child. Breaks JSON output!", pchName ); + } + } + } + // Also, in debug assert last child ptr looks correct + Assert( m_pLastChild == pCurLastChild ); +#endif + + + m_pLastChild = pCurLastChild->m_pNextPeer = pNewNode; + return m_pLastChild; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Set a child node's string value +//----------------------------------------------------------------------------- +void CWebAPIValues::SetChildStringValue( const char *pchChildName, const char *pchValue ) +{ + CreateChildObject( pchChildName )->SetStringValue( pchValue ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Set a child node's int32 value +//----------------------------------------------------------------------------- +void CWebAPIValues::SetChildInt32Value( const char *pchChildName, int32 nValue ) +{ + CreateChildObject( pchChildName )->SetInt32Value( nValue ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Set a child node's uint32 value +//----------------------------------------------------------------------------- +void CWebAPIValues::SetChildUInt32Value( const char *pchChildName, uint32 unValue ) +{ + CreateChildObject( pchChildName )->SetUInt32Value( unValue ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Set a child node's int64 value +//----------------------------------------------------------------------------- +void CWebAPIValues::SetChildInt64Value ( const char *pchChildName, int64 lValue ) +{ + CreateChildObject( pchChildName )->SetInt64Value( lValue ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Set a child node's uint64 value +//----------------------------------------------------------------------------- +void CWebAPIValues::SetChildUInt64Value( const char *pchChildName, uint64 ulValue ) +{ + CreateChildObject( pchChildName )->SetUInt64Value( ulValue ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Set a child node's double value +//----------------------------------------------------------------------------- +void CWebAPIValues::SetChildDoubleValue( const char *pchChildName, double flValue ) +{ + CreateChildObject( pchChildName )->SetDoubleValue( flValue ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Set a child node's binary blob value +//----------------------------------------------------------------------------- +void CWebAPIValues::SetChildBinaryValue( const char *pchChildName, const uint8 *pValue, uint32 unBytes ) +{ + CreateChildObject( pchChildName )->SetBinaryValue( pValue, unBytes ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Set a child node's boolean value +//----------------------------------------------------------------------------- +void CWebAPIValues::SetChildBoolValue( const char *pchChildName, bool bValue ) +{ + CreateChildObject( pchChildName )->SetBoolValue( bValue ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Set a child node's boolean value +//----------------------------------------------------------------------------- +void CWebAPIValues::SetChildNullValue( const char *pchChildName ) +{ + CreateChildObject( pchChildName )->SetNullValue(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Get a child node's int32 value or return the default if the node doesn't exist +//----------------------------------------------------------------------------- +int32 CWebAPIValues::GetChildInt32Value( const char *pchChildName, int32 nDefault ) const +{ + const CWebAPIValues *pChild = FindChild( pchChildName ); + if( pChild ) + return pChild->GetInt32Value(); + else + return nDefault; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get a child node's uint32 value or return the default if the node doesn't exist +//----------------------------------------------------------------------------- +uint32 CWebAPIValues::GetChildUInt32Value( const char *pchChildName, uint32 unDefault ) const +{ + const CWebAPIValues *pChild = FindChild( pchChildName ); + if( pChild ) + return pChild->GetUInt32Value(); + else + return unDefault; +} + + + +//----------------------------------------------------------------------------- +// Purpose: Get a child node's int64 value or return the default if the node doesn't exist +//----------------------------------------------------------------------------- +int64 CWebAPIValues::GetChildInt64Value( const char *pchChildName, int64 lDefault ) const +{ + const CWebAPIValues *pChild = FindChild( pchChildName ); + if( pChild ) + return pChild->GetInt64Value(); + else + return lDefault; +} + + + +//----------------------------------------------------------------------------- +// Purpose: Get a child node's uint64 value or return the default if the node doesn't exist +//----------------------------------------------------------------------------- +uint64 CWebAPIValues::GetChildUInt64Value( const char *pchChildName, uint64 ulDefault ) const +{ + const CWebAPIValues *pChild = FindChild( pchChildName ); + if( pChild ) + return pChild->GetUInt64Value(); + else + return ulDefault; +} + + + +//----------------------------------------------------------------------------- +// Purpose: Get a child node's double value or return the default if the node doesn't exist +//----------------------------------------------------------------------------- +double CWebAPIValues::GetChildDoubleValue( const char *pchChildName, double flDefault ) const +{ + const CWebAPIValues *pChild = FindChild( pchChildName ); + if( pChild ) + return pChild->GetDoubleValue(); + else + return flDefault; +} + + + +//----------------------------------------------------------------------------- +// Purpose: Get a child node's string value or return the default if the node doesn't exist +//----------------------------------------------------------------------------- +void CWebAPIValues::GetChildStringValue( CUtlString &stringOut, const char *pchChildName, const char *pchDefault ) const +{ + const CWebAPIValues *pChild = FindChild( pchChildName ); + if( pChild ) + { + pChild->GetStringValue( stringOut ); + } + else + { + stringOut = pchDefault; + } +} + + + +//----------------------------------------------------------------------------- +// Purpose: Get a child node's binary blob value (returns false if the child wasn't found) +//----------------------------------------------------------------------------- +bool CWebAPIValues::BGetChildBinaryValue( CUtlBuffer &bufferOut, const char *pchChildName ) const +{ + const CWebAPIValues *pChild = FindChild( pchChildName ); + if( pChild ) + { + pChild->GetBinaryValue( bufferOut ); + return true; + } + else + { + return false; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Get a child node's binary blob value (returns false if the child wasn't found) +//----------------------------------------------------------------------------- +bool CWebAPIValues::IsChildNullValue( const char *pchChildName ) const +{ + const CWebAPIValues *pChild = FindChild( pchChildName ); + if( pChild ) + return pChild->IsNullValue(); + else + return false; +} + + + +//----------------------------------------------------------------------------- +// Purpose: Get a child node's bool value or return the default if the node doesn't exist +//----------------------------------------------------------------------------- +bool CWebAPIValues::GetChildBoolValue( const char *pchChildName, bool bDefault ) const +{ + const CWebAPIValues *pChild = FindChild( pchChildName ); + if( pChild ) + return pChild->GetBoolValue(); + else + return bDefault; +} + + +//----------------------------------------------------------------------------- +// Purpose: Find first matching child by name, O(N) on number of children, this class isn't designed for searching +//----------------------------------------------------------------------------- +CWebAPIValues * CWebAPIValues::FindChild( const char *pchName ) +{ + CWebAPIValues *pCurLastChild = m_pFirstChild; + + while ( pCurLastChild ) + { + if ( Q_stricmp( pCurLastChild->GetName(), pchName ) == 0 ) + return pCurLastChild; + + pCurLastChild = pCurLastChild->m_pNextPeer; + } + + return NULL; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the first child of this node +//----------------------------------------------------------------------------- +CWebAPIValues * CWebAPIValues::GetFirstChild() +{ + return m_pFirstChild; +} + + +//----------------------------------------------------------------------------- +// Purpose: Call this on the returned value from GetFirstChild() or a previous GetNextChild() call to +// proceed to the next child of the parent GetFirstChild() was originally called on. +//----------------------------------------------------------------------------- +CWebAPIValues * CWebAPIValues::GetNextChild() +{ + return m_pNextPeer; +} + + +//----------------------------------------------------------------------------- +// Purpose: Call this on any node to return the parent of that node +//----------------------------------------------------------------------------- +CWebAPIValues * CWebAPIValues::GetParent() +{ + return m_pParent; +} + +//----------------------------------------------------------------------------- +// Purpose: Deletes a child node by name +//----------------------------------------------------------------------------- +void CWebAPIValues::DeleteChild( const char *pchName ) +{ + CWebAPIValues *pChild = NULL; // child we're examining, could be NULL at exit if we don't find it + CWebAPIValues *pPrev = NULL; // previous sibling, or NULL + + for ( pChild = m_pFirstChild; pChild != NULL; pPrev = pChild, pChild = pChild->m_pNextPeer ) + { + if ( !Q_stricmp( pChild->GetName(), pchName ) ) + { + if ( pChild == m_pFirstChild ) + { + // first child, fixup parent's pointer to take pChild out + Assert( pPrev == NULL ); + m_pFirstChild = pChild->m_pNextPeer; + } + else + { + // not first child, fixup sibling's pointer to take pChild out + Assert( pPrev != NULL ); + pPrev->m_pNextPeer = pChild->m_pNextPeer; + } + + // clean up next ptr on child so we don't double free + pChild->m_pNextPeer = NULL; + + // fixup last child pointer if pChild is the last child + if ( pChild == m_pLastChild ) + { + m_pLastChild = pPrev; + } + + break; + } + } + + // debug only, check that there is no child by the specified name any more and that it was excised OK + AssertMsg( FindChild( pchName ) == NULL, "cwebapivalues deleted child is still lurking" ); + Assert( pChild == NULL || pChild->m_pNextPeer == NULL ); + + // now take the removed child out of the heap + delete pChild; +} + +//----------------------------------------------------------------------------- +// Purpose: Emits JSON formatted representation of values +// +// when bEmitOldStyleArrays is true, arrays are emitted as a child of a singlet object, and any empty arrays +// will be emitted as having a single null member. +// +// when bEmitOldStyleArrays is false, arrays are emitted bare (no subobject) and empty arrays +// are emitted empty. +//----------------------------------------------------------------------------- +bool CWebAPIValues::BEmitJSONRecursive( const CWebAPIValues *pCurrent, CUtlBuffer &outputBuffer, int nTabLevel, size_t unMaxResultSize, bool bEmitOldStyleArrays ) +{ + bool bSuccess = true; + + + while( pCurrent ) + { + // don't let the buffer grow until it consumes all available memory + if( unMaxResultSize && (size_t)outputBuffer.TellMaxPut() > unMaxResultSize ) + { + return false; + } + + // Can't emit nameless nodes in JSON. Nodes should always have a name. + Assert( pCurrent->GetName() ); + if ( pCurrent->GetName() ) + { + for( int i=0; i < nTabLevel; ++i ) + outputBuffer.PutChar ( '\t' ); + + if ( !pCurrent->m_pParent || pCurrent->m_pParent->GetType() != k_EWebAPIValueType_NumericArray ) + { + EmitJSONString( outputBuffer, pCurrent->GetName() ); + outputBuffer.PutChar( ':' ); + outputBuffer.PutChar( ' ' ); + } + + if ( pCurrent->m_eValueType == k_EWebAPIValueType_Object || pCurrent->m_eValueType == k_EWebAPIValueType_NumericArray ) + { + + if( bEmitOldStyleArrays || pCurrent->m_eValueType == k_EWebAPIValueType_Object ) + { + outputBuffer.PutChar( '{' ); + outputBuffer.PutChar( '\n' ); + } + + if ( pCurrent->m_eValueType == k_EWebAPIValueType_NumericArray ) + { + if( bEmitOldStyleArrays ) + { + for( int i=0; i < nTabLevel+1; ++i ) + outputBuffer.PutChar ( '\t' ); + + EmitJSONString( outputBuffer, (const char *)pCurrent->m_pStringBuffer->Base() + pCurrent->m_nArrayChildElementNamePos ); + outputBuffer.PutChar( ':' ); + outputBuffer.PutChar( ' ' ); + } + + outputBuffer.PutChar( '[' ); + outputBuffer.PutChar( '\n' ); + } + + // First add any children + if ( pCurrent->m_pFirstChild ) + { + int nChildTabLevel = nTabLevel+1; + if ( pCurrent->m_eValueType == k_EWebAPIValueType_NumericArray && bEmitOldStyleArrays ) + ++nChildTabLevel; + + bSuccess = BEmitJSONRecursive( pCurrent->m_pFirstChild, outputBuffer, nChildTabLevel, unMaxResultSize, bEmitOldStyleArrays ); + if ( !bSuccess ) + return false; + } + else if ( bEmitOldStyleArrays ) + { + for( int i=0; i < nTabLevel + 1; ++i ) + outputBuffer.PutChar ( '\t' ); + + outputBuffer.Put( "null", 4 ); + } + + outputBuffer.PutChar( '\n' ); + + for( int i=0; i < nTabLevel; ++i ) + outputBuffer.PutChar ( '\t' ); + + if ( pCurrent->m_eValueType == k_EWebAPIValueType_NumericArray ) + { + if( bEmitOldStyleArrays ) + outputBuffer.PutChar( '\t' ); + outputBuffer.PutChar( ']' ); + outputBuffer.PutChar( '\n' ); + + for( int i=0; i < nTabLevel; ++i ) + outputBuffer.PutChar ( '\t' ); + } + + if( bEmitOldStyleArrays || pCurrent->m_eValueType == k_EWebAPIValueType_Object ) + { + outputBuffer.PutChar( '}' ); + } + } + else + { + + switch ( pCurrent->m_eValueType ) + { + case k_EWebAPIValueType_Int32: + { + CNumStr numStr( pCurrent->m_nValue ); + outputBuffer.Put( numStr.String(), Q_strlen( numStr.String() ) ); + } + break; + case k_EWebAPIValueType_Int64: + { + CNumStr numStr( pCurrent->m_lValue ); + outputBuffer.Put( numStr.String(), Q_strlen( numStr.String() ) ); + } + break; + case k_EWebAPIValueType_UInt32: + { + CNumStr numStr( pCurrent->m_unValue ); + outputBuffer.Put( numStr.String(), Q_strlen( numStr.String() ) ); + } + break; + case k_EWebAPIValueType_UInt64: + { + CNumStr numStr( pCurrent->m_ulValue ); + outputBuffer.Put( numStr.String(), Q_strlen( numStr.String() ) ); + } + break; + case k_EWebAPIValueType_Double: + { + CNumStr numStr( pCurrent->m_flValue ); + outputBuffer.Put( numStr.String(), Q_strlen( numStr.String() ) ); + } + break; + + case k_EWebAPIValueType_String: + { + if ( pCurrent->m_nStrValuePos < 0 ) + outputBuffer.Put( "null", 4 ); + else + EmitJSONString( outputBuffer, (const char *)pCurrent->m_pStringBuffer->Base() + pCurrent->m_nStrValuePos ); + } + break; + + case k_EWebAPIValueType_Bool: + { + if ( !pCurrent->m_bValue ) + outputBuffer.Put( "false", 5 ); + else + outputBuffer.Put( "true", 4 ); + } + break; + + case k_EWebAPIValueType_Null: + { + outputBuffer.Put( "null", 4 ); + } + break; + + case k_EWebAPIValueType_BinaryBlob: + { + if ( pCurrent->m_BinaryValue.m_unBytes == 0 ) + outputBuffer.Put( "null", 4 ); + else + { + CUtlMemory<char> buffEncoded; + DbgVerify( Base64EncodeIntoUTLMemory( (const uint8 *)pCurrent->m_pStringBuffer->Base() + pCurrent->m_BinaryValue.m_nDataPos, pCurrent->m_BinaryValue.m_unBytes, buffEncoded ) ); + EmitJSONString( outputBuffer, buffEncoded.Base() ); + } + } + break; + default: + break; + } + } + } + + // Now, check for any peers + if ( bSuccess && pCurrent->m_pNextPeer ) + { + outputBuffer.PutChar( ',' ); + outputBuffer.PutChar( '\n' ); + + pCurrent = pCurrent->m_pNextPeer; + } + else + { + // We're done, or failing early + pCurrent = NULL; + } + } + + return bSuccess && outputBuffer.IsValid(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Emits KeyValues .vdf style formatted representation of values +//----------------------------------------------------------------------------- +bool CWebAPIValues::BEmitVDFRecursive( const CWebAPIValues *pCurrent, CUtlBuffer &outputBuffer, int nTabLevel, uint32 nArrayElement, size_t unMaxResultSize, bool bIncludeArrayElementName ) +{ + bool bSuccess = true; + + // We can have lots of peers, so this is an optimization to avoid tail recursion and resulting stack-overflows! + while( pCurrent ) + { + // don't let the buffer grow until it consumes all available memory + if( unMaxResultSize && (size_t)outputBuffer.TellMaxPut() > unMaxResultSize ) + { + return false; + } + if ( pCurrent->GetName() ) + { + for( int i=0; i < nTabLevel; ++i ) + outputBuffer.PutChar ( '\t' ); + + // Open node, special naming when inside arrays + if ( pCurrent->m_pParent && pCurrent->m_pParent->GetType() == k_EWebAPIValueType_NumericArray ) + { + CNumStr numStr( nArrayElement ); + numStr.AddQuotes(); + outputBuffer.Put( numStr.String(), Q_strlen( numStr.String() ) ); + } + else + { + EmitVDFString( outputBuffer, pCurrent->GetName() ); + } + + if ( pCurrent->m_eValueType == k_EWebAPIValueType_Object || pCurrent->m_eValueType == k_EWebAPIValueType_NumericArray ) + { + outputBuffer.PutChar( '\n' ); + + for( int i=0; i < nTabLevel; ++i ) + outputBuffer.PutChar ( '\t' ); + + outputBuffer.PutChar( '{' ); + outputBuffer.PutChar( '\n' ); + + if ( pCurrent->m_eValueType == k_EWebAPIValueType_NumericArray && bIncludeArrayElementName ) + { + for( int i=0; i < nTabLevel+1; ++i ) + outputBuffer.PutChar ( '\t' ); + + EmitVDFString( outputBuffer, (const char *)pCurrent->m_pStringBuffer->Base() + pCurrent->m_nArrayChildElementNamePos ); + outputBuffer.PutChar( '\n' ); + + for( int i=0; i < nTabLevel+1; ++i ) + outputBuffer.PutChar ( '\t' ); + + outputBuffer.PutChar( '{' ); + outputBuffer.PutChar( '\n' ); + } + + if ( pCurrent->m_pFirstChild ) + { + int nChildTabLevel = nTabLevel+1; + if ( pCurrent->m_eValueType == k_EWebAPIValueType_NumericArray && bIncludeArrayElementName ) + ++nChildTabLevel; + + bSuccess = BEmitVDFRecursive( pCurrent->m_pFirstChild, outputBuffer, nChildTabLevel, 0, unMaxResultSize, bIncludeArrayElementName ); + if ( !bSuccess ) + return false; + } + + if ( pCurrent->m_eValueType == k_EWebAPIValueType_NumericArray && bIncludeArrayElementName ) + { + outputBuffer.PutChar( '\n' ); + + for( int i=0; i < nTabLevel+1; ++i ) + outputBuffer.PutChar ( '\t' ); + + outputBuffer.PutChar( '}' ); + } + + outputBuffer.PutChar( '\n' ); + + for( int i=0; i < nTabLevel; ++i ) + outputBuffer.PutChar ( '\t' ); + + outputBuffer.PutChar( '}' ); + } + else + { + outputBuffer.PutChar( '\t' ); + + switch ( pCurrent->m_eValueType ) + { + case k_EWebAPIValueType_Int32: + { + CNumStr numStr( pCurrent->m_nValue ); + numStr.AddQuotes(); + outputBuffer.Put( numStr.String(), Q_strlen( numStr.String() ) ); + } + break; + case k_EWebAPIValueType_Int64: + { + CNumStr numStr( pCurrent->m_lValue ); + numStr.AddQuotes(); + outputBuffer.Put( numStr.String(), Q_strlen( numStr.String() ) ); + } + break; + case k_EWebAPIValueType_UInt32: + { + CNumStr numStr( pCurrent->m_unValue ); + numStr.AddQuotes(); + outputBuffer.Put( numStr.String(), Q_strlen( numStr.String() ) ); + } + break; + case k_EWebAPIValueType_UInt64: + { + CNumStr numStr( pCurrent->m_ulValue ); + numStr.AddQuotes(); + outputBuffer.Put( numStr.String(), Q_strlen( numStr.String() ) ); + } + break; + case k_EWebAPIValueType_Double: + { + CNumStr numStr( pCurrent->m_flValue ); + numStr.AddQuotes(); + outputBuffer.Put( numStr.String(), Q_strlen( numStr.String() ) ); + } + break; + + case k_EWebAPIValueType_String: + { + EmitVDFString( outputBuffer, pCurrent->m_nStrValuePos >= 0 ? ( (const char *)pCurrent->m_pStringBuffer->Base() + pCurrent->m_nStrValuePos ) : NULL ); + } + break; + + case k_EWebAPIValueType_Bool: + { + if ( !pCurrent->m_bValue ) + outputBuffer.Put( "\"0\"", 3 ); + else + outputBuffer.Put( "\"1\"", 3 ); + } + break; + + case k_EWebAPIValueType_Null: + { + outputBuffer.Put ("\"\"", 2 ); + } + break; + + case k_EWebAPIValueType_BinaryBlob: + { + if ( pCurrent->m_BinaryValue.m_unBytes == 0 ) + outputBuffer.Put( "\"\"", 2 ); + else + { + CUtlMemory<char> buffEncoded; + DbgVerify( Base64EncodeIntoUTLMemory( (const uint8 *)pCurrent->m_pStringBuffer->Base() + pCurrent->m_BinaryValue.m_nDataPos, pCurrent->m_BinaryValue.m_unBytes, buffEncoded ) ); + EmitVDFString( outputBuffer, buffEncoded.Base() ); + } + } + break; + default: + break; + } + } + } + + // Now, check for any peers + if ( bSuccess && pCurrent->m_pNextPeer ) + { + outputBuffer.PutChar( '\n' ); + pCurrent = pCurrent->m_pNextPeer; + nArrayElement += 1; + } + else + { + // We're done, or failing early + pCurrent = NULL; + } + } + + return bSuccess && outputBuffer.IsValid(); +} + + +//----------------------------------------------------------------------------- +// Purpose: Emits XML formatted representation of values +//----------------------------------------------------------------------------- +bool CWebAPIValues::BEmitXMLRecursive( const CWebAPIValues *pCurrent, CUtlBuffer &outputBuffer, int nTabLevel, size_t unMaxResultSize ) +{ + bool bSuccess = true; + + while( pCurrent ) + { + + // don't let the buffer grow until it consumes all available memory + if( unMaxResultSize && (size_t)outputBuffer.TellMaxPut() > unMaxResultSize ) + { + return false; + } + + // Can't emit nameless nodes in XML. Nodes should always have a name. + Assert( pCurrent->GetName() ); + if ( pCurrent->GetName() ) + { + for( int i=0; i < nTabLevel; ++i ) + outputBuffer.PutChar ( '\t' ); + + // Open node + outputBuffer.PutChar( '<' ); + EmitXMLString( outputBuffer, pCurrent->GetName() ); + outputBuffer.PutChar( '>' ); + + if ( pCurrent->m_eValueType == k_EWebAPIValueType_Object || pCurrent->m_eValueType == k_EWebAPIValueType_NumericArray ) + { + // First add any children + if ( pCurrent->m_pFirstChild ) + { + outputBuffer.PutChar( '\n' ); + + bSuccess = BEmitXMLRecursive( pCurrent->m_pFirstChild, outputBuffer, nTabLevel+1, unMaxResultSize ); + if ( !bSuccess ) + return false; + + outputBuffer.PutChar( '\n' ); + + // Return to correct tab level, for when we close element below + for( int i=0; i < nTabLevel; ++i ) + outputBuffer.PutChar ( '\t' ); + } + } + else + { + + switch ( pCurrent->m_eValueType ) + { + case k_EWebAPIValueType_Int32: + { + CNumStr numStr( pCurrent->m_nValue ); + outputBuffer.Put( numStr.String(), Q_strlen( numStr.String() ) ); + } + break; + case k_EWebAPIValueType_Int64: + { + CNumStr numStr( pCurrent->m_lValue ); + outputBuffer.Put( numStr.String(), Q_strlen( numStr.String() ) ); + } + break; + case k_EWebAPIValueType_UInt32: + { + CNumStr numStr( pCurrent->m_unValue ); + outputBuffer.Put( numStr.String(), Q_strlen( numStr.String() ) ); + } + break; + case k_EWebAPIValueType_UInt64: + { + CNumStr numStr( pCurrent->m_ulValue ); + outputBuffer.Put( numStr.String(), Q_strlen( numStr.String() ) ); + } + break; + case k_EWebAPIValueType_Double: + { + CNumStr numStr( pCurrent->m_flValue ); + outputBuffer.Put( numStr.String(), Q_strlen( numStr.String() ) ); + } + break; + + case k_EWebAPIValueType_String: + { + if ( pCurrent->m_nStrValuePos < 0 ) + outputBuffer.Put( "null", 4 ); + else + EmitXMLString( outputBuffer, (const char *)pCurrent->m_pStringBuffer->Base() + pCurrent->m_nStrValuePos ); + } + break; + + case k_EWebAPIValueType_Bool: + { + if ( !pCurrent->m_bValue ) + outputBuffer.Put( "false", 5 ); + else + outputBuffer.Put( "true", 4 ); + } + break; + + case k_EWebAPIValueType_Null: + { + outputBuffer.Put ("null", 4 ); + } + break; + + case k_EWebAPIValueType_BinaryBlob: + { + if ( pCurrent->m_BinaryValue.m_unBytes == 0 ) + outputBuffer.Put( "null", 4 ); + else + { + CUtlMemory<char> buffEncoded; + DbgVerify( Base64EncodeIntoUTLMemory( (const uint8 *)pCurrent->m_pStringBuffer->Base() + pCurrent->m_BinaryValue.m_nDataPos, pCurrent->m_BinaryValue.m_unBytes, buffEncoded ) ); + EmitXMLString( outputBuffer, buffEncoded.Base() ); + } + } + break; + default: + break; + } + } + } + + // Close element + outputBuffer.PutChar( '<' ); + outputBuffer.PutChar( '/' ); + EmitXMLString( outputBuffer, pCurrent->GetName() ); + outputBuffer.PutChar( '>' ); + + // Now, check for any peers + if ( bSuccess && pCurrent->m_pNextPeer ) + { + outputBuffer.PutChar( '\n' ); + pCurrent = pCurrent->m_pNextPeer; + } + else + { + // We're done, or failing early + pCurrent = NULL; + } + } + + return bSuccess && outputBuffer.IsValid(); +} + + +struct JSONParserContext_t +{ + CWebAPIValues *m_pCurrentNode; + CWebAPIValues *m_pRootNode; + CUtlString m_sChildName; + bool m_bIsKey; +}; + +static int JSONParserCallback(void* void_ctx, int type, const JSON_value* value) +{ + JSONParserContext_t *ctx = (JSONParserContext_t *)void_ctx; + CWebAPIValues *pCreatedNode = NULL; + switch(type) + { + case JSON_T_ARRAY_BEGIN: + // handle the root node + if( !ctx->m_pRootNode ) + { + Assert( !ctx->m_pCurrentNode ); + ctx->m_pRootNode = ctx->m_pCurrentNode = new CWebAPIValues( ctx->m_sChildName, "e" ); + break; + } + + if( ctx->m_pCurrentNode->IsArray() ) + { + Assert( ctx->m_sChildName.IsEmpty() ); + ctx->m_pCurrentNode = ctx->m_pCurrentNode->AddChildArrayToArray( "e" ); + } + else + { + Assert( !ctx->m_sChildName.IsEmpty() ); + ctx->m_pCurrentNode = ctx->m_pCurrentNode->CreateChildArray( ctx->m_sChildName, "e" ); + ctx->m_sChildName.Clear(); + } + break; + + case JSON_T_ARRAY_END: + ctx->m_pCurrentNode = ctx->m_pCurrentNode->GetParent(); + break; + + case JSON_T_OBJECT_BEGIN: + // this might be the start of the root object itself + if( !ctx->m_pRootNode ) + { + ctx->m_pRootNode = ctx->m_pCurrentNode = new CWebAPIValues( ctx->m_sChildName ); + break; + } + + if( ctx->m_pCurrentNode->IsArray() ) + { + Assert( ctx->m_sChildName.IsEmpty() ); + ctx->m_pCurrentNode = ctx->m_pCurrentNode->AddChildObjectToArray(); + } + else + { + Assert( !ctx->m_sChildName.IsEmpty() ); + ctx->m_pCurrentNode = ctx->m_pCurrentNode->CreateChildObject( ctx->m_sChildName ); + ctx->m_sChildName.Clear(); + } + break; + + case JSON_T_OBJECT_END: + ctx->m_pCurrentNode = ctx->m_pCurrentNode->GetParent(); + break; + + case JSON_T_KEY: + ctx->m_sChildName = value->vu.str.value; + break; + + case JSON_T_INTEGER: + case JSON_T_FLOAT: + case JSON_T_NULL: + case JSON_T_TRUE: + case JSON_T_FALSE: + case JSON_T_STRING: + // create the new node for this value + if( ctx->m_pCurrentNode->IsArray() ) + { + pCreatedNode = ctx->m_pCurrentNode->AddChildObjectToArray(); + } + else + { + pCreatedNode = ctx->m_pCurrentNode->CreateChildObject( ctx->m_sChildName ); + ctx->m_sChildName.Clear(); + } + + // set the actual value + switch( type ) + { + case JSON_T_INTEGER: + + // try to figure out what type to use + if( value->vu.integer_value >= 0 ) + { + // unsigned + if( value->vu.integer_value <= UINT_MAX ) + { + pCreatedNode->SetUInt32Value( (uint32)value->vu.integer_value ); + } + else + { + pCreatedNode->SetUInt64Value( (uint64)value->vu.integer_value ); + } + } + else + { + // signed + if( value->vu.integer_value >= INT_MIN ) + { + pCreatedNode->SetInt32Value( (int32)value->vu.integer_value ); + } + else + { + pCreatedNode->SetInt64Value( value->vu.integer_value ); + } + } + break; + + case JSON_T_FLOAT: + pCreatedNode->SetDoubleValue( value->vu.float_value ); + break; + + case JSON_T_NULL: + pCreatedNode->SetNullValue(); + break; + + case JSON_T_TRUE: + pCreatedNode->SetBoolValue( true ); + break; + + case JSON_T_FALSE: + pCreatedNode->SetBoolValue( false ); + break; + + case JSON_T_STRING: + pCreatedNode->SetStringValue( value->vu.str.value ); + break; + + } + break; + + default: + Assert( false ); + break; + } + + return 1; +} + + +// parses JSON into a tree of CWebAPIValues nodes with this as the root +CWebAPIValues *CWebAPIValues::ParseJSON( CUtlBuffer &inputBuffer ) +{ + // + // if there's nothing to parse, just early out + inputBuffer.EatWhiteSpace(); + if( inputBuffer.GetBytesRemaining() == 0 ) + return NULL; + + // if the first character is the start of a string, + // wrap the whole thing in an object so we can parse it. + // We'll unwrap it at the end + char cFirst = *(char *)inputBuffer.PeekGet(); + bool bWrapContent = cFirst == '\"'; + + JSON_config config; + + struct JSON_parser_struct* jc = NULL; + + init_JSON_config(&config); + + JSONParserContext_t context; + context.m_pCurrentNode = NULL; + context.m_pRootNode = NULL; + context.m_bIsKey = false; + + config.depth = 19; + config.callback = &JSONParserCallback; + config.allow_comments = 1; + config.handle_floats_manually = 0; + config.callback_ctx = &context; + config.malloc = malloc; + config.free = free; + + jc = new_JSON_parser(&config); + + bool bSuccess = true; + if( bWrapContent ) + JSON_parser_char( jc, '{' ); + while( inputBuffer.GetBytesRemaining( ) > 0 ) + { + if( !JSON_parser_char( jc, (unsigned char)inputBuffer.GetChar() ) ) + { + bSuccess = false; + break; + } + } + + if( bWrapContent ) + { + JSON_parser_char( jc, '}' ); + } + + if( bSuccess ) + { + if (!JSON_parser_done(jc)) + { + bSuccess = false; + } + } + + delete_JSON_parser(jc); + + // unwrap the root node + if( bWrapContent && bSuccess ) + { + CWebAPIValues *pWrapRoot = context.m_pRootNode; + if( !pWrapRoot ) + { + bSuccess = false; + } + else + { + CWebAPIValues *pRealRoot = pWrapRoot->GetFirstChild(); + if( !pRealRoot ) + { + bSuccess = false; + } + else + { + if( pRealRoot->GetNextChild() ) + { + bSuccess = false; + } + else + { + pWrapRoot->m_pFirstChild = NULL; + pRealRoot->m_pParent = NULL; + context.m_pRootNode = pRealRoot; + + delete pWrapRoot; + } + } + } + } + + if( bSuccess ) + { + return context.m_pRootNode; + } + else + { + delete context.m_pRootNode; + return NULL; + } +} + + +CWebAPIValues *CWebAPIValues::ParseJSON( const char *pchJSONString ) +{ + CUtlBuffer bufJSON( pchJSONString, Q_strlen( pchJSONString), CUtlBuffer::TEXT_BUFFER | CUtlBuffer::READ_ONLY ); + return ParseJSON( bufJSON ); +} + + +void CWebAPIValues::CopyFrom( const CWebAPIValues *pSource ) +{ + switch( pSource->GetType() ) + { + case k_EWebAPIValueType_Int32: + SetInt32Value( pSource->GetInt32Value() ); + break; + case k_EWebAPIValueType_Int64: + SetInt64Value( pSource->GetInt64Value() ); + break; + case k_EWebAPIValueType_UInt32: + SetUInt32Value( pSource->GetUInt32Value() ); + break; + case k_EWebAPIValueType_UInt64: + SetUInt64Value( pSource->GetUInt64Value() ); + break; + case k_EWebAPIValueType_Double: + SetDoubleValue( pSource->GetDoubleValue() ); + break; + + case k_EWebAPIValueType_String: + { + CUtlString sValue; + pSource->GetStringValue( sValue ); + SetStringValue( sValue ); + } + break; + + case k_EWebAPIValueType_Bool: + SetBoolValue( pSource->GetBoolValue() ); + break; + + case k_EWebAPIValueType_Null: + SetNullValue( ); + break; + + case k_EWebAPIValueType_BinaryBlob: + { + CUtlBuffer bufValue; + pSource->GetBinaryValue( bufValue ); + SetBinaryValue( (uint8 *)bufValue.Base(), bufValue.TellMaxPut() ); + } + break; + + case k_EWebAPIValueType_Object: + { + ClearValue(); + m_eValueType = k_EWebAPIValueType_Object; + + const CWebAPIValues *pSourceChild = pSource->GetFirstChild(); + while( pSourceChild ) + { + CWebAPIValues *pChild; + if( pSourceChild->IsArray() ) + pChild = CreateChildArray( pSourceChild->GetName(), pSourceChild->GetElementName() ); + else + pChild = CreateChildObject( pSourceChild->GetName() ); + + pChild->CopyFrom( pSourceChild ); + pSourceChild = pSourceChild->GetNextChild(); + } + } + break; + + case k_EWebAPIValueType_NumericArray: + { + // the type should have been set when this node was first created + Assert( m_eValueType == k_EWebAPIValueType_NumericArray ); + m_eValueType = k_EWebAPIValueType_NumericArray; + + const CWebAPIValues *pSourceChild = pSource->GetFirstChild(); + while( pSourceChild ) + { + CWebAPIValues *pChild; + if( pSourceChild->IsArray() ) + pChild = AddChildArrayToArray( pSourceChild->GetElementName() ); + else + pChild = AddChildObjectToArray( ); + + pChild->CopyFrom( pSourceChild ); + pSourceChild = pSourceChild->GetNextChild(); + } + } + break; + + default: + AssertMsg( false, "Unknown type in CWebAPIValues::CopyFrom" ); + break; + } +} + +//----------------------------------------------------------------------------- +// Purpose: Fills a WebAPI key values from a corresponding protobuf message. +//----------------------------------------------------------------------------- +bool ProtoBufHelper::RecursiveAddProtoBufToWebAPIValues( CWebAPIValues *pWebAPIRoot, const ::google::protobuf::Message & msg ) +{ + using ::google::protobuf::FieldDescriptor; + const ::google::protobuf::Descriptor *pDescriptor = msg.GetDescriptor(); + const ::google::protobuf::Reflection *pReflection = msg.GetReflection(); + + for ( int iField = 0; iField < pDescriptor->field_count(); iField++ ) + { + const ::google::protobuf::FieldDescriptor *pField = pDescriptor->field( iField ); + const char *pFieldName = pField->name().c_str(); + + if ( pField->is_repeated() ) + { + if ( pReflection->FieldSize( msg, pField ) == 0 ) + { + continue; // No need to create the array if it is empty + // If the field has been disabled externally, it has been cleared already (and thus will be skipped) + } + + // bugbug: Is there a way to get this from the google API? + const char *pchTypeName = "unknown_type"; + switch ( pField->cpp_type() ) + { + case FieldDescriptor::CPPTYPE_INT32: pchTypeName = "int32"; break; + case FieldDescriptor::CPPTYPE_INT64: pchTypeName = "int64"; break; + case FieldDescriptor::CPPTYPE_UINT32: pchTypeName = "uint32"; break; + case FieldDescriptor::CPPTYPE_DOUBLE: pchTypeName = "double"; break; + case FieldDescriptor::CPPTYPE_FLOAT: pchTypeName = "float"; break; + case FieldDescriptor::CPPTYPE_BOOL: pchTypeName = "bool"; break; + case FieldDescriptor::CPPTYPE_ENUM: pchTypeName = "enum"; break; + case FieldDescriptor::CPPTYPE_STRING: + { + if ( pField->type() == FieldDescriptor::TYPE_STRING ) + { + pchTypeName = "string"; + } + else + { + AssertMsg1( pField->type() == FieldDescriptor::TYPE_BYTES, "Unrecognized field type: %d", pField->type() ); + pchTypeName = "bytes"; + } + break; + } + case FieldDescriptor::CPPTYPE_MESSAGE: pchTypeName = "message"; break; + } + + CWebAPIValues *pContainer = pWebAPIRoot->CreateChildArray( pFieldName, pchTypeName ); + for ( int iRepeated = 0; iRepeated < pReflection->FieldSize( msg, pField ); iRepeated++ ) + { + switch ( pField->cpp_type() ) + { + case FieldDescriptor::CPPTYPE_INT32: pContainer->SetChildInt32Value( NULL, pReflection->GetRepeatedInt32( msg, pField, iRepeated ) ); break; + case FieldDescriptor::CPPTYPE_INT64: pContainer->SetChildInt64Value( NULL, pReflection->GetRepeatedInt64( msg, pField, iRepeated ) ); break; + case FieldDescriptor::CPPTYPE_UINT32: pContainer->SetChildUInt32Value( NULL, pReflection->GetRepeatedUInt32( msg, pField, iRepeated ) ); break; + case FieldDescriptor::CPPTYPE_UINT64: pContainer->SetChildUInt64Value( NULL, pReflection->GetRepeatedUInt64( msg, pField, iRepeated ) ); break; + case FieldDescriptor::CPPTYPE_DOUBLE: pContainer->SetChildDoubleValue( NULL, pReflection->GetRepeatedDouble( msg, pField, iRepeated ) ); break; + case FieldDescriptor::CPPTYPE_FLOAT: pContainer->SetChildDoubleValue( NULL, pReflection->GetRepeatedFloat( msg, pField, iRepeated ) ); break; + case FieldDescriptor::CPPTYPE_BOOL: pContainer->SetChildBoolValue( NULL, pReflection->GetRepeatedBool( msg, pField, iRepeated ) ); break; + case FieldDescriptor::CPPTYPE_ENUM: pContainer->SetChildInt32Value( NULL, pReflection->GetRepeatedEnum( msg, pField, iRepeated )->number() ); break; + case FieldDescriptor::CPPTYPE_STRING: + { + const std::string &strValue = pReflection->GetRepeatedString( msg, pField, iRepeated ); + if ( pField->type() == FieldDescriptor::TYPE_STRING ) + { + pContainer->SetChildStringValue( NULL, strValue.c_str() ); + } + else + { + // Binary blobs are automatically encoded in Base64 when converted to string by Web request + pContainer->SetChildBinaryValue( NULL, (const uint8 *)strValue.c_str(), strValue.size() ); + } + break; + } + case FieldDescriptor::CPPTYPE_MESSAGE: + { + const ::google::protobuf::Message &subMsg = pReflection->GetRepeatedMessage( msg, pField, iRepeated ); + CWebAPIValues *pChild = pContainer->CreateChildObject( NULL ); + if ( RecursiveAddProtoBufToWebAPIValues( pChild, subMsg ) == false ) + { + return false; + } + break; + } + default: + AssertMsg1( false, "Unknown cpp_type %d", pField->cpp_type() ); + return false; + } + } + } + else + { + if ( ( ( pReflection->HasField( msg, pField ) == false ) && ( pField->has_default_value() == false ) ) + // || pField->options().GetExtension( is_field_disabled_externally ) // Steam vesion supports this + ) + { + continue; // If no field set and no default value set, there is no need to send the field + // Or it is externally disabled (it has been cleared already but there still may be a default value set) + } + + switch ( pField->cpp_type() ) + { + case FieldDescriptor::CPPTYPE_INT32: pWebAPIRoot->SetChildInt32Value( pFieldName, pReflection->GetInt32( msg, pField ) ); break; + case FieldDescriptor::CPPTYPE_INT64: pWebAPIRoot->SetChildInt64Value( pFieldName, pReflection->GetInt64( msg, pField ) ); break; + case FieldDescriptor::CPPTYPE_UINT32: pWebAPIRoot->SetChildUInt32Value( pFieldName, pReflection->GetUInt32( msg, pField ) ); break; + case FieldDescriptor::CPPTYPE_UINT64: pWebAPIRoot->SetChildUInt64Value( pFieldName, pReflection->GetUInt64( msg, pField ) ); break; + case FieldDescriptor::CPPTYPE_DOUBLE: pWebAPIRoot->SetChildDoubleValue( pFieldName, pReflection->GetDouble( msg, pField ) ); break; + case FieldDescriptor::CPPTYPE_FLOAT: pWebAPIRoot->SetChildDoubleValue( pFieldName, pReflection->GetFloat( msg, pField ) ); break; + case FieldDescriptor::CPPTYPE_BOOL: pWebAPIRoot->SetChildBoolValue( pFieldName, pReflection->GetBool( msg, pField ) ); break; + case FieldDescriptor::CPPTYPE_ENUM: pWebAPIRoot->SetChildInt32Value( pFieldName, pReflection->GetEnum( msg, pField )->number() ); break; + case FieldDescriptor::CPPTYPE_STRING: + { + const std::string &strValue = pReflection->GetString( msg, pField ); + if ( pField->type() == FieldDescriptor::TYPE_STRING ) + { + pWebAPIRoot->SetChildStringValue( pFieldName, strValue.c_str() ); + } + else + { + AssertMsg1( pField->type() == FieldDescriptor::TYPE_BYTES, "Unrecognized field type: %d", pField->type() ); + // Binary blobs are automatically encoded in Base64 when converted to string by Web request + pWebAPIRoot->SetChildBinaryValue( pFieldName, (const uint8 *)strValue.c_str(), strValue.size() ); + } + break; + } + case FieldDescriptor::CPPTYPE_MESSAGE: + { + CWebAPIValues *pChild = pWebAPIRoot->CreateChildObject( pFieldName ); +#undef GetMessage // Work around unfortunate Microsoft macro + const ::google::protobuf::Message &subMsg = pReflection->GetMessage( msg, pField ); +#ifdef UNICODE +#define GetMessage GetMessageW +#else +#define GetMessage GetMessageA +#endif // !UNICODE + if ( RecursiveAddProtoBufToWebAPIValues( pChild, subMsg ) == false ) + { + return false; + } + break; + } + default: + AssertMsg1( false, "Unknown cpp_type %d", pField->cpp_type() ); + return false; + } + } + } + return true; +} + diff --git a/gcsdk/workthreadpool.cpp b/gcsdk/workthreadpool.cpp new file mode 100644 index 0000000..95294cb --- /dev/null +++ b/gcsdk/workthreadpool.cpp @@ -0,0 +1,751 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//============================================================================= + + +#include "stdafx.h" +#include "tslist.h" +#include <workthreadpool.h> +#include <gclogger.h> + +#include "tier0/memdbgon.h" + +namespace GCSDK { + +IWorkThreadPoolSignal *CWorkThreadPool::sm_pWorkItemsCompletedSignal = NULL; + +//----------------------------------------------------------------------------- +// Purpose: CWorkThread constructors +//----------------------------------------------------------------------------- +CWorkThread::CWorkThread( CWorkThreadPool *pThreadPool ) +: m_pThreadPool( pThreadPool ), + m_bExitThread( false ), + m_bFinished( false ) +{ +} + +CWorkThread::CWorkThread( CWorkThreadPool *pThreadPool, const char *pszName ) +: m_pThreadPool( pThreadPool ), + m_bExitThread( false ), + m_bFinished( false ) +{ + SetName( pszName ); +} + +//----------------------------------------------------------------------------- +// Purpose: Tell work thread pool not to set event on every item added (SetEvent is very expensive) +//----------------------------------------------------------------------------- +void CWorkThreadPool::SetNeverSetEventOnAdd( bool bNeverSet ) +{ + bool bWasSet = m_bNeverSetOnAdd; + m_bNeverSetOnAdd = bNeverSet; + + // In case of disabling set right away to make sure if we have pending work we execute it now with no latency + if ( bWasSet && !m_bNeverSetOnAdd ) + m_EventNewWorkItem.Set(); +} + + + +//----------------------------------------------------------------------------- +// Purpose: performs the work loop for the thread, waits for work, +// notifies the owner (the pool) as it completes work and before it exits +//----------------------------------------------------------------------------- +int CWorkThread::Run() +{ + // manage our thread pool's statistics + ++m_pThreadPool->m_cThreadsRunning; + +#ifdef _SERVER + g_CompletionPortManager.AssociateCallingThreadWithIOCP(); +#endif + + OnStart(); + +#if 0 // need to port over new vprof code +#if defined( VPROF_ENABLED ) + CVProfile *pProfile = GetVProfProfileForCurrentThread(); +#endif +#endif + + CWorkThreadPool *pPool = m_pThreadPool; + + int nIterations = 0; + const int nMaxFastIterations = 4; + while ( !m_bExitThread ) + { +#if 0 // game vprof doesn't yet support TLS'd vprof instances, until new vprof code is ported +#if defined( VPROF_ENABLED ) + if ( pProfile ) + pProfile->MarkFrame( GetName() ); +#endif +#endif + pPool->m_cActiveThreads++; + + nIterations = 0; + while ( (pPool->BNeverSetEventOnAdd() && nIterations < nMaxFastIterations) || nIterations == 0 ) + { + // process any items which have arrived + CWorkItem *pWorkItem = pPool->GetNextWorkItemToProcess( ); + while ( pWorkItem ) + { +#if 0 + pPool->m_StatWaitTime.Update( pWorkItem->WaitingTime() ); +#endif + if ( pWorkItem->HasTimedOut() ) + { + pWorkItem->m_bCanceled = true; + } + else + { + // call the work item to do its work + pWorkItem->m_bCanceled = false; + + CFastTimer fastTimer; + fastTimer.Start(); + pWorkItem->m_bRunning = true; + bool bSuccess = pWorkItem->ThreadProcess( this ); + pWorkItem->m_bRunning = false; + fastTimer.End(); + CCycleCount cycleCount = fastTimer.GetDuration(); + pWorkItem->SetCycleCount(cycleCount); +#if 0 + pPool->m_StatExecutionTime.Update( cycleCount.GetUlMicroseconds() ); +#endif + if ( bSuccess ) + pPool->m_cSuccesses ++; + else + pWorkItem->m_bResubmit ? pPool->m_cRetries++ : pPool->m_cFailures++; + } + + // do we need to resubmit this item? + if ( pWorkItem->m_bResubmit ) + { + pWorkItem->m_bResubmit = false; + pWorkItem->m_bCanceled = false; + // put it at the tail of the incoming queue + pPool->AddWorkItem( pWorkItem ); + pWorkItem->Release(); // dec since AddWorkItem added 1 more again + } + else + { + // put it in the outgoing queue + pPool->OnWorkItemCompleted( pWorkItem ); + } + + // If we are flagged as exiting don't try to get more work, we need to exit right away and orphan the work + // to avoid blocking shutdown. + if ( !m_bExitThread ) + { + // get the next work item (if any) + pWorkItem = pPool->GetNextWorkItemToProcess( ); + } + else + { + pWorkItem = NULL; + } + +#if 0 // game vprof doesn't yet support TLS'd vprof instances, until new vprof code is ported +#if defined( VPROF_ENABLED ) + if ( pProfile && pWorkItem ) + pProfile->MarkFrame( GetName() ); +#endif +#endif + } + + if ( m_bExitThread ) + break; + + ++nIterations; + if ( pPool->BNeverSetEventOnAdd() && nIterations < nMaxFastIterations ) + { + VPROF_BUDGET( "CWorkThread -- Sleep", VPROF_BUDGETGROUP_SLEEPING ); + ThreadSleep( 2 ); + } + } + + pPool->m_cActiveThreads--; + + // wait for a new work item to arrive in the queue, check the counts first just to be sure + { + VPROF_BUDGET( "CWorkThread -- Sleep", VPROF_BUDGETGROUP_SLEEPING ); +#ifdef _SERVER + if ( pPool->BNeverSetEventOnAdd() ) + pPool->m_EventNewWorkItem.Wait( 15 ); + else + pPool->m_EventNewWorkItem.Wait( 50 ); +#else + pPool->m_EventNewWorkItem.Wait( 50 ); +#endif + } + } + + // Since we are exiting, we must have been signaled to shutdown, and we should signal any remaining threads + // since each signal wakes only one thread. + pPool->m_EventNewWorkItem.Set(); + + m_bFinished = true; + + // updates stats + --m_pThreadPool->m_cThreadsRunning; + + return EXIT_SUCCESS; +} + + +//----------------------------------------------------------------------------- +// Purpose: Construct a new CWorkThreadPool object +//----------------------------------------------------------------------------- +CWorkThreadPool::CWorkThreadPool( const char *pszThreadName ) + : +#if 0 + m_StatWaitTime( 100 ), + m_StatExecutionTime( 100 ), +#endif + m_bThreadsInitialized( false ), + m_cThreadsRunning( 0 ), + m_cActiveThreads( 0 ), + m_bMayHaveJobTimeouts( false ), + m_bExiting( false ), + m_bAutoCreateThreads( false ), + m_cMaxThreads( 0 ), + m_cFailures( 0 ), + m_cSuccesses( 0 ), + m_pWorkThreadConstructor( NULL ), + m_ulLastCompletedSequenceNumber( 0 ), + m_ulLastUsedSequenceNumber( 0 ), + m_ulLastDispatchedSequenceNumber( 0 ), + m_bEnsureOutputOrdering( false ), + m_bNeverSetOnAdd( false ) +{ + Assert( pszThreadName != NULL ); + Q_strncpy( m_szThreadNamePfx, pszThreadName, sizeof( m_szThreadNamePfx ) ); + m_LimitTimerCreateNewThreads.SetLimit( 1 ); + + m_pTSQueueToProcess = new CTSQueue< CWorkItem* >; + m_pTSQueueCompleted = new CTSQueue< CWorkItem* >; +} + + +//----------------------------------------------------------------------------- +// Purpose: destructor; does assertion checks to make sure we weere shut down cleanly +// cleans up even if we weren't cleanly stopped +//----------------------------------------------------------------------------- +CWorkThreadPool::~CWorkThreadPool() +{ + // If you hit this you probably didn't call StopWorkThreads() first + AssertMsg1( ( !m_bThreadsInitialized || m_bExiting ) && 0 == m_cThreadsRunning, + "CWorkThreadPool::~CWorkThreadPool(): Thread pool %s shutdown incorrectly.\n", + m_szThreadNamePfx ); + + if ( m_WorkThreads.Count() ) + { + StopWorkThreads(); + Assert( 0 == m_WorkThreads.Count() ); + } + + Assert( 0 == m_cThreadsRunning ); + + // WARNING: We need to release any items left in the queues + CWorkItem *pWorkItem = NULL; + if ( m_pTSQueueCompleted->Count() > 0 ) + { + EmitWarning( SPEW_THREADS, 2, "CWorkThreadPool::~CWorkThreadPool: work complete queue not empty, %d items discarded.\n", m_pTSQueueCompleted->Count() ); + pWorkItem = NULL; + while ( m_pTSQueueCompleted->PopItem( &pWorkItem ) ) + { + while( pWorkItem->Release() ) + { + /* nothing */ + } + } + } + + if ( m_pTSQueueToProcess->Count() > 0 ) + { + EmitWarning( SPEW_THREADS, 2, "CWorkThreadPool::~CWorkThreadPool: work processing queue not empty: %d items discarded.\n", m_pTSQueueToProcess->Count() ); + while ( m_pTSQueueToProcess->PopItem( &pWorkItem ) ) + { + while( pWorkItem->Release() ) + { + /* nothing */ + } + } + } + + delete m_pTSQueueToProcess; + delete m_pTSQueueCompleted; +} + + +#if 0 +//----------------------------------------------------------------------------- +// Purpose: estimate the current backlog time using previous execution time, +// the number of outstanding items, and the number of running threads +//----------------------------------------------------------------------------- +uint64 CWorkThreadPool::GetCurrentBacklogTime() const +{ + if ( m_WorkThreads.Count() == 0 ) + return 0; + return ( m_pTSQueueToProcess->Count() * m_StatExecutionTime.GetUlAvg() ) / m_WorkThreads.Count(); +} +#endif + + +int CWorkThreadPool::AddWorkThread( CWorkThread *pThread ) +{ + AUTO_LOCK( m_WorkThreadMutex ); + Assert( pThread ); + return m_WorkThreads.AddToTail( pThread ); +} + + +void CWorkThreadPool::StartWorkThread( CWorkThread *pWorkThread, int iName ) +{ + char rgchThreadName[32]; + Q_snprintf( rgchThreadName, sizeof( rgchThreadName ), "%s:%d", m_szThreadNamePfx, iName ); + pWorkThread->SetName( rgchThreadName ); + if ( !pWorkThread->Start() ) + EmitError( SPEW_THREADS, "CWorkThreadPool::StartWorkThread: Thread creation failed.\n" ); +} + + +void CWorkThreadPool::StartWorkThreads() +{ + m_bThreadsInitialized = true; + if ( 0 == m_WorkThreads.Count() ) + { + EmitWarning( SPEW_THREADS, 2, "CWorkThreadPool::StartWorkThreads: called with no threads in the pool, this is probably a bug.\n" ); + return; + } + m_bExiting = false; + m_cThreadsRunning = 0; + AUTO_LOCK( m_WorkThreadMutex ); + FOR_EACH_VEC( m_WorkThreads, i ) + { + StartWorkThread( m_WorkThreads[i], i ); + } + + // XXX why? + while ( m_cThreadsRunning == (uint) 0 ) + { + ThreadSleep( 1 ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: stops whatever work threads we're running +// this must be called before the thread pool object is destroyed +//----------------------------------------------------------------------------- +void CWorkThreadPool::StopWorkThreads() +{ + // indicate that we're shutting down; + // don't accept more work in this thread + m_bExiting = true; + + AUTO_LOCK( m_WorkThreadMutex ); + + FOR_EACH_VEC( m_WorkThreads, i ) + { + m_WorkThreads[i]->m_bExitThread = true; + m_WorkThreads[i]->Cancel(); + } + + // loop until all threads are dead + while ( true ) + { + // This thread already holds the mutex; recursive try-lock should always succeed + DbgVerify( BTryDeleteExitedWorkerThreads() ); + + if ( m_WorkThreads.Count() == 0 ) + break; + + // Keep waking up threads until they're all dead. + m_EventNewWorkItem.Set(); + +#ifdef _PS3 + // call to abort any running call to gethostbyname(). + // this is called over all the remaining work threads, while + // waiting for the rest of the work threads to finish so that they won't + // spuriously block on new calls to gethostbyname() as the + // sys_net_abort_resolver call only stops the next call to the + // network API, not any future calls. + + FOR_EACH_VEC( m_WorkThreads, iPS3 ) + { + // PS3 hack to abort gethostbyname() calls that may be blocking... + sys_net_abort_resolver( m_WorkThreads[ iPS3 ]->GetThreadID(), SYS_NET_ABORT_STRICT_CHECK ); + } +#endif + + const uint k_uJoinTimeoutMillisec = 10000; // 10 seconds seems pretty arbitrary. + + CWorkThread *pWorkThread = m_WorkThreads[0]; + bool bJoined = pWorkThread->Join( k_uJoinTimeoutMillisec ); + if ( !bJoined ) + { + // Print thread id as a pointer for cross-platform compatibility + EmitWarning( SPEW_THREADS, 2, "Thread \"%s\" (ID %p) failed to shut down", pWorkThread->GetName(), (void*)pWorkThread->GetThreadID() ); + } + else + { + // Succesful join means that the thread has terminated. + if ( !pWorkThread->m_bFinished ) + { + // This would be a logic error in the thread proc if it ever tripped. + AssertMsg( false, "pWorkThread->m_bFinished is false but thread is not running" ); + // Recover by flagging the thread as potentially eligable for deletion, since it's dead. + pWorkThread->m_bFinished = true; + } + } + } + + Assert( m_WorkThreads.Count() == 0 && m_cThreadsRunning == (uint32) 0 ); +} + + +//----------------------------------------------------------------------------- +// Purpose: sees if we have a non-zero number of work threads, +// or a non-zero number of active threads +//----------------------------------------------------------------------------- +bool CWorkThreadPool::HasWorkItemsToProcess() const +{ + return ( m_pTSQueueToProcess->Count() > 0 ) + || ( m_cActiveThreads > 0 ); +} + + +//----------------------------------------------------------------------------- +// Purpose: sets dynamic thread construction +//----------------------------------------------------------------------------- +void CWorkThreadPool::SetWorkThreadAutoConstruct( int cMaxThreads, IWorkThreadFactory *pWorkThreadConstructor ) +{ + AUTO_LOCK( m_WorkThreadMutex ); + + m_bThreadsInitialized = true; + m_bAutoCreateThreads = true; + m_cMaxThreads = MAX( 1, cMaxThreads ); + m_pWorkThreadConstructor = pWorkThreadConstructor; + + // If we have too many threads now, mark some to exit next time they loop. + for ( int i = m_cMaxThreads; i < m_WorkThreads.Count(); i++ ) + { + m_WorkThreads[i]->m_bExitThread = true; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds a work item +// Output: true if successful, +// false if a low priority work item is not added due to a busy system +// false if this work pool is shutting down and work isn't being accepted +// NOTE: Adding normal priority items should always succeed +//----------------------------------------------------------------------------- +bool CWorkThreadPool::AddWorkItem( CWorkItem *pWorkItem ) +{ + Assert( !m_bExiting ); + if ( m_bExiting ) + return false; + + if ( m_bEnsureOutputOrdering ) + { + AssertMsg( pWorkItem->m_bResubmit == false, "CWorkThreadPool can't support item auto resubmission when ensuring output ordering" ); + } + + // if we're in auto-create mode, make sure we have enough threads running + if ( m_bAutoCreateThreads && m_WorkThreads.Count() < m_cMaxThreads ) + { + int cPendingItems = m_pTSQueueToProcess->Count(); + + // we shouldn't get more than 12 items queued per already existing thread, otherwise we + // want to create a new thread to help us keep up. + if ( m_WorkThreads.Count() < 1 || m_WorkThreads.Count() * 12 < ( cPendingItems + 1 ) ) + { + if ( m_WorkThreads.Count() >= 2 && !m_LimitTimerCreateNewThreads.BLimitReached() ) + { + // Don't create more yet, we don't want to create them too fast + } + else + { + // create another thread + CWorkThread *pWorkThread = NULL; + if ( m_pWorkThreadConstructor ) + { + pWorkThread = m_pWorkThreadConstructor->CreateWorkerThread( this ); + } + else + { + pWorkThread = new CWorkThread( this ); + } + if( pWorkThread != NULL ) + { + int iName = AddWorkThread( pWorkThread ); + StartWorkThread( pWorkThread, iName ); + } + m_LimitTimerCreateNewThreads.SetLimit( 250*k_nThousand ); + } + } + } + + // + // Do we actually have any threads ? If creating threads can fail, then maybe we don't ! + // In that case, this WorkItem is not going to run ! + // + if ( m_WorkThreads.Count() == 0 ) + { + Assert(false); + return false ; + } + + + // WARNING: We need to call pWorkItem AddRef() and Release() at all entry/exit points for the thread pool system. + pWorkItem->AddRef(); + + pWorkItem->m_ulSequenceNumber = (++m_ulLastUsedSequenceNumber); + m_pTSQueueToProcess->PushItem( pWorkItem ); + + if ( !BNeverSetEventOnAdd() && m_cActiveThreads == 0 ) + { + VPROF_BUDGET( "SetEvent()", VPROF_BUDGETGROUP_THREADINGMAIN ); + m_EventNewWorkItem.Set(); + } + + return true; +} + + +CWorkItem *CWorkThreadPool::GetNextCompletedWorkItem( ) +{ + CWorkItem *pWorkItem = NULL; + + // Use a while loop just in case ref counts get screwed up and an item gets deleted when we release our reference to it + while ( m_pTSQueueCompleted->PopItem( &pWorkItem ) ) + { + // WARNING: We need to call workitem AddRef() and Release() at all entry/exit points for the thread pool system. + // Release() returns the current refcount of the object (after decrementing it by one) and should be non-zero unless the + // the caller has released it already. + if ( pWorkItem != NULL && pWorkItem->Release() > 0 ) + { + return pWorkItem; + } + } + + return NULL; +} + + +//----------------------------------------------------------------------------- +// Purpose: gets the next work item to process. This non-blocking function +// returns NULL immediately if there's nothing left in the queue. +// otherwise, a pointer to the next CWorkItem. +//----------------------------------------------------------------------------- +CWorkItem *CWorkThreadPool::GetNextWorkItemToProcess( ) +{ + CWorkItem *pWorkItem = NULL; + + if ( m_pTSQueueToProcess->Count() && m_pTSQueueToProcess->PopItem( &pWorkItem ) ) + { + return pWorkItem; + } + + return NULL; +} + + +bool CWorkThreadPool::BDispatchCompletedWorkItems( const CLimitTimer &limitTimer, CJobMgr *pJobMgr ) +{ + BTryDeleteExitedWorkerThreads(); + + CWorkItem *pWorkItem = GetNextCompletedWorkItem( ); + while ( pWorkItem != NULL ) + { + uint64 ulSequenceNumber = pWorkItem->m_ulSequenceNumber; + // NOTE: despite its name, this YIELDS - the target job + // is resumed, and we resume here. + if ( !pWorkItem->DispatchCompletedWorkItem( pJobMgr ) ) + { + EmitWarning( SPEW_THREADS, 2, "Work Item for Work Pool %s completed but job no longer existed to notify\n", m_szThreadNamePfx == NULL ? "UNKNOWN" :m_szThreadNamePfx ); + AssertMsg1( m_bMayHaveJobTimeouts, "Work Item for Work Pool %s completed but job no longer existed to notify", m_szThreadNamePfx == NULL ? "UNKNOWN" :m_szThreadNamePfx ); + } + + // pWorkItem was released by DispatchCompletedWorkItem + m_ulLastDispatchedSequenceNumber = ulSequenceNumber; + if ( limitTimer.BLimitReached() ) + break; + + pWorkItem = GetNextCompletedWorkItem( ); + } + + return ( GetCompletedWorkItemCount() > 0 ); +} + + +//----------------------------------------------------------------------------- +// Purpose: delete any thread objects that have exited +// we'll make sure the thread has actually ended; +// if they haven't, they'll remain in the threads to delete list +//----------------------------------------------------------------------------- +bool CWorkThreadPool::BTryDeleteExitedWorkerThreads() +{ + if ( m_WorkThreadMutex.TryLock() ) + { + if ( m_cThreadsRunning < (uint) m_WorkThreads.Count() ) + { + FOR_EACH_VEC_BACK( m_WorkThreads, i ) + { + CWorkThread *pWorkThread = m_WorkThreads[i]; + if ( pWorkThread->m_bFinished && !pWorkThread->IsThreadRunning() ) + { + m_WorkThreads.FastRemove( i ); + delete pWorkThread; + } + } + } + m_WorkThreadMutex.Unlock(); + return true; + } + return false; +} + + +bool CWorkItem::DispatchCompletedWorkItem( CJobMgr *pJobMgr ) +{ + // Check if this work item needs to signal a job + if ( pJobMgr && k_GIDNil != m_JobID ) + { + if ( !pJobMgr->BRouteWorkItemCompletedIfExists( m_JobID, m_bCanceled ) ) + return false; + } + else if ( k_GIDNil != m_JobID ) + { + // This should never happen since we have already released our reference to the work item + // and the calling job should have released its ref when it exited + AssertMsg( false, "CWorkItem::DispatchCompletedWorkItem: got a work item with no job ID" ); + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Called by the worker thread when it finishes an individual work item +// This function will see if our work is meant to be well-ordred; if so, +// it will do the necessary work to ensure ordering. +// +// It adds the item to the completed work item list so +// the pool owner can retrieve it and checks to see if any threads +// deserve to be shut down. +//----------------------------------------------------------------------------- +void CWorkThreadPool::OnWorkItemCompleted( CWorkItem *pWorkItem ) +{ + if ( sm_pWorkItemsCompletedSignal != NULL ) + sm_pWorkItemsCompletedSignal->Signal(); + + if ( !m_bEnsureOutputOrdering ) + { + // Since we aren't locking this sequence number could get screwed up a bit, but it's + // pretty meaningless if ensure output ordering if off anyway... + m_ulLastCompletedSequenceNumber = pWorkItem->m_ulSequenceNumber; + m_pTSQueueCompleted->PushItem( pWorkItem ); + } + else + { + // In the ordered case we need to lock completely here since we'll be moving around between + // various data structures and also need to ensure the ordering of items in the TS queue + m_MutexOnItemCompletedOrdered.Lock(); + if ( m_ulLastCompletedSequenceNumber + 1 == pWorkItem->m_ulSequenceNumber ) + { + m_ulLastCompletedSequenceNumber = pWorkItem->m_ulSequenceNumber; + m_pTSQueueCompleted->PushItem( pWorkItem ); + + // We walk the vector multiple times, but it should be very short as items are likely to come in + // close to in order, just mixed up a little if we have lots of threads or one item is much more + // costly than others. + bool bFoundNext = false; + do + { + bFoundNext = false; + FOR_EACH_VEC( m_vecCompletedAndWaiting, i ) + { + CWorkItem *pWaiting = m_vecCompletedAndWaiting[i]; + if ( m_ulLastCompletedSequenceNumber + 1 == pWaiting->m_ulSequenceNumber ) + { + m_ulLastCompletedSequenceNumber = pWaiting->m_ulSequenceNumber; + m_pTSQueueCompleted->PushItem( pWaiting ); + m_vecCompletedAndWaiting.FastRemove( i ); + bFoundNext = true; + break; + } + } + } while ( bFoundNext == true ); + } + else + { + m_vecCompletedAndWaiting.AddToTail( pWorkItem ); + } + m_MutexOnItemCompletedOrdered.Unlock(); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: return the count of items we've queued to process +//----------------------------------------------------------------------------- +int CWorkThreadPool::GetWorkItemToProcessCount() const +{ + return m_pTSQueueToProcess->Count(); +} + + +//----------------------------------------------------------------------------- +// Purpose: return the count of items we've completed but not notified the consumer about +//----------------------------------------------------------------------------- +int CWorkThreadPool::GetCompletedWorkItemCount() const +{ + int nCount = m_pTSQueueCompleted->Count(); + return nCount; +} + + +#ifdef DBGFLAG_VALIDATE +//----------------------------------------------------------------------------- +// Purpose: Validates memory +//----------------------------------------------------------------------------- +void CWorkThreadPool::Validate( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE(); + AUTO_LOCK( m_WorkThreadMutex ); + + ValidateObj( m_WorkThreads ); + FOR_EACH_VEC( m_WorkThreads, iWorkThread ) + { + m_WorkThreads[ iWorkThread ]->Suspend(); + ValidatePtr( m_WorkThreads[ iWorkThread ] ); + } + + ValidateAlignedPtr( m_pTSQueueCompleted ); + ValidateAlignedPtr( m_pTSQueueToProcess ); + ValidateObj( m_vecCompletedAndWaiting ); + FOR_EACH_VEC( m_vecCompletedAndWaiting, j ) + { + ValidatePtr( m_vecCompletedAndWaiting.Element( j ) ); + } + + FOR_EACH_VEC( m_WorkThreads, iWorkThread ) + { + m_WorkThreads[ iWorkThread ]->Resume(); + } + +#if 0 + ValidateObj( m_StatExecutionTime ); + ValidateObj( m_StatWaitTime ); +#endif +} +#endif // DBGFLAG_VALIDATE + +} // namespace GCSDK |