diff options
| author | FluorescentCIAAfricanAmerican <[email protected]> | 2020-04-22 12:56:21 -0400 |
|---|---|---|
| committer | FluorescentCIAAfricanAmerican <[email protected]> | 2020-04-22 12:56:21 -0400 |
| commit | 3bf9df6b2785fa6d951086978a3e66f49427166a (patch) | |
| tree | 2c0f1f0c63c4832882bc93814ebd2c2b1c6224e5 /gcsdk/gcjob.cpp | |
| download | archived-source-engine-2018-hl2-src-master.tar.xz archived-source-engine-2018-hl2-src-master.zip | |
Diffstat (limited to 'gcsdk/gcjob.cpp')
| -rw-r--r-- | gcsdk/gcjob.cpp | 1590 |
1 files changed, 1590 insertions, 0 deletions
diff --git a/gcsdk/gcjob.cpp b/gcsdk/gcjob.cpp new file mode 100644 index 0000000..f70c249 --- /dev/null +++ b/gcsdk/gcjob.cpp @@ -0,0 +1,1590 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +#include "stdafx.h" + +#include "msgprotobuf.h" +#include "smartptr.h" +#include "rtime.h" +#include "gcsdk/gcreportprinter.h" +#include <winsock.h> + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +namespace GCSDK +{ + +GCConVar cv_webapi_result_size_limit( "webapi_result_size_limit", "50000000" ); +GCConVar cv_webapi_serialize_threads( "webapi_serialize_threads", "1", 0, "Switch for serializing webAPI responses on threads" ); +static GCConVar webapi_account_tracking( "webapi_account_tracking", "1", "Controls whether or not account tracking stats are collected for web api usage" ); +static GCConVar webapi_kill_switch( "webapi_kill_switch", "0", "When set to zero this will block no web api calls, when set to 1 this will block all web api except those sent by steam. To block steam, use the webapi_enable_steam_<priority> controls" ); +static GCConVar webapi_kill_switch_error_response( "webapi_kill_switch_error_response", "1", "Determines if a response should be sent when the kill switch kills a web api request" ); +static GCConVar webapi_rate_limit_calls_per_min( "webapi_rate_limit_calls_per_min", "600", "Determines how many messages can be sent from an account via the web api per minute. <0 disables this limiting" ); +static GCConVar webapi_rate_limit_mb_per_min( "webapi_rate_limit_mb_per_min", "20", "Determines how many megabytes of data can be sent to an account via the web api per minute. <0 disables this limiting" ); +static GCConVar webapi_elevated_rate_limit_calls_per_min( "webapi_elevated_rate_limit_calls_per_min", "-1", "Determines how many messages can be sent from an account via the web api per minute for elevated accounts. <0 disables this limiting" ); +static GCConVar webapi_elevated_rate_limit_mb_per_min( "webapi_elevated_rate_limit_mb_per_min", "-1", "Determines how many megabytes of data can be sent to an account via the web api per minute for elevated accounts. <0 disables this limiting" ); +static GCConVar webapi_ip_rate_limit( "webapi_ip_rate_limit", "1", "Controls whether or not we rate limit based upon IPs or just accounts" ); + +//----------------------------------------------------------------------------- +// CWebAPIAccountTracker +// +// Utility that tracks web api calls based upon accesses made by various users +//----------------------------------------------------------------------------- +class CWebAPIAccountTracker +{ +public: + //called when a web api request is made to track the call. A return value of true indicates that it should be allowed, + //false indicates it should be blocked + bool TrackUser( AccountID_t nID, uint32 nIP ); + //called once the size of the response is known and will track bandwidth and caller attributed to the provided function + void TrackFunction( AccountID_t nID, uint32 nIP, const char* pszFunction, uint32 nResponseSize ); + + //called to reset all permissions to default + void ResetAccountPermissions(); + //called to associate a permission level with an account + void SetAccountPermission( AccountID_t nID, EWebAPIAccountLevel eLevel ); + + //completely resets accumulated stats + void ResetStats(); + //resets just the profile stats + void ResetProfileStats(); + + //different caller report filters that can be used + enum EDumpCaller + { + eDumpCaller_All, + eDumpCaller_Blocked, + eDumpCaller_Status, + eDumpCaller_Calls, + }; + + //different reports that can be provided + void DumpTotalCallers( EDumpCaller eFilter, const char* pszFunctionFilter = NULL ) const; + void DumpTotalIPs( EDumpCaller eFilter, const char* pszFunctionFilter = NULL ) const; + void DumpCaller( AccountID_t nID ) const; + void DumpIP( uint32 nIP ) const; + void DumpFunctions() const; + void DumpProfile( bool bAllTime ) const; + void DumpSteamServers() const; + + //given a steam server name, this will return the identifier of that server + uint32 GetSteamServerID( const char* pszServer ); + + //for steam level requests, we have a priority provided, and to track stats separately we break them up to different account IDs instead of just zero + static const uint32 k_nSteamIP_High = 2; + static const uint32 k_nSteamIP_Normal = 1; + static const uint32 k_nSteamIP_Low = 0; + +private: + + //called to get the starting time of this rate interval + RTime32 GetRateIntervalStart( AccountID_t nCaller ) const; + + struct SCallerStats + { + SCallerStats() : + m_nBlockedCalls( 0 ), + m_eLevel( eWebAPIAccountLevel_RateLimited ) + { + ResetRateInterval( 0 ); + } + + void ResetRateInterval( RTime32 nStart ) + { + m_nRateIntervalStartTime = nStart; + m_nRateIntervalCalls = 0; + m_nRateIntervalBytes = 0; + } + + //the most recent rate interval that we received a message from the user (used to expire old counts) + RTime32 m_nRateIntervalStartTime; + //how many messages have been sent within this rate interval (used for rate limiting) + uint32 m_nRateIntervalCalls; + //how many bytes have been sent for this account during this interval + uint32 m_nRateIntervalBytes; + //total number of blocked calls + uint32 m_nBlockedCalls; + //flags associated with this caller, used to block/whitelist, etc + EWebAPIAccountLevel m_eLevel; + }; + + struct SFunctionStats + { + //track the number of calls and bandwidth. The profile versions are separate and used for displaying profiles over a window + uint32 m_nTotalCalls; + uint32 m_nProfileCalls; + uint32 m_nMaxBytes; + uint32 m_nProfileMaxBytes; + uint64 m_nTotalBytes; + uint64 m_nProfileBytes; + + //a map of who has called us, which consists of the caller account and IP as the key + struct SCaller + { + bool operator==( const SCaller& rhs ) const { return m_nAccountID == rhs.m_nAccountID && m_nIP == rhs.m_nIP; } + AccountID_t m_nAccountID; + uint32 m_nIP; + }; + struct SCalls + { + uint32 m_nCalls; + uint64 m_nBytes; + }; + CUtlHashMapLarge< SCaller, SCalls > m_Callers; + }; + + //a structure used to simplify reporting so a vector can just be built of these, and then provided to the report function which will handle sorting it and displaying + struct SReportRow + { + SReportRow( const char* pszFunction, uint32 nCalls, uint64 nSize ) : + m_pszFunction( pszFunction ), + m_nCalls( nCalls ), + m_nSize( nSize ) + {} + + const char* m_pszFunction; + uint32 m_nCalls; + uint64 m_nSize; + }; + + struct SSteamServer + { + CUtlString m_sName; + uint32 m_nID; + }; + + //called to find an existing user, or create one if not in the list already + SCallerStats* CreateAccountUser( AccountID_t nID, RTime32 nRateIntervalStart ); + SCallerStats* CreateIPUser( uint32 nIP, RTime32 nRateIntervalStart ); + + //called to print a report of the provided report rows as either an ID list or a function list. This will re-sort the provided vector + static void PrintReport( const CUtlVector< SReportRow >& vec ); + + //how many seconds are in a rate interval + static const uint32 knRateIntervalTimeS = 60; + + CUtlHashMapLarge< AccountID_t, SCallerStats > m_AccountCallers; + CUtlHashMapLarge< uint32, SCallerStats > m_IPCallers; + CUtlHashMapLarge< uintp, SFunctionStats* > m_Functions; + CUtlHashMapLarge< const char*, SSteamServer*, CaseSensitiveStrEquals, MurmurHash3ConstCharPtr > m_SteamServers; + CJobTime m_ProfileTime; +}; + +//our global profiler +static CWebAPIAccountTracker g_WebAPIAccountTracker; + +void WebAPIAccount_ResetAllPermissions() +{ + g_WebAPIAccountTracker.ResetAccountPermissions(); +} + +void WebAPIAccount_SetPermission( AccountID_t nID, EWebAPIAccountLevel eLevel ) +{ + g_WebAPIAccountTracker.SetAccountPermission( nID, eLevel ); +} + +bool WebAPIAccount_BTrackUserAndValidate( AccountID_t nID, uint32 unIP ) +{ + return g_WebAPIAccountTracker.TrackUser( nID, unIP ); +} + +RTime32 CWebAPIAccountTracker::GetRateIntervalStart( AccountID_t nCaller ) const +{ + //we shift the time by the account ID so that all users don't wrap at the same time which can cause a temporary surge in web API requests + RTime32 curTime = CRTime::RTime32TimeCur() + ( nCaller % knRateIntervalTimeS ); + return curTime - ( curTime % knRateIntervalTimeS ); +} + +CWebAPIAccountTracker::SCallerStats* CWebAPIAccountTracker::CreateAccountUser( AccountID_t nID, RTime32 nRateIntervalStart ) +{ + int nIndex = m_AccountCallers.Find( nID ); + if( nIndex == m_AccountCallers.InvalidIndex() ) + { + nIndex = m_AccountCallers.Insert( nID ); + SCallerStats& caller = m_AccountCallers[ nIndex ]; + caller.m_nRateIntervalStartTime = nRateIntervalStart; + + //account ID is always unrestricted! + if( nID == 0 ) + caller.m_eLevel = eWebAPIAccountLevel_Unlimited; + } + + return &m_AccountCallers[ nIndex ]; +} + +CWebAPIAccountTracker::SCallerStats* CWebAPIAccountTracker::CreateIPUser( uint32 nIP, RTime32 nRateIntervalStart ) +{ + int nIndex = m_IPCallers.Find( nIP ); + if( nIndex == m_IPCallers.InvalidIndex() ) + { + nIndex = m_IPCallers.Insert( nIP ); + SCallerStats& caller = m_IPCallers[ nIndex ]; + caller.m_nRateIntervalStartTime = nRateIntervalStart; + } + + return &m_IPCallers[ nIndex ]; +} + +uint32 CWebAPIAccountTracker::GetSteamServerID( const char* pszServer ) +{ + int nIndex = m_SteamServers.Find( pszServer ); + if( nIndex == m_SteamServers.InvalidIndex() ) + { + SSteamServer* pServer = new SSteamServer; + pServer->m_sName = pszServer; + pServer->m_nID = m_SteamServers.Count(); + m_SteamServers.Insert( pServer->m_sName, pServer ); + return pServer->m_nID; + } + + return m_SteamServers[ nIndex ]->m_nID; +} + +void CWebAPIAccountTracker::DumpSteamServers() const +{ + CGCReportPrinter rp; + rp.AddStringColumn( "ID" ); + rp.AddStringColumn( "Server" ); + + FOR_EACH_MAP_FAST( m_SteamServers, nServer ) + { + const uint32 nID = m_SteamServers[ nServer ]->m_nID; + rp.StrValue( CFmtStr( "%u.%u.%u.%u", iptod( nID << 8 ) ) ); + rp.StrValue( m_SteamServers[ nServer ]->m_sName ); + rp.CommitRow(); + } + + rp.SortReport( "ID", false ); + rp.PrintReport( SPEW_CONSOLE ); +} + +//determines what the resulting account level access should be based upon the access rights of the IP address and the account +static EWebAPIAccountLevel DetermineAccessLevel( EWebAPIAccountLevel eAccount, EWebAPIAccountLevel eIP ) +{ + //unrestricted users should always be allowed, regardless of the IP range that they are making requests from, same with unlimited IP addresses + if( ( eAccount == eWebAPIAccountLevel_Unlimited ) || ( eIP == eWebAPIAccountLevel_Unlimited ) ) + return eWebAPIAccountLevel_Unlimited; + + //otherwise, if either is blocked, then block + if( ( eAccount == eWebAPIAccountLevel_Blocked ) || ( eIP == eWebAPIAccountLevel_Blocked ) ) + return eWebAPIAccountLevel_Blocked; + + //now we are dealing with default case versus elevated. Elevated wins over default + if( ( eAccount == eWebAPIAccountLevel_Elevated ) || ( eIP == eWebAPIAccountLevel_Elevated ) ) + return eWebAPIAccountLevel_Elevated; + + //default + return eWebAPIAccountLevel_RateLimited; +} + +bool CWebAPIAccountTracker::TrackUser( AccountID_t nID, uint32 nIP ) +{ + if( !webapi_account_tracking.GetBool() ) + return true; + + //first off update their aggregate caller stats + { + //what is our current time, and at what time did this rate interval start + const RTime32 rateIntervalStart = GetRateIntervalStart( nID ); + + //see if this account is completely blocked + SCallerStats* pAccountCaller = CreateAccountUser( nID, rateIntervalStart ); + SCallerStats* pIPCaller = CreateIPUser( nIP, rateIntervalStart ); + + //determine what our policy should be based upon the access level of the IP and the user + EWebAPIAccountLevel eAccessLevel = DetermineAccessLevel( pAccountCaller->m_eLevel, pIPCaller->m_eLevel ); + + //if we are blocked, just bail now + if( eAccessLevel == eWebAPIAccountLevel_Blocked ) + { + pAccountCaller->m_nBlockedCalls++; + pIPCaller->m_nBlockedCalls++; + return false; + } + + //reset the rate interval tracking + if( pAccountCaller->m_nRateIntervalStartTime < rateIntervalStart ) + pAccountCaller->ResetRateInterval( rateIntervalStart ); + if( pIPCaller->m_nRateIntervalStartTime < rateIntervalStart ) + pIPCaller->ResetRateInterval( rateIntervalStart ); + + //now handle rate limiting + if( ( eAccessLevel == eWebAPIAccountLevel_RateLimited ) || ( eAccessLevel == eWebAPIAccountLevel_Elevated ) ) + { + //determine the rate we want to limit + int32 nCallsPerMin = ( eAccessLevel == eWebAPIAccountLevel_RateLimited ) ? webapi_rate_limit_calls_per_min.GetInt() : webapi_elevated_rate_limit_calls_per_min.GetInt(); + int32 nBytesPerMin = ( ( eAccessLevel == eWebAPIAccountLevel_RateLimited ) ? webapi_rate_limit_mb_per_min.GetInt() : webapi_elevated_rate_limit_mb_per_min.GetInt() ) * 1024 * 1024; + + //see if this account is rate limited + if( ( eAccessLevel == eWebAPIAccountLevel_RateLimited ) ) + { + bool bAllow = true; + + //see if we are being limited based upon call rate limiting (tracking based upon ip and account) Note that + //we don't return until we've dones stat tracking for both so the reports are accurate and capture it at both levels + if( ( nCallsPerMin >= 0 && pAccountCaller->m_nRateIntervalCalls >= ( uint32 )nCallsPerMin ) || + ( nBytesPerMin >= 0 && pAccountCaller->m_nRateIntervalBytes >= ( uint32 )nBytesPerMin ) ) + { + pAccountCaller->m_nBlockedCalls++; + bAllow = false; + } + + if( webapi_ip_rate_limit.GetBool() ) + { + if( ( nCallsPerMin >= 0 && pIPCaller->m_nRateIntervalCalls >= ( uint32 )nCallsPerMin ) || + ( nBytesPerMin >= 0 && pIPCaller->m_nRateIntervalBytes >= ( uint32 )nBytesPerMin ) ) + { + pIPCaller->m_nBlockedCalls++; + bAllow = false; + } + } + + if( !bAllow ) + return false; + } + } + } + + return true; +} + +void CWebAPIAccountTracker::TrackFunction( AccountID_t nID, uint32 nIP, const char* pszFunction, uint32 nResponseSize ) +{ + if( !webapi_account_tracking.GetBool() ) + return; + + //update the bytes for that user + { + int nCallerIndex = m_AccountCallers.Find( nID ); + if( nCallerIndex != m_AccountCallers.InvalidIndex() ) + { + SCallerStats& caller = m_AccountCallers[ nCallerIndex ]; + caller.m_nRateIntervalBytes += nResponseSize; + caller.m_nRateIntervalCalls++; + } + } + + //update the bytes for that user and for their IP + { + int nCallerIndex = m_IPCallers.Find( nIP ); + if( nCallerIndex != m_IPCallers.InvalidIndex() ) + { + SCallerStats& caller = m_IPCallers[ nCallerIndex ]; + caller.m_nRateIntervalBytes += nResponseSize; + caller.m_nRateIntervalCalls++; + } + } + + //now update the function specific stats + { + int nFunctionIndex = m_Functions.Find( ( uintp )pszFunction ); + if( nFunctionIndex == m_Functions.InvalidIndex() ) + { + SFunctionStats* pNewStats = new SFunctionStats; + pNewStats->m_nTotalCalls = 0; + pNewStats->m_nProfileCalls = 0; + pNewStats->m_nTotalBytes = 0; + pNewStats->m_nProfileBytes = 0; + pNewStats->m_nMaxBytes = 0; + pNewStats->m_nProfileMaxBytes = 0; + nFunctionIndex = m_Functions.Insert( ( uintp )pszFunction, pNewStats ); + } + + //update our stats + SFunctionStats& function = *m_Functions[ nFunctionIndex ]; + function.m_nTotalCalls++; + function.m_nProfileCalls++; + function.m_nTotalBytes += nResponseSize; + function.m_nProfileBytes += nResponseSize; + function.m_nMaxBytes = MAX( function.m_nMaxBytes, nResponseSize ); + function.m_nProfileMaxBytes = MAX( function.m_nProfileMaxBytes, nResponseSize ); + + //update caller stats + { + struct SFunctionStats::SCaller caller; + caller.m_nAccountID = nID; + caller.m_nIP = nIP; + + int nCallerIndex = function.m_Callers.Find( caller ); + if( nCallerIndex == function.m_Callers.InvalidIndex() ) + { + nCallerIndex = function.m_Callers.Insert( caller ); + function.m_Callers[ nCallerIndex ].m_nCalls = 1; + function.m_Callers[ nCallerIndex ].m_nBytes = nResponseSize; + } + else + { + function.m_Callers[ nCallerIndex ].m_nCalls++; + function.m_Callers[ nCallerIndex ].m_nBytes += nResponseSize; + } + } + } +} + +void CWebAPIAccountTracker::SetAccountPermission( AccountID_t nID, EWebAPIAccountLevel eLevel ) +{ + SCallerStats* pCaller = CreateAccountUser( nID, GetRateIntervalStart( nID ) ); + pCaller->m_eLevel = eLevel; +} + +void CWebAPIAccountTracker::ResetAccountPermissions() +{ + FOR_EACH_MAP_FAST( m_AccountCallers, nCaller ) + { + m_AccountCallers[ nCaller ].m_eLevel = eWebAPIAccountLevel_RateLimited; + } + FOR_EACH_MAP_FAST( m_IPCallers, nCaller ) + { + m_IPCallers[ nCaller ].m_eLevel = eWebAPIAccountLevel_RateLimited; + } +} + +void CWebAPIAccountTracker::ResetStats() +{ + FOR_EACH_MAP_FAST( m_AccountCallers, nCaller ) + { + m_AccountCallers[ nCaller ].ResetRateInterval( GetRateIntervalStart( m_AccountCallers.Key( nCaller ) ) ); + m_AccountCallers[ nCaller ].m_nBlockedCalls = 0; + } + FOR_EACH_MAP_FAST( m_IPCallers, nCaller ) + { + m_IPCallers[ nCaller ].ResetRateInterval( GetRateIntervalStart( m_IPCallers.Key( nCaller ) ) ); + m_IPCallers[ nCaller ].m_nBlockedCalls = 0; + } + m_Functions.PurgeAndDeleteElements(); +} + +void CWebAPIAccountTracker::ResetProfileStats() +{ + FOR_EACH_MAP_FAST( m_Functions, nFunction ) + { + m_Functions[ nFunction ]->m_nProfileCalls = 0; + m_Functions[ nFunction ]->m_nProfileBytes = 0; + m_Functions[ nFunction ]->m_nProfileMaxBytes = 0; + } + m_ProfileTime.SetToJobTime(); +} + +static const int k_cSteamIDRenderedMaxLen = 36; + +//----------------------------------------------------------------------------- +// Purpose: Renders the steam ID to a buffer with an admin console link. NOTE: for convenience of +// calling code, this code returns a pointer to a static buffer and is NOT thread-safe. +// Output: buffer with rendered Steam ID +//----------------------------------------------------------------------------- +static const char * CSteamID_RenderLink( const CSteamID & steamID ) +{ + // longest length of returned string is k_cBufLen + // <link cmd="steamid64 %llu"></link> => 30 + 20 == 50 + // 50 + k_cSteamIDRenderedMaxLen + 1 + const int k_cBufLen = 50 + k_cSteamIDRenderedMaxLen + 1; + + const int k_cBufs = 4; // # of static bufs to use (so people can compose output with multiple calls to RenderLink() ) + static char rgchBuf[k_cBufs][k_cBufLen]; + static int nBuf = 0; + char * pchBuf = rgchBuf[nBuf]; // get pointer to current static buf + nBuf++; // use next buffer for next call to this method + nBuf %= k_cBufs; + + Q_snprintf( pchBuf, k_cBufLen, "<link cmd=\"steamid64 %llu\">%s</link>", steamID.ConvertToUint64(), steamID.Render() ); + return pchBuf; +} + + +//----------------------------------------------------------------------------- +// Purpose: Renders the passed-in steam ID to a buffer with admin console link. NOTE: for convenience +// of calling code, this code returns a pointer to a static buffer and is NOT thread-safe. +// Input: 64-bit representation of Steam ID to render +// Output: buffer with rendered Steam ID link +//----------------------------------------------------------------------------- +static const char * CSteamID_RenderLink( uint64 ulSteamID ) +{ + CSteamID steamID( ulSteamID ); + return CSteamID_RenderLink( steamID ); +} + + + +void CWebAPIAccountTracker::DumpCaller( AccountID_t nID ) const +{ + const CSteamID steamID = GGCInterface()->ConstructSteamIDForClient( nID ); + //cache the account name here so we don't yield while we have indices + CUtlString sPersona = GGCBase()->YieldingGetPersonaName( steamID, "[Unknown]" ); + + //dump high level user stats + int nCallerIndex = m_AccountCallers.Find( nID ); + if( nCallerIndex == m_AccountCallers.InvalidIndex() ) + { + EG_MSG( SPEW_CONSOLE, "User %u not found in any web api calls\n", nID ); + return; + } + + //a map of IP addresses that have been used by this account + CUtlHashMapLarge< uint32, SFunctionStats::SCalls > ipCalls; + + //now each function they called + uint64 nTotalBytes = 0; + uint32 nTotalCalls = 0; + CUtlVector< SReportRow > vFuncs; + FOR_EACH_MAP_FAST( m_Functions, nFunc ) + { + //add up how many calls they made to this function across all IPs + uint64 nFnBytes = 0; + uint32 nFnCalls = 0; + FOR_EACH_MAP_FAST( m_Functions[ nFunc ]->m_Callers, nCaller ) + { + const CWebAPIAccountTracker::SFunctionStats::SCaller& caller = m_Functions[ nFunc ]->m_Callers.Key( nCaller ); + if( caller.m_nAccountID == nID ) + { + const CWebAPIAccountTracker::SFunctionStats::SCalls& calls = m_Functions[ nFunc ]->m_Callers[ nCaller ]; + nFnBytes += calls.m_nBytes; + nFnCalls += calls.m_nCalls; + + int nIPIndex = ipCalls.Find( caller.m_nIP ); + if( nIPIndex == ipCalls.InvalidIndex() ) + { + SFunctionStats::SCalls toAdd; + toAdd.m_nBytes = calls.m_nBytes; + toAdd.m_nCalls = calls.m_nCalls; + ipCalls.Insert( caller.m_nIP, toAdd ); + } + else + { + ipCalls[ nIPIndex ].m_nBytes += calls.m_nBytes; + ipCalls[ nIPIndex ].m_nBytes += calls.m_nCalls; + } + } + } + + if( nFnCalls > 0 ) + { + vFuncs.AddToTail( SReportRow( ( const char* )m_Functions.Key( nFunc ), nFnCalls, nFnBytes ) ); + } + nTotalBytes += nFnBytes; + nTotalCalls += nFnCalls; + } + + const SCallerStats& caller = m_AccountCallers[ nCallerIndex ]; + EG_MSG( SPEW_CONSOLE, "---------------------------------------------------\n" ); + EG_MSG( SPEW_CONSOLE, "User %s: \"%s\"\n", CSteamID_RenderLink( steamID ), sPersona.String() ); + double fTotalMB = nTotalBytes / ( 1024.0 * 1024.0 ); + double fMBPerHour = fTotalMB / ( GGCBase()->GetGCUpTime() / 3600.0 ); + double fCallsPerHour = nTotalCalls / ( GGCBase()->GetGCUpTime() / 3600.0 ); + EG_MSG( SPEW_CONSOLE, "\tAccess: %u, Total Calls: %u, Blocked calls: %u, Total: %.2fMB, MB/h: %.2f, Calls/h: %.0f\n", caller.m_eLevel, nTotalCalls, caller.m_nBlockedCalls, fTotalMB, fMBPerHour, fCallsPerHour ); + //don't let someone accidentally change Steam's access! + if( nID != 0 ) + { + if( caller.m_eLevel == eWebAPIAccountLevel_RateLimited ) + { + EG_MSG( SPEW_CONSOLE, "\t<link cmd=\"webapi_account_set_access %u %d\">[Block Account]</link>", nID, eWebAPIAccountLevel_Blocked ); + EG_MSG( SPEW_CONSOLE, "\t<link cmd=\"webapi_account_set_access %u %d\">[Elevate Account]</link>\n", nID, eWebAPIAccountLevel_Elevated ); + } + else if( caller.m_eLevel == eWebAPIAccountLevel_Blocked ) + { + EG_MSG( SPEW_CONSOLE, "\t<link cmd=\"webapi_account_set_access %u %d\">[Unblock Account]</link>\n", nID, eWebAPIAccountLevel_RateLimited ); + } + else if( caller.m_eLevel == eWebAPIAccountLevel_Elevated ) + { + EG_MSG( SPEW_CONSOLE, "\t<link cmd=\"webapi_account_set_access %u %d\">[Demote Account]</link>\n", nID, eWebAPIAccountLevel_RateLimited ); + } + } + + //print a report of the IP addresses that they are calling from + { + CGCReportPrinter rp; + rp.AddStringColumn( "IP" ); + rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB ); + + FOR_EACH_MAP_FAST( ipCalls, nIP ) + { + rp.StrValue( CFmtStr( "%u.%u.%u.%u", iptod( ipCalls.Key( nIP ) ) ), CFmtStr( "webapi_account_dump_ip %u", ipCalls.Key( nIP ) ) ); + rp.IntValue( ipCalls[ nIP ].m_nCalls ); + rp.IntValue( ipCalls[ nIP ].m_nBytes ); + rp.CommitRow(); + } + + rp.SortReport( "MB" ); + rp.PrintReport( SPEW_CONSOLE ); + } + + //and print a report of all the functions that they've called + PrintReport( vFuncs ); +} + +void CWebAPIAccountTracker::DumpIP( uint32 nIP ) const +{ + //dump high level user stats + int nCallerIndex = m_IPCallers.Find( nIP ); + if( nCallerIndex == m_IPCallers.InvalidIndex() ) + { + EG_MSG( SPEW_CONSOLE, "IP %u not found in any web api calls\n", nIP ); + return; + } + + //a map of IP addresses that have been used by this account + CUtlHashMapLarge< AccountID_t, SFunctionStats::SCalls > accountCalls; + + //now each function they called + uint64 nTotalBytes = 0; + uint32 nTotalCalls = 0; + CUtlVector< SReportRow > vFuncs; + FOR_EACH_MAP_FAST( m_Functions, nFunc ) + { + //add up how many calls they made to this function across all IPs + uint64 nFnBytes = 0; + uint32 nFnCalls = 0; + FOR_EACH_MAP_FAST( m_Functions[ nFunc ]->m_Callers, nCaller ) + { + const CWebAPIAccountTracker::SFunctionStats::SCaller& caller = m_Functions[ nFunc ]->m_Callers.Key( nCaller ); + if( caller.m_nIP == nIP ) + { + const CWebAPIAccountTracker::SFunctionStats::SCalls& calls = m_Functions[ nFunc ]->m_Callers[ nCaller ]; + nFnBytes += calls.m_nBytes; + nFnCalls += calls.m_nCalls; + + int nAccountIndex = accountCalls.Find( caller.m_nAccountID ); + if( nAccountIndex == accountCalls.InvalidIndex() ) + { + SFunctionStats::SCalls toAdd; + toAdd.m_nBytes = calls.m_nBytes; + toAdd.m_nCalls = calls.m_nCalls; + accountCalls.Insert( caller.m_nAccountID, toAdd ); + GGCBase()->PreloadPersonaName( GGCInterface()->ConstructSteamIDForClient( caller.m_nAccountID ) ); + } + else + { + accountCalls[ nAccountIndex ].m_nBytes += calls.m_nBytes; + accountCalls[ nAccountIndex ].m_nBytes += calls.m_nCalls; + } + } + } + + if( nFnCalls > 0 ) + { + vFuncs.AddToTail( SReportRow( ( const char* )m_Functions.Key( nFunc ), nFnCalls, nFnBytes ) ); + } + nTotalBytes += nFnBytes; + nTotalCalls += nFnCalls; + } + + const SCallerStats& caller = m_IPCallers[ nCallerIndex ]; + EG_MSG( SPEW_CONSOLE, "---------------------------------------------------\n" ); + EG_MSG( SPEW_CONSOLE, "IP %u.%u.%u.%u\n", iptod( nIP ) ); + double fTotalMB = nTotalBytes / ( 1024.0 * 1024.0 ); + double fMBPerHour = fTotalMB / ( GGCBase()->GetGCUpTime() / 3600.0 ); + double fCallsPerHour = nTotalCalls / ( GGCBase()->GetGCUpTime() / 3600.0 ); + EG_MSG( SPEW_CONSOLE, "\tAccess: %u, Total Calls: %u, Blocked calls: %u, Total: %.2fMB, MB/h: %.2f, Calls/h: %.0f\n", caller.m_eLevel, nTotalCalls, caller.m_nBlockedCalls, fTotalMB, fMBPerHour, fCallsPerHour ); + + //print a report of the accounts that they are calling from + { + CGCReportPrinter rp; + rp.AddSteamIDColumn( "Account" ); + rp.AddStringColumn( "Persona" ); + rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB ); + + FOR_EACH_MAP_FAST( accountCalls, nAccount ) + { + CSteamID steamID = GGCInterface()->ConstructSteamIDForClient( accountCalls.Key( nAccount ) ); + rp.SteamIDValue( steamID, CFmtStr( "webapi_account_dump_caller %u", steamID.GetAccountID() ) ); + rp.StrValue( GGCBase()->YieldingGetPersonaName( steamID, "[unknown]" ), CFmtStr( "webapi_account_dump_caller %u", steamID.GetAccountID() ) ); + rp.IntValue( accountCalls[ nAccount ].m_nCalls ); + rp.IntValue( accountCalls[ nAccount ].m_nBytes ); + rp.CommitRow(); + } + + rp.SortReport( "MB" ); + rp.PrintReport( SPEW_CONSOLE ); + } + + //and print a report of all the functions that they've called + PrintReport( vFuncs ); +} + +struct SCallerReportStats +{ + uint32 m_nFunctions; + uint32 m_nCalls; + uint64 m_nBytes; +}; + +void CWebAPIAccountTracker::DumpTotalCallers( EDumpCaller eFilter, const char* pszFunctionFilter ) const +{ + //accumulate stats for each unique caller + CUtlHashMapLarge< AccountID_t, SCallerReportStats > mapCallers; + FOR_EACH_MAP_FAST( m_Functions, nCurrFunction ) + { + //handle filtering out functions we don't care about + const char* pszFunctionName = ( const char* )m_Functions.Key( nCurrFunction ); + if( pszFunctionFilter && ( V_stristr( pszFunctionName, pszFunctionFilter ) == NULL ) ) + continue; + + const SFunctionStats& function = *m_Functions[ nCurrFunction ]; + FOR_EACH_MAP_FAST( function.m_Callers, nCurrCaller ) + { + const AccountID_t key = function.m_Callers.Key( nCurrCaller ).m_nAccountID; + + //add this account + int nStatIndex = mapCallers.Find( key ); + if( nStatIndex == mapCallers.InvalidIndex() ) + { + SCallerReportStats stats; + stats.m_nFunctions = 0; + stats.m_nCalls = 0; + stats.m_nBytes = 0; + nStatIndex = mapCallers.Insert( key, stats ); + GGCBase()->PreloadPersonaName( GGCInterface()->ConstructSteamIDForClient( key ) ); + } + + mapCallers[ nStatIndex ].m_nFunctions += 1; + mapCallers[ nStatIndex ].m_nCalls += function.m_Callers[ nCurrCaller ].m_nCalls; + mapCallers[ nStatIndex ].m_nBytes += function.m_Callers[ nCurrCaller ].m_nBytes; + } + } + + CGCReportPrinter rp; + rp.AddStringColumn( "Account" ); + rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB ); + rp.AddIntColumn( "Blocked", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "Access", CGCReportPrinter::eSummary_None ); + rp.AddIntColumn( "APIs", CGCReportPrinter::eSummary_None ); + rp.AddIntColumn( "MB/h", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB ); + rp.AddIntColumn( "Calls/h", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "KB/c", CGCReportPrinter::eSummary_None ); + rp.AddStringColumn( "UserName" ); + + const double fUpTimeHrs = MAX( GGCBase()->GetGCUpTime() / 3600.0, 0.01 ); + + //now show the report + FOR_EACH_MAP_FAST( m_AccountCallers, nCurrCaller ) + { + //apply filters to our results + if( ( eFilter == eDumpCaller_Blocked ) && ( m_AccountCallers[ nCurrCaller ].m_nBlockedCalls == 0 ) ) + continue; + if( ( eFilter == eDumpCaller_Status ) && ( m_AccountCallers[ nCurrCaller ].m_eLevel == eWebAPIAccountLevel_RateLimited ) ) + continue; + + const AccountID_t accountID = m_AccountCallers.Key( nCurrCaller ); + const SCallerReportStats* pStats = NULL; + int nCollectedStats = mapCallers.Find( accountID ); + if( nCollectedStats != mapCallers.InvalidIndex() ) + { + pStats = &mapCallers[ nCollectedStats ]; + } + + //filter out users that didn't make any calls if appropriate + if( ( eFilter == eDumpCaller_Calls ) && ( !pStats || ( pStats->m_nCalls == 0 ) ) ) + continue; + + const CSteamID steamID( GGCInterface()->ConstructSteamIDForClient( accountID ) ); + rp.StrValue( steamID.Render() , CFmtStr( "webapi_account_dump_caller %u", accountID ) ); + rp.IntValue( ( pStats ) ? pStats->m_nCalls : 0 ); + rp.IntValue( ( pStats ) ? pStats->m_nBytes : 0 ); + rp.IntValue( m_AccountCallers[ nCurrCaller ].m_nBlockedCalls ); + rp.IntValue( m_AccountCallers[ nCurrCaller ].m_eLevel ); + rp.IntValue( ( pStats ) ? pStats->m_nFunctions : 0 ); + rp.IntValue( ( int64 )( ( ( pStats ) ? pStats->m_nBytes : 0 ) / fUpTimeHrs ) ); + rp.IntValue( ( int64 )( ( ( pStats ) ? pStats->m_nCalls : 0 ) / fUpTimeHrs ) ); + rp.IntValue( ( pStats && pStats->m_nCalls > 0 ) ? ( pStats->m_nBytes / pStats->m_nCalls ) / 1024 : 0 ); + rp.StrValue( GGCBase()->YieldingGetPersonaName( steamID, "[unknown]" ) ); + rp.CommitRow(); + } + + const char* pszSort = "MB/h"; + if( eFilter == eDumpCaller_Blocked ) + pszSort = "Blocked"; + else if( eFilter == eDumpCaller_Status ) + pszSort = "Access"; + + rp.SortReport( pszSort ); + rp.PrintReport( SPEW_CONSOLE ); +} + +void CWebAPIAccountTracker::DumpTotalIPs( EDumpCaller eFilter, const char* pszFunctionFilter ) const +{ + //accumulate stats for each unique caller + CUtlHashMapLarge< uint32, SCallerReportStats > mapIPs; + FOR_EACH_MAP_FAST( m_Functions, nCurrFunction ) + { + //handle filtering out functions we don't care about + const char* pszFunctionName = ( const char* )m_Functions.Key( nCurrFunction ); + if( pszFunctionFilter && ( V_stristr( pszFunctionName, pszFunctionFilter ) == NULL ) ) + continue; + + const SFunctionStats& function = *m_Functions[ nCurrFunction ]; + FOR_EACH_MAP_FAST( function.m_Callers, nCurrCaller ) + { + const uint32 key = function.m_Callers.Key( nCurrCaller ).m_nIP; + + //add this account + int nStatIndex = mapIPs.Find( key ); + if( nStatIndex == mapIPs.InvalidIndex() ) + { + SCallerReportStats stats; + stats.m_nFunctions = 0; + stats.m_nCalls = 0; + stats.m_nBytes = 0; + nStatIndex = mapIPs.Insert( key, stats ); + } + + mapIPs[ nStatIndex ].m_nFunctions += 1; + mapIPs[ nStatIndex ].m_nCalls += function.m_Callers[ nCurrCaller ].m_nCalls; + mapIPs[ nStatIndex ].m_nBytes += function.m_Callers[ nCurrCaller ].m_nBytes; + } + } + + CGCReportPrinter rp; + rp.AddStringColumn( "IP" ); + rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB ); + rp.AddIntColumn( "Blocked", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "Access", CGCReportPrinter::eSummary_None ); + rp.AddIntColumn( "APIs", CGCReportPrinter::eSummary_None ); + rp.AddIntColumn( "MB/h", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB ); + rp.AddIntColumn( "Calls/h", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "KB/c", CGCReportPrinter::eSummary_None ); + + const double fUpTimeHrs = MAX( GGCBase()->GetGCUpTime() / 3600.0, 0.01 ); + + //now show the report + FOR_EACH_MAP_FAST( m_IPCallers, nCurrCaller ) + { + //apply filters to our results + if( ( eFilter == eDumpCaller_Blocked ) && ( m_IPCallers[ nCurrCaller ].m_nBlockedCalls == 0 ) ) + continue; + if( ( eFilter == eDumpCaller_Status ) && ( m_IPCallers[ nCurrCaller ].m_eLevel == eWebAPIAccountLevel_RateLimited ) ) + continue; + + const uint32 nIP = m_IPCallers.Key( nCurrCaller ); + const SCallerReportStats* pStats = NULL; + int nCollectedStats = mapIPs.Find( nIP ); + if( nCollectedStats != mapIPs.InvalidIndex() ) + { + pStats = &mapIPs[ nCollectedStats ]; + } + + //filter out users that didn't make any calls if appropriate + if( ( eFilter == eDumpCaller_Calls ) && ( !pStats || ( pStats->m_nCalls == 0 ) ) ) + continue; + + rp.StrValue( CFmtStr( "%u.%u.%u.%u", iptod( nIP ) ), CFmtStr( "webapi_account_dump_ip %u", nIP ) ); + rp.IntValue( ( pStats ) ? pStats->m_nCalls : 0 ); + rp.IntValue( ( pStats ) ? pStats->m_nBytes : 0 ); + rp.IntValue( m_IPCallers[ nCurrCaller ].m_nBlockedCalls ); + rp.IntValue( m_IPCallers[ nCurrCaller ].m_eLevel ); + rp.IntValue( ( pStats ) ? pStats->m_nFunctions : 0 ); + rp.IntValue( ( int64 )( ( ( pStats ) ? pStats->m_nBytes : 0 ) / fUpTimeHrs ) ); + rp.IntValue( ( int64 )( ( ( pStats ) ? pStats->m_nCalls : 0 ) / fUpTimeHrs ) ); + rp.IntValue( ( pStats && pStats->m_nCalls > 0 ) ? ( pStats->m_nBytes / pStats->m_nCalls ) / 1024 : 0 ); + rp.CommitRow(); + } + + const char* pszSort = "MB/h"; + if( eFilter == eDumpCaller_Blocked ) + pszSort = "Blocked"; + else if( eFilter == eDumpCaller_Status ) + pszSort = "Access"; + + rp.SortReport( pszSort ); + rp.PrintReport( SPEW_CONSOLE ); +} + +void CWebAPIAccountTracker::DumpFunctions() const +{ + CUtlVector< SReportRow > vFuncs; + vFuncs.EnsureCapacity( m_Functions.Count() ); + + FOR_EACH_MAP_FAST( m_Functions, i ) + { + vFuncs.AddToTail( SReportRow( ( const char* )m_Functions.Key( i ), m_Functions[ i ]->m_nTotalCalls, m_Functions[ i ]->m_nTotalBytes ) ); + } + PrintReport( vFuncs ); +} + +void CWebAPIAccountTracker::DumpProfile( bool bAllTime ) const +{ + //accumulate totals so we can do percentage + uint32 nTotalCalls = 0; + uint64 nTotalBytes = 0; + FOR_EACH_MAP_FAST( m_Functions, nFunction ) + { + if( bAllTime ) + { + nTotalCalls += m_Functions[ nFunction ]->m_nTotalCalls; + nTotalBytes += m_Functions[ nFunction ]->m_nTotalBytes; + } + else + { + nTotalCalls += m_Functions[ nFunction ]->m_nProfileCalls; + nTotalBytes += m_Functions[ nFunction ]->m_nProfileBytes; + } + } + + //determine how much time we are covering, and come up with a scale to normalize the values + uint64 nSampleMicroS = ( bAllTime ) ? ( uint64 )GGCBase()->GetGCUpTime() * 1000000 : m_ProfileTime.CServerMicroSecsPassed(); + double fToPS = ( nSampleMicroS > 0 ) ? 1000000.0 / ( double )nSampleMicroS : 1.0; + + EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "Web API Profile: Sampled %.2f seconds\n", nSampleMicroS / ( 1000.0 * 1000.0 ) ); + + CGCReportPrinter rp; + rp.AddStringColumn( "Web API Name" ); + rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB ); + rp.AddFloatColumn( "%", CGCReportPrinter::eSummary_Total, 1 ); + rp.AddFloatColumn( "KBPS", CGCReportPrinter::eSummary_Total, 1 ); + rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total ); + rp.AddFloatColumn( "%", CGCReportPrinter::eSummary_Total, 1 ); + rp.AddFloatColumn( "CallsPS", CGCReportPrinter::eSummary_Total, 1 ); + rp.AddIntColumn( "MaxKB", CGCReportPrinter::eSummary_Max ); + + FOR_EACH_MAP_FAST( m_Functions, nFunction ) + { + const SFunctionStats* pFunc = m_Functions[ nFunction ]; + uint32 nCalls = ( bAllTime ) ? pFunc->m_nTotalCalls : pFunc->m_nProfileCalls; + uint32 nMax = ( bAllTime ) ? pFunc->m_nMaxBytes : pFunc->m_nProfileMaxBytes; + uint64 nBytes = ( bAllTime ) ? pFunc->m_nTotalBytes : pFunc->m_nProfileBytes; + + rp.StrValue( ( const char* )m_Functions.Key( nFunction ) ); + rp.IntValue( nBytes ); + rp.FloatValue( ( 100.0 * nBytes ) / nTotalBytes ); + rp.FloatValue( ( nBytes / 1024.0 ) * fToPS ); + rp.IntValue( nCalls ); + rp.FloatValue( ( 100.0 * nCalls ) / nTotalCalls ); + rp.FloatValue( nCalls * fToPS ); + rp.IntValue( nMax / 1024 ); + rp.CommitRow(); + } + + rp.SortReport( "KBPS" ); + rp.PrintReport( SPEW_CONSOLE ); +} + +void CWebAPIAccountTracker::PrintReport( const CUtlVector< SReportRow >& vec ) +{ + CGCReportPrinter rp; + + //now print it out based upon the type + rp.AddStringColumn( "Function" ); + rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total ); + rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB ); + rp.AddFloatColumn( "Calls/h", CGCReportPrinter::eSummary_Total, 0 ); + rp.AddFloatColumn( "MB/h", CGCReportPrinter::eSummary_Total ); + + const double fUpTimeHrs = MAX( GGCBase()->GetGCUpTime() / 3600.0, 0.01 ); + + for( int i = 0; i < vec.Count(); i++ ) + { + rp.StrValue( vec[ i ].m_pszFunction, CFmtStr( "webapi_account_dump_function_callers %s", vec[ i ].m_pszFunction ) ); + rp.IntValue( vec[ i ].m_nCalls ); + rp.IntValue( vec[ i ].m_nSize ); + rp.FloatValue( vec[ i ].m_nCalls / fUpTimeHrs ); + rp.FloatValue( ( vec[ i ].m_nSize / ( 1024.0 * 1024.0 ) ) / fUpTimeHrs ); + rp.CommitRow(); + } + + rp.SortReport( "MB/h" ); + rp.PrintReport( SPEW_CONSOLE ); +} + + +GC_CON_COMMAND( webapi_account_dump_steam_servers, "Dumps the ID listings of the various steam servers encoded in the IP address of Steam requests" ) +{ + g_WebAPIAccountTracker.DumpSteamServers(); +} + +GC_CON_COMMAND( webapi_account_dump_callers, "Dumps the most frequent callers of web api's for the current run of the GC" ) +{ + g_WebAPIAccountTracker.DumpTotalCallers( CWebAPIAccountTracker::eDumpCaller_Calls ); +} + +GC_CON_COMMAND( webapi_account_dump_ips, "Dumps the most frequent ip callers of web api's for the current run of the GC" ) +{ + g_WebAPIAccountTracker.DumpTotalIPs( CWebAPIAccountTracker::eDumpCaller_Calls ); +} + +GC_CON_COMMAND( webapi_account_dump_blocked_callers, "Dumps the callers that have been blocked and how many calls have been blocked" ) +{ + g_WebAPIAccountTracker.DumpTotalCallers( CWebAPIAccountTracker::eDumpCaller_Blocked ); +} + +GC_CON_COMMAND( webapi_account_dump_caller_access, "Dumps the access rights of any caller that is not the default rate limiting" ) +{ + g_WebAPIAccountTracker.DumpTotalCallers( CWebAPIAccountTracker::eDumpCaller_Status ); +} + +GC_CON_COMMAND( webapi_account_dump_functions, "Dumps the most frequently called web api functions" ) +{ + g_WebAPIAccountTracker.DumpFunctions(); +} + +GC_CON_COMMAND_PARAMS( webapi_account_dump_function_callers, 1, "<function name> - Dumps the most frequent callers of functions that match the provided substring" ) +{ + g_WebAPIAccountTracker.DumpTotalCallers( CWebAPIAccountTracker::eDumpCaller_Calls, args[ 1 ] ); +} + +GC_CON_COMMAND_PARAMS( webapi_account_dump_caller, 1, "<caller account> - Dumps the functions that the provided account ID has been calling the most" ) +{ + g_WebAPIAccountTracker.DumpCaller( ( AccountID_t )V_atoui64( args[ 1 ] ) ); +} + +GC_CON_COMMAND_PARAMS( webapi_account_dump_ip, 1, "<ip> - Dumps the functions that the provided ip has been calling the most" ) +{ + g_WebAPIAccountTracker.DumpIP( ( AccountID_t )V_atoui64( args[ 1 ] ) ); +} + +GC_CON_COMMAND( webapi_account_reset_stats, "Forces a reset of all stats collected for web api account stats" ) +{ + g_WebAPIAccountTracker.ResetStats(); +} + +//utility class for dumping out the profile results after time has expired +static void DumpWebAPIProfile() +{ + g_WebAPIAccountTracker.DumpProfile( false ); +} + +GC_CON_COMMAND_PARAMS( webapi_profile, 1, "<seconds to profile> Turns on web api profiling for N seconds and dumps the results" ) +{ + float fSeconds = MAX( 1.0f, atof( args[ 1 ] ) ); + g_WebAPIAccountTracker.ResetProfileStats(); + static CGlobalScheduledFunction s_DumpProfile; + s_DumpProfile.ScheduleMS( DumpWebAPIProfile, fSeconds * 1000.0f ); +} + +//console commands to control web API profiling +GC_CON_COMMAND( webapi_profile_reset, "Turns on web api profiling" ) +{ + g_WebAPIAccountTracker.ResetProfileStats(); +} + +GC_CON_COMMAND( webapi_profile_dump, "Displays stats collected while web api profiling was enabled" ) +{ + g_WebAPIAccountTracker.DumpProfile( false ); +} + +GC_CON_COMMAND( webapi_profile_dump_total, "Displays stats collected while web api profiling was enabled" ) +{ + g_WebAPIAccountTracker.DumpProfile( true ); +} + +//----------------------------------------------------------------------------- +// Purpose: Sends a message and waits for a response +// Input: steamIDTarget - The entity this message is going to +// msgOut - The message to send +// nTimeoutSec - Number of seconds to wait for a response +// pMsgIn - Pointer to the message that will contain the response +// eMsg - The type of message the response should be +// Returns: True is the response was received, false otherwise. The contents +// of pMsgIn will be valid only if the function returns true. +//----------------------------------------------------------------------------- +bool CGCJob::BYldSendMessageAndGetReply( CSteamID &steamIDTarget, CGCMsgBase &msgOut, uint nTimeoutSec, CGCMsgBase *pMsgIn, MsgType_t eMsg ) +{ + IMsgNetPacket *pNetPacket = NULL; + + if ( !BYldSendMessageAndGetReply( steamIDTarget, msgOut, nTimeoutSec, &pNetPacket ) ) + return false; + + pMsgIn->SetPacket( pNetPacket ); + + if ( pMsgIn->Hdr().m_eMsg != eMsg ) + return false; + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Sends a message and waits for a response +// Input: steamIDTarget - The entity this message is going to +// msgOut - The message to send +// nTimeoutSec - Number of seconds to wait for a response +// ppNetPackets - Pointer to a IMsgNetPacket pointer which will contain +// the response +// Returns: True is the response was received, false otherwise. *ppNetPacket +// will point to a valid packet only if the function returns true. +//----------------------------------------------------------------------------- +bool CGCJob::BYldSendMessageAndGetReply( CSteamID &steamIDTarget, CGCMsgBase &msgOut, uint nTimeoutSec, IMsgNetPacket **ppNetPacket ) +{ + msgOut.ExpectingReply( GetJobID() ); + + if ( !m_pGC->BSendGCMsgToClient( steamIDTarget, msgOut ) ) + return false; + + SetJobTimeout( nTimeoutSec ); + return BYieldingWaitForMsg( ppNetPacket ); +} + +//----------------------------------------------------------------------------- +// Purpose: BYldSendMessageAndGetReply, ProtoBuf edition +//----------------------------------------------------------------------------- +bool CGCJob::BYldSendMessageAndGetReply( CSteamID &steamIDTarget, CProtoBufMsgBase &msgOut, uint nTimeoutSec, CProtoBufMsgBase *pMsgIn, MsgType_t eMsg ) +{ + IMsgNetPacket *pNetPacket = NULL; + + if ( !BYldSendMessageAndGetReply( steamIDTarget, msgOut, nTimeoutSec, &pNetPacket ) ) + return false; + + pMsgIn->InitFromPacket( pNetPacket ); + + if ( pMsgIn->GetEMsg() != eMsg ) + return false; + + return true; +} + +bool CGCJob::BYldSendMessageAndGetReply( CSteamID &steamIDTarget, CProtoBufMsgBase &msgOut, uint nTimeoutSec, IMsgNetPacket **ppNetPacket ) +{ + msgOut.ExpectingReply( GetJobID() ); + + if ( !m_pGC->BSendGCMsgToClient( steamIDTarget, msgOut ) ) + return false; + + SetJobTimeout( nTimeoutSec ); + return BYieldingWaitForMsg( ppNetPacket ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CGCWGJob::CGCWGJob( CGCBase *pGCBase ) +: CGCJob( pGCBase ), + m_steamID( k_steamIDNil ), + m_pWebApiFunc( NULL ) +{ +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CGCWGJob::~CGCWGJob() +{ + +} + + +//----------------------------------------------------------------------------- +// Purpose: Receive a k_EMsgWGRequest and manage keyvalues serialization/deserialization +//----------------------------------------------------------------------------- +bool CGCWGJob::BYieldingRunGCJob( IMsgNetPacket * pNetPacket ) +{ + CGCMsg<MsgGCWGRequest_t> msg( pNetPacket ); + + CUtlString strRequestName; + KeyValuesAD pkvRequest( "request" ); + KeyValuesAD pkvResponse( "response" ); + + m_steamID = CSteamID( msg.Body().m_ulSteamID ); + + msg.BReadStr( &strRequestName ); + + //deserialize KV + m_bufRequest.Clear(); + m_bufRequest.Put( msg.PubReadCur(), msg.Body().m_cubKeyValues ); + KVPacker packer; + if ( !packer.ReadAsBinary( pkvRequest, m_bufRequest ) ) + { + AssertMsg( false, "Failed to deserialize key values from WG request" ); + CGCWGJobMgr::SendErrorMessage( msg, "Failed to deserialize key values from WG request", k_EResultInvalidParam ); + return false; + } + + if( !BVerifyParams( msg, pkvRequest, m_pWebApiFunc ) ) + return false; + + bool bRet = BYieldingRunJobFromRequest( pkvRequest, pkvResponse ); + + + // request failed, set success for wg + if ( pkvResponse->IsEmpty( "success" ) ) + { + pkvResponse->SetInt( "success", bRet ? k_EResultOK : k_EResultFail ); + } + + // send response msg + CGCWGJobMgr::SendResponse( msg, pkvResponse, bRet ); + + return true; +} + +bool CGCWGJob::BVerifyParams( const CGCMsg<MsgGCWGRequest_t> & msg, KeyValues *pkvRequest, const WebApiFunc_t * pWebApiFunc ) +{ + // we've found the function; now validate the call + for ( int i = 0; i < Q_ARRAYSIZE( pWebApiFunc->m_rgParams ); i++ ) + { + if ( !pWebApiFunc->m_rgParams[i].m_pchParam ) + break; + + // just simple validation for now; make sure the key exists + if ( !pWebApiFunc->m_rgParams[i].m_bOptional && !pkvRequest->FindKey( pWebApiFunc->m_rgParams[i].m_pchParam ) ) + { + CGCWGJobMgr::SendErrorMessage( msg, + CFmtStr( "Error: Missing parameter '%s' from web request '%s'\n", pWebApiFunc->m_rgParams[i].m_pchParam, pWebApiFunc->m_pchRequestName ), + k_EResultInvalidParam ); + EmitWarning( SPEW_GC, 2, "Error: Missing parameter '%s' from web request '%s'\n", pWebApiFunc->m_rgParams[i].m_pchParam, pWebApiFunc->m_pchRequestName ); + return false; + } + } + + return true; +} + + +void CGCWGJob::SetErrorMessage( KeyValues *pkvErr, const char *pchErrorMsg, int32 nResult ) +{ + CGCWGJobMgr::SetErrorMessage( pkvErr, pchErrorMsg, nResult ); +} + + +//----------------------------------------------------------------------------- +// CGCJobVerifySession - A job that asks steam if a given user is still connected +// and cleans up the session if the user is gone +//----------------------------------------------------------------------------- +bool CGCJobVerifySession::BYieldingRunGCJob() +{ + if ( !m_pGC->BYieldingLockSteamID( m_steamID, __FILE__, __LINE__ ) ) + return false; + + m_pGC->YieldingRequestSession( m_steamID ); + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CWebAPIJob::CWebAPIJob( CGCBase *pGC, EWebAPIOutputFormat eDefaultOutputFormat ) +: CGCJob( pGC ), m_eDefaultOutputFormat( eDefaultOutputFormat ) +{ +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CWebAPIJob::~CWebAPIJob() +{ +} + +//----------------------------------------------------------------------------- +// Called to handle converting a web api response to a serialized result and free the object as well on a background thread + +class CEmitWebAPIData +{ +public: + CEmitWebAPIData( CMsgHttpRequest* pRequest) : m_bResult( false ), m_Request( pRequest ) {} + + //inputs + CHTTPRequest m_Request; + + //outputs + CHTTPResponse m_Response; + std::string m_sSerializedResponse; + bool m_bResult; +}; + +//the worker thread function that is responsible for serializing the response from web api values to a text buffer, freeing the web api value tree, and serializing the message to a protobuf for +//direct sending. If this function fails (a false response value) the message will not be serialized +static void ThreadedEmitFormattedOutputWrapperAndFreeResponse( CWebAPIResponse *pResponse, CEmitWebAPIData* pEmitData, EWebAPIOutputFormat eDefaultFormat, size_t unMaxResultSize ) +{ + // parse the output type that we want the result to be in + EWebAPIOutputFormat eOutputFormat = eDefaultFormat; + const char *pszParamOutput = pEmitData->m_Request.GetGETParamString( "format", NULL ); + if ( !pszParamOutput ) + pszParamOutput = pEmitData->m_Request.GetPOSTParamString( "format", NULL ); + + if ( pszParamOutput ) + { + if ( Q_stricmp( pszParamOutput, "xml" ) == 0 ) + eOutputFormat = k_EWebAPIOutputFormat_XML; + else if ( Q_stricmp( pszParamOutput, "vdf" ) == 0 ) + eOutputFormat = k_EWebAPIOutputFormat_VDF; + else if ( Q_stricmp( pszParamOutput, "json" ) == 0 ) + eOutputFormat = k_EWebAPIOutputFormat_JSON; + } + + pEmitData->m_bResult = pResponse->BEmitFormattedOutput( eOutputFormat, *( pEmitData->m_Response.GetBodyBuffer() ), unMaxResultSize ); + delete pResponse; + + //update the response code on the output (can probably be done elsewhere), but must be done before we pack the message below + switch( eOutputFormat ) + { + case k_EWebAPIOutputFormat_JSON: + pEmitData->m_Response.SetResponseHeaderValue( "content-type", "application/json; charset=UTF-8" ); + break; + case k_EWebAPIOutputFormat_XML: + pEmitData->m_Response.SetResponseHeaderValue( "content-type", "text/xml; charset=UTF-8" ); + break; + case k_EWebAPIOutputFormat_VDF: + pEmitData->m_Response.SetResponseHeaderValue( "content-type", "text/vdf; charset=UTF-8" ); + break; + default: + break; + } + + //if successful, we can go ahead and convert this all the way into a completely serialized form for sending over the wire + if( pEmitData->m_bResult ) + { + CMsgHttpResponse msgResponse; + pEmitData->m_Response.SerializeIntoProtoBuf( msgResponse ); + msgResponse.SerializeToString( &pEmitData->m_sSerializedResponse ); + } +} + +//called to respond to a web api request with the specified response value +static void WebAPIRespondToRequest( const char* pszName, uint32 nSenderIP, const CHTTPResponse& response, const CProtoBufMsg< CMsgWebAPIRequest >& msg ) +{ + VPROF_BUDGET( "WebAPI - sending result", VPROF_BUDGETGROUP_STEAM ); + CProtoBufMsg<CMsgHttpResponse> msgResponse( k_EGCMsgWebAPIJobRequestHttpResponse, msg ); + response.SerializeIntoProtoBuf( msgResponse.Body() ); + GGCBase()->BReplyToMessage( msgResponse, msg ); + + //track this message in the web API response + g_WebAPIAccountTracker.TrackFunction( msg.Body().api_key().account_id(), nSenderIP, pszName, msgResponse.Body().body().size() ); +} + +//responses to the web api message with an error code +static void WebAPIRespondWithError( const char* pszName, uint32 nSenderIP, const CProtoBufMsg< CMsgWebAPIRequest >& msg, EHTTPStatusCode statusCode ) +{ + CHTTPResponse response; + response.SetStatusCode( statusCode ); + WebAPIRespondToRequest( pszName, nSenderIP, response, msg ); +} + +//parses an IP address out of the provided string. This will be the last IP address in the list +static uint32 ParseIPAddrFromForwardHeader( const char* pszHeader ) +{ + //find the last comma in the string, our IP address follows that + const char* pszStart = V_strrchr( pszHeader, ',' ); + //if no match, then we just have the ip address, otherwise advance past the comma + if( !pszStart ) + pszStart = pszHeader; + else + pszStart++; + + //skip leading spaces + while( V_isspace( *pszStart ) ) + pszStart++; + + return ntohl( inet_addr( pszStart ) ); +} + +//----------------------------------------------------------------------------- +// Purpose: Receive a k_EMsgWebAPIJobRequest and manage serialization/deserialization to +// web request/response objects +//----------------------------------------------------------------------------- +bool CWebAPIJob::BYieldingRunJobFromMsg( IMsgNetPacket * pNetPacket ) +{ + VPROF_BUDGET( "WebAPI", VPROF_BUDGETGROUP_STEAM ); + + CProtoBufMsg<CMsgWebAPIRequest> msg( pNetPacket ); + + //make sure all the required parameters were present + bool bMsgParsedOK = msg.Body().has_api_key() + && msg.Body().has_interface_name() + && msg.Body().has_method_name() + && msg.Body().has_request() + && msg.Body().has_version(); + + if( !bMsgParsedOK ) + { + WebAPIRespondWithError( GetName(), 0, msg, k_EHTTPStatusCode400BadRequest ); + return true; + } + + //determine the account that sent this request + const AccountID_t nSenderAccountID = msg.Body().api_key().account_id(); + uint32 nSenderIP = 0; + + //if this isn't a system request, try and identify the IP address of the sender so we can rate limit accordingly + if( nSenderAccountID != 0 ) + { + const int nNumHeaders = msg.Body().request().headers_size(); + for( int nHeader = 0; nHeader < nNumHeaders; nHeader++ ) + { + if( strcmp( msg.Body().request().headers( nHeader ).name().c_str(), "X-Forwarded-For" ) != 0 ) + continue; + + //this is our IP address + nSenderIP = ParseIPAddrFromForwardHeader( msg.Body().request().headers( nHeader ).value().c_str() ); + } + + //see if we have the kill switch turned on + if( webapi_kill_switch.GetBool() ) + { + if( webapi_kill_switch_error_response.GetBool() ) + WebAPIRespondWithError( GetName(), nSenderIP, msg, k_EHTTPStatusCode503ServiceUnavailable ); + return true; + } + } + else + { +// !FIXME! DOTAMERGE +// //determine the priority of this steam request +// uint32 nPriority; +// nSenderIP = GetSteamRequestIPAddress( msg.Body().request(), nPriority ); +// //and allow for a kill switch based upon this priority +// if( ( ( nPriority == CWebAPIAccountTracker::k_nSteamIP_Low ) && !webapi_enable_steam_low.GetBool() ) || +// ( ( nPriority == CWebAPIAccountTracker::k_nSteamIP_Normal ) && !webapi_enable_steam_normal.GetBool() ) || +// ( ( nPriority == CWebAPIAccountTracker::k_nSteamIP_High ) && !webapi_enable_steam_high.GetBool() ) ) +// { +// if( webapi_kill_switch_error_response.GetBool() ) +// WebAPIRespondWithError( GetName(), 0, msg ); +// return true; +// } + } + + //track stats for this account, and handle rate limiting + if( !g_WebAPIAccountTracker.TrackUser( nSenderAccountID, nSenderIP ) ) + { + if( webapi_kill_switch_error_response.GetBool() ) + WebAPIRespondWithError( GetName(), nSenderIP, msg, k_EHTTPStatusCode429TooManyRequests ); + return true; + } + + //allocate the data that we'll use to fill out the request and send it to the background thread for work + CPlainAutoPtr< CEmitWebAPIData > pEmitData( new CEmitWebAPIData( const_cast< CMsgHttpRequest* >( &msg.Body().request() ) ) ); + CHTTPRequest& request = pEmitData->m_Request; + CHTTPResponse& response = pEmitData->m_Response; + + { + VPROF_BUDGET( "WebAPI - Prepare msg", VPROF_BUDGETGROUP_STEAM ); + + m_webAPIKey.DeserializeFromProtoBuf( msg.Body().api_key() ); + } + + CPlainAutoPtr< CWebAPIResponse > pwebAPIResponse( new CWebAPIResponse() ); + + { + VPROF_BUDGET( "WebAPI - Process msg", VPROF_BUDGETGROUP_STEAM ); + if( !BYieldingRunJobFromAPIRequest( msg.Body().interface_name().c_str(), msg.Body().method_name().c_str(), msg.Body().version(), &request, &response, pwebAPIResponse.Get() ) ) + { + //error executing our job + WebAPIRespondWithError( GetName(), nSenderIP, msg, k_EHTTPStatusCode500InternalServerError ); + return false; + } + } + +// !FIXME! DOTAMERGE +// //see if they want to re-route this request +// if( m_nRerouteRequest >= 0 ) +// { +// if( ( uint32 )m_nRerouteRequest == m_pGC->GetGCDirIndex() ) +// { +// AssertMsg( false, "Error: WebAPI %s attempting to re-route a web api message to itself (%d)", GetName(), m_nRerouteRequest ); +// } +// else +// { +// //route to the other GC and discard this message +// CProtoBufMsg< CMsgGCMsgWebAPIJobRequestForwardResponse > msgRoute( k_EGCMsgWebAPIJobRequestForwardResponse ); +// msgRoute.Body().set_dir_index( m_nRerouteRequest ); +// m_pGC->BReplyToMessage( msgRoute, msg ); +// } +// return true; +// } + + VPROF_BUDGET( "WebAPI - Emitting result", VPROF_BUDGETGROUP_STEAM ); + + response.SetStatusCode( pwebAPIResponse->GetStatusCode() ); + response.SetExpirationHeaderDeltaFromNow( pwebAPIResponse->GetExpirationSeconds() ); + if( pwebAPIResponse->GetLastModified() ) + response.SetHeaderTimeValue( "last-modified", pwebAPIResponse->GetLastModified() ); + + // if we aren't allowed to have a message body on this, simply send the result back now + if( !CHTTPUtil::BStatusCodeAllowsBody( pwebAPIResponse->GetStatusCode() ) ) + { + AssertMsg( pwebAPIResponse->GetRootValue() == NULL, "Response HTTP status code %d doesn't allow a body, but one was present", pwebAPIResponse->GetStatusCode() ); + //since we didn't have a body to serialize, we need to handle sending just the response + WebAPIRespondToRequest( GetName(), nSenderIP, response, msg ); + return true; + } + + //let the job convert the formatting and free our response for us (quite costly). Note that we detach the web API response since this job will free it + bool bThreadFuncSucceeded = BYieldingWaitForThreadFunc( CreateFunctor( ThreadedEmitFormattedOutputWrapperAndFreeResponse, pwebAPIResponse.Detach(), pEmitData.Get(), m_eDefaultOutputFormat, (size_t)cv_webapi_result_size_limit.GetInt() ) ); + + //if we called the function successfully and had a valid result, just send back the preserialized body + if( bThreadFuncSucceeded && pEmitData->m_bResult ) + { + m_pGC->BReplyToMessageWithPreSerializedBody( k_EGCMsgWebAPIJobRequestHttpResponse, msg, ( const byte* )pEmitData->m_sSerializedResponse.c_str(), pEmitData->m_sSerializedResponse.size() ); + g_WebAPIAccountTracker.TrackFunction( nSenderAccountID, nSenderIP, GetName(), pEmitData->m_sSerializedResponse.size() ); + } + else + { + //we failed to generate a response, see if we ran out of space + if( response.GetBodyBuffer()->TellMaxPut() > cv_webapi_result_size_limit.GetInt() ) + { + // !FIXME! DOTAMERGE + //CGCAlertInfo alert( "WebAPIResponseSize", "WebAPI request %s failed to emit because it exceeded %d characters", request.GetURL(), cv_webapi_result_size_limit.GetInt() ); + // + //switch( request.GetEHTTPMethod() ) + //{ + //case k_EHTTPMethodGET: + // { + // const uint32 nNumParams = request.GetGETParamCount(); + // for( uint32 nParam = 0; nParam < nNumParams; nParam++ ) + // { + // alert.AddExtendedInfoLine( "%s=%s", request.GetGETParamName( nParam ), request.GetGETParamValue( nParam ) ); + // } + // } + // break; + //case k_EHTTPMethodPOST: + // { + // const uint32 nNumParams = request.GetPOSTParamCount(); + // for( uint32 nParam = 0; nParam < nNumParams; nParam++ ) + // { + // alert.AddExtendedInfoLine( "%s=%s", request.GetPOSTParamName( nParam ), request.GetPOSTParamValue( nParam ) ); + // } + // } + // break; + //} + // + //GGCBase()->PostAlert( alert ); + GGCBase()->PostAlert( k_EAlertTypeInfo, false, CFmtStr( "WebAPI request %s failed to emit because it exceeded %d characters", request.GetURL(), cv_webapi_result_size_limit.GetInt() ) ); + } + else + { + EG_WARNING( SPEW_GC, "WebAPI %s - request %s failed to emit for unknown reason\n", GetName(), request.GetURL() ); + } + //make sure that they get an error code back + WebAPIRespondWithError( GetName(), nSenderIP, msg, k_EHTTPStatusCode500InternalServerError ); + } + + return true; +} + +void CWebAPIJob::AddLocalizedString( CWebAPIValues *pOutDefn, const char *pchFieldName, const char *pchKeyName, ELanguage eLang, bool bReturnTokenIfNotFound ) +{ + // NULL keys we just skip + if( !pchKeyName ) + return; + + const char *pchValue; + if( eLang == k_Lang_None ) + { + pchValue = pchKeyName; + } + else + { + pchValue = GGCBase()->LocalizeToken( pchKeyName, eLang, bReturnTokenIfNotFound ); + } + + if( pchValue ) + { + pOutDefn->CreateChildObject( pchFieldName )->SetStringValue( pchValue ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: A wrapper to call BEmitFormattedOutput and pass out a return value +// since functors don't make return values available. +//----------------------------------------------------------------------------- +/*static*/ void CWebAPIJob::ThreadedEmitFormattedOutputWrapper( CWebAPIResponse *pResponse, EWebAPIOutputFormat eFormat, CUtlBuffer *poutputBuffer, size_t unMaxResultSize, bool *pbResult ) +{ + *pbResult = pResponse->BEmitFormattedOutput( eFormat, *poutputBuffer, unMaxResultSize ); +} + + +} // namespace GCSDK |