From 3bf9df6b2785fa6d951086978a3e66f49427166a Mon Sep 17 00:00:00 2001 From: FluorescentCIAAfricanAmerican <0934gj3049fk@protonmail.com> Date: Wed, 22 Apr 2020 12:56:21 -0400 Subject: 1 --- devtools/diffmemstats/diffmemstats.cpp | 444 +++++++++++++++++++++++++++++++++ devtools/diffmemstats/diffmemstats.sln | 20 ++ devtools/diffmemstats/diffmemstats.vpc | 31 +++ 3 files changed, 495 insertions(+) create mode 100644 devtools/diffmemstats/diffmemstats.cpp create mode 100644 devtools/diffmemstats/diffmemstats.sln create mode 100644 devtools/diffmemstats/diffmemstats.vpc (limited to 'devtools/diffmemstats') diff --git a/devtools/diffmemstats/diffmemstats.cpp b/devtools/diffmemstats/diffmemstats.cpp new file mode 100644 index 0000000..2058fc1 --- /dev/null +++ b/devtools/diffmemstats/diffmemstats.cpp @@ -0,0 +1,444 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// DiffMemStats.cpp : processes two "memstats.txt" memory dumps from the game into a 'diff' text file +// which Excel will display in legible form (useful for tracking mem leaks) + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +typedef map CItemMap; +typedef pair CDelta; + +struct Sequence +{ + const char *m_name; + float *m_values; + float m_maxDelta; + float m_endDelta; +}; + + +// The number of chains which will be output (the top N, after sorting and skipping) +int gNumSequencesToOutput = 16; + +// The number of chains which will be skipped before output (the top N, after sorting) +int gNumSequencesToSkip = 0; + +// If this is true, we sort chains by their maximum delta from the starting value, +// otherwise we sort by the change from the start to the end value +bool gSortByMaxChange = false; + +// If this is true, we use the absolute value of deltas for sorting +// (so large negative deltas rank high as well as large position ones) +bool gSortByAbsDeltas = false; + +// By default, we expect the input sequence of memstats files to be in chronological +// order and from the same play session - if true, this relaxes that restriction: +bool gAllowArbitraryInputSequence = false; + +// Output deltas from the first value in each sequence, rather than the current value +// (outputs N-1 values instead of N) +bool gOutputDeltas = false; + +// If this is true, output absolute values (by default, output values are relative to +// those in the first input file - the first value in each sequence is subtracted out) +bool gOutputAbsolute = false; + +// Output MB instead of KB +bool gOutputMB = false; + + +bool GetQuotedString( ifstream & file, string & item ) +{ + // Skip the opening quote + char ch; + while ( file.get( ch ) && ( ch != '"' ) ); + // Get the string + getline( file, item, '"' ); + // Skip the comma + file.get( ch ); + + return ( !file.eof() ); +} + +string & CleanupName( string & name ) +{ + // Strip everything before "\src\", to make memstats files from different peoples' machines compatible + const char * pSrc = strstr( name.c_str(), "src\\" ); + if ( pSrc && ( pSrc != name.c_str() ) ) + { + string strippedName( pSrc ); + name = strippedName; + } + return name; +} + +bool ParseFile( char *pszFile, CItemMap *pResult ) +{ + ifstream file; + string item; + float size; + + // Is this a CSV file? + bool bIsCSV = !!strstr( pszFile, ".csv" ); + + pResult->clear(); + file.open( pszFile ); + if ( !file.is_open() ) + { + printf( "Failed to open %s\n", pszFile ); + return false; + } + + // Skip the header + getline( file, item ); + + float maxEntrySize = 0; + while ( !file.eof() ) + { + if ( bIsCSV ) + { + // Comma-separated data + if ( !GetQuotedString( file, item ) ) + break; + } + else + { + // Tab-delimited data + getline( file, item, '\t' ); + if ( !item.length() ) + break; + } + file >> size; + maxEntrySize = max( maxEntrySize, size ); + pResult->insert( make_pair( CleanupName( item ), size ) ); + getline( file, item ); // skip the end of line + } + + // XBox 360 has 512MB of RAM, so we can tell if data is in MB or KB + // (and it's pretty unlikely we have no allocation entries greater than 512KB!) + bool bInputDataInKB = ( maxEntrySize > 512 ); + + // Convert the output to either KB or MB, as requested + float multiplier = 1.0f; + if ( bInputDataInKB && gOutputMB ) + multiplier = 1.0f / 1024.0f; + else if ( !bInputDataInKB && !gOutputMB ) + multiplier = 1024.0f; + CItemMap::iterator p1; + for ( p1 = pResult->begin(); p1 != pResult->end(); p1++ ) + { + p1->second = p1->second*multiplier; + } + + return ( pResult->size() > 0 ); +} + +bool FillMissingEntries( CItemMap *items, int numItems, int * numAllocations ) +{ + // First, generate a list of all unique allocations + CItemMap allAllocations; + CItemMap::const_iterator p1, p2; + for ( int i = 0; i < numItems; i++ ) + { + for ( p1 = items[i].begin(); p1 != items[i].end(); p1++ ) + { + p2 = allAllocations.find( p1->first ); + if ( p2 == allAllocations.end() ) + allAllocations.insert( make_pair( p1->first, 0.0f ) ); + } + } + + // Determine how many sequences we have in total + *numAllocations = (int)allAllocations.size(); + + // Now make sure each allocation is present in every CItemMap. Where absent, assign the + // previous known value, and where there is no known value assign zero. + // 'Validity' requires that a given allocation will always be present in CItemMaps after + // the first one in which it occurs (this is what you would get if the input files represent + // memdumps in chronological order, all from the same play session). + bool isValid = true; + for ( p1 = allAllocations.begin(); p1 != allAllocations.end(); p1++ ) + { + float curValue = 0.0f; + bool foundFirstOccurrence = false; + for ( int i = 0;i < numItems; i++ ) + { + p2 = items[i].find( p1->first ); + if ( p2 != items[i].end() ) + { + // Entry already present, update current value + curValue = p2->second; + foundFirstOccurrence = true; + } + else + { + // Entry missing, add it (and check validity) + items[i].insert( make_pair( p1->first, curValue ) ); + if ( foundFirstOccurrence ) + isValid = false; + } + } + } + + return isValid; +} + +bool CompareSequence( const Sequence * & lhs, const Sequence * & rhs ) +{ + if ( gSortByMaxChange ) + return ( lhs->m_maxDelta > rhs->m_maxDelta ); + else + return ( lhs->m_endDelta > rhs->m_endDelta ); +} + +vector & CreateSequences( CItemMap *items, int numItems ) +{ + // Create a vector of Sequence objects, each of which holds the + // sequence of 'Allocation Size' values for each allocation + vector & sequences = *new vector(); + + CItemMap::const_iterator p1, p2; + for ( p1 = items[0].begin(); p1 != items[0].end(); p1++ ) + { + Sequence * seq = new Sequence; + seq->m_name = p1->first.c_str(); + seq->m_values = new float[ numItems ]; + float startVal = p1->second; + float maxDelta = 0.0f; + float endDelta = 0.0f; + for ( int i = 0; i < numItems; i++ ) + { + p2 = items[ i ].find( seq->m_name ); + assert( p2 != items[i].end() ); + if ( p2 != items[i].end() ) + { + seq->m_values[i] = p2->second; + float delta = p2->second - startVal; + if ( gSortByAbsDeltas ) + delta = fabs( delta ); + if ( delta > maxDelta ) + maxDelta = delta; + endDelta = delta; + } + } + seq->m_endDelta = endDelta; + seq->m_maxDelta = maxDelta; + sequences.push_back( seq ); + } + + // Now sort the sequences vector + sort( sequences.begin(), sequences.end(), CompareSequence ); + + return sequences; +} + + +void Usage() +{ + printf( "diffmemstats is used for hunting down memory leaks\n" ); + printf( "\n" ); + printf( " USAGE: diffmemstats [options] [, ...]\n" ); + printf( "\n" ); + printf( "Input is a sequence of memstats.txt files (saved from game using 'mem_dump')\n" ); + printf( "and output is a single tab-separated text file, where each line represents a\n" ); + printf( "given allocation's size as it varies over time through the memstats sequence\n" ); + printf( "(lines are sorted by maximum change over time - see sortend/sortmax options).\n" ); + printf( "This text file can then be graphed in Excel using a 'stacked column' chart.\n" ); + printf( "\n" ); + printf( "NOTE: input files must be in chronological order, from a SINGLE play session\n" ); + printf( " (unless -allowmismatch is specified).\n" ); + printf( "\n" ); + printf( "options:\n" ); + printf( "[-numchains:N] the top N sequences are output (default: 16)\n" ); + printf( "[-skipchains:M] skip the top M sequences before output (default: 0)\n" ); + printf( "[-delta] output deltas between adjacent values in each sequence\n" ); + printf( " (the first delta for each sequence will always be zero)\n" ); + printf( "[-absolute] output absolute values (default is to subtract out the\n" ); + printf( " first value in each sequence), overridden by '-delta'\n" ); + printf( "[-sortend] sort sequences by start-to-end change (default)\n" ); + printf( "[-sortmax] sort sequences by start-to-max-value change\n" ); + printf( "[-sortabs] sort by absolute change values\n" ); + printf( "[-allowmismatch] don't check that the input file sequence is in\n" ); + printf( " chronological order and from the same play session\n" ); + printf( "[-mb] output values in MB (default is KB)\n" ); +} + +bool ParseOption( _TCHAR* option ) +{ + if ( option[0] != '-' ) + return false; + + option++; + + int numChains, numRead = sscanf( option, "numchains:%d", &numChains ); + if ( numRead == 1 ) + { + if ( numChains >= 0 ) + { + gNumSequencesToOutput = numChains; + return true; + } + return false; + } + + int skipChains, numRead2 = sscanf( option, "skipchains:%d", &skipChains ); + if ( numRead2 == 1 ) + { + if ( skipChains >= 0 ) + { + gNumSequencesToSkip = skipChains; + return true; + } + return false; + } + + if ( !stricmp( option, "delta" ) ) + { + gOutputDeltas = true; + return true; + } + + if ( !stricmp( option, "absolute" ) ) + { + gOutputAbsolute = true; + return true; + } + + if ( !stricmp( option, "sortend" ) ) + { + gSortByMaxChange = false; + return true; + } + + if ( !stricmp( option, "sortmax" ) ) + { + gSortByMaxChange = true; + return true; + } + + if ( !stricmp( option, "sortabs" ) ) + { + gSortByAbsDeltas = true; + return true; + } + + if ( !stricmp( option, "allowmismatch" ) ) + { + gAllowArbitraryInputSequence = true; + return true; + } + + if ( !stricmp( option, "mb" ) ) + { + gOutputMB = true; + return true; + } + + return false; +} + +// NOTE: this app doesn't bother with little things like freeing memory +int _tmain(int argc, _TCHAR* argv[]) +{ + if ( argc < 3 ) + { + Usage(); + return 1; + } + + // Grab options + int numOptions = 0; + argv++; + while ( argv[0][0] == '-' ) + { + if ( !ParseOption( argv[0] ) ) + { + Usage(); + return 1; + } + numOptions++; + argv++; + } + + + + +// TODO: allow the user to pass a starting filename and have the program figure out the sequence of files +// in that folder (using Aaron's naming scheme: ___.txt) + + + + int numFiles = argc - 1 - numOptions; + CItemMap *items = new CItemMap[ numFiles ]; + string *names = new string[ numFiles ]; + for ( int i = 0; i < numFiles; i++ ) + { + strlwr( argv[0] ); + if ( !ParseFile( argv[0], &items[i] ) ) + return 1; + + // Create a label for each column of output data + string name = argv[0]; + if ( ( name.find( ".csv" ) == ( name.length() - 4 ) ) || + ( name.find( ".txt" ) == ( name.length() - 4 ) ) ) + { + name = name.substr( 0, name.length() - 4 ); + } + names[ i ] = ( gOutputDeltas ? "[delta] " : "[size] " ) + name; + argv++; + } + + // Generate missing entries (i.e. make it so that each allocation + // occurs in every CItemMap, so we have a sequence of 'numFiles' value, duplicating ) + int numAllocations = 0; + bool isValidSequence = FillMissingEntries( items, numFiles, &numAllocations ); + if ( !isValidSequence && !gAllowArbitraryInputSequence ) + { + printf( "ERROR: input files did not all come from the same play session, or are in the wrong order (to allow this, specify -allowmismatch)\n" ); + return 1; + } + + // Create a vector of Sequence objects, each of which holds the sequence of 'size' + // values for each allocation. The vector is sorted based on max change from the + // start value, or the start-to-end change (gSortByMaxChange). + vector & sequences = CreateSequences( items, numFiles ); + + // Headings + printf( "Allocation type" ); + for ( int i = 0; i < numFiles; i++ ) + { + printf( "\t%s", names[ i ].c_str() ); + } + printf("\n"); + + for ( int i = gNumSequencesToSkip; (i < (gNumSequencesToSkip + gNumSequencesToOutput)) && (i < numAllocations); i++ ) + { + const Sequence & seq = *sequences.at(i); + printf( seq.m_name ); + for ( int j = 0; j < numFiles; j++ ) + { + // Subtract out either the first (want change since the sequence start) + // or the prior value (want change from one value to the next). + int base = 0; + if ( gOutputDeltas && ( j > 0 ) ) + base = j - 1; + float baseVal = seq.m_values[base]; + if ( gOutputAbsolute && !gOutputDeltas ) + baseVal = 0.0f; + printf( "\t%.2f", (seq.m_values[j] - baseVal) ); + } + printf( "\n" ); + } + + return 0; +} + diff --git a/devtools/diffmemstats/diffmemstats.sln b/devtools/diffmemstats/diffmemstats.sln new file mode 100644 index 0000000..563b8b2 --- /dev/null +++ b/devtools/diffmemstats/diffmemstats.sln @@ -0,0 +1,20 @@ + +Microsoft Visual Studio Solution File, Format Version 9.00 +# Visual Studio 2005 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "diffmemstats", "diffmemstats.vcproj", "{0E8C25AB-098A-4699-999E-8FD7EE989EFD}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Release|Win32 = Release|Win32 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {0E8C25AB-098A-4699-999E-8FD7EE989EFD}.Debug|Win32.ActiveCfg = Debug|Win32 + {0E8C25AB-098A-4699-999E-8FD7EE989EFD}.Debug|Win32.Build.0 = Debug|Win32 + {0E8C25AB-098A-4699-999E-8FD7EE989EFD}.Release|Win32.ActiveCfg = Release|Win32 + {0E8C25AB-098A-4699-999E-8FD7EE989EFD}.Release|Win32.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/devtools/diffmemstats/diffmemstats.vpc b/devtools/diffmemstats/diffmemstats.vpc new file mode 100644 index 0000000..425bc33 --- /dev/null +++ b/devtools/diffmemstats/diffmemstats.vpc @@ -0,0 +1,31 @@ +//----------------------------------------------------------------------------- +// DIFFMEMSTATS.VPC +// +// Project Script +//----------------------------------------------------------------------------- + +$Macro SRCDIR "..\.." +$Macro OUTBINDIR "$SRCDIR\devtools\bin" + +$Include "$SRCDIR\vpc_scripts\source_exe_con_base.vpc" + +$Configuration +{ + $Compiler + { + $AdditionalIncludeDirectories "$BASE;$SRCDIR\x360xdk\include\win32\vs2005" + $AdditionalOptions "/EHsc" + } + + $Linker + { + } +} + +$Project "diffmemstats" +{ + $Folder "Source Files" + { + $File "diffmemstats.cpp" + } +} -- cgit v1.2.3