summaryrefslogtreecommitdiff
path: root/devtools/diffmemstats
diff options
context:
space:
mode:
Diffstat (limited to 'devtools/diffmemstats')
-rw-r--r--devtools/diffmemstats/diffmemstats.cpp444
-rw-r--r--devtools/diffmemstats/diffmemstats.sln20
-rw-r--r--devtools/diffmemstats/diffmemstats.vpc31
3 files changed, 495 insertions, 0 deletions
diff --git a/devtools/diffmemstats/diffmemstats.cpp b/devtools/diffmemstats/diffmemstats.cpp
new file mode 100644
index 0000000..2058fc1
--- /dev/null
+++ b/devtools/diffmemstats/diffmemstats.cpp
@@ -0,0 +1,444 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+// DiffMemStats.cpp : processes two "memstats<n>.txt" memory dumps from the game into a 'diff' text file
+// which Excel will display in legible form (useful for tracking mem leaks)
+
+#include <math.h>
+#include <iostream>
+#include <tchar.h>
+#include <assert.h>
+#include <algorithm>
+#include <fstream>
+#include <string>
+#include <vector>
+#include <map>
+
+using namespace std;
+
+typedef map<string,float> CItemMap;
+typedef pair<string, float> CDelta;
+
+struct Sequence
+{
+ const char *m_name;
+ float *m_values;
+ float m_maxDelta;
+ float m_endDelta;
+};
+
+
+// The number of chains which will be output (the top N, after sorting and skipping)
+int gNumSequencesToOutput = 16;
+
+// The number of chains which will be skipped before output (the top N, after sorting)
+int gNumSequencesToSkip = 0;
+
+// If this is true, we sort chains by their maximum delta from the starting value,
+// otherwise we sort by the change from the start to the end value
+bool gSortByMaxChange = false;
+
+// If this is true, we use the absolute value of deltas for sorting
+// (so large negative deltas rank high as well as large position ones)
+bool gSortByAbsDeltas = false;
+
+// By default, we expect the input sequence of memstats files to be in chronological
+// order and from the same play session - if true, this relaxes that restriction:
+bool gAllowArbitraryInputSequence = false;
+
+// Output deltas from the first value in each sequence, rather than the current value
+// (outputs N-1 values instead of N)
+bool gOutputDeltas = false;
+
+// If this is true, output absolute values (by default, output values are relative to
+// those in the first input file - the first value in each sequence is subtracted out)
+bool gOutputAbsolute = false;
+
+// Output MB instead of KB
+bool gOutputMB = false;
+
+
+bool GetQuotedString( ifstream & file, string & item )
+{
+ // Skip the opening quote
+ char ch;
+ while ( file.get( ch ) && ( ch != '"' ) );
+ // Get the string
+ getline( file, item, '"' );
+ // Skip the comma
+ file.get( ch );
+
+ return ( !file.eof() );
+}
+
+string & CleanupName( string & name )
+{
+ // Strip everything before "\src\", to make memstats files from different peoples' machines compatible
+ const char * pSrc = strstr( name.c_str(), "src\\" );
+ if ( pSrc && ( pSrc != name.c_str() ) )
+ {
+ string strippedName( pSrc );
+ name = strippedName;
+ }
+ return name;
+}
+
+bool ParseFile( char *pszFile, CItemMap *pResult )
+{
+ ifstream file;
+ string item;
+ float size;
+
+ // Is this a CSV file?
+ bool bIsCSV = !!strstr( pszFile, ".csv" );
+
+ pResult->clear();
+ file.open( pszFile );
+ if ( !file.is_open() )
+ {
+ printf( "Failed to open %s\n", pszFile );
+ return false;
+ }
+
+ // Skip the header
+ getline( file, item );
+
+ float maxEntrySize = 0;
+ while ( !file.eof() )
+ {
+ if ( bIsCSV )
+ {
+ // Comma-separated data
+ if ( !GetQuotedString( file, item ) )
+ break;
+ }
+ else
+ {
+ // Tab-delimited data
+ getline( file, item, '\t' );
+ if ( !item.length() )
+ break;
+ }
+ file >> size;
+ maxEntrySize = max( maxEntrySize, size );
+ pResult->insert( make_pair( CleanupName( item ), size ) );
+ getline( file, item ); // skip the end of line
+ }
+
+ // XBox 360 has 512MB of RAM, so we can tell if data is in MB or KB
+ // (and it's pretty unlikely we have no allocation entries greater than 512KB!)
+ bool bInputDataInKB = ( maxEntrySize > 512 );
+
+ // Convert the output to either KB or MB, as requested
+ float multiplier = 1.0f;
+ if ( bInputDataInKB && gOutputMB )
+ multiplier = 1.0f / 1024.0f;
+ else if ( !bInputDataInKB && !gOutputMB )
+ multiplier = 1024.0f;
+ CItemMap::iterator p1;
+ for ( p1 = pResult->begin(); p1 != pResult->end(); p1++ )
+ {
+ p1->second = p1->second*multiplier;
+ }
+
+ return ( pResult->size() > 0 );
+}
+
+bool FillMissingEntries( CItemMap *items, int numItems, int * numAllocations )
+{
+ // First, generate a list of all unique allocations
+ CItemMap allAllocations;
+ CItemMap::const_iterator p1, p2;
+ for ( int i = 0; i < numItems; i++ )
+ {
+ for ( p1 = items[i].begin(); p1 != items[i].end(); p1++ )
+ {
+ p2 = allAllocations.find( p1->first );
+ if ( p2 == allAllocations.end() )
+ allAllocations.insert( make_pair( p1->first, 0.0f ) );
+ }
+ }
+
+ // Determine how many sequences we have in total
+ *numAllocations = (int)allAllocations.size();
+
+ // Now make sure each allocation is present in every CItemMap. Where absent, assign the
+ // previous known value, and where there is no known value assign zero.
+ // 'Validity' requires that a given allocation will always be present in CItemMaps after
+ // the first one in which it occurs (this is what you would get if the input files represent
+ // memdumps in chronological order, all from the same play session).
+ bool isValid = true;
+ for ( p1 = allAllocations.begin(); p1 != allAllocations.end(); p1++ )
+ {
+ float curValue = 0.0f;
+ bool foundFirstOccurrence = false;
+ for ( int i = 0;i < numItems; i++ )
+ {
+ p2 = items[i].find( p1->first );
+ if ( p2 != items[i].end() )
+ {
+ // Entry already present, update current value
+ curValue = p2->second;
+ foundFirstOccurrence = true;
+ }
+ else
+ {
+ // Entry missing, add it (and check validity)
+ items[i].insert( make_pair( p1->first, curValue ) );
+ if ( foundFirstOccurrence )
+ isValid = false;
+ }
+ }
+ }
+
+ return isValid;
+}
+
+bool CompareSequence( const Sequence * & lhs, const Sequence * & rhs )
+{
+ if ( gSortByMaxChange )
+ return ( lhs->m_maxDelta > rhs->m_maxDelta );
+ else
+ return ( lhs->m_endDelta > rhs->m_endDelta );
+}
+
+vector<const Sequence *> & CreateSequences( CItemMap *items, int numItems )
+{
+ // Create a vector of Sequence objects, each of which holds the
+ // sequence of 'Allocation Size' values for each allocation
+ vector<const Sequence *> & sequences = *new vector<const Sequence *>();
+
+ CItemMap::const_iterator p1, p2;
+ for ( p1 = items[0].begin(); p1 != items[0].end(); p1++ )
+ {
+ Sequence * seq = new Sequence;
+ seq->m_name = p1->first.c_str();
+ seq->m_values = new float[ numItems ];
+ float startVal = p1->second;
+ float maxDelta = 0.0f;
+ float endDelta = 0.0f;
+ for ( int i = 0; i < numItems; i++ )
+ {
+ p2 = items[ i ].find( seq->m_name );
+ assert( p2 != items[i].end() );
+ if ( p2 != items[i].end() )
+ {
+ seq->m_values[i] = p2->second;
+ float delta = p2->second - startVal;
+ if ( gSortByAbsDeltas )
+ delta = fabs( delta );
+ if ( delta > maxDelta )
+ maxDelta = delta;
+ endDelta = delta;
+ }
+ }
+ seq->m_endDelta = endDelta;
+ seq->m_maxDelta = maxDelta;
+ sequences.push_back( seq );
+ }
+
+ // Now sort the sequences vector
+ sort( sequences.begin(), sequences.end(), CompareSequence );
+
+ return sequences;
+}
+
+
+void Usage()
+{
+ printf( "diffmemstats is used for hunting down memory leaks\n" );
+ printf( "\n" );
+ printf( " USAGE: diffmemstats [options] <file1> <file2> [<file3>, ...]\n" );
+ printf( "\n" );
+ printf( "Input is a sequence of memstats<n>.txt files (saved from game using 'mem_dump')\n" );
+ printf( "and output is a single tab-separated text file, where each line represents a\n" );
+ printf( "given allocation's size as it varies over time through the memstats sequence\n" );
+ printf( "(lines are sorted by maximum change over time - see sortend/sortmax options).\n" );
+ printf( "This text file can then be graphed in Excel using a 'stacked column' chart.\n" );
+ printf( "\n" );
+ printf( "NOTE: input files must be in chronological order, from a SINGLE play session\n" );
+ printf( " (unless -allowmismatch is specified).\n" );
+ printf( "\n" );
+ printf( "options:\n" );
+ printf( "[-numchains:N] the top N sequences are output (default: 16)\n" );
+ printf( "[-skipchains:M] skip the top M sequences before output (default: 0)\n" );
+ printf( "[-delta] output deltas between adjacent values in each sequence\n" );
+ printf( " (the first delta for each sequence will always be zero)\n" );
+ printf( "[-absolute] output absolute values (default is to subtract out the\n" );
+ printf( " first value in each sequence), overridden by '-delta'\n" );
+ printf( "[-sortend] sort sequences by start-to-end change (default)\n" );
+ printf( "[-sortmax] sort sequences by start-to-max-value change\n" );
+ printf( "[-sortabs] sort by absolute change values\n" );
+ printf( "[-allowmismatch] don't check that the input file sequence is in\n" );
+ printf( " chronological order and from the same play session\n" );
+ printf( "[-mb] output values in MB (default is KB)\n" );
+}
+
+bool ParseOption( _TCHAR* option )
+{
+ if ( option[0] != '-' )
+ return false;
+
+ option++;
+
+ int numChains, numRead = sscanf( option, "numchains:%d", &numChains );
+ if ( numRead == 1 )
+ {
+ if ( numChains >= 0 )
+ {
+ gNumSequencesToOutput = numChains;
+ return true;
+ }
+ return false;
+ }
+
+ int skipChains, numRead2 = sscanf( option, "skipchains:%d", &skipChains );
+ if ( numRead2 == 1 )
+ {
+ if ( skipChains >= 0 )
+ {
+ gNumSequencesToSkip = skipChains;
+ return true;
+ }
+ return false;
+ }
+
+ if ( !stricmp( option, "delta" ) )
+ {
+ gOutputDeltas = true;
+ return true;
+ }
+
+ if ( !stricmp( option, "absolute" ) )
+ {
+ gOutputAbsolute = true;
+ return true;
+ }
+
+ if ( !stricmp( option, "sortend" ) )
+ {
+ gSortByMaxChange = false;
+ return true;
+ }
+
+ if ( !stricmp( option, "sortmax" ) )
+ {
+ gSortByMaxChange = true;
+ return true;
+ }
+
+ if ( !stricmp( option, "sortabs" ) )
+ {
+ gSortByAbsDeltas = true;
+ return true;
+ }
+
+ if ( !stricmp( option, "allowmismatch" ) )
+ {
+ gAllowArbitraryInputSequence = true;
+ return true;
+ }
+
+ if ( !stricmp( option, "mb" ) )
+ {
+ gOutputMB = true;
+ return true;
+ }
+
+ return false;
+}
+
+// NOTE: this app doesn't bother with little things like freeing memory
+int _tmain(int argc, _TCHAR* argv[])
+{
+ if ( argc < 3 )
+ {
+ Usage();
+ return 1;
+ }
+
+ // Grab options
+ int numOptions = 0;
+ argv++;
+ while ( argv[0][0] == '-' )
+ {
+ if ( !ParseOption( argv[0] ) )
+ {
+ Usage();
+ return 1;
+ }
+ numOptions++;
+ argv++;
+ }
+
+
+
+
+// TODO: allow the user to pass a starting filename and have the program figure out the sequence of files
+// in that folder (using Aaron's naming scheme: <map_name>_<mmdd>_<hhmmss>_<count>.txt)
+
+
+
+ int numFiles = argc - 1 - numOptions;
+ CItemMap *items = new CItemMap[ numFiles ];
+ string *names = new string[ numFiles ];
+ for ( int i = 0; i < numFiles; i++ )
+ {
+ strlwr( argv[0] );
+ if ( !ParseFile( argv[0], &items[i] ) )
+ return 1;
+
+ // Create a label for each column of output data
+ string name = argv[0];
+ if ( ( name.find( ".csv" ) == ( name.length() - 4 ) ) ||
+ ( name.find( ".txt" ) == ( name.length() - 4 ) ) )
+ {
+ name = name.substr( 0, name.length() - 4 );
+ }
+ names[ i ] = ( gOutputDeltas ? "[delta] " : "[size] " ) + name;
+ argv++;
+ }
+
+ // Generate missing entries (i.e. make it so that each allocation
+ // occurs in every CItemMap, so we have a sequence of 'numFiles' value, duplicating )
+ int numAllocations = 0;
+ bool isValidSequence = FillMissingEntries( items, numFiles, &numAllocations );
+ if ( !isValidSequence && !gAllowArbitraryInputSequence )
+ {
+ printf( "ERROR: input files did not all come from the same play session, or are in the wrong order (to allow this, specify -allowmismatch)\n" );
+ return 1;
+ }
+
+ // Create a vector of Sequence objects, each of which holds the sequence of 'size'
+ // values for each allocation. The vector is sorted based on max change from the
+ // start value, or the start-to-end change (gSortByMaxChange).
+ vector<const Sequence *> & sequences = CreateSequences( items, numFiles );
+
+ // Headings
+ printf( "Allocation type" );
+ for ( int i = 0; i < numFiles; i++ )
+ {
+ printf( "\t%s", names[ i ].c_str() );
+ }
+ printf("\n");
+
+ for ( int i = gNumSequencesToSkip; (i < (gNumSequencesToSkip + gNumSequencesToOutput)) && (i < numAllocations); i++ )
+ {
+ const Sequence & seq = *sequences.at(i);
+ printf( seq.m_name );
+ for ( int j = 0; j < numFiles; j++ )
+ {
+ // Subtract out either the first (want change since the sequence start)
+ // or the prior value (want change from one value to the next).
+ int base = 0;
+ if ( gOutputDeltas && ( j > 0 ) )
+ base = j - 1;
+ float baseVal = seq.m_values[base];
+ if ( gOutputAbsolute && !gOutputDeltas )
+ baseVal = 0.0f;
+ printf( "\t%.2f", (seq.m_values[j] - baseVal) );
+ }
+ printf( "\n" );
+ }
+
+ return 0;
+}
+
diff --git a/devtools/diffmemstats/diffmemstats.sln b/devtools/diffmemstats/diffmemstats.sln
new file mode 100644
index 0000000..563b8b2
--- /dev/null
+++ b/devtools/diffmemstats/diffmemstats.sln
@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 9.00
+# Visual Studio 2005
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "diffmemstats", "diffmemstats.vcproj", "{0E8C25AB-098A-4699-999E-8FD7EE989EFD}"
+EndProject
+Global
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|Win32 = Debug|Win32
+ Release|Win32 = Release|Win32
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {0E8C25AB-098A-4699-999E-8FD7EE989EFD}.Debug|Win32.ActiveCfg = Debug|Win32
+ {0E8C25AB-098A-4699-999E-8FD7EE989EFD}.Debug|Win32.Build.0 = Debug|Win32
+ {0E8C25AB-098A-4699-999E-8FD7EE989EFD}.Release|Win32.ActiveCfg = Release|Win32
+ {0E8C25AB-098A-4699-999E-8FD7EE989EFD}.Release|Win32.Build.0 = Release|Win32
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+EndGlobal
diff --git a/devtools/diffmemstats/diffmemstats.vpc b/devtools/diffmemstats/diffmemstats.vpc
new file mode 100644
index 0000000..425bc33
--- /dev/null
+++ b/devtools/diffmemstats/diffmemstats.vpc
@@ -0,0 +1,31 @@
+//-----------------------------------------------------------------------------
+// DIFFMEMSTATS.VPC
+//
+// Project Script
+//-----------------------------------------------------------------------------
+
+$Macro SRCDIR "..\.."
+$Macro OUTBINDIR "$SRCDIR\devtools\bin"
+
+$Include "$SRCDIR\vpc_scripts\source_exe_con_base.vpc"
+
+$Configuration
+{
+ $Compiler
+ {
+ $AdditionalIncludeDirectories "$BASE;$SRCDIR\x360xdk\include\win32\vs2005"
+ $AdditionalOptions "/EHsc"
+ }
+
+ $Linker
+ {
+ }
+}
+
+$Project "diffmemstats"
+{
+ $Folder "Source Files"
+ {
+ $File "diffmemstats.cpp"
+ }
+}
generated by cgit v1.2.3 (git 2.25.1) at 2026-06-06 23:31:31 +0000