diff options
| author | FluorescentCIAAfricanAmerican <[email protected]> | 2020-04-22 12:56:21 -0400 |
|---|---|---|
| committer | FluorescentCIAAfricanAmerican <[email protected]> | 2020-04-22 12:56:21 -0400 |
| commit | 3bf9df6b2785fa6d951086978a3e66f49427166a (patch) | |
| tree | 2c0f1f0c63c4832882bc93814ebd2c2b1c6224e5 /tier1 | |
| download | archived-source-engine-2018-hl2-src-master.tar.xz archived-source-engine-2018-hl2-src-master.zip | |
Diffstat (limited to 'tier1')
51 files changed, 29592 insertions, 0 deletions
diff --git a/tier1/KeyValues.cpp b/tier1/KeyValues.cpp new file mode 100644 index 0000000..3e00331 --- /dev/null +++ b/tier1/KeyValues.cpp @@ -0,0 +1,3209 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +// +//=============================================================================// + +#if defined( _WIN32 ) && !defined( _X360 ) +#include <windows.h> // for WideCharToMultiByte and MultiByteToWideChar +#elif defined(POSIX) +#include <wchar.h> // wcslen() +#define _alloca alloca +#define _wtoi(arg) wcstol(arg, NULL, 10) +#define _wtoi64(arg) wcstoll(arg, NULL, 10) +#endif + +#include <KeyValues.h> +#include "filesystem.h" +#include <vstdlib/IKeyValuesSystem.h> +#include "tier0/icommandline.h" +#include "tier0/vprof_telemetry.h" +#include <Color.h> +#include <stdlib.h> +#include "tier0/dbg.h" +#include "tier0/mem.h" +#include "utlbuffer.h" +#include "utlhash.h" +#include "utlvector.h" +#include "utlqueue.h" +#include "UtlSortVector.h" +#include "convar.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include <tier0/memdbgon.h> + +static const char * s_LastFileLoadingFrom = "unknown"; // just needed for error messages + +// Statics for the growable string table +int (*KeyValues::s_pfGetSymbolForString)( const char *name, bool bCreate ) = &KeyValues::GetSymbolForStringClassic; +const char *(*KeyValues::s_pfGetStringForSymbol)( int symbol ) = &KeyValues::GetStringForSymbolClassic; +CKeyValuesGrowableStringTable *KeyValues::s_pGrowableStringTable = NULL; + +#define KEYVALUES_TOKEN_SIZE 4096 +static char s_pTokenBuf[KEYVALUES_TOKEN_SIZE]; + + +#define INTERNALWRITE( pData, len ) InternalWrite( filesystem, f, pBuf, pData, len ) + + +// a simple class to keep track of a stack of valid parsed symbols +const int MAX_ERROR_STACK = 64; +class CKeyValuesErrorStack +{ +public: + CKeyValuesErrorStack() : m_pFilename("NULL"), m_errorIndex(0), m_maxErrorIndex(0) {} + + void SetFilename( const char *pFilename ) + { + m_pFilename = pFilename; + m_maxErrorIndex = 0; + } + + // entering a new keyvalues block, save state for errors + // Not save symbols instead of pointers because the pointers can move! + int Push( int symName ) + { + if ( m_errorIndex < MAX_ERROR_STACK ) + { + m_errorStack[m_errorIndex] = symName; + } + m_errorIndex++; + m_maxErrorIndex = max( m_maxErrorIndex, (m_errorIndex-1) ); + return m_errorIndex-1; + } + + // exiting block, error isn't in this block, remove. + void Pop() + { + m_errorIndex--; + Assert(m_errorIndex>=0); + } + + // Allows you to keep the same stack level, but change the name as you parse peers + void Reset( int stackLevel, int symName ) + { + Assert( stackLevel >= 0 ); + Assert( stackLevel < m_errorIndex ); + if ( stackLevel < MAX_ERROR_STACK ) + m_errorStack[stackLevel] = symName; + } + + // Hit an error, report it and the parsing stack for context + void ReportError( const char *pError ) + { + bool bSpewCR = false; + + Warning( "KeyValues Error: %s in file %s\n", pError, m_pFilename ); + for ( int i = 0; i < m_maxErrorIndex; i++ ) + { + if ( i < MAX_ERROR_STACK && m_errorStack[i] != INVALID_KEY_SYMBOL ) + { + if ( i < m_errorIndex ) + { + Warning( "%s, ", KeyValues::CallGetStringForSymbol(m_errorStack[i]) ); + } + else + { + Warning( "(*%s*), ", KeyValues::CallGetStringForSymbol(m_errorStack[i]) ); + } + + bSpewCR = true; + } + } + + if ( bSpewCR ) + Warning( "\n" ); + } + +private: + int m_errorStack[MAX_ERROR_STACK]; + const char *m_pFilename; + int m_errorIndex; + int m_maxErrorIndex; +} g_KeyValuesErrorStack; + + +// a simple helper that creates stack entries as it goes in & out of scope +class CKeyErrorContext +{ +public: + CKeyErrorContext( KeyValues *pKv ) + { + Init( pKv->GetNameSymbol() ); + } + + ~CKeyErrorContext() + { + g_KeyValuesErrorStack.Pop(); + } + CKeyErrorContext( int symName ) + { + Init( symName ); + } + void Reset( int symName ) + { + g_KeyValuesErrorStack.Reset( m_stackLevel, symName ); + } + int GetStackLevel() const + { + return m_stackLevel; + } +private: + void Init( int symName ) + { + m_stackLevel = g_KeyValuesErrorStack.Push( symName ); + } + + int m_stackLevel; +}; + +// Uncomment this line to hit the ~CLeakTrack assert to see what's looking like it's leaking +// #define LEAKTRACK + +#ifdef LEAKTRACK + +class CLeakTrack +{ +public: + CLeakTrack() + { + } + ~CLeakTrack() + { + if ( keys.Count() != 0 ) + { + Assert( 0 ); + } + } + + struct kve + { + KeyValues *kv; + char name[ 256 ]; + }; + + void AddKv( KeyValues *kv, char const *name ) + { + kve k; + Q_strncpy( k.name, name ? name : "NULL", sizeof( k.name ) ); + k.kv = kv; + + keys.AddToTail( k ); + } + + void RemoveKv( KeyValues *kv ) + { + int c = keys.Count(); + for ( int i = 0; i < c; i++ ) + { + if ( keys[i].kv == kv ) + { + keys.Remove( i ); + break; + } + } + } + + CUtlVector< kve > keys; +}; + +static CLeakTrack track; + +#define TRACK_KV_ADD( ptr, name ) track.AddKv( ptr, name ) +#define TRACK_KV_REMOVE( ptr ) track.RemoveKv( ptr ) + +#else + +#define TRACK_KV_ADD( ptr, name ) +#define TRACK_KV_REMOVE( ptr ) + +#endif + + +//----------------------------------------------------------------------------- +// Purpose: An arbitrarily growable string table for KeyValues key names. +// See the comment in the header for more info. +//----------------------------------------------------------------------------- +class CKeyValuesGrowableStringTable +{ +public: + // Constructor + CKeyValuesGrowableStringTable() : + #ifdef PLATFORM_64BITS + m_vecStrings( 0, 4 * 512 * 1024 ) + #else + m_vecStrings( 0, 512 * 1024 ) + #endif + , m_hashLookup( 2048, 0, 0, m_Functor, m_Functor ) + { + m_vecStrings.AddToTail( '\0' ); + } + + // Translates a string to an index + int GetSymbolForString( const char *name, bool bCreate = true ) + { + AUTO_LOCK( m_mutex ); + + // Put the current details into our hash functor + m_Functor.SetCurString( name ); + m_Functor.SetCurStringBase( (const char *)m_vecStrings.Base() ); + + if ( bCreate ) + { + bool bInserted = false; + UtlHashHandle_t hElement = m_hashLookup.Insert( -1, &bInserted ); + if ( bInserted ) + { + int iIndex = m_vecStrings.AddMultipleToTail( V_strlen( name ) + 1, name ); + m_hashLookup[ hElement ] = iIndex; + } + + return m_hashLookup[ hElement ]; + } + else + { + UtlHashHandle_t hElement = m_hashLookup.Find( -1 ); + if ( m_hashLookup.IsValidHandle( hElement ) ) + return m_hashLookup[ hElement ]; + else + return -1; + } + } + + // Translates an index back to a string + const char *GetStringForSymbol( int symbol ) + { + return (const char *)m_vecStrings.Base() + symbol; + } + +private: + + // A class plugged into CUtlHash that allows us to change the behavior of the table + // and store only the index in the table. + class CLookupFunctor + { + public: + CLookupFunctor() : m_pchCurString( NULL ), m_pchCurBase( NULL ) {} + + // Sets what we are currently inserting or looking for. + void SetCurString( const char *pchCurString ) { m_pchCurString = pchCurString; } + void SetCurStringBase( const char *pchCurBase ) { m_pchCurBase = pchCurBase; } + + // The compare function. + bool operator()( int nLhs, int nRhs ) const + { + const char *pchLhs = nLhs > 0 ? m_pchCurBase + nLhs : m_pchCurString; + const char *pchRhs = nRhs > 0 ? m_pchCurBase + nRhs : m_pchCurString; + + return ( 0 == V_stricmp( pchLhs, pchRhs ) ); + } + + // The hash function. + unsigned int operator()( int nItem ) const + { + return HashStringCaseless( m_pchCurString ); + } + + private: + const char *m_pchCurString; + const char *m_pchCurBase; + }; + + CThreadFastMutex m_mutex; + CLookupFunctor m_Functor; + CUtlHash<int, CLookupFunctor &, CLookupFunctor &> m_hashLookup; + CUtlVector<char> m_vecStrings; +}; + + +//----------------------------------------------------------------------------- +// Purpose: Sets whether the KeyValues system should use an arbitrarily growable +// string table. See the comment in the header for more info. +//----------------------------------------------------------------------------- +void KeyValues::SetUseGrowableStringTable( bool bUseGrowableTable ) +{ + if ( bUseGrowableTable ) + { + s_pfGetStringForSymbol = &(KeyValues::GetStringForSymbolGrowable); + s_pfGetSymbolForString = &(KeyValues::GetSymbolForStringGrowable); + + if ( NULL == s_pGrowableStringTable ) + { + s_pGrowableStringTable = new CKeyValuesGrowableStringTable; + } + } + else + { + s_pfGetStringForSymbol = &(KeyValues::GetStringForSymbolClassic); + s_pfGetSymbolForString = &(KeyValues::GetSymbolForStringClassic); + + delete s_pGrowableStringTable; + s_pGrowableStringTable = NULL; + } +} + +//----------------------------------------------------------------------------- +// Purpose: Bodys of the function pointers used for interacting with the key +// name string table +//----------------------------------------------------------------------------- +int KeyValues::GetSymbolForStringClassic( const char *name, bool bCreate ) +{ + return KeyValuesSystem()->GetSymbolForString( name, bCreate ); +} + +const char *KeyValues::GetStringForSymbolClassic( int symbol ) +{ + return KeyValuesSystem()->GetStringForSymbol( symbol ); +} + +int KeyValues::GetSymbolForStringGrowable( const char *name, bool bCreate ) +{ + return s_pGrowableStringTable->GetSymbolForString( name, bCreate ); +} + +const char *KeyValues::GetStringForSymbolGrowable( int symbol ) +{ + return s_pGrowableStringTable->GetStringForSymbol( symbol ); +} + + + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +KeyValues::KeyValues( const char *setName ) +{ + TRACK_KV_ADD( this, setName ); + + Init(); + SetName ( setName ); +} + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +KeyValues::KeyValues( const char *setName, const char *firstKey, const char *firstValue ) +{ + TRACK_KV_ADD( this, setName ); + + Init(); + SetName( setName ); + SetString( firstKey, firstValue ); +} + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +KeyValues::KeyValues( const char *setName, const char *firstKey, const wchar_t *firstValue ) +{ + TRACK_KV_ADD( this, setName ); + + Init(); + SetName( setName ); + SetWString( firstKey, firstValue ); +} + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +KeyValues::KeyValues( const char *setName, const char *firstKey, int firstValue ) +{ + TRACK_KV_ADD( this, setName ); + + Init(); + SetName( setName ); + SetInt( firstKey, firstValue ); +} + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +KeyValues::KeyValues( const char *setName, const char *firstKey, const char *firstValue, const char *secondKey, const char *secondValue ) +{ + TRACK_KV_ADD( this, setName ); + + Init(); + SetName( setName ); + SetString( firstKey, firstValue ); + SetString( secondKey, secondValue ); +} + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +KeyValues::KeyValues( const char *setName, const char *firstKey, int firstValue, const char *secondKey, int secondValue ) +{ + TRACK_KV_ADD( this, setName ); + + Init(); + SetName( setName ); + SetInt( firstKey, firstValue ); + SetInt( secondKey, secondValue ); +} + +//----------------------------------------------------------------------------- +// Purpose: Initialize member variables +//----------------------------------------------------------------------------- +void KeyValues::Init() +{ + m_iKeyName = INVALID_KEY_SYMBOL; + m_iDataType = TYPE_NONE; + + m_pSub = NULL; + m_pPeer = NULL; + m_pChain = NULL; + + m_sValue = NULL; + m_wsValue = NULL; + m_pValue = NULL; + + m_bHasEscapeSequences = false; + m_bEvaluateConditionals = true; + + // for future proof + memset( unused, 0, sizeof(unused) ); +} + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +KeyValues::~KeyValues() +{ + TRACK_KV_REMOVE( this ); + + RemoveEverything(); +} + +//----------------------------------------------------------------------------- +// Purpose: remove everything +//----------------------------------------------------------------------------- +void KeyValues::RemoveEverything() +{ + KeyValues *dat; + KeyValues *datNext = NULL; + for ( dat = m_pSub; dat != NULL; dat = datNext ) + { + datNext = dat->m_pPeer; + dat->m_pPeer = NULL; + delete dat; + } + + for ( dat = m_pPeer; dat && dat != this; dat = datNext ) + { + datNext = dat->m_pPeer; + dat->m_pPeer = NULL; + delete dat; + } + + delete [] m_sValue; + m_sValue = NULL; + delete [] m_wsValue; + m_wsValue = NULL; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *f - +//----------------------------------------------------------------------------- + +void KeyValues::RecursiveSaveToFile( CUtlBuffer& buf, int indentLevel, bool sortKeys /*= false*/, bool bAllowEmptyString /*= false*/ ) +{ + RecursiveSaveToFile( NULL, FILESYSTEM_INVALID_HANDLE, &buf, indentLevel, sortKeys, bAllowEmptyString ); +} + +//----------------------------------------------------------------------------- +// Adds a chain... if we don't find stuff in this keyvalue, we'll look +// in the one we're chained to. +//----------------------------------------------------------------------------- + +void KeyValues::ChainKeyValue( KeyValues* pChain ) +{ + m_pChain = pChain; +} + +//----------------------------------------------------------------------------- +// Purpose: Get the name of the current key section +//----------------------------------------------------------------------------- +const char *KeyValues::GetName( void ) const +{ + return s_pfGetStringForSymbol( m_iKeyName ); +} + +//----------------------------------------------------------------------------- +// Purpose: Read a single token from buffer (0 terminated) +//----------------------------------------------------------------------------- +#pragma warning (disable:4706) +const char *KeyValues::ReadToken( CUtlBuffer &buf, bool &wasQuoted, bool &wasConditional ) +{ + wasQuoted = false; + wasConditional = false; + + if ( !buf.IsValid() ) + return NULL; + + // eating white spaces and remarks loop + while ( true ) + { + buf.EatWhiteSpace(); + if ( !buf.IsValid() ) + return NULL; // file ends after reading whitespaces + + // stop if it's not a comment; a new token starts here + if ( !buf.EatCPPComment() ) + break; + } + + const char *c = (const char*)buf.PeekGet( sizeof(char), 0 ); + if ( !c ) + return NULL; + + // read quoted strings specially + if ( *c == '\"' ) + { + wasQuoted = true; + buf.GetDelimitedString( m_bHasEscapeSequences ? GetCStringCharConversion() : GetNoEscCharConversion(), + s_pTokenBuf, KEYVALUES_TOKEN_SIZE ); + return s_pTokenBuf; + } + + if ( *c == '{' || *c == '}' ) + { + // it's a control char, just add this one char and stop reading + s_pTokenBuf[0] = *c; + s_pTokenBuf[1] = 0; + buf.SeekGet( CUtlBuffer::SEEK_CURRENT, 1 ); + return s_pTokenBuf; + } + + // read in the token until we hit a whitespace or a control character + bool bReportedError = false; + bool bConditionalStart = false; + int nCount = 0; + while ( ( c = (const char*)buf.PeekGet( sizeof(char), 0 ) ) ) + { + // end of file + if ( *c == 0 ) + break; + + // break if any control character appears in non quoted tokens + if ( *c == '"' || *c == '{' || *c == '}' ) + break; + + if ( *c == '[' ) + bConditionalStart = true; + + if ( *c == ']' && bConditionalStart ) + { + wasConditional = true; + } + + // break on whitespace + if ( isspace(*c) ) + break; + + if (nCount < (KEYVALUES_TOKEN_SIZE-1) ) + { + s_pTokenBuf[nCount++] = *c; // add char to buffer + } + else if ( !bReportedError ) + { + bReportedError = true; + g_KeyValuesErrorStack.ReportError(" ReadToken overflow" ); + } + + buf.SeekGet( CUtlBuffer::SEEK_CURRENT, 1 ); + } + s_pTokenBuf[ nCount ] = 0; + return s_pTokenBuf; +} +#pragma warning (default:4706) + + + +//----------------------------------------------------------------------------- +// Purpose: if parser should translate escape sequences ( /n, /t etc), set to true +//----------------------------------------------------------------------------- +void KeyValues::UsesEscapeSequences(bool state) +{ + m_bHasEscapeSequences = state; +} + + +//----------------------------------------------------------------------------- +// Purpose: if parser should evaluate conditional blocks ( [$WINDOWS] etc. ) +//----------------------------------------------------------------------------- +void KeyValues::UsesConditionals(bool state) +{ + m_bEvaluateConditionals = state; +} + + +//----------------------------------------------------------------------------- +// Purpose: Load keyValues from disk +//----------------------------------------------------------------------------- +bool KeyValues::LoadFromFile( IBaseFileSystem *filesystem, const char *resourceName, const char *pathID, bool refreshCache ) +{ + TM_ZONE_DEFAULT( TELEMETRY_LEVEL0 ); + TM_ZONE_DEFAULT_PARAM( TELEMETRY_LEVEL0, resourceName ); + + Assert(filesystem); +#ifdef WIN32 + Assert( IsX360() || ( IsPC() && _heapchk() == _HEAPOK ) ); +#endif + +#ifdef STAGING_ONLY + static bool s_bCacheEnabled = !!CommandLine()->FindParm( "-enable_keyvalues_cache" ); + const bool bUseCache = s_bCacheEnabled && ( s_pfGetSymbolForString == KeyValues::GetSymbolForStringClassic ); +#else + /* + People are cheating with the keyvalue cache enabled by doing the below, so disable it. + + For example if one is to allow a blue demoman texture on sv_pure they + change it to this, "$basetexture" "temp/demoman_blue". Remember to move the + demoman texture to the temp folder in the materials folder. It will likely + not be there so make a new folder for it. Once the directory in the + demoman_blue vmt is changed to the temp folder and the vtf texture is in + the temp folder itself you are finally done. + + I packed my mods into a vpk but I don't think it's required. Once in game + you must create a server via the create server button and select the map + that will load the custom texture before you join a valve server. I suggest + you only do this with player textures and such as they are always loaded. + After you load the map you join the valve server and the textures should + appear and work on valve servers. + + This can be done on any sv_pure 1 server but it depends on what is type of + files are allowed. All valve servers allow temp files so that is the + example I used here." + + So all vmt's files can bypass sv_pure 1. And I believe this mod is mostly + made of vmt files, so valve's sv_pure 1 bull is pretty redundant. + */ + const bool bUseCache = false; +#endif + + // If pathID is null, we cannot cache the result because that has a weird iterate-through-a-bunch-of-locations behavior. + const bool bUseCacheForRead = bUseCache && !refreshCache && pathID != NULL; + const bool bUseCacheForWrite = bUseCache && pathID != NULL; + + COM_TimestampedLog( "KeyValues::LoadFromFile(%s%s%s): Begin", pathID ? pathID : "", pathID && resourceName ? "/" : "", resourceName ? resourceName : "" ); + + // Keep a cache of keyvalues, try to load it here. + if ( bUseCacheForRead && KeyValuesSystem()->LoadFileKeyValuesFromCache( this, resourceName, pathID, filesystem ) ) { + COM_TimestampedLog( "KeyValues::LoadFromFile(%s%s%s): End / CacheHit", pathID ? pathID : "", pathID && resourceName ? "/" : "", resourceName ? resourceName : "" ); + return true; + } + + FileHandle_t f = filesystem->Open(resourceName, "rb", pathID); + if ( !f ) + { + COM_TimestampedLog("KeyValues::LoadFromFile(%s%s%s): End / FileNotFound", pathID ? pathID : "", pathID && resourceName ? "/" : "", resourceName ? resourceName : ""); + return false; + } + + s_LastFileLoadingFrom = (char*)resourceName; + + // load file into a null-terminated buffer + int fileSize = filesystem->Size( f ); + unsigned bufSize = ((IFileSystem *)filesystem)->GetOptimalReadSize( f, fileSize + 2 ); + + char *buffer = (char*)((IFileSystem *)filesystem)->AllocOptimalReadBuffer( f, bufSize ); + Assert( buffer ); + + // read into local buffer + bool bRetOK = ( ((IFileSystem *)filesystem)->ReadEx( buffer, bufSize, fileSize, f ) != 0 ); + + filesystem->Close( f ); // close file after reading + + if ( bRetOK ) + { + buffer[fileSize] = 0; // null terminate file as EOF + buffer[fileSize+1] = 0; // double NULL terminating in case this is a unicode file + bRetOK = LoadFromBuffer( resourceName, buffer, filesystem ); + } + + // The cache relies on the KeyValuesSystem string table, which will only be valid if we're + // using classic mode. + if ( bUseCacheForWrite && bRetOK ) + { + KeyValuesSystem()->AddFileKeyValuesToCache( this, resourceName, pathID ); + } + + ( (IFileSystem *)filesystem )->FreeOptimalReadBuffer( buffer ); + + COM_TimestampedLog("KeyValues::LoadFromFile(%s%s%s): End / Success", pathID ? pathID : "", pathID && resourceName ? "/" : "", resourceName ? resourceName : ""); + + return bRetOK; +} + +//----------------------------------------------------------------------------- +// Purpose: Save the keyvalues to disk +// Creates the path to the file if it doesn't exist +//----------------------------------------------------------------------------- +bool KeyValues::SaveToFile( IBaseFileSystem *filesystem, const char *resourceName, const char *pathID, bool sortKeys /*= false*/, bool bAllowEmptyString /*= false*/, bool bCacheResult /*= false*/ ) +{ + // create a write file + FileHandle_t f = filesystem->Open(resourceName, "wb", pathID); + + if ( f == FILESYSTEM_INVALID_HANDLE ) + { + DevMsg(1, "KeyValues::SaveToFile: couldn't open file \"%s\" in path \"%s\".\n", + resourceName?resourceName:"NULL", pathID?pathID:"NULL" ); + return false; + } + + KeyValuesSystem()->InvalidateCacheForFile( resourceName, pathID ); + if ( bCacheResult ) { + KeyValuesSystem()->AddFileKeyValuesToCache( this, resourceName, pathID ); + } + RecursiveSaveToFile(filesystem, f, NULL, 0, sortKeys, bAllowEmptyString ); + filesystem->Close(f); + + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Write out a set of indenting +//----------------------------------------------------------------------------- +void KeyValues::WriteIndents( IBaseFileSystem *filesystem, FileHandle_t f, CUtlBuffer *pBuf, int indentLevel ) +{ + for ( int i = 0; i < indentLevel; i++ ) + { + INTERNALWRITE( "\t", 1 ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: Write out a string where we convert the double quotes to backslash double quote +//----------------------------------------------------------------------------- +void KeyValues::WriteConvertedString( IBaseFileSystem *filesystem, FileHandle_t f, CUtlBuffer *pBuf, const char *pszString ) +{ + // handle double quote chars within the string + // the worst possible case is that the whole string is quotes + int len = Q_strlen(pszString); + char *convertedString = (char *) _alloca ((len + 1) * sizeof(char) * 2); + int j=0; + for (int i=0; i <= len; i++) + { + if (pszString[i] == '\"') + { + convertedString[j] = '\\'; + j++; + } + else if ( m_bHasEscapeSequences && pszString[i] == '\\' ) + { + convertedString[j] = '\\'; + j++; + } + convertedString[j] = pszString[i]; + j++; + } + + INTERNALWRITE(convertedString, Q_strlen(convertedString)); +} + + +void KeyValues::InternalWrite( IBaseFileSystem *filesystem, FileHandle_t f, CUtlBuffer *pBuf, const void *pData, int len ) +{ + if ( filesystem ) + { + filesystem->Write( pData, len, f ); + } + + if ( pBuf ) + { + pBuf->Put( pData, len ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: Save keyvalues from disk, if subkey values are detected, calls +// itself to save those +//----------------------------------------------------------------------------- +void KeyValues::RecursiveSaveToFile( IBaseFileSystem *filesystem, FileHandle_t f, CUtlBuffer *pBuf, int indentLevel, bool sortKeys, bool bAllowEmptyString ) +{ + // write header + WriteIndents( filesystem, f, pBuf, indentLevel ); + INTERNALWRITE("\"", 1); + WriteConvertedString(filesystem, f, pBuf, GetName()); + INTERNALWRITE("\"\n", 2); + WriteIndents( filesystem, f, pBuf, indentLevel ); + INTERNALWRITE("{\n", 2); + + // loop through all our keys writing them to disk + if ( sortKeys ) + { + CUtlSortVector< KeyValues*, CUtlSortVectorKeyValuesByName > vecSortedKeys; + + for ( KeyValues *dat = m_pSub; dat != NULL; dat = dat->m_pPeer ) + { + vecSortedKeys.InsertNoSort(dat); + } + vecSortedKeys.RedoSort(); + + FOR_EACH_VEC( vecSortedKeys, i ) + { + SaveKeyToFile( vecSortedKeys[i], filesystem, f, pBuf, indentLevel, sortKeys, bAllowEmptyString ); + } + } + else + { + for ( KeyValues *dat = m_pSub; dat != NULL; dat = dat->m_pPeer ) + SaveKeyToFile( dat, filesystem, f, pBuf, indentLevel, sortKeys, bAllowEmptyString ); + } + + // write tail + WriteIndents(filesystem, f, pBuf, indentLevel); + INTERNALWRITE("}\n", 2); +} + +void KeyValues::SaveKeyToFile( KeyValues *dat, IBaseFileSystem *filesystem, FileHandle_t f, CUtlBuffer *pBuf, int indentLevel, bool sortKeys, bool bAllowEmptyString ) +{ + if ( dat->m_pSub ) + { + dat->RecursiveSaveToFile( filesystem, f, pBuf, indentLevel + 1, sortKeys, bAllowEmptyString ); + } + else + { + // only write non-empty keys + + switch (dat->m_iDataType) + { + case TYPE_STRING: + { + if ( dat->m_sValue && ( bAllowEmptyString || *(dat->m_sValue) ) ) + { + WriteIndents(filesystem, f, pBuf, indentLevel + 1); + INTERNALWRITE("\"", 1); + WriteConvertedString(filesystem, f, pBuf, dat->GetName()); + INTERNALWRITE("\"\t\t\"", 4); + + WriteConvertedString(filesystem, f, pBuf, dat->m_sValue); + + INTERNALWRITE("\"\n", 2); + } + break; + } + case TYPE_WSTRING: + { + if ( dat->m_wsValue ) + { + static char buf[KEYVALUES_TOKEN_SIZE]; + // make sure we have enough space + int result = Q_UnicodeToUTF8( dat->m_wsValue, buf, KEYVALUES_TOKEN_SIZE); + if (result) + { + WriteIndents(filesystem, f, pBuf, indentLevel + 1); + INTERNALWRITE("\"", 1); + INTERNALWRITE(dat->GetName(), Q_strlen(dat->GetName())); + INTERNALWRITE("\"\t\t\"", 4); + + WriteConvertedString(filesystem, f, pBuf, buf); + + INTERNALWRITE("\"\n", 2); + } + } + break; + } + + case TYPE_INT: + { + WriteIndents(filesystem, f, pBuf, indentLevel + 1); + INTERNALWRITE("\"", 1); + INTERNALWRITE(dat->GetName(), Q_strlen(dat->GetName())); + INTERNALWRITE("\"\t\t\"", 4); + + char buf[32]; + Q_snprintf(buf, sizeof( buf ), "%d", dat->m_iValue); + + INTERNALWRITE(buf, Q_strlen(buf)); + INTERNALWRITE("\"\n", 2); + break; + } + + case TYPE_UINT64: + { + WriteIndents(filesystem, f, pBuf, indentLevel + 1); + INTERNALWRITE("\"", 1); + INTERNALWRITE(dat->GetName(), Q_strlen(dat->GetName())); + INTERNALWRITE("\"\t\t\"", 4); + + char buf[32]; + // write "0x" + 16 char 0-padded hex encoded 64 bit value +#ifdef WIN32 + Q_snprintf( buf, sizeof( buf ), "0x%016I64X", *( (uint64 *)dat->m_sValue ) ); +#else + Q_snprintf( buf, sizeof( buf ), "0x%016llX", *( (uint64 *)dat->m_sValue ) ); +#endif + + INTERNALWRITE(buf, Q_strlen(buf)); + INTERNALWRITE("\"\n", 2); + break; + } + + case TYPE_FLOAT: + { + WriteIndents(filesystem, f, pBuf, indentLevel + 1); + INTERNALWRITE("\"", 1); + INTERNALWRITE(dat->GetName(), Q_strlen(dat->GetName())); + INTERNALWRITE("\"\t\t\"", 4); + + char buf[48]; + Q_snprintf(buf, sizeof( buf ), "%f", dat->m_flValue); + + INTERNALWRITE(buf, Q_strlen(buf)); + INTERNALWRITE("\"\n", 2); + break; + } + case TYPE_COLOR: + DevMsg(1, "KeyValues::RecursiveSaveToFile: TODO, missing code for TYPE_COLOR.\n"); + break; + + default: + break; + } + } +} + +//----------------------------------------------------------------------------- +// Purpose: looks up a key by symbol name +//----------------------------------------------------------------------------- +KeyValues *KeyValues::FindKey(int keySymbol) const +{ + for (KeyValues *dat = m_pSub; dat != NULL; dat = dat->m_pPeer) + { + if (dat->m_iKeyName == keySymbol) + return dat; + } + + return NULL; +} + +//----------------------------------------------------------------------------- +// Purpose: Find a keyValue, create it if it is not found. +// Set bCreate to true to create the key if it doesn't already exist +// (which ensures a valid pointer will be returned) +//----------------------------------------------------------------------------- +KeyValues *KeyValues::FindKey(const char *keyName, bool bCreate) +{ + // return the current key if a NULL subkey is asked for + if (!keyName || !keyName[0]) + return this; + + // look for '/' characters deliminating sub fields + char szBuf[256]; + const char *subStr = strchr(keyName, '/'); + const char *searchStr = keyName; + + // pull out the substring if it exists + if (subStr) + { + int size = subStr - keyName; + Q_memcpy( szBuf, keyName, size ); + szBuf[size] = 0; + searchStr = szBuf; + } + + // lookup the symbol for the search string + HKeySymbol iSearchStr = s_pfGetSymbolForString( searchStr, bCreate ); + + if ( iSearchStr == INVALID_KEY_SYMBOL ) + { + // not found, couldn't possibly be in key value list + return NULL; + } + + KeyValues *lastItem = NULL; + KeyValues *dat; + // find the searchStr in the current peer list + for (dat = m_pSub; dat != NULL; dat = dat->m_pPeer) + { + lastItem = dat; // record the last item looked at (for if we need to append to the end of the list) + + // symbol compare + if (dat->m_iKeyName == iSearchStr) + { + break; + } + } + + if ( !dat && m_pChain ) + { + dat = m_pChain->FindKey(keyName, false); + } + + // make sure a key was found + if (!dat) + { + if (bCreate) + { + // we need to create a new key + dat = new KeyValues( searchStr ); +// Assert(dat != NULL); + + dat->UsesEscapeSequences( m_bHasEscapeSequences != 0 ); // use same format as parent + dat->UsesConditionals( m_bEvaluateConditionals != 0 ); + + // insert new key at end of list + if (lastItem) + { + lastItem->m_pPeer = dat; + } + else + { + m_pSub = dat; + } + dat->m_pPeer = NULL; + + // a key graduates to be a submsg as soon as it's m_pSub is set + // this should be the only place m_pSub is set + m_iDataType = TYPE_NONE; + } + else + { + return NULL; + } + } + + // if we've still got a subStr we need to keep looking deeper in the tree + if ( subStr ) + { + // recursively chain down through the paths in the string + return dat->FindKey(subStr + 1, bCreate); + } + + return dat; +} + +//----------------------------------------------------------------------------- +// Purpose: Create a new key, with an autogenerated name. +// Name is guaranteed to be an integer, of value 1 higher than the highest +// other integer key name +//----------------------------------------------------------------------------- +KeyValues *KeyValues::CreateNewKey() +{ + int newID = 1; + + // search for any key with higher values + KeyValues *pLastChild = NULL; + for (KeyValues *dat = m_pSub; dat != NULL; dat = dat->m_pPeer) + { + // case-insensitive string compare + int val = atoi(dat->GetName()); + if (newID <= val) + { + newID = val + 1; + } + + pLastChild = dat; + } + + char buf[12]; + Q_snprintf( buf, sizeof(buf), "%d", newID ); + + return CreateKeyUsingKnownLastChild( buf, pLastChild ); +} + + +//----------------------------------------------------------------------------- +// Create a key +//----------------------------------------------------------------------------- +KeyValues* KeyValues::CreateKey( const char *keyName ) +{ + KeyValues *pLastChild = FindLastSubKey(); + return CreateKeyUsingKnownLastChild( keyName, pLastChild ); +} + +//----------------------------------------------------------------------------- +KeyValues* KeyValues::CreateKeyUsingKnownLastChild( const char *keyName, KeyValues *pLastChild ) +{ + // Create a new key + KeyValues* dat = new KeyValues( keyName ); + + dat->UsesEscapeSequences( m_bHasEscapeSequences != 0 ); // use same format as parent does + dat->UsesConditionals( m_bEvaluateConditionals != 0 ); + + // add into subkey list + AddSubkeyUsingKnownLastChild( dat, pLastChild ); + + return dat; +} + +//----------------------------------------------------------------------------- +void KeyValues::AddSubkeyUsingKnownLastChild( KeyValues *pSubkey, KeyValues *pLastChild ) +{ + // Make sure the subkey isn't a child of some other keyvalues + Assert( pSubkey != NULL ); + Assert( pSubkey->m_pPeer == NULL ); + + // Empty child list? + if ( pLastChild == NULL ) + { + Assert( m_pSub == NULL ); + m_pSub = pSubkey; + } + else + { + Assert( m_pSub != NULL ); + Assert( pLastChild->m_pPeer == NULL ); + +// // In debug, make sure that they really do know which child is the last one +// #ifdef _DEBUG +// KeyValues *pTempDat = m_pSub; +// while ( pTempDat->GetNextKey() != NULL ) +// { +// pTempDat = pTempDat->GetNextKey(); +// } +// Assert( pTempDat == pLastChild ); +// #endif + + pLastChild->SetNextKey( pSubkey ); + } +} + + +//----------------------------------------------------------------------------- +// Adds a subkey. Make sure the subkey isn't a child of some other keyvalues +//----------------------------------------------------------------------------- +void KeyValues::AddSubKey( KeyValues *pSubkey ) +{ + // Make sure the subkey isn't a child of some other keyvalues + Assert( pSubkey != NULL ); + Assert( pSubkey->m_pPeer == NULL ); + + // add into subkey list + if ( m_pSub == NULL ) + { + m_pSub = pSubkey; + } + else + { + KeyValues *pTempDat = m_pSub; + while ( pTempDat->GetNextKey() != NULL ) + { + pTempDat = pTempDat->GetNextKey(); + } + + pTempDat->SetNextKey( pSubkey ); + } +} + + + +//----------------------------------------------------------------------------- +// Purpose: Remove a subkey from the list +//----------------------------------------------------------------------------- +void KeyValues::RemoveSubKey(KeyValues *subKey) +{ + if (!subKey) + return; + + // check the list pointer + if (m_pSub == subKey) + { + m_pSub = subKey->m_pPeer; + } + else + { + // look through the list + KeyValues *kv = m_pSub; + while (kv->m_pPeer) + { + if (kv->m_pPeer == subKey) + { + kv->m_pPeer = subKey->m_pPeer; + break; + } + + kv = kv->m_pPeer; + } + } + + subKey->m_pPeer = NULL; +} + + + +//----------------------------------------------------------------------------- +// Purpose: Locate last child. Returns NULL if we have no children +//----------------------------------------------------------------------------- +KeyValues *KeyValues::FindLastSubKey() +{ + + // No children? + if ( m_pSub == NULL ) + return NULL; + + // Scan for the last one + KeyValues *pLastChild = m_pSub; + while ( pLastChild->m_pPeer ) + pLastChild = pLastChild->m_pPeer; + return pLastChild; +} + +//----------------------------------------------------------------------------- +// Purpose: Sets this key's peer to the KeyValues passed in +//----------------------------------------------------------------------------- +void KeyValues::SetNextKey( KeyValues *pDat ) +{ + m_pPeer = pDat; +} + + +KeyValues* KeyValues::GetFirstTrueSubKey() +{ + KeyValues *pRet = m_pSub; + while ( pRet && pRet->m_iDataType != TYPE_NONE ) + pRet = pRet->m_pPeer; + + return pRet; +} + +KeyValues* KeyValues::GetNextTrueSubKey() +{ + KeyValues *pRet = m_pPeer; + while ( pRet && pRet->m_iDataType != TYPE_NONE ) + pRet = pRet->m_pPeer; + + return pRet; +} + +KeyValues* KeyValues::GetFirstValue() +{ + KeyValues *pRet = m_pSub; + while ( pRet && pRet->m_iDataType == TYPE_NONE ) + pRet = pRet->m_pPeer; + + return pRet; +} + +KeyValues* KeyValues::GetNextValue() +{ + KeyValues *pRet = m_pPeer; + while ( pRet && pRet->m_iDataType == TYPE_NONE ) + pRet = pRet->m_pPeer; + + return pRet; +} + + +//----------------------------------------------------------------------------- +// Purpose: Get the integer value of a keyName. Default value is returned +// if the keyName can't be found. +//----------------------------------------------------------------------------- +int KeyValues::GetInt( const char *keyName, int defaultValue ) +{ + KeyValues *dat = FindKey( keyName, false ); + if ( dat ) + { + switch ( dat->m_iDataType ) + { + case TYPE_STRING: + return atoi(dat->m_sValue); + case TYPE_WSTRING: + return _wtoi(dat->m_wsValue); + case TYPE_FLOAT: + return (int)dat->m_flValue; + case TYPE_UINT64: + // can't convert, since it would lose data + Assert(0); + return 0; + case TYPE_INT: + case TYPE_PTR: + default: + return dat->m_iValue; + }; + } + return defaultValue; +} + +//----------------------------------------------------------------------------- +// Purpose: Get the integer value of a keyName. Default value is returned +// if the keyName can't be found. +//----------------------------------------------------------------------------- +uint64 KeyValues::GetUint64( const char *keyName, uint64 defaultValue ) +{ + KeyValues *dat = FindKey( keyName, false ); + if ( dat ) + { + switch ( dat->m_iDataType ) + { + case TYPE_STRING: + return (uint64)Q_atoi64(dat->m_sValue); + case TYPE_WSTRING: + return _wtoi64(dat->m_wsValue); + case TYPE_FLOAT: + return (int)dat->m_flValue; + case TYPE_UINT64: + return *((uint64 *)dat->m_sValue); + case TYPE_INT: + case TYPE_PTR: + default: + return dat->m_iValue; + }; + } + return defaultValue; +} + +//----------------------------------------------------------------------------- +// Purpose: Get the pointer value of a keyName. Default value is returned +// if the keyName can't be found. +//----------------------------------------------------------------------------- +void *KeyValues::GetPtr( const char *keyName, void *defaultValue ) +{ + KeyValues *dat = FindKey( keyName, false ); + if ( dat ) + { + switch ( dat->m_iDataType ) + { + case TYPE_PTR: + return dat->m_pValue; + + case TYPE_WSTRING: + case TYPE_STRING: + case TYPE_FLOAT: + case TYPE_INT: + case TYPE_UINT64: + default: + return NULL; + }; + } + return defaultValue; +} + +//----------------------------------------------------------------------------- +// Purpose: Get the float value of a keyName. Default value is returned +// if the keyName can't be found. +//----------------------------------------------------------------------------- +float KeyValues::GetFloat( const char *keyName, float defaultValue ) +{ + KeyValues *dat = FindKey( keyName, false ); + if ( dat ) + { + switch ( dat->m_iDataType ) + { + case TYPE_STRING: + return (float)atof(dat->m_sValue); + case TYPE_WSTRING: +#ifdef WIN32 + return (float) _wtof(dat->m_wsValue); // no wtof +#else + Assert( !"impl me" ); + return 0.0; +#endif + case TYPE_FLOAT: + return dat->m_flValue; + case TYPE_INT: + return (float)dat->m_iValue; + case TYPE_UINT64: + return (float)(*((uint64 *)dat->m_sValue)); + case TYPE_PTR: + default: + return 0.0f; + }; + } + return defaultValue; +} + +//----------------------------------------------------------------------------- +// Purpose: Get the string pointer of a keyName. Default value is returned +// if the keyName can't be found. +//----------------------------------------------------------------------------- +const char *KeyValues::GetString( const char *keyName, const char *defaultValue ) +{ + KeyValues *dat = FindKey( keyName, false ); + if ( dat ) + { + // convert the data to string form then return it + char buf[64]; + switch ( dat->m_iDataType ) + { + case TYPE_FLOAT: + Q_snprintf( buf, sizeof( buf ), "%f", dat->m_flValue ); + SetString( keyName, buf ); + break; + case TYPE_PTR: + Q_snprintf( buf, sizeof( buf ), "%lld", (int64)(size_t)dat->m_pValue ); + SetString( keyName, buf ); + break; + case TYPE_INT: + Q_snprintf( buf, sizeof( buf ), "%d", dat->m_iValue ); + SetString( keyName, buf ); + break; + case TYPE_UINT64: + Q_snprintf( buf, sizeof( buf ), "%lld", *((uint64 *)(dat->m_sValue)) ); + SetString( keyName, buf ); + break; + + case TYPE_WSTRING: + { + // convert the string to char *, set it for future use, and return it + char wideBuf[512]; + int result = Q_UnicodeToUTF8(dat->m_wsValue, wideBuf, 512); + if ( result ) + { + // note: this will copy wideBuf + SetString( keyName, wideBuf ); + } + else + { + return defaultValue; + } + break; + } + case TYPE_STRING: + break; + default: + return defaultValue; + }; + + return dat->m_sValue; + } + return defaultValue; +} + + +const wchar_t *KeyValues::GetWString( const char *keyName, const wchar_t *defaultValue) +{ + KeyValues *dat = FindKey( keyName, false ); + if ( dat ) + { + wchar_t wbuf[64]; + switch ( dat->m_iDataType ) + { + case TYPE_FLOAT: + swprintf(wbuf, Q_ARRAYSIZE(wbuf), L"%f", dat->m_flValue); + SetWString( keyName, wbuf); + break; + case TYPE_PTR: + swprintf( wbuf, Q_ARRAYSIZE(wbuf), L"%lld", (int64)(size_t)dat->m_pValue ); + SetWString( keyName, wbuf ); + break; + case TYPE_INT: + swprintf( wbuf, Q_ARRAYSIZE(wbuf), L"%d", dat->m_iValue ); + SetWString( keyName, wbuf ); + break; + case TYPE_UINT64: + { + swprintf( wbuf, Q_ARRAYSIZE(wbuf), L"%lld", *((uint64 *)(dat->m_sValue)) ); + SetWString( keyName, wbuf ); + } + break; + + case TYPE_WSTRING: + break; + case TYPE_STRING: + { + int bufSize = Q_strlen(dat->m_sValue) + 1; + wchar_t *pWBuf = new wchar_t[ bufSize ]; + int result = Q_UTF8ToUnicode(dat->m_sValue, pWBuf, bufSize * sizeof( wchar_t ) ); + if ( result >= 0 ) // may be a zero length string + { + SetWString( keyName, pWBuf); + } + else + { + delete [] pWBuf; + return defaultValue; + } + delete [] pWBuf; + break; + } + default: + return defaultValue; + }; + + return (const wchar_t* )dat->m_wsValue; + } + return defaultValue; +} + +//----------------------------------------------------------------------------- +// Purpose: Get a bool interpretation of the key. +//----------------------------------------------------------------------------- +bool KeyValues::GetBool( const char *keyName, bool defaultValue, bool* optGotDefault ) +{ + if ( FindKey( keyName ) ) + { + if ( optGotDefault ) + (*optGotDefault) = false; + return 0 != GetInt( keyName, 0 ); + } + + if ( optGotDefault ) + (*optGotDefault) = true; + + return defaultValue; +} + +//----------------------------------------------------------------------------- +// Purpose: Gets a color +//----------------------------------------------------------------------------- +Color KeyValues::GetColor( const char *keyName ) +{ + Color color(0, 0, 0, 0); + KeyValues *dat = FindKey( keyName, false ); + if ( dat ) + { + if ( dat->m_iDataType == TYPE_COLOR ) + { + color[0] = dat->m_Color[0]; + color[1] = dat->m_Color[1]; + color[2] = dat->m_Color[2]; + color[3] = dat->m_Color[3]; + } + else if ( dat->m_iDataType == TYPE_FLOAT ) + { + color[0] = dat->m_flValue; + } + else if ( dat->m_iDataType == TYPE_INT ) + { + color[0] = dat->m_iValue; + } + else if ( dat->m_iDataType == TYPE_STRING ) + { + // parse the colors out of the string + float a = 0.0f, b = 0.0f, c = 0.0f, d = 0.0f; + sscanf(dat->m_sValue, "%f %f %f %f", &a, &b, &c, &d); + color[0] = (unsigned char)a; + color[1] = (unsigned char)b; + color[2] = (unsigned char)c; + color[3] = (unsigned char)d; + } + } + return color; +} + +//----------------------------------------------------------------------------- +// Purpose: Sets a color +//----------------------------------------------------------------------------- +void KeyValues::SetColor( const char *keyName, Color value) +{ + KeyValues *dat = FindKey( keyName, true ); + + if ( dat ) + { + dat->m_iDataType = TYPE_COLOR; + dat->m_Color[0] = value[0]; + dat->m_Color[1] = value[1]; + dat->m_Color[2] = value[2]; + dat->m_Color[3] = value[3]; + } +} + +void KeyValues::SetStringValue( char const *strValue ) +{ + // delete the old value + delete [] m_sValue; + // make sure we're not storing the WSTRING - as we're converting over to STRING + delete [] m_wsValue; + m_wsValue = NULL; + + if (!strValue) + { + // ensure a valid value + strValue = ""; + } + + // allocate memory for the new value and copy it in + int len = Q_strlen( strValue ); + m_sValue = new char[len + 1]; + Q_memcpy( m_sValue, strValue, len+1 ); + + m_iDataType = TYPE_STRING; +} + +//----------------------------------------------------------------------------- +// Purpose: Set the string value of a keyName. +//----------------------------------------------------------------------------- +void KeyValues::SetString( const char *keyName, const char *value ) +{ + KeyValues *dat = FindKey( keyName, true ); + + if ( dat ) + { + if ( dat->m_iDataType == TYPE_STRING && dat->m_sValue == value ) + { + return; + } + + // delete the old value + delete [] dat->m_sValue; + // make sure we're not storing the WSTRING - as we're converting over to STRING + delete [] dat->m_wsValue; + dat->m_wsValue = NULL; + + if (!value) + { + // ensure a valid value + value = ""; + } + + // allocate memory for the new value and copy it in + int len = Q_strlen( value ); + dat->m_sValue = new char[len + 1]; + Q_memcpy( dat->m_sValue, value, len+1 ); + + dat->m_iDataType = TYPE_STRING; + } +} + +//----------------------------------------------------------------------------- +// Purpose: Set the string value of a keyName. +//----------------------------------------------------------------------------- +void KeyValues::SetWString( const char *keyName, const wchar_t *value ) +{ + KeyValues *dat = FindKey( keyName, true ); + if ( dat ) + { + // delete the old value + delete [] dat->m_wsValue; + // make sure we're not storing the STRING - as we're converting over to WSTRING + delete [] dat->m_sValue; + dat->m_sValue = NULL; + + if (!value) + { + // ensure a valid value + value = L""; + } + + // allocate memory for the new value and copy it in + int len = Q_wcslen( value ); + dat->m_wsValue = new wchar_t[len + 1]; + Q_memcpy( dat->m_wsValue, value, (len+1) * sizeof(wchar_t) ); + + dat->m_iDataType = TYPE_WSTRING; + } +} + +//----------------------------------------------------------------------------- +// Purpose: Set the integer value of a keyName. +//----------------------------------------------------------------------------- +void KeyValues::SetInt( const char *keyName, int value ) +{ + KeyValues *dat = FindKey( keyName, true ); + + if ( dat ) + { + dat->m_iValue = value; + dat->m_iDataType = TYPE_INT; + } +} + +//----------------------------------------------------------------------------- +// Purpose: Set the integer value of a keyName. +//----------------------------------------------------------------------------- +void KeyValues::SetUint64( const char *keyName, uint64 value ) +{ + KeyValues *dat = FindKey( keyName, true ); + + if ( dat ) + { + // delete the old value + delete [] dat->m_sValue; + // make sure we're not storing the WSTRING - as we're converting over to STRING + delete [] dat->m_wsValue; + dat->m_wsValue = NULL; + + dat->m_sValue = new char[sizeof(uint64)]; + *((uint64 *)dat->m_sValue) = value; + dat->m_iDataType = TYPE_UINT64; + } +} + +//----------------------------------------------------------------------------- +// Purpose: Set the float value of a keyName. +//----------------------------------------------------------------------------- +void KeyValues::SetFloat( const char *keyName, float value ) +{ + KeyValues *dat = FindKey( keyName, true ); + + if ( dat ) + { + dat->m_flValue = value; + dat->m_iDataType = TYPE_FLOAT; + } +} + +void KeyValues::SetName( const char * setName ) +{ + m_iKeyName = s_pfGetSymbolForString( setName, true ); +} + +//----------------------------------------------------------------------------- +// Purpose: Set the pointer value of a keyName. +//----------------------------------------------------------------------------- +void KeyValues::SetPtr( const char *keyName, void *value ) +{ + KeyValues *dat = FindKey( keyName, true ); + + if ( dat ) + { + dat->m_pValue = value; + dat->m_iDataType = TYPE_PTR; + } +} + +//----------------------------------------------------------------------------- +// Purpose: Copies the tree from the other KeyValues into this one, recursively +// beginning with the root specified by rootSrc. +//----------------------------------------------------------------------------- +void KeyValues::CopyKeyValuesFromRecursive( const KeyValues& rootSrc ) +{ + // This code used to be recursive, which was more elegant. Unfortunately, it also blew the stack for large + // KeyValues. So now we have the iterative version which is uglier but doesn't blow the stack. + // This uses breadth-first traversal. + + struct CopyStruct + { + KeyValues* dst; + const KeyValues* src; + }; + + char tmp[256]; + KeyValues* localDst = NULL; + + CUtlQueue<CopyStruct> nodeQ; + nodeQ.Insert({ this, &rootSrc }); + + while ( nodeQ.Count() > 0 ) + { + CopyStruct cs = nodeQ.RemoveAtHead(); + + // Process all the siblings of the current node. If anyone has a child, add it to the queue. + while (cs.src) + { + Assert( (cs.src != NULL) == (cs.dst != NULL) ); + + // Copy the node contents + cs.dst->CopyKeyValue( *cs.src, sizeof(tmp), tmp ); + + // Add children to the queue to process later. + if (cs.src->m_pSub) { + cs.dst->m_pSub = localDst = new KeyValues( NULL ); + nodeQ.Insert({ localDst, cs.src->m_pSub }); + } + + // Process siblings until we hit the end of the line. + if (cs.src->m_pPeer) { + cs.dst->m_pPeer = new KeyValues( NULL ); + } + else { + cs.dst->m_pPeer = NULL; + } + + // Advance to the next peer. + cs.src = cs.src->m_pPeer; + cs.dst = cs.dst->m_pPeer; + } + } +} + +//----------------------------------------------------------------------------- +// Purpose: Copies a single KeyValue from src to this, using the provided temporary +// buffer if the keytype requires it. Does NOT recurse. +//----------------------------------------------------------------------------- +void KeyValues::CopyKeyValue( const KeyValues& src, size_t tmpBufferSizeB, char* tmpBuffer ) +{ + m_iKeyName = src.GetNameSymbol(); + + if ( src.m_pSub ) + return; + + m_iDataType = src.m_iDataType; + + switch( src.m_iDataType ) + { + case TYPE_NONE: + break; + case TYPE_STRING: + if( src.m_sValue ) + { + int len = Q_strlen(src.m_sValue) + 1; + m_sValue = new char[len]; + Q_strncpy( m_sValue, src.m_sValue, len ); + } + break; + case TYPE_INT: + { + m_iValue = src.m_iValue; + Q_snprintf( tmpBuffer, tmpBufferSizeB, "%d", m_iValue ); + int len = Q_strlen(tmpBuffer) + 1; + m_sValue = new char[len]; + Q_strncpy( m_sValue, tmpBuffer, len ); + } + break; + case TYPE_FLOAT: + { + m_flValue = src.m_flValue; + Q_snprintf( tmpBuffer, tmpBufferSizeB, "%f", m_flValue ); + int len = Q_strlen(tmpBuffer) + 1; + m_sValue = new char[len]; + Q_strncpy( m_sValue, tmpBuffer, len ); + } + break; + case TYPE_PTR: + { + m_pValue = src.m_pValue; + } + break; + case TYPE_UINT64: + { + m_sValue = new char[sizeof(uint64)]; + Q_memcpy( m_sValue, src.m_sValue, sizeof(uint64) ); + } + break; + case TYPE_COLOR: + { + m_Color[0] = src.m_Color[0]; + m_Color[1] = src.m_Color[1]; + m_Color[2] = src.m_Color[2]; + m_Color[3] = src.m_Color[3]; + } + break; + + default: + { + // do nothing . .what the heck is this? + Assert( 0 ); + } + break; + } +} + +KeyValues& KeyValues::operator=( const KeyValues& src ) +{ + RemoveEverything(); + Init(); // reset all values + CopyKeyValuesFromRecursive( src ); + return *this; +} + + +//----------------------------------------------------------------------------- +// Make a new copy of all subkeys, add them all to the passed-in keyvalues +//----------------------------------------------------------------------------- +void KeyValues::CopySubkeys( KeyValues *pParent ) const +{ + // recursively copy subkeys + // Also maintain ordering.... + KeyValues *pPrev = NULL; + for ( KeyValues *sub = m_pSub; sub != NULL; sub = sub->m_pPeer ) + { + // take a copy of the subkey + KeyValues *dat = sub->MakeCopy(); + + // add into subkey list + if (pPrev) + { + pPrev->m_pPeer = dat; + } + else + { + pParent->m_pSub = dat; + } + dat->m_pPeer = NULL; + pPrev = dat; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Makes a copy of the whole key-value pair set +//----------------------------------------------------------------------------- +KeyValues *KeyValues::MakeCopy( void ) const +{ + KeyValues *newKeyValue = new KeyValues(GetName()); + + newKeyValue->UsesEscapeSequences( m_bHasEscapeSequences != 0 ); + newKeyValue->UsesConditionals( m_bEvaluateConditionals != 0 ); + + // copy data + newKeyValue->m_iDataType = m_iDataType; + switch ( m_iDataType ) + { + case TYPE_STRING: + { + if ( m_sValue ) + { + int len = Q_strlen( m_sValue ); + Assert( !newKeyValue->m_sValue ); + newKeyValue->m_sValue = new char[len + 1]; + Q_memcpy( newKeyValue->m_sValue, m_sValue, len+1 ); + } + } + break; + case TYPE_WSTRING: + { + if ( m_wsValue ) + { + int len = Q_wcslen( m_wsValue ); + newKeyValue->m_wsValue = new wchar_t[len+1]; + Q_memcpy( newKeyValue->m_wsValue, m_wsValue, (len+1)*sizeof(wchar_t)); + } + } + break; + + case TYPE_INT: + newKeyValue->m_iValue = m_iValue; + break; + + case TYPE_FLOAT: + newKeyValue->m_flValue = m_flValue; + break; + + case TYPE_PTR: + newKeyValue->m_pValue = m_pValue; + break; + + case TYPE_COLOR: + newKeyValue->m_Color[0] = m_Color[0]; + newKeyValue->m_Color[1] = m_Color[1]; + newKeyValue->m_Color[2] = m_Color[2]; + newKeyValue->m_Color[3] = m_Color[3]; + break; + + case TYPE_UINT64: + newKeyValue->m_sValue = new char[sizeof(uint64)]; + Q_memcpy( newKeyValue->m_sValue, m_sValue, sizeof(uint64) ); + break; + }; + + // recursively copy subkeys + CopySubkeys( newKeyValue ); + return newKeyValue; +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +KeyValues *KeyValues::MakeCopy( bool copySiblings ) const +{ + KeyValues* rootDest = MakeCopy(); + if ( !copySiblings ) + return rootDest; + + const KeyValues* curSrc = GetNextKey(); + KeyValues* curDest = rootDest; + while (curSrc) { + curDest->SetNextKey( curSrc->MakeCopy() ); + curDest = curDest->GetNextKey(); + curSrc = curSrc->GetNextKey(); + } + + return rootDest; +} + +//----------------------------------------------------------------------------- +// Purpose: Check if a keyName has no value assigned to it. +//----------------------------------------------------------------------------- +bool KeyValues::IsEmpty(const char *keyName) +{ + KeyValues *dat = FindKey(keyName, false); + if (!dat) + return true; + + if (dat->m_iDataType == TYPE_NONE && dat->m_pSub == NULL) + return true; + + return false; +} + +//----------------------------------------------------------------------------- +// Purpose: Clear out all subkeys, and the current value +//----------------------------------------------------------------------------- +void KeyValues::Clear( void ) +{ + delete m_pSub; + m_pSub = NULL; + m_iDataType = TYPE_NONE; +} + +//----------------------------------------------------------------------------- +// Purpose: Get the data type of the value stored in a keyName +//----------------------------------------------------------------------------- +KeyValues::types_t KeyValues::GetDataType(const char *keyName) +{ + KeyValues *dat = FindKey(keyName, false); + if (dat) + return (types_t)dat->m_iDataType; + + return TYPE_NONE; +} + +//----------------------------------------------------------------------------- +// Purpose: Deletion, ensures object gets deleted from correct heap +//----------------------------------------------------------------------------- +void KeyValues::deleteThis() +{ + delete this; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : includedKeys - +//----------------------------------------------------------------------------- +void KeyValues::AppendIncludedKeys( CUtlVector< KeyValues * >& includedKeys ) +{ + // Append any included keys, too... + KeyValues *insertSpot = this; + int includeCount = includedKeys.Count(); + for ( int i = 0; i < includeCount; i++ ) + { + KeyValues *kv = includedKeys[ i ]; + Assert( kv ); + + while ( insertSpot->GetNextKey() ) + { + insertSpot = insertSpot->GetNextKey(); + } + + insertSpot->SetNextKey( kv ); + } +} + +void KeyValues::ParseIncludedKeys( char const *resourceName, const char *filetoinclude, + IBaseFileSystem* pFileSystem, const char *pPathID, CUtlVector< KeyValues * >& includedKeys ) +{ + Assert( resourceName ); + Assert( filetoinclude ); + Assert( pFileSystem ); + + // Load it... + if ( !pFileSystem ) + { + return; + } + + // Get relative subdirectory + char fullpath[ 512 ]; + Q_strncpy( fullpath, resourceName, sizeof( fullpath ) ); + + // Strip off characters back to start or first / + int len = Q_strlen( fullpath ); + for (;;) + { + if ( len <= 0 ) + { + break; + } + + if ( fullpath[ len - 1 ] == '\\' || + fullpath[ len - 1 ] == '/' ) + { + break; + } + + // zero it + fullpath[ len - 1 ] = 0; + --len; + } + + // Append included file + Q_strncat( fullpath, filetoinclude, sizeof( fullpath ), COPY_ALL_CHARACTERS ); + + KeyValues *newKV = new KeyValues( fullpath ); + + // CUtlSymbol save = s_CurrentFileSymbol; // did that had any use ??? + + newKV->UsesEscapeSequences( m_bHasEscapeSequences != 0 ); // use same format as parent + newKV->UsesConditionals( m_bEvaluateConditionals != 0 ); + + if ( newKV->LoadFromFile( pFileSystem, fullpath, pPathID ) ) + { + includedKeys.AddToTail( newKV ); + } + else + { + DevMsg( "KeyValues::ParseIncludedKeys: Couldn't load included keyvalue file %s\n", fullpath ); + newKV->deleteThis(); + } + + // s_CurrentFileSymbol = save; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : baseKeys - +//----------------------------------------------------------------------------- +void KeyValues::MergeBaseKeys( CUtlVector< KeyValues * >& baseKeys ) +{ + int includeCount = baseKeys.Count(); + int i; + for ( i = 0; i < includeCount; i++ ) + { + KeyValues *kv = baseKeys[ i ]; + Assert( kv ); + + RecursiveMergeKeyValues( kv ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : baseKV - keyvalues we're basing ourselves on +//----------------------------------------------------------------------------- +void KeyValues::RecursiveMergeKeyValues( KeyValues *baseKV ) +{ + // Merge ourselves + // we always want to keep our value, so nothing to do here + + // Now merge our children + for ( KeyValues *baseChild = baseKV->m_pSub; baseChild != NULL; baseChild = baseChild->m_pPeer ) + { + // for each child in base, see if we have a matching kv + + bool bFoundMatch = false; + + // If we have a child by the same name, merge those keys + for ( KeyValues *newChild = m_pSub; newChild != NULL; newChild = newChild->m_pPeer ) + { + if ( !Q_strcmp( baseChild->GetName(), newChild->GetName() ) ) + { + newChild->RecursiveMergeKeyValues( baseChild ); + bFoundMatch = true; + break; + } + } + + // If not merged, append this key + if ( !bFoundMatch ) + { + KeyValues *dat = baseChild->MakeCopy(); + Assert( dat ); + AddSubKey( dat ); + } + } +} + +//----------------------------------------------------------------------------- +// Returns whether a keyvalues conditional evaluates to true or false +// Needs more flexibility with conditionals, checking convars would be nice. +//----------------------------------------------------------------------------- +bool EvaluateConditional( const char *str ) +{ + if ( !str ) + return false; + + if ( *str == '[' ) + str++; + + bool bNot = false; // should we negate this command? + if ( *str == '!' ) + bNot = true; + + if ( Q_stristr( str, "$X360" ) ) + return IsX360() ^ bNot; + + if ( Q_stristr( str, "$WIN32" ) ) + return IsPC() ^ bNot; // hack hack - for now WIN32 really means IsPC + + if ( Q_stristr( str, "$WINDOWS" ) ) + return IsWindows() ^ bNot; + + if ( Q_stristr( str, "$OSX" ) ) + return IsOSX() ^ bNot; + + if ( Q_stristr( str, "$LINUX" ) ) + return IsLinux() ^ bNot; + + if ( Q_stristr( str, "$POSIX" ) ) + return IsPosix() ^ bNot; + + return false; +} + + +//----------------------------------------------------------------------------- +// Read from a buffer... +//----------------------------------------------------------------------------- +bool KeyValues::LoadFromBuffer( char const *resourceName, CUtlBuffer &buf, IBaseFileSystem* pFileSystem, const char *pPathID ) +{ + KeyValues *pPreviousKey = NULL; + KeyValues *pCurrentKey = this; + CUtlVector< KeyValues * > includedKeys; + CUtlVector< KeyValues * > baseKeys; + bool wasQuoted; + bool wasConditional; + g_KeyValuesErrorStack.SetFilename( resourceName ); + do + { + bool bAccepted = true; + + // the first thing must be a key + const char *s = ReadToken( buf, wasQuoted, wasConditional ); + if ( !buf.IsValid() || !s || *s == 0 ) + break; + + if ( !Q_stricmp( s, "#include" ) ) // special include macro (not a key name) + { + s = ReadToken( buf, wasQuoted, wasConditional ); + // Name of subfile to load is now in s + + if ( !s || *s == 0 ) + { + g_KeyValuesErrorStack.ReportError("#include is NULL " ); + } + else + { + ParseIncludedKeys( resourceName, s, pFileSystem, pPathID, includedKeys ); + } + + continue; + } + else if ( !Q_stricmp( s, "#base" ) ) + { + s = ReadToken( buf, wasQuoted, wasConditional ); + // Name of subfile to load is now in s + + if ( !s || *s == 0 ) + { + g_KeyValuesErrorStack.ReportError("#base is NULL " ); + } + else + { + ParseIncludedKeys( resourceName, s, pFileSystem, pPathID, baseKeys ); + } + + continue; + } + + if ( !pCurrentKey ) + { + pCurrentKey = new KeyValues( s ); + Assert( pCurrentKey ); + + pCurrentKey->UsesEscapeSequences( m_bHasEscapeSequences != 0 ); // same format has parent use + pCurrentKey->UsesConditionals( m_bEvaluateConditionals != 0 ); + + if ( pPreviousKey ) + { + pPreviousKey->SetNextKey( pCurrentKey ); + } + } + else + { + pCurrentKey->SetName( s ); + } + + // get the '{' + s = ReadToken( buf, wasQuoted, wasConditional ); + + if ( wasConditional ) + { + bAccepted = !m_bEvaluateConditionals || EvaluateConditional( s ); + + // Now get the '{' + s = ReadToken( buf, wasQuoted, wasConditional ); + } + + if ( s && *s == '{' && !wasQuoted ) + { + // header is valid so load the file + pCurrentKey->RecursiveLoadFromBuffer( resourceName, buf ); + } + else + { + g_KeyValuesErrorStack.ReportError("LoadFromBuffer: missing {" ); + } + + if ( !bAccepted ) + { + if ( pPreviousKey ) + { + pPreviousKey->SetNextKey( NULL ); + } + pCurrentKey->Clear(); + } + else + { + pPreviousKey = pCurrentKey; + pCurrentKey = NULL; + } + } while ( buf.IsValid() ); + + AppendIncludedKeys( includedKeys ); + { + // delete included keys! + int i; + for ( i = includedKeys.Count() - 1; i > 0; i-- ) + { + KeyValues *kv = includedKeys[ i ]; + kv->deleteThis(); + } + } + + MergeBaseKeys( baseKeys ); + { + // delete base keys! + int i; + for ( i = baseKeys.Count() - 1; i >= 0; i-- ) + { + KeyValues *kv = baseKeys[ i ]; + kv->deleteThis(); + } + } + + g_KeyValuesErrorStack.SetFilename( "" ); + + return true; +} + + +//----------------------------------------------------------------------------- +// Read from a buffer... +//----------------------------------------------------------------------------- +bool KeyValues::LoadFromBuffer( char const *resourceName, const char *pBuffer, IBaseFileSystem* pFileSystem, const char *pPathID ) +{ + if ( !pBuffer ) + return true; + + COM_TimestampedLog("KeyValues::LoadFromBuffer(%s%s%s): Begin", pPathID ? pPathID : "", pPathID && resourceName ? "/" : "", resourceName ? resourceName : ""); + + int nLen = Q_strlen( pBuffer ); + CUtlBuffer buf( pBuffer, nLen, CUtlBuffer::READ_ONLY | CUtlBuffer::TEXT_BUFFER ); + + // Translate Unicode files into UTF-8 before proceeding + if ( nLen > 2 && (uint8)pBuffer[0] == 0xFF && (uint8)pBuffer[1] == 0xFE ) + { + int nUTF8Len = V_UnicodeToUTF8( (wchar_t*)(pBuffer+2), NULL, 0 ); + char *pUTF8Buf = new char[nUTF8Len]; + V_UnicodeToUTF8( (wchar_t*)(pBuffer+2), pUTF8Buf, nUTF8Len ); + buf.AssumeMemory( pUTF8Buf, nUTF8Len, nUTF8Len, CUtlBuffer::READ_ONLY | CUtlBuffer::TEXT_BUFFER ); + } + + bool retVal = LoadFromBuffer( resourceName, buf, pFileSystem, pPathID ); + + COM_TimestampedLog("KeyValues::LoadFromBuffer(%s%s%s): End", pPathID ? pPathID : "", pPathID && resourceName ? "/" : "", resourceName ? resourceName : ""); + + return retVal; +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void KeyValues::RecursiveLoadFromBuffer( char const *resourceName, CUtlBuffer &buf ) +{ + CKeyErrorContext errorReport(this); + bool wasQuoted; + bool wasConditional; + if ( errorReport.GetStackLevel() > 100 ) + { + g_KeyValuesErrorStack.ReportError( "RecursiveLoadFromBuffer: recursion overflow" ); + return; + } + + // keep this out of the stack until a key is parsed + CKeyErrorContext errorKey( INVALID_KEY_SYMBOL ); + + // Locate the last child. (Almost always, we will not have any children.) + // We maintain the pointer to the last child here, so we don't have to re-locate + // it each time we append the next subkey, which causes O(N^2) time + KeyValues *pLastChild = FindLastSubKey();; + + // Keep parsing until we hit the closing brace which terminates this block, or a parse error + while ( 1 ) + { + bool bAccepted = true; + + // get the key name + const char * name = ReadToken( buf, wasQuoted, wasConditional ); + + if ( !name ) // EOF stop reading + { + g_KeyValuesErrorStack.ReportError("RecursiveLoadFromBuffer: got EOF instead of keyname" ); + break; + } + + if ( !*name ) // empty token, maybe "" or EOF + { + g_KeyValuesErrorStack.ReportError("RecursiveLoadFromBuffer: got empty keyname" ); + break; + } + + if ( *name == '}' && !wasQuoted ) // top level closed, stop reading + break; + + // Always create the key; note that this could potentially + // cause some duplication, but that's what we want sometimes + KeyValues *dat = CreateKeyUsingKnownLastChild( name, pLastChild ); + + errorKey.Reset( dat->GetNameSymbol() ); + + // get the value + const char * value = ReadToken( buf, wasQuoted, wasConditional ); + + if ( wasConditional && value ) + { + bAccepted = !m_bEvaluateConditionals || EvaluateConditional( value ); + + // get the real value + value = ReadToken( buf, wasQuoted, wasConditional ); + } + + if ( !value ) + { + g_KeyValuesErrorStack.ReportError("RecursiveLoadFromBuffer: got NULL key" ); + break; + } + + if ( *value == '}' && !wasQuoted ) + { + g_KeyValuesErrorStack.ReportError("RecursiveLoadFromBuffer: got } in key" ); + break; + } + + if ( *value == '{' && !wasQuoted ) + { + // this isn't a key, it's a section + errorKey.Reset( INVALID_KEY_SYMBOL ); + // sub value list + dat->RecursiveLoadFromBuffer( resourceName, buf ); + } + else + { + if ( wasConditional ) + { + g_KeyValuesErrorStack.ReportError("RecursiveLoadFromBuffer: got conditional between key and value" ); + break; + } + + if (dat->m_sValue) + { + delete[] dat->m_sValue; + dat->m_sValue = NULL; + } + + int len = Q_strlen( value ); + + // Here, let's determine if we got a float or an int.... + char* pIEnd; // pos where int scan ended + char* pFEnd; // pos where float scan ended + const char* pSEnd = value + len ; // pos where token ends + + int ival = strtol( value, &pIEnd, 10 ); + float fval = (float)strtod( value, &pFEnd ); + bool bOverflow = ( ival == LONG_MAX || ival == LONG_MIN ) && errno == ERANGE; +#ifdef POSIX + // strtod supports hex representation in strings under posix but we DON'T + // want that support in keyvalues, so undo it here if needed + if ( len > 1 && tolower(value[1]) == 'x' ) + { + fval = 0.0f; + pFEnd = (char *)value; + } +#endif + + if ( *value == 0 ) + { + dat->m_iDataType = TYPE_STRING; + } + else if ( ( 18 == len ) && ( value[0] == '0' ) && ( value[1] == 'x' ) ) + { + // an 18-byte value prefixed with "0x" (followed by 16 hex digits) is an int64 value + int64 retVal = 0; + for( int i=2; i < 2 + 16; i++ ) + { + char digit = value[i]; + if ( digit >= 'a' ) + digit -= 'a' - ( '9' + 1 ); + else + if ( digit >= 'A' ) + digit -= 'A' - ( '9' + 1 ); + retVal = ( retVal * 16 ) + ( digit - '0' ); + } + dat->m_sValue = new char[sizeof(uint64)]; + *((uint64 *)dat->m_sValue) = retVal; + dat->m_iDataType = TYPE_UINT64; + } + else if ( (pFEnd > pIEnd) && (pFEnd == pSEnd) ) + { + dat->m_flValue = fval; + dat->m_iDataType = TYPE_FLOAT; + } + else if (pIEnd == pSEnd && !bOverflow) + { + dat->m_iValue = ival; + dat->m_iDataType = TYPE_INT; + } + else + { + dat->m_iDataType = TYPE_STRING; + } + + if (dat->m_iDataType == TYPE_STRING) + { + // copy in the string information + dat->m_sValue = new char[len+1]; + Q_memcpy( dat->m_sValue, value, len+1 ); + } + + // Look ahead one token for a conditional tag + int prevPos = buf.TellGet(); + const char *peek = ReadToken( buf, wasQuoted, wasConditional ); + if ( wasConditional ) + { + bAccepted = !m_bEvaluateConditionals || EvaluateConditional( peek ); + } + else + { + buf.SeekGet( CUtlBuffer::SEEK_HEAD, prevPos ); + } + } + + Assert( dat->m_pPeer == NULL ); + if ( bAccepted ) + { + Assert( pLastChild == NULL || pLastChild->m_pPeer == dat ); + pLastChild = dat; + } + else + { + //this->RemoveSubKey( dat ); + if ( pLastChild == NULL ) + { + Assert( m_pSub == dat ); + m_pSub = NULL; + } + else + { + Assert( pLastChild->m_pPeer == dat ); + pLastChild->m_pPeer = NULL; + } + + dat->deleteThis(); + dat = NULL; + } + } +} + + + +// writes KeyValue as binary data to buffer +bool KeyValues::WriteAsBinary( CUtlBuffer &buffer ) +{ + if ( buffer.IsText() ) // must be a binary buffer + return false; + + if ( !buffer.IsValid() ) // must be valid, no overflows etc + return false; + + // Write subkeys: + + // loop through all our peers + for ( KeyValues *dat = this; dat != NULL; dat = dat->m_pPeer ) + { + // write type + buffer.PutUnsignedChar( dat->m_iDataType ); + + // write name + buffer.PutString( dat->GetName() ); + + // write type + switch (dat->m_iDataType) + { + case TYPE_NONE: + { + dat->m_pSub->WriteAsBinary( buffer ); + break; + } + case TYPE_STRING: + { + if (dat->m_sValue && *(dat->m_sValue)) + { + buffer.PutString( dat->m_sValue ); + } + else + { + buffer.PutString( "" ); + } + break; + } + case TYPE_WSTRING: + { + Assert( !"TYPE_WSTRING" ); + break; + } + + case TYPE_INT: + { + buffer.PutInt( dat->m_iValue ); + break; + } + + case TYPE_UINT64: + { + buffer.PutDouble( *((double *)dat->m_sValue) ); + break; + } + + case TYPE_FLOAT: + { + buffer.PutFloat( dat->m_flValue ); + break; + } + case TYPE_COLOR: + { + buffer.PutUnsignedChar( dat->m_Color[0] ); + buffer.PutUnsignedChar( dat->m_Color[1] ); + buffer.PutUnsignedChar( dat->m_Color[2] ); + buffer.PutUnsignedChar( dat->m_Color[3] ); + break; + } + case TYPE_PTR: + { + buffer.PutUnsignedInt( (int)dat->m_pValue ); + } + + default: + break; + } + } + + // write tail, marks end of peers + buffer.PutUnsignedChar( TYPE_NUMTYPES ); + + return buffer.IsValid(); +} + +// read KeyValues from binary buffer, returns true if parsing was successful +bool KeyValues::ReadAsBinary( CUtlBuffer &buffer, int nStackDepth ) +{ + if ( buffer.IsText() ) // must be a binary buffer + return false; + + if ( !buffer.IsValid() ) // must be valid, no overflows etc + return false; + + RemoveEverything(); // remove current content + Init(); // reset + + if ( nStackDepth > 100 ) + { + AssertMsgOnce( false, "KeyValues::ReadAsBinary() stack depth > 100\n" ); + return false; + } + + KeyValues *dat = this; + types_t type = (types_t)buffer.GetUnsignedChar(); + + // loop through all our peers + while ( true ) + { + if ( type == TYPE_NUMTYPES ) + break; // no more peers + + dat->m_iDataType = type; + + { + char token[KEYVALUES_TOKEN_SIZE]; + buffer.GetString( token ); + token[KEYVALUES_TOKEN_SIZE-1] = 0; + dat->SetName( token ); + } + + switch ( type ) + { + case TYPE_NONE: + { + dat->m_pSub = new KeyValues(""); + dat->m_pSub->ReadAsBinary( buffer, nStackDepth + 1 ); + break; + } + case TYPE_STRING: + { + char token[KEYVALUES_TOKEN_SIZE]; + buffer.GetString( token ); + token[KEYVALUES_TOKEN_SIZE-1] = 0; + + int len = Q_strlen( token ); + dat->m_sValue = new char[len + 1]; + Q_memcpy( dat->m_sValue, token, len+1 ); + + break; + } + case TYPE_WSTRING: + { + Assert( !"TYPE_WSTRING" ); + break; + } + + case TYPE_INT: + { + dat->m_iValue = buffer.GetInt(); + break; + } + + case TYPE_UINT64: + { + dat->m_sValue = new char[sizeof(uint64)]; + *((uint64 *)dat->m_sValue) = buffer.GetInt64(); + break; + } + + case TYPE_FLOAT: + { + dat->m_flValue = buffer.GetFloat(); + break; + } + case TYPE_COLOR: + { + dat->m_Color[0] = buffer.GetUnsignedChar(); + dat->m_Color[1] = buffer.GetUnsignedChar(); + dat->m_Color[2] = buffer.GetUnsignedChar(); + dat->m_Color[3] = buffer.GetUnsignedChar(); + break; + } + case TYPE_PTR: + { + dat->m_pValue = (void*)buffer.GetUnsignedInt(); + } + + default: + break; + } + + if ( !buffer.IsValid() ) // error occured + return false; + + type = (types_t)buffer.GetUnsignedChar(); + + if ( type == TYPE_NUMTYPES ) + break; + + // new peer follows + dat->m_pPeer = new KeyValues(""); + dat = dat->m_pPeer; + } + + return buffer.IsValid(); +} + +#include "tier0/memdbgoff.h" + +//----------------------------------------------------------------------------- +// Purpose: memory allocator +//----------------------------------------------------------------------------- +void *KeyValues::operator new( size_t iAllocSize ) +{ + MEM_ALLOC_CREDIT(); + return KeyValuesSystem()->AllocKeyValuesMemory( (int)iAllocSize ); +} + +void *KeyValues::operator new( size_t iAllocSize, int nBlockUse, const char *pFileName, int nLine ) +{ + MemAlloc_PushAllocDbgInfo( pFileName, nLine ); + void *p = KeyValuesSystem()->AllocKeyValuesMemory( (int)iAllocSize ); + MemAlloc_PopAllocDbgInfo(); + return p; +} + +//----------------------------------------------------------------------------- +// Purpose: deallocator +//----------------------------------------------------------------------------- +void KeyValues::operator delete( void *pMem ) +{ + KeyValuesSystem()->FreeKeyValuesMemory(pMem); +} + +void KeyValues::operator delete( void *pMem, int nBlockUse, const char *pFileName, int nLine ) +{ + KeyValuesSystem()->FreeKeyValuesMemory(pMem); +} + +void KeyValues::UnpackIntoStructure( KeyValuesUnpackStructure const *pUnpackTable, void *pDest, size_t DestSizeInBytes ) +{ +#ifdef DBGFLAG_ASSERT + void *pDestEnd = ( char * )pDest + DestSizeInBytes + 1; +#endif + + uint8 *dest=(uint8 *) pDest; + while( pUnpackTable->m_pKeyName ) + { + uint8 *dest_field=dest+pUnpackTable->m_nFieldOffset; + KeyValues *find_it=FindKey( pUnpackTable->m_pKeyName ); + + switch( pUnpackTable->m_eDataType ) + { + case UNPACK_TYPE_FLOAT: + { + Assert( dest_field + sizeof( float ) < pDestEnd ); + + float default_value=(pUnpackTable->m_pKeyDefault)?atof(pUnpackTable->m_pKeyDefault):0.0; + *( ( float *) dest_field)=GetFloat( pUnpackTable->m_pKeyName, default_value ); + break; + } + break; + + case UNPACK_TYPE_VECTOR: + { + Assert( dest_field + sizeof( Vector ) < pDestEnd ); + + Vector *dest_v=(Vector *) dest_field; + char const *src_string= + GetString( pUnpackTable->m_pKeyName, pUnpackTable->m_pKeyDefault ); + if ( (!src_string) || + ( sscanf(src_string,"%f %f %f", + &(dest_v->x), &(dest_v->y), &(dest_v->z)) != 3)) + dest_v->Init( 0, 0, 0 ); + } + break; + + case UNPACK_TYPE_FOUR_FLOATS: + { + Assert( dest_field + sizeof( float ) * 4 < pDestEnd ); + + float *dest_f=(float *) dest_field; + char const *src_string= + GetString( pUnpackTable->m_pKeyName, pUnpackTable->m_pKeyDefault ); + if ( (!src_string) || + ( sscanf(src_string,"%f %f %f %f", + dest_f,dest_f+1,dest_f+2,dest_f+3)) != 4) + memset( dest_f, 0, 4*sizeof(float) ); + } + break; + + case UNPACK_TYPE_TWO_FLOATS: + { + Assert( dest_field + sizeof( float ) * 2 < pDestEnd ); + + float *dest_f=(float *) dest_field; + char const *src_string= + GetString( pUnpackTable->m_pKeyName, pUnpackTable->m_pKeyDefault ); + if ( (!src_string) || + ( sscanf(src_string,"%f %f", + dest_f,dest_f+1)) != 2) + memset( dest_f, 0, 2*sizeof(float) ); + } + break; + + case UNPACK_TYPE_STRING: + { + Assert( dest_field + pUnpackTable->m_nFieldSize < pDestEnd ); + + char *dest_s=(char *) dest_field; + strncpy( dest_s, GetString( pUnpackTable->m_pKeyName, + pUnpackTable->m_pKeyDefault ), + pUnpackTable->m_nFieldSize ); + + } + break; + + case UNPACK_TYPE_INT: + { + Assert( dest_field + sizeof( int ) < pDestEnd ); + + int *dest_i=(int *) dest_field; + int default_int=0; + if ( pUnpackTable->m_pKeyDefault) + default_int = atoi( pUnpackTable->m_pKeyDefault ); + *(dest_i)=GetInt( pUnpackTable->m_pKeyName, default_int ); + } + break; + + case UNPACK_TYPE_VECTOR_COLOR: + { + Assert( dest_field + sizeof( Vector ) < pDestEnd ); + + Vector *dest_v=(Vector *) dest_field; + if (find_it) + { + Color c=GetColor( pUnpackTable->m_pKeyName ); + dest_v->x = c.r(); + dest_v->y = c.g(); + dest_v->z = c.b(); + } + else + { + if ( pUnpackTable->m_pKeyDefault ) + sscanf(pUnpackTable->m_pKeyDefault,"%f %f %f", + &(dest_v->x), &(dest_v->y), &(dest_v->z)); + else + dest_v->Init( 0, 0, 0 ); + } + *(dest_v) *= (1.0/255); + } + } + pUnpackTable++; + } +} + +//----------------------------------------------------------------------------- +// Helper function for processing a keyvalue tree for console resolution support. +// Alters key/values for easier console video resolution support. +// If running SD (640x480), the presence of "???_lodef" creates or slams "???". +// If running HD (1280x720), the presence of "???_hidef" creates or slams "???". +//----------------------------------------------------------------------------- +bool KeyValues::ProcessResolutionKeys( const char *pResString ) +{ + if ( !pResString ) + { + // not for pc, console only + return false; + } + + KeyValues *pSubKey = GetFirstSubKey(); + if ( !pSubKey ) + { + // not a block + return false; + } + + for ( ; pSubKey != NULL; pSubKey = pSubKey->GetNextKey() ) + { + // recursively descend each sub block + pSubKey->ProcessResolutionKeys( pResString ); + + // check to see if our substring is present + if ( Q_stristr( pSubKey->GetName(), pResString ) != NULL ) + { + char normalKeyName[128]; + V_strncpy( normalKeyName, pSubKey->GetName(), sizeof( normalKeyName ) ); + + // substring must match exactly, otherwise keys like "_lodef" and "_lodef_wide" would clash. + char *pString = Q_stristr( normalKeyName, pResString ); + if ( pString && !Q_stricmp( pString, pResString ) ) + { + *pString = '\0'; + + // find and delete the original key (if any) + KeyValues *pKey = FindKey( normalKeyName ); + if ( pKey ) + { + // remove the key + RemoveSubKey( pKey ); + } + + // rename the marked key + pSubKey->SetName( normalKeyName ); + } + } + } + + return true; +} + + + +// +// KeyValues dumping implementation +// +bool KeyValues::Dump( IKeyValuesDumpContext *pDump, int nIndentLevel /* = 0 */, bool bSorted /*= false*/ ) +{ + if ( !pDump->KvBeginKey( this, nIndentLevel ) ) + return false; + + if ( bSorted ) + { + CUtlSortVector< KeyValues*, CUtlSortVectorKeyValuesByName > vecSortedKeys; + + // Dump values + for ( KeyValues *val = this ? GetFirstValue() : NULL; val; val = val->GetNextValue() ) + { + vecSortedKeys.InsertNoSort( val ); + } + vecSortedKeys.RedoSort(); + + FOR_EACH_VEC( vecSortedKeys, i ) + { + if ( !pDump->KvWriteValue( vecSortedKeys[i], nIndentLevel + 1 ) ) + return false; + } + + vecSortedKeys.Purge(); + + // Dump subkeys + for ( KeyValues *sub = this ? GetFirstTrueSubKey() : NULL; sub; sub = sub->GetNextTrueSubKey() ) + { + vecSortedKeys.InsertNoSort( sub ); + } + vecSortedKeys.RedoSort(); + + FOR_EACH_VEC( vecSortedKeys, i ) + { + if ( !vecSortedKeys[i]->Dump( pDump, nIndentLevel + 1, bSorted ) ) + return false; + } + } + else + { + // Dump values + for ( KeyValues *val = this ? GetFirstValue() : NULL; val; val = val->GetNextValue() ) + { + if ( !pDump->KvWriteValue( val, nIndentLevel + 1 ) ) + return false; + } + + // Dump subkeys + for ( KeyValues *sub = this ? GetFirstTrueSubKey() : NULL; sub; sub = sub->GetNextTrueSubKey() ) + { + if ( !sub->Dump( pDump, nIndentLevel + 1 ) ) + return false; + } + } + + return pDump->KvEndKey( this, nIndentLevel ); +} + +bool IKeyValuesDumpContextAsText::KvBeginKey( KeyValues *pKey, int nIndentLevel ) +{ + if ( pKey ) + { + return + KvWriteIndent( nIndentLevel ) && + KvWriteText( pKey->GetName() ) && + KvWriteText( "\n" ) && + KvWriteIndent( nIndentLevel ) && + KvWriteText( "{\n" ); + } + else + { + return + KvWriteIndent( nIndentLevel ) && + KvWriteText( "<< NULL >>\n" ); + } +} + +bool IKeyValuesDumpContextAsText::KvWriteValue( KeyValues *val, int nIndentLevel ) +{ + if ( !val ) + { + return + KvWriteIndent( nIndentLevel ) && + KvWriteText( "<< NULL >>\n" ); + } + + if ( !KvWriteIndent( nIndentLevel ) ) + return false; + + if ( !KvWriteText( val->GetName() ) ) + return false; + + if ( !KvWriteText( " " ) ) + return false; + + switch ( val->GetDataType() ) + { + case KeyValues::TYPE_STRING: + { + if ( !KvWriteText( val->GetString() ) ) + return false; + } + break; + + case KeyValues::TYPE_INT: + { + int n = val->GetInt(); + char *chBuffer = ( char * ) stackalloc( 128 ); + V_snprintf( chBuffer, 128, "int( %d = 0x%X )", n, n ); + if ( !KvWriteText( chBuffer ) ) + return false; + } + break; + + case KeyValues::TYPE_FLOAT: + { + float fl = val->GetFloat(); + char *chBuffer = ( char * ) stackalloc( 128 ); + V_snprintf( chBuffer, 128, "float( %f )", fl ); + if ( !KvWriteText( chBuffer ) ) + return false; + } + break; + + case KeyValues::TYPE_PTR: + { + void *ptr = val->GetPtr(); + char *chBuffer = ( char * ) stackalloc( 128 ); + V_snprintf( chBuffer, 128, "ptr( 0x%p )", ptr ); + if ( !KvWriteText( chBuffer ) ) + return false; + } + break; + + case KeyValues::TYPE_WSTRING: + { + wchar_t const *wsz = val->GetWString(); + int nLen = V_wcslen( wsz ); + int numBytes = nLen*2 + 64; + char *chBuffer = ( char * ) stackalloc( numBytes ); + V_snprintf( chBuffer, numBytes, "%ls [wstring, len = %d]", wsz, nLen ); + if ( !KvWriteText( chBuffer ) ) + return false; + } + break; + + case KeyValues::TYPE_UINT64: + { + uint64 n = val->GetUint64(); + char *chBuffer = ( char * ) stackalloc( 128 ); + V_snprintf( chBuffer, 128, "u64( %lld = 0x%llX )", n, n ); + if ( !KvWriteText( chBuffer ) ) + return false; + } + break; + + default: + break; + { + int n = val->GetDataType(); + char *chBuffer = ( char * ) stackalloc( 128 ); + V_snprintf( chBuffer, 128, "??kvtype[%d]", n ); + if ( !KvWriteText( chBuffer ) ) + return false; + } + break; + } + + return KvWriteText( "\n" ); +} + +bool IKeyValuesDumpContextAsText::KvEndKey( KeyValues *pKey, int nIndentLevel ) +{ + if ( pKey ) + { + return + KvWriteIndent( nIndentLevel ) && + KvWriteText( "}\n" ); + } + else + { + return true; + } +} + +bool IKeyValuesDumpContextAsText::KvWriteIndent( int nIndentLevel ) +{ + int numIndentBytes = ( nIndentLevel * 2 + 1 ); + char *pchIndent = ( char * ) stackalloc( numIndentBytes ); + memset( pchIndent, ' ', numIndentBytes - 1 ); + pchIndent[ numIndentBytes - 1 ] = 0; + return KvWriteText( pchIndent ); +} + + +bool CKeyValuesDumpContextAsDevMsg::KvBeginKey( KeyValues *pKey, int nIndentLevel ) +{ + static ConVarRef r_developer( "developer" ); + if ( r_developer.IsValid() && r_developer.GetInt() < m_nDeveloperLevel ) + // If "developer" is not the correct level, then avoid evaluating KeyValues tree early + return false; + else + return IKeyValuesDumpContextAsText::KvBeginKey( pKey, nIndentLevel ); +} + +bool CKeyValuesDumpContextAsDevMsg::KvWriteText( char const *szText ) +{ + if ( m_nDeveloperLevel > 0 ) + { + DevMsg( m_nDeveloperLevel, "%s", szText ); + } + else + { + Msg( "%s", szText ); + } + return true; +} diff --git a/tier1/NetAdr.cpp b/tier1/NetAdr.cpp new file mode 100644 index 0000000..00514d6 --- /dev/null +++ b/tier1/NetAdr.cpp @@ -0,0 +1,378 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// NetAdr.cpp: implementation of the CNetAdr class. +// +//===========================================================================// +#if defined( _WIN32 ) && !defined( _X360 ) +#include <windows.h> +#endif + +#include "tier0/dbg.h" +#include "netadr.h" +#include "tier1/strtools.h" + +#if defined( _WIN32 ) && !defined( _X360 ) +#define WIN32_LEAN_AND_MEAN +#include <winsock.h> +typedef int socklen_t; +#elif !defined( _X360 ) +#include <netinet/in.h> // ntohs() +#include <netdb.h> // gethostbyname() +#include <sys/socket.h> // getsockname() +#endif + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +////////////////////////////////////////////////////////////////////// +// Construction/Destruction +////////////////////////////////////////////////////////////////////// + +bool netadr_t::CompareAdr (const netadr_t &a, bool onlyBase) const +{ + if ( a.type != type ) + return false; + + if ( type == NA_LOOPBACK ) + return true; + + if ( type == NA_BROADCAST ) + return true; + + if ( type == NA_IP ) + { + if ( !onlyBase && (port != a.port) ) + return false; + + if ( a.ip[0] == ip[0] && a.ip[1] == ip[1] && a.ip[2] == ip[2] && a.ip[3] == ip[3] ) + return true; + } + + return false; +} + +bool netadr_t::CompareClassBAdr (const netadr_t &a) const +{ + if ( a.type != type ) + return false; + + if ( type == NA_LOOPBACK ) + return true; + + if ( type == NA_IP ) + { + if (a.ip[0] == ip[0] && a.ip[1] == ip[1] ) + return true; + } + + return false; +} + +bool netadr_t::CompareClassCAdr (const netadr_t &a) const +{ + if ( a.type != type ) + return false; + + if ( type == NA_LOOPBACK ) + return true; + + if ( type == NA_IP ) + { + if (a.ip[0] == ip[0] && a.ip[1] == ip[1] && a.ip[2] == ip[2] ) + return true; + } + + return false; +} +// reserved addresses are not routeable, so they can all be used in a LAN game +bool netadr_t::IsReservedAdr () const +{ + if ( type == NA_LOOPBACK ) + return true; + + if ( type == NA_IP ) + { + if ( (ip[0] == 10) || // 10.x.x.x is reserved + (ip[0] == 127) || // 127.x.x.x + (ip[0] == 172 && ip[1] >= 16 && ip[1] <= 31) || // 172.16.x.x - 172.31.x.x + (ip[0] == 192 && ip[1] >= 168) ) // 192.168.x.x + return true; + } + return false; +} + +const char * netadr_t::ToString( bool onlyBase ) const +{ + // Select a static buffer + static char s[4][64]; + static int slot = 0; + int useSlot = ( slot++ ) % 4; + + // Render into it + ToString( s[useSlot], sizeof(s[0]), onlyBase ); + + // Pray the caller uses it before it gets clobbered + return s[useSlot]; +} + +void netadr_t::ToString( char *pchBuffer, uint32 unBufferSize, bool onlyBase ) const +{ + + if (type == NA_LOOPBACK) + { + V_strncpy( pchBuffer, "loopback", unBufferSize ); + } + else if (type == NA_BROADCAST) + { + V_strncpy( pchBuffer, "broadcast", unBufferSize ); + } + else if (type == NA_IP) + { + if ( onlyBase ) + { + V_snprintf( pchBuffer, unBufferSize, "%i.%i.%i.%i", ip[0], ip[1], ip[2], ip[3]); + } + else + { + V_snprintf( pchBuffer, unBufferSize, "%i.%i.%i.%i:%i", ip[0], ip[1], ip[2], ip[3], ntohs(port)); + } + } + else + { + V_strncpy( pchBuffer, "unknown", unBufferSize ); + } +} + +bool netadr_t::IsLocalhost() const +{ + // are we 127.0.0.1 ? + return (ip[0] == 127) && (ip[1] == 0) && (ip[2] == 0) && (ip[3] == 1); +} + +bool netadr_t::IsLoopback() const +{ + // are we useding engine loopback buffers + return type == NA_LOOPBACK; +} + +void netadr_t::Clear() +{ + ip[0] = ip[1] = ip[2] = ip[3] = 0; + port = 0; + type = NA_NULL; +} + +void netadr_t::SetIP(uint8 b1, uint8 b2, uint8 b3, uint8 b4) +{ + ip[0] = b1; + ip[1] = b2; + ip[2] = b3; + ip[3] = b4; +} + +void netadr_t::SetIP(uint unIP) +{ + *((uint*)ip) = BigLong( unIP ); +} + +void netadr_t::SetType(netadrtype_t newtype) +{ + type = newtype; +} + +netadrtype_t netadr_t::GetType() const +{ + return type; +} + +unsigned short netadr_t::GetPort() const +{ + return BigShort( port ); +} + +unsigned int netadr_t::GetIPNetworkByteOrder() const +{ + return *(unsigned int *)&ip; +} + +unsigned int netadr_t::GetIPHostByteOrder() const +{ + return BigDWord( GetIPNetworkByteOrder() ); +} + +void netadr_t::ToSockadr (struct sockaddr * s) const +{ + Q_memset ( s, 0, sizeof(struct sockaddr)); + + if (type == NA_BROADCAST) + { + ((struct sockaddr_in*)s)->sin_family = AF_INET; + ((struct sockaddr_in*)s)->sin_port = port; + ((struct sockaddr_in*)s)->sin_addr.s_addr = INADDR_BROADCAST; + } + else if (type == NA_IP) + { + ((struct sockaddr_in*)s)->sin_family = AF_INET; + ((struct sockaddr_in*)s)->sin_addr.s_addr = *(int *)&ip; + ((struct sockaddr_in*)s)->sin_port = port; + } + else if (type == NA_LOOPBACK ) + { + ((struct sockaddr_in*)s)->sin_family = AF_INET; + ((struct sockaddr_in*)s)->sin_port = port; + ((struct sockaddr_in*)s)->sin_addr.s_addr = INADDR_LOOPBACK ; + } +} + +bool netadr_t::SetFromSockadr(const struct sockaddr * s) +{ + if (s->sa_family == AF_INET) + { + type = NA_IP; + *(int *)&ip = ((struct sockaddr_in *)s)->sin_addr.s_addr; + port = ((struct sockaddr_in *)s)->sin_port; + return true; + } + else + { + Clear(); + return false; + } +} + +bool netadr_t::IsValid() const +{ + return ( (port !=0 ) && (type != NA_NULL) && + ( ip[0] != 0 || ip[1] != 0 || ip[2] != 0 || ip[3] != 0 ) ); +} + +bool netadr_t::IsBaseAdrValid() const +{ + return ( (type != NA_NULL) && + ( ip[0] != 0 || ip[1] != 0 || ip[2] != 0 || ip[3] != 0 ) ); +} + +#ifdef _WIN32 +#undef SetPort // get around stupid WINSPOOL.H macro +#endif + +void netadr_t::SetPort(unsigned short newport) +{ + port = BigShort( newport ); +} + +bool netadr_t::SetFromString( const char *pch, bool bUseDNS ) +{ + Clear(); + type = NA_IP; + + Assert( pch ); // invalid to call this with NULL pointer; fix your code bug! + if ( !pch ) // but let's not crash + return false; + + char address[ 128 ]; + V_strcpy_safe( address, pch ); + + if ( !V_strnicmp( address, "loopback", 8 ) ) + { + char newaddress[ 128 ]; + type = NA_LOOPBACK; + V_strcpy_safe( newaddress, "127.0.0.1" ); + V_strcat_safe( newaddress, address + 8 ); // copy anything after "loopback" + + V_strcpy_safe( address, newaddress ); + } + + if ( !V_strnicmp( address, "localhost", 9 ) ) + { + V_memcpy( address, "127.0.0.1", 9 ); // Note use of memcpy allows us to keep the colon and rest of string since localhost and 127.0.0.1 are both 9 characters. + } + + // Starts with a number and has a dot + if ( address[0] >= '0' && + address[0] <= '9' && + strchr( address, '.' ) ) + { + int n1 = -1, n2 = -1, n3 = -1, n4 = -1, n5 = 0; // set port to 0 if we don't parse one + int nRes = sscanf( address, "%d.%d.%d.%d:%d", &n1, &n2, &n3, &n4, &n5 ); + if ( + nRes < 4 + || n1 < 0 || n1 > 255 + || n2 < 0 || n2 > 255 + || n3 < 0 || n3 > 255 + || n4 < 0 || n4 > 255 + || n5 < 0 || n5 > 65535 + ) + return false; + SetIP( n1, n2, n3, n4 ); + SetPort( ( uint16 ) n5 ); + return true; + } + + if ( bUseDNS ) + { +// X360TBD: + // dgoodenough - since this is skipped on X360, seems reasonable to skip as well on PS3 + // PS3_BUILDFIX + // FIXME - Leap of faith, this works without asserting on X360, so I assume it will on PS3 +#if !defined( _X360 ) && !defined( _PS3 ) + // Null out the colon if there is one + char *pchColon = strchr( address, ':' ); + if ( pchColon ) + { + *pchColon = 0; + } + + // DNS it base name + struct hostent *h = gethostbyname( address ); + if ( !h ) + return false; + + SetIP( ntohl( *(int *)h->h_addr_list[0] ) ); + + // Set Port to whatever was specified after the colon + if ( pchColon ) + { + SetPort( V_atoi( ++pchColon ) ); + } + return true; +#else + Assert( 0 ); + return false; +#endif + } + + return false; +} + +bool netadr_t::operator<(const netadr_t &netadr) const +{ + if ( *((uint *)netadr.ip) < *((uint *)ip) ) + return true; + else if ( *((uint *)netadr.ip) > *((uint *)ip) ) + return false; + return ( netadr.port < port ); +} + + +void netadr_t::SetFromSocket( int hSocket ) +{ + // dgoodenough - since this is skipped on X360, seems reasonable to skip as well on PS3 + // PS3_BUILDFIX + // FIXME - Leap of faith, this works without asserting on X360, so I assume it will on PS3 +#if !defined( _X360 ) && !defined( _PS3 ) + Clear(); + type = NA_IP; + + struct sockaddr address; + socklen_t namelen = sizeof(address); + if ( getsockname( hSocket, (struct sockaddr *)&address, &namelen) == 0 ) + { + SetFromSockadr( &address ); + } +#else + Assert(0); +#endif +} diff --git a/tier1/bitbuf.cpp b/tier1/bitbuf.cpp new file mode 100644 index 0000000..52a25e4 --- /dev/null +++ b/tier1/bitbuf.cpp @@ -0,0 +1,1490 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +// +//=============================================================================// + +#include "bitbuf.h" +#include "coordsize.h" +#include "mathlib/vector.h" +#include "mathlib/mathlib.h" +#include "tier1/strtools.h" +#include "bitvec.h" + +// FIXME: Can't use this until we get multithreaded allocations in tier0 working for tools +// This is used by VVIS and fails to link +// NOTE: This must be the last file included!!! +//#include "tier0/memdbgon.h" + +#ifdef _X360 +// mandatory ... wary of above comment and isolating, tier0 is built as MT though +#include "tier0/memdbgon.h" +#endif + +#if _WIN32 +#define FAST_BIT_SCAN 1 +#if _X360 +#define CountLeadingZeros(x) _CountLeadingZeros(x) +inline unsigned int CountTrailingZeros( unsigned int elem ) +{ + // this implements CountTrailingZeros() / BitScanForward() + unsigned int mask = elem-1; + unsigned int comp = ~elem; + elem = mask & comp; + return (32 - _CountLeadingZeros(elem)); +} +#else +#include <intrin.h> +#pragma intrinsic(_BitScanReverse) +#pragma intrinsic(_BitScanForward) + +inline unsigned int CountLeadingZeros(unsigned int x) +{ + unsigned long firstBit; + if ( _BitScanReverse(&firstBit,x) ) + return 31 - firstBit; + return 32; +} +inline unsigned int CountTrailingZeros(unsigned int elem) +{ + unsigned long out; + if ( _BitScanForward(&out, elem) ) + return out; + return 32; +} + +#endif +#else +#define FAST_BIT_SCAN 0 +#endif + + +static BitBufErrorHandler g_BitBufErrorHandler = 0; + +inline int BitForBitnum(int bitnum) +{ + return GetBitForBitnum(bitnum); +} + +void InternalBitBufErrorHandler( BitBufErrorType errorType, const char *pDebugName ) +{ + if ( g_BitBufErrorHandler ) + g_BitBufErrorHandler( errorType, pDebugName ); +} + + +void SetBitBufErrorHandler( BitBufErrorHandler fn ) +{ + g_BitBufErrorHandler = fn; +} + + +// #define BB_PROFILING + +unsigned long g_LittleBits[32]; + +// Precalculated bit masks for WriteUBitLong. Using these tables instead of +// doing the calculations gives a 33% speedup in WriteUBitLong. +unsigned long g_BitWriteMasks[32][33]; + +// (1 << i) - 1 +unsigned long g_ExtraMasks[33]; + +class CBitWriteMasksInit +{ +public: + CBitWriteMasksInit() + { + for( unsigned int startbit=0; startbit < 32; startbit++ ) + { + for( unsigned int nBitsLeft=0; nBitsLeft < 33; nBitsLeft++ ) + { + unsigned int endbit = startbit + nBitsLeft; + g_BitWriteMasks[startbit][nBitsLeft] = BitForBitnum(startbit) - 1; + if(endbit < 32) + g_BitWriteMasks[startbit][nBitsLeft] |= ~(BitForBitnum(endbit) - 1); + } + } + + for ( unsigned int maskBit=0; maskBit < 32; maskBit++ ) + g_ExtraMasks[maskBit] = BitForBitnum(maskBit) - 1; + g_ExtraMasks[32] = ~0ul; + + for ( unsigned int littleBit=0; littleBit < 32; littleBit++ ) + StoreLittleDWord( &g_LittleBits[littleBit], 0, 1u<<littleBit ); + } +}; +static CBitWriteMasksInit g_BitWriteMasksInit; + + +// ---------------------------------------------------------------------------------------- // +// bf_write +// ---------------------------------------------------------------------------------------- // + +bf_write::bf_write() +{ + m_pData = NULL; + m_nDataBytes = 0; + m_nDataBits = -1; // set to -1 so we generate overflow on any operation + m_iCurBit = 0; + m_bOverflow = false; + m_bAssertOnOverflow = true; + m_pDebugName = NULL; +} + +bf_write::bf_write( const char *pDebugName, void *pData, int nBytes, int nBits ) +{ + m_bAssertOnOverflow = true; + m_pDebugName = pDebugName; + StartWriting( pData, nBytes, 0, nBits ); +} + +bf_write::bf_write( void *pData, int nBytes, int nBits ) +{ + m_bAssertOnOverflow = true; + m_pDebugName = NULL; + StartWriting( pData, nBytes, 0, nBits ); +} + +void bf_write::StartWriting( void *pData, int nBytes, int iStartBit, int nBits ) +{ + // Make sure it's dword aligned and padded. + Assert( (nBytes % 4) == 0 ); + Assert(((unsigned long)pData & 3) == 0); + + // The writing code will overrun the end of the buffer if it isn't dword aligned, so truncate to force alignment + nBytes &= ~3; + + m_pData = (unsigned long*)pData; + m_nDataBytes = nBytes; + + if ( nBits == -1 ) + { + m_nDataBits = nBytes << 3; + } + else + { + Assert( nBits <= nBytes*8 ); + m_nDataBits = nBits; + } + + m_iCurBit = iStartBit; + m_bOverflow = false; +} + +void bf_write::Reset() +{ + m_iCurBit = 0; + m_bOverflow = false; +} + + +void bf_write::SetAssertOnOverflow( bool bAssert ) +{ + m_bAssertOnOverflow = bAssert; +} + + +const char* bf_write::GetDebugName() +{ + return m_pDebugName; +} + + +void bf_write::SetDebugName( const char *pDebugName ) +{ + m_pDebugName = pDebugName; +} + + +void bf_write::SeekToBit( int bitPos ) +{ + m_iCurBit = bitPos; +} + + +// Sign bit comes first +void bf_write::WriteSBitLong( int data, int numbits ) +{ + // Force the sign-extension bit to be correct even in the case of overflow. + int nValue = data; + int nPreserveBits = ( 0x7FFFFFFF >> ( 32 - numbits ) ); + int nSignExtension = ( nValue >> 31 ) & ~nPreserveBits; + nValue &= nPreserveBits; + nValue |= nSignExtension; + + AssertMsg2( nValue == data, "WriteSBitLong: 0x%08x does not fit in %d bits", data, numbits ); + + WriteUBitLong( nValue, numbits, false ); +} + +void bf_write::WriteVarInt32( uint32 data ) +{ + // Check if align and we have room, slow path if not + if ( (m_iCurBit & 7) == 0 && (m_iCurBit + bitbuf::kMaxVarint32Bytes * 8 ) <= m_nDataBits) + { + uint8 *target = ((uint8*)m_pData) + (m_iCurBit>>3); + + target[0] = static_cast<uint8>(data | 0x80); + if ( data >= (1 << 7) ) + { + target[1] = static_cast<uint8>((data >> 7) | 0x80); + if ( data >= (1 << 14) ) + { + target[2] = static_cast<uint8>((data >> 14) | 0x80); + if ( data >= (1 << 21) ) + { + target[3] = static_cast<uint8>((data >> 21) | 0x80); + if ( data >= (1 << 28) ) + { + target[4] = static_cast<uint8>(data >> 28); + m_iCurBit += 5 * 8; + return; + } + else + { + target[3] &= 0x7F; + m_iCurBit += 4 * 8; + return; + } + } + else + { + target[2] &= 0x7F; + m_iCurBit += 3 * 8; + return; + } + } + else + { + target[1] &= 0x7F; + m_iCurBit += 2 * 8; + return; + } + } + else + { + target[0] &= 0x7F; + m_iCurBit += 1 * 8; + return; + } + } + else // Slow path + { + while ( data > 0x7F ) + { + WriteUBitLong( (data & 0x7F) | 0x80, 8 ); + data >>= 7; + } + WriteUBitLong( data & 0x7F, 8 ); + } +} + +void bf_write::WriteVarInt64( uint64 data ) +{ + // Check if align and we have room, slow path if not + if ( (m_iCurBit & 7) == 0 && (m_iCurBit + bitbuf::kMaxVarintBytes * 8 ) <= m_nDataBits ) + { + uint8 *target = ((uint8*)m_pData) + (m_iCurBit>>3); + + // Splitting into 32-bit pieces gives better performance on 32-bit + // processors. + uint32 part0 = static_cast<uint32>(data ); + uint32 part1 = static_cast<uint32>(data >> 28); + uint32 part2 = static_cast<uint32>(data >> 56); + + int size; + + // Here we can't really optimize for small numbers, since the data is + // split into three parts. Cheking for numbers < 128, for instance, + // would require three comparisons, since you'd have to make sure part1 + // and part2 are zero. However, if the caller is using 64-bit integers, + // it is likely that they expect the numbers to often be very large, so + // we probably don't want to optimize for small numbers anyway. Thus, + // we end up with a hardcoded binary search tree... + if ( part2 == 0 ) + { + if ( part1 == 0 ) + { + if ( part0 < (1 << 14) ) + { + if ( part0 < (1 << 7) ) + { + size = 1; goto size1; + } + else + { + size = 2; goto size2; + } + } + else + { + if ( part0 < (1 << 21) ) + { + size = 3; goto size3; + } + else + { + size = 4; goto size4; + } + } + } + else + { + if ( part1 < (1 << 14) ) + { + if ( part1 < (1 << 7) ) + { + size = 5; goto size5; + } + else + { + size = 6; goto size6; + } + } + else + { + if ( part1 < (1 << 21) ) + { + size = 7; goto size7; + } + else + { + size = 8; goto size8; + } + } + } + } + else + { + if ( part2 < (1 << 7) ) + { + size = 9; goto size9; + } + else + { + size = 10; goto size10; + } + } + + AssertFatalMsg( false, "Can't get here." ); + + size10: target[9] = static_cast<uint8>((part2 >> 7) | 0x80); + size9 : target[8] = static_cast<uint8>((part2 ) | 0x80); + size8 : target[7] = static_cast<uint8>((part1 >> 21) | 0x80); + size7 : target[6] = static_cast<uint8>((part1 >> 14) | 0x80); + size6 : target[5] = static_cast<uint8>((part1 >> 7) | 0x80); + size5 : target[4] = static_cast<uint8>((part1 ) | 0x80); + size4 : target[3] = static_cast<uint8>((part0 >> 21) | 0x80); + size3 : target[2] = static_cast<uint8>((part0 >> 14) | 0x80); + size2 : target[1] = static_cast<uint8>((part0 >> 7) | 0x80); + size1 : target[0] = static_cast<uint8>((part0 ) | 0x80); + + target[size-1] &= 0x7F; + m_iCurBit += size * 8; + } + else // slow path + { + while ( data > 0x7F ) + { + WriteUBitLong( (data & 0x7F) | 0x80, 8 ); + data >>= 7; + } + WriteUBitLong( data & 0x7F, 8 ); + } +} + +void bf_write::WriteSignedVarInt32( int32 data ) +{ + WriteVarInt32( bitbuf::ZigZagEncode32( data ) ); +} + +void bf_write::WriteSignedVarInt64( int64 data ) +{ + WriteVarInt64( bitbuf::ZigZagEncode64( data ) ); +} + +int bf_write::ByteSizeVarInt32( uint32 data ) +{ + int size = 1; + while ( data > 0x7F ) { + size++; + data >>= 7; + } + return size; +} + +int bf_write::ByteSizeVarInt64( uint64 data ) +{ + int size = 1; + while ( data > 0x7F ) { + size++; + data >>= 7; + } + return size; +} + +int bf_write::ByteSizeSignedVarInt32( int32 data ) +{ + return ByteSizeVarInt32( bitbuf::ZigZagEncode32( data ) ); +} + +int bf_write::ByteSizeSignedVarInt64( int64 data ) +{ + return ByteSizeVarInt64( bitbuf::ZigZagEncode64( data ) ); +} + +void bf_write::WriteBitLong(unsigned int data, int numbits, bool bSigned) +{ + if(bSigned) + WriteSBitLong((int)data, numbits); + else + WriteUBitLong(data, numbits); +} + +bool bf_write::WriteBits(const void *pInData, int nBits) +{ +#if defined( BB_PROFILING ) + VPROF( "bf_write::WriteBits" ); +#endif + + unsigned char *pOut = (unsigned char*)pInData; + int nBitsLeft = nBits; + + // Bounds checking.. + if ( (m_iCurBit+nBits) > m_nDataBits ) + { + SetOverflowFlag(); + CallErrorHandler( BITBUFERROR_BUFFER_OVERRUN, GetDebugName() ); + return false; + } + + // Align output to dword boundary + while (((unsigned long)pOut & 3) != 0 && nBitsLeft >= 8) + { + + WriteUBitLong( *pOut, 8, false ); + ++pOut; + nBitsLeft -= 8; + } + + if ( IsPC() && (nBitsLeft >= 32) && (m_iCurBit & 7) == 0 ) + { + // current bit is byte aligned, do block copy + int numbytes = nBitsLeft >> 3; + int numbits = numbytes << 3; + + Q_memcpy( (char*)m_pData+(m_iCurBit>>3), pOut, numbytes ); + pOut += numbytes; + nBitsLeft -= numbits; + m_iCurBit += numbits; + } + + // X360TBD: Can't write dwords in WriteBits because they'll get swapped + if ( IsPC() && nBitsLeft >= 32 ) + { + unsigned long iBitsRight = (m_iCurBit & 31); + unsigned long iBitsLeft = 32 - iBitsRight; + unsigned long bitMaskLeft = g_BitWriteMasks[iBitsRight][32]; + unsigned long bitMaskRight = g_BitWriteMasks[0][iBitsRight]; + + unsigned long *pData = &m_pData[m_iCurBit>>5]; + + // Read dwords. + while(nBitsLeft >= 32) + { + unsigned long curData = *(unsigned long*)pOut; + pOut += sizeof(unsigned long); + + *pData &= bitMaskLeft; + *pData |= curData << iBitsRight; + + pData++; + + if ( iBitsLeft < 32 ) + { + curData >>= iBitsLeft; + *pData &= bitMaskRight; + *pData |= curData; + } + + nBitsLeft -= 32; + m_iCurBit += 32; + } + } + + + // write remaining bytes + while ( nBitsLeft >= 8 ) + { + WriteUBitLong( *pOut, 8, false ); + ++pOut; + nBitsLeft -= 8; + } + + // write remaining bits + if ( nBitsLeft ) + { + WriteUBitLong( *pOut, nBitsLeft, false ); + } + + return !IsOverflowed(); +} + + +bool bf_write::WriteBitsFromBuffer( bf_read *pIn, int nBits ) +{ + // This could be optimized a little by + while ( nBits > 32 ) + { + WriteUBitLong( pIn->ReadUBitLong( 32 ), 32 ); + nBits -= 32; + } + + WriteUBitLong( pIn->ReadUBitLong( nBits ), nBits ); + return !IsOverflowed() && !pIn->IsOverflowed(); +} + + +void bf_write::WriteBitAngle( float fAngle, int numbits ) +{ + int d; + unsigned int mask; + unsigned int shift; + + shift = BitForBitnum(numbits); + mask = shift - 1; + + d = (int)( (fAngle / 360.0) * shift ); + d &= mask; + + WriteUBitLong((unsigned int)d, numbits); +} + +void bf_write::WriteBitCoordMP( const float f, bool bIntegral, bool bLowPrecision ) +{ +#if defined( BB_PROFILING ) + VPROF( "bf_write::WriteBitCoordMP" ); +#endif + int signbit = (f <= -( bLowPrecision ? COORD_RESOLUTION_LOWPRECISION : COORD_RESOLUTION )); + int intval = (int)abs(f); + int fractval = bLowPrecision ? + ( abs((int)(f*COORD_DENOMINATOR_LOWPRECISION)) & (COORD_DENOMINATOR_LOWPRECISION-1) ) : + ( abs((int)(f*COORD_DENOMINATOR)) & (COORD_DENOMINATOR-1) ); + + bool bInBounds = intval < (1 << COORD_INTEGER_BITS_MP ); + + unsigned int bits, numbits; + + if ( bIntegral ) + { + // Integer encoding: in-bounds bit, nonzero bit, optional sign bit + integer value bits + if ( intval ) + { + // Adjust the integers from [1..MAX_COORD_VALUE] to [0..MAX_COORD_VALUE-1] + --intval; + bits = intval * 8 + signbit * 4 + 2 + bInBounds; + numbits = 3 + (bInBounds ? COORD_INTEGER_BITS_MP : COORD_INTEGER_BITS); + } + else + { + bits = bInBounds; + numbits = 2; + } + } + else + { + // Float encoding: in-bounds bit, integer bit, sign bit, fraction value bits, optional integer value bits + if ( intval ) + { + // Adjust the integers from [1..MAX_COORD_VALUE] to [0..MAX_COORD_VALUE-1] + --intval; + bits = intval * 8 + signbit * 4 + 2 + bInBounds; + bits += bInBounds ? (fractval << (3+COORD_INTEGER_BITS_MP)) : (fractval << (3+COORD_INTEGER_BITS)); + numbits = 3 + (bInBounds ? COORD_INTEGER_BITS_MP : COORD_INTEGER_BITS) + + (bLowPrecision ? COORD_FRACTIONAL_BITS_MP_LOWPRECISION : COORD_FRACTIONAL_BITS); + } + else + { + bits = fractval * 8 + signbit * 4 + 0 + bInBounds; + numbits = 3 + (bLowPrecision ? COORD_FRACTIONAL_BITS_MP_LOWPRECISION : COORD_FRACTIONAL_BITS); + } + } + + WriteUBitLong( bits, numbits ); +} + +void bf_write::WriteBitCoord (const float f) +{ +#if defined( BB_PROFILING ) + VPROF( "bf_write::WriteBitCoord" ); +#endif + int signbit = (f <= -COORD_RESOLUTION); + int intval = (int)abs(f); + int fractval = abs((int)(f*COORD_DENOMINATOR)) & (COORD_DENOMINATOR-1); + + + // Send the bit flags that indicate whether we have an integer part and/or a fraction part. + WriteOneBit( intval ); + WriteOneBit( fractval ); + + if ( intval || fractval ) + { + // Send the sign bit + WriteOneBit( signbit ); + + // Send the integer if we have one. + if ( intval ) + { + // Adjust the integers from [1..MAX_COORD_VALUE] to [0..MAX_COORD_VALUE-1] + intval--; + WriteUBitLong( (unsigned int)intval, COORD_INTEGER_BITS ); + } + + // Send the fraction if we have one + if ( fractval ) + { + WriteUBitLong( (unsigned int)fractval, COORD_FRACTIONAL_BITS ); + } + } +} + +void bf_write::WriteBitVec3Coord( const Vector& fa ) +{ + int xflag, yflag, zflag; + + xflag = (fa[0] >= COORD_RESOLUTION) || (fa[0] <= -COORD_RESOLUTION); + yflag = (fa[1] >= COORD_RESOLUTION) || (fa[1] <= -COORD_RESOLUTION); + zflag = (fa[2] >= COORD_RESOLUTION) || (fa[2] <= -COORD_RESOLUTION); + + WriteOneBit( xflag ); + WriteOneBit( yflag ); + WriteOneBit( zflag ); + + if ( xflag ) + WriteBitCoord( fa[0] ); + if ( yflag ) + WriteBitCoord( fa[1] ); + if ( zflag ) + WriteBitCoord( fa[2] ); +} + +void bf_write::WriteBitNormal( float f ) +{ + int signbit = (f <= -NORMAL_RESOLUTION); + + // NOTE: Since +/-1 are valid values for a normal, I'm going to encode that as all ones + unsigned int fractval = abs( (int)(f*NORMAL_DENOMINATOR) ); + + // clamp.. + if (fractval > NORMAL_DENOMINATOR) + fractval = NORMAL_DENOMINATOR; + + // Send the sign bit + WriteOneBit( signbit ); + + // Send the fractional component + WriteUBitLong( fractval, NORMAL_FRACTIONAL_BITS ); +} + +void bf_write::WriteBitVec3Normal( const Vector& fa ) +{ + int xflag, yflag; + + xflag = (fa[0] >= NORMAL_RESOLUTION) || (fa[0] <= -NORMAL_RESOLUTION); + yflag = (fa[1] >= NORMAL_RESOLUTION) || (fa[1] <= -NORMAL_RESOLUTION); + + WriteOneBit( xflag ); + WriteOneBit( yflag ); + + if ( xflag ) + WriteBitNormal( fa[0] ); + if ( yflag ) + WriteBitNormal( fa[1] ); + + // Write z sign bit + int signbit = (fa[2] <= -NORMAL_RESOLUTION); + WriteOneBit( signbit ); +} + +void bf_write::WriteBitAngles( const QAngle& fa ) +{ + // FIXME: + Vector tmp( fa.x, fa.y, fa.z ); + WriteBitVec3Coord( tmp ); +} + +void bf_write::WriteChar(int val) +{ + WriteSBitLong(val, sizeof(char) << 3); +} + +void bf_write::WriteByte(int val) +{ + WriteUBitLong(val, sizeof(unsigned char) << 3); +} + +void bf_write::WriteShort(int val) +{ + WriteSBitLong(val, sizeof(short) << 3); +} + +void bf_write::WriteWord(int val) +{ + WriteUBitLong(val, sizeof(unsigned short) << 3); +} + +void bf_write::WriteLong(long val) +{ + WriteSBitLong(val, sizeof(long) << 3); +} + +void bf_write::WriteLongLong(int64 val) +{ + uint *pLongs = (uint*)&val; + + // Insert the two DWORDS according to network endian + const short endianIndex = 0x0100; + byte *idx = (byte*)&endianIndex; + WriteUBitLong(pLongs[*idx++], sizeof(long) << 3); + WriteUBitLong(pLongs[*idx], sizeof(long) << 3); +} + +void bf_write::WriteFloat(float val) +{ + // Pre-swap the float, since WriteBits writes raw data + LittleFloat( &val, &val ); + + WriteBits(&val, sizeof(val) << 3); +} + +bool bf_write::WriteBytes( const void *pBuf, int nBytes ) +{ + return WriteBits(pBuf, nBytes << 3); +} + +bool bf_write::WriteString(const char *pStr) +{ + if(pStr) + { + do + { + WriteChar( *pStr ); + ++pStr; + } while( *(pStr-1) != 0 ); + } + else + { + WriteChar( 0 ); + } + + return !IsOverflowed(); +} + +// ---------------------------------------------------------------------------------------- // +// bf_read +// ---------------------------------------------------------------------------------------- // + +bf_read::bf_read() +{ + m_pData = NULL; + m_nDataBytes = 0; + m_nDataBits = -1; // set to -1 so we overflow on any operation + m_iCurBit = 0; + m_bOverflow = false; + m_bAssertOnOverflow = true; + m_pDebugName = NULL; +} + +bf_read::bf_read( const void *pData, int nBytes, int nBits ) +{ + m_bAssertOnOverflow = true; + StartReading( pData, nBytes, 0, nBits ); +} + +bf_read::bf_read( const char *pDebugName, const void *pData, int nBytes, int nBits ) +{ + m_bAssertOnOverflow = true; + m_pDebugName = pDebugName; + StartReading( pData, nBytes, 0, nBits ); +} + +void bf_read::StartReading( const void *pData, int nBytes, int iStartBit, int nBits ) +{ + // Make sure we're dword aligned. + Assert(((size_t)pData & 3) == 0); + + m_pData = (unsigned char*)pData; + m_nDataBytes = nBytes; + + if ( nBits == -1 ) + { + m_nDataBits = m_nDataBytes << 3; + } + else + { + Assert( nBits <= nBytes*8 ); + m_nDataBits = nBits; + } + + m_iCurBit = iStartBit; + m_bOverflow = false; +} + +void bf_read::Reset() +{ + m_iCurBit = 0; + m_bOverflow = false; +} + +void bf_read::SetAssertOnOverflow( bool bAssert ) +{ + m_bAssertOnOverflow = bAssert; +} + +void bf_read::SetDebugName( const char *pName ) +{ + m_pDebugName = pName; +} + +void bf_read::SetOverflowFlag() +{ + if ( m_bAssertOnOverflow ) + { + Assert( false ); + } + m_bOverflow = true; +} + +unsigned int bf_read::CheckReadUBitLong(int numbits) +{ + // Ok, just read bits out. + int i, nBitValue; + unsigned int r = 0; + + for(i=0; i < numbits; i++) + { + nBitValue = ReadOneBitNoCheck(); + r |= nBitValue << i; + } + m_iCurBit -= numbits; + + return r; +} + +void bf_read::ReadBits(void *pOutData, int nBits) +{ +#if defined( BB_PROFILING ) + VPROF( "bf_read::ReadBits" ); +#endif + + unsigned char *pOut = (unsigned char*)pOutData; + int nBitsLeft = nBits; + + + // align output to dword boundary + while( ((size_t)pOut & 3) != 0 && nBitsLeft >= 8 ) + { + *pOut = (unsigned char)ReadUBitLong(8); + ++pOut; + nBitsLeft -= 8; + } + + // X360TBD: Can't read dwords in ReadBits because they'll get swapped + if ( IsPC() ) + { + // read dwords + while ( nBitsLeft >= 32 ) + { + *((unsigned long*)pOut) = ReadUBitLong(32); + pOut += sizeof(unsigned long); + nBitsLeft -= 32; + } + } + + // read remaining bytes + while ( nBitsLeft >= 8 ) + { + *pOut = ReadUBitLong(8); + ++pOut; + nBitsLeft -= 8; + } + + // read remaining bits + if ( nBitsLeft ) + { + *pOut = ReadUBitLong(nBitsLeft); + } + +} + +int bf_read::ReadBitsClamped_ptr(void *pOutData, size_t outSizeBytes, size_t nBits) +{ + size_t outSizeBits = outSizeBytes * 8; + size_t readSizeBits = nBits; + int skippedBits = 0; + if ( readSizeBits > outSizeBits ) + { + // Should we print a message when we clamp the data being read? Only + // in debug builds I think. + AssertMsg( 0, "Oversized network packet received, and clamped." ); + readSizeBits = outSizeBits; + skippedBits = (int)( nBits - outSizeBits ); + // What should we do in this case, which should only happen if nBits + // is negative for some reason? + //if ( skippedBits < 0 ) + // return 0; + } + + ReadBits( pOutData, readSizeBits ); + SeekRelative( skippedBits ); + + // Return the number of bits actually read. + return (int)readSizeBits; +} + +float bf_read::ReadBitAngle( int numbits ) +{ + float fReturn; + int i; + float shift; + + shift = (float)( BitForBitnum(numbits) ); + + i = ReadUBitLong( numbits ); + fReturn = (float)i * (360.0 / shift); + + return fReturn; +} + +unsigned int bf_read::PeekUBitLong( int numbits ) +{ + unsigned int r; + int i, nBitValue; +#ifdef BIT_VERBOSE + int nShifts = numbits; +#endif + + bf_read savebf; + + savebf = *this; // Save current state info + + r = 0; + for(i=0; i < numbits; i++) + { + nBitValue = ReadOneBit(); + + // Append to current stream + if ( nBitValue ) + { + r |= BitForBitnum(i); + } + } + + *this = savebf; + +#ifdef BIT_VERBOSE + Con_Printf( "PeekBitLong: %i %i\n", nShifts, (unsigned int)r ); +#endif + + return r; +} + +unsigned int bf_read::ReadUBitLongNoInline( int numbits ) +{ + return ReadUBitLong( numbits ); +} + +unsigned int bf_read::ReadUBitVarInternal( int encodingType ) +{ + m_iCurBit -= 4; + // int bits = { 4, 8, 12, 32 }[ encodingType ]; + int bits = 4 + encodingType*4 + (((2 - encodingType) >> 31) & 16); + return ReadUBitLong( bits ); +} + +// Append numbits least significant bits from data to the current bit stream +int bf_read::ReadSBitLong( int numbits ) +{ + unsigned int r = ReadUBitLong(numbits); + unsigned int s = 1 << (numbits-1); + if (r >= s) + { + // sign-extend by removing sign bit and then subtracting sign bit again + r = r - s - s; + } + return r; +} + +uint32 bf_read::ReadVarInt32() +{ + uint32 result = 0; + int count = 0; + uint32 b; + + do + { + if ( count == bitbuf::kMaxVarint32Bytes ) + { + return result; + } + b = ReadUBitLong( 8 ); + result |= (b & 0x7F) << (7 * count); + ++count; + } while (b & 0x80); + + return result; +} + +uint64 bf_read::ReadVarInt64() +{ + uint64 result = 0; + int count = 0; + uint64 b; + + do + { + if ( count == bitbuf::kMaxVarintBytes ) + { + return result; + } + b = ReadUBitLong( 8 ); + result |= static_cast<uint64>(b & 0x7F) << (7 * count); + ++count; + } while (b & 0x80); + + return result; +} + +int32 bf_read::ReadSignedVarInt32() +{ + uint32 value = ReadVarInt32(); + return bitbuf::ZigZagDecode32( value ); +} + +int64 bf_read::ReadSignedVarInt64() +{ + uint32 value = ReadVarInt64(); + return bitbuf::ZigZagDecode64( value ); +} + +unsigned int bf_read::ReadBitLong(int numbits, bool bSigned) +{ + if(bSigned) + return (unsigned int)ReadSBitLong(numbits); + else + return ReadUBitLong(numbits); +} + + +// Basic Coordinate Routines (these contain bit-field size AND fixed point scaling constants) +float bf_read::ReadBitCoord (void) +{ +#if defined( BB_PROFILING ) + VPROF( "bf_read::ReadBitCoord" ); +#endif + int intval=0,fractval=0,signbit=0; + float value = 0.0; + + + // Read the required integer and fraction flags + intval = ReadOneBit(); + fractval = ReadOneBit(); + + // If we got either parse them, otherwise it's a zero. + if ( intval || fractval ) + { + // Read the sign bit + signbit = ReadOneBit(); + + // If there's an integer, read it in + if ( intval ) + { + // Adjust the integers from [0..MAX_COORD_VALUE-1] to [1..MAX_COORD_VALUE] + intval = ReadUBitLong( COORD_INTEGER_BITS ) + 1; + } + + // If there's a fraction, read it in + if ( fractval ) + { + fractval = ReadUBitLong( COORD_FRACTIONAL_BITS ); + } + + // Calculate the correct floating point value + value = intval + ((float)fractval * COORD_RESOLUTION); + + // Fixup the sign if negative. + if ( signbit ) + value = -value; + } + + return value; +} + +float bf_read::ReadBitCoordMP( bool bIntegral, bool bLowPrecision ) +{ +#if defined( BB_PROFILING ) + VPROF( "bf_read::ReadBitCoordMP" ); +#endif + // BitCoordMP float encoding: inbounds bit, integer bit, sign bit, optional int bits, float bits + // BitCoordMP integer encoding: inbounds bit, integer bit, optional sign bit, optional int bits. + // int bits are always encoded as (value - 1) since zero is handled by the integer bit + + // With integer-only encoding, the presence of the third bit depends on the second + int flags = ReadUBitLong(3 - bIntegral); + enum { INBOUNDS=1, INTVAL=2, SIGN=4 }; + + if ( bIntegral ) + { + if ( flags & INTVAL ) + { + // Read the third bit and the integer portion together at once + unsigned int bits = ReadUBitLong( (flags & INBOUNDS) ? COORD_INTEGER_BITS_MP+1 : COORD_INTEGER_BITS+1 ); + // Remap from [0,N] to [1,N+1] + int intval = (bits >> 1) + 1; + return (bits & 1) ? -intval : intval; + } + return 0.f; + } + + static const float mul_table[4] = + { + 1.f/(1<<COORD_FRACTIONAL_BITS), + -1.f/(1<<COORD_FRACTIONAL_BITS), + 1.f/(1<<COORD_FRACTIONAL_BITS_MP_LOWPRECISION), + -1.f/(1<<COORD_FRACTIONAL_BITS_MP_LOWPRECISION) + }; + //equivalent to: float multiply = mul_table[ ((flags & SIGN) ? 1 : 0) + bLowPrecision*2 ]; + float multiply = *(float*)((uintptr_t)&mul_table[0] + (flags & 4) + bLowPrecision*8); + + static const unsigned char numbits_table[8] = + { + COORD_FRACTIONAL_BITS, + COORD_FRACTIONAL_BITS, + COORD_FRACTIONAL_BITS + COORD_INTEGER_BITS, + COORD_FRACTIONAL_BITS + COORD_INTEGER_BITS_MP, + COORD_FRACTIONAL_BITS_MP_LOWPRECISION, + COORD_FRACTIONAL_BITS_MP_LOWPRECISION, + COORD_FRACTIONAL_BITS_MP_LOWPRECISION + COORD_INTEGER_BITS, + COORD_FRACTIONAL_BITS_MP_LOWPRECISION + COORD_INTEGER_BITS_MP + }; + unsigned int bits = ReadUBitLong( numbits_table[ (flags & (INBOUNDS|INTVAL)) + bLowPrecision*4 ] ); + + if ( flags & INTVAL ) + { + // Shuffle the bits to remap the integer portion from [0,N] to [1,N+1] + // and then paste in front of the fractional parts so we only need one + // int-to-float conversion. + + uint fracbitsMP = bits >> COORD_INTEGER_BITS_MP; + uint fracbits = bits >> COORD_INTEGER_BITS; + + uint intmaskMP = ((1<<COORD_INTEGER_BITS_MP)-1); + uint intmask = ((1<<COORD_INTEGER_BITS)-1); + + uint selectNotMP = (flags & INBOUNDS) - 1; + + fracbits -= fracbitsMP; + fracbits &= selectNotMP; + fracbits += fracbitsMP; + + intmask -= intmaskMP; + intmask &= selectNotMP; + intmask += intmaskMP; + + uint intpart = (bits & intmask) + 1; + uint intbitsLow = intpart << COORD_FRACTIONAL_BITS_MP_LOWPRECISION; + uint intbits = intpart << COORD_FRACTIONAL_BITS; + uint selectNotLow = (uint)bLowPrecision - 1; + + intbits -= intbitsLow; + intbits &= selectNotLow; + intbits += intbitsLow; + + bits = fracbits | intbits; + } + + return (int)bits * multiply; +} + +unsigned int bf_read::ReadBitCoordBits (void) +{ +#if defined( BB_PROFILING ) + VPROF( "bf_read::ReadBitCoordBits" ); +#endif + + unsigned int flags = ReadUBitLong(2); + if ( flags == 0 ) + return 0; + + static const int numbits_table[3] = + { + COORD_INTEGER_BITS + 1, + COORD_FRACTIONAL_BITS + 1, + COORD_INTEGER_BITS + COORD_FRACTIONAL_BITS + 1 + }; + return ReadUBitLong( numbits_table[ flags-1 ] ) * 4 + flags; +} + +unsigned int bf_read::ReadBitCoordMPBits( bool bIntegral, bool bLowPrecision ) +{ +#if defined( BB_PROFILING ) + VPROF( "bf_read::ReadBitCoordMPBits" ); +#endif + + unsigned int flags = ReadUBitLong(2); + enum { INBOUNDS=1, INTVAL=2 }; + int numbits = 0; + + if ( bIntegral ) + { + if ( flags & INTVAL ) + { + numbits = (flags & INBOUNDS) ? (1 + COORD_INTEGER_BITS_MP) : (1 + COORD_INTEGER_BITS); + } + else + { + return flags; // no extra bits + } + } + else + { + static const unsigned char numbits_table[8] = + { + 1 + COORD_FRACTIONAL_BITS, + 1 + COORD_FRACTIONAL_BITS, + 1 + COORD_FRACTIONAL_BITS + COORD_INTEGER_BITS, + 1 + COORD_FRACTIONAL_BITS + COORD_INTEGER_BITS_MP, + 1 + COORD_FRACTIONAL_BITS_MP_LOWPRECISION, + 1 + COORD_FRACTIONAL_BITS_MP_LOWPRECISION, + 1 + COORD_FRACTIONAL_BITS_MP_LOWPRECISION + COORD_INTEGER_BITS, + 1 + COORD_FRACTIONAL_BITS_MP_LOWPRECISION + COORD_INTEGER_BITS_MP + }; + numbits = numbits_table[ flags + bLowPrecision*4 ]; + } + + return flags + ReadUBitLong(numbits)*4; +} + +void bf_read::ReadBitVec3Coord( Vector& fa ) +{ + int xflag, yflag, zflag; + + // This vector must be initialized! Otherwise, If any of the flags aren't set, + // the corresponding component will not be read and will be stack garbage. + fa.Init( 0, 0, 0 ); + + xflag = ReadOneBit(); + yflag = ReadOneBit(); + zflag = ReadOneBit(); + + if ( xflag ) + fa[0] = ReadBitCoord(); + if ( yflag ) + fa[1] = ReadBitCoord(); + if ( zflag ) + fa[2] = ReadBitCoord(); +} + +float bf_read::ReadBitNormal (void) +{ + // Read the sign bit + int signbit = ReadOneBit(); + + // Read the fractional part + unsigned int fractval = ReadUBitLong( NORMAL_FRACTIONAL_BITS ); + + // Calculate the correct floating point value + float value = (float)fractval * NORMAL_RESOLUTION; + + // Fixup the sign if negative. + if ( signbit ) + value = -value; + + return value; +} + +void bf_read::ReadBitVec3Normal( Vector& fa ) +{ + int xflag = ReadOneBit(); + int yflag = ReadOneBit(); + + if (xflag) + fa[0] = ReadBitNormal(); + else + fa[0] = 0.0f; + + if (yflag) + fa[1] = ReadBitNormal(); + else + fa[1] = 0.0f; + + // The first two imply the third (but not its sign) + int znegative = ReadOneBit(); + + float fafafbfb = fa[0] * fa[0] + fa[1] * fa[1]; + if (fafafbfb < 1.0f) + fa[2] = sqrt( 1.0f - fafafbfb ); + else + fa[2] = 0.0f; + + if (znegative) + fa[2] = -fa[2]; +} + +void bf_read::ReadBitAngles( QAngle& fa ) +{ + Vector tmp; + ReadBitVec3Coord( tmp ); + fa.Init( tmp.x, tmp.y, tmp.z ); +} + +int64 bf_read::ReadLongLong() +{ + int64 retval; + uint *pLongs = (uint*)&retval; + + // Read the two DWORDs according to network endian + const short endianIndex = 0x0100; + byte *idx = (byte*)&endianIndex; + pLongs[*idx++] = ReadUBitLong(sizeof(long) << 3); + pLongs[*idx] = ReadUBitLong(sizeof(long) << 3); + + return retval; +} + +float bf_read::ReadFloat() +{ + float ret; + Assert( sizeof(ret) == 4 ); + ReadBits(&ret, 32); + + // Swap the float, since ReadBits reads raw data + LittleFloat( &ret, &ret ); + return ret; +} + +bool bf_read::ReadBytes(void *pOut, int nBytes) +{ + ReadBits(pOut, nBytes << 3); + return !IsOverflowed(); +} + +bool bf_read::ReadString( char *pStr, int maxLen, bool bLine, int *pOutNumChars ) +{ + Assert( maxLen != 0 ); + + bool bTooSmall = false; + int iChar = 0; + while(1) + { + char val = ReadChar(); + if ( val == 0 ) + break; + else if ( bLine && val == '\n' ) + break; + + if ( iChar < (maxLen-1) ) + { + pStr[iChar] = val; + ++iChar; + } + else + { + bTooSmall = true; + } + } + + // Make sure it's null-terminated. + Assert( iChar < maxLen ); + pStr[iChar] = 0; + + if ( pOutNumChars ) + *pOutNumChars = iChar; + + return !IsOverflowed() && !bTooSmall; +} + + +char* bf_read::ReadAndAllocateString( bool *pOverflow ) +{ + char str[2048]; + + int nChars; + bool bOverflow = !ReadString( str, sizeof( str ), false, &nChars ); + if ( pOverflow ) + *pOverflow = bOverflow; + + // Now copy into the output and return it; + char *pRet = new char[ nChars + 1 ]; + for ( int i=0; i <= nChars; i++ ) + pRet[i] = str[i]; + + return pRet; +} + +void bf_read::ExciseBits( int startbit, int bitstoremove ) +{ + int endbit = startbit + bitstoremove; + int remaining_to_end = m_nDataBits - endbit; + + bf_write temp; + temp.StartWriting( (void *)m_pData, m_nDataBits << 3, startbit ); + + Seek( endbit ); + + for ( int i = 0; i < remaining_to_end; i++ ) + { + temp.WriteOneBit( ReadOneBit() ); + } + + Seek( startbit ); + + m_nDataBits -= bitstoremove; + m_nDataBytes = m_nDataBits >> 3; +} + +int bf_read::CompareBitsAt( int offset, bf_read * RESTRICT other, int otherOffset, int numbits ) RESTRICT +{ + extern unsigned long g_ExtraMasks[33]; + + if ( numbits == 0 ) + return 0; + + int overflow1 = offset + numbits > m_nDataBits; + int overflow2 = otherOffset + numbits > other->m_nDataBits; + + int x = overflow1 | overflow2; + if ( x != 0 ) + return x; + + unsigned int iStartBit1 = offset & 31u; + unsigned int iStartBit2 = otherOffset & 31u; + unsigned long *pData1 = (unsigned long*)m_pData + (offset >> 5); + unsigned long *pData2 = (unsigned long*)other->m_pData + (otherOffset >> 5); + unsigned long *pData1End = pData1 + ((offset + numbits - 1) >> 5); + unsigned long *pData2End = pData2 + ((otherOffset + numbits - 1) >> 5); + + while ( numbits > 32 ) + { + x = LoadLittleDWord( (unsigned long*)pData1, 0 ) >> iStartBit1; + x ^= LoadLittleDWord( (unsigned long*)pData1, 1 ) << (32 - iStartBit1); + x ^= LoadLittleDWord( (unsigned long*)pData2, 0 ) >> iStartBit2; + x ^= LoadLittleDWord( (unsigned long*)pData2, 1 ) << (32 - iStartBit2); + if ( x != 0 ) + { + return x; + } + ++pData1; + ++pData2; + numbits -= 32; + } + + x = LoadLittleDWord( (unsigned long*)pData1, 0 ) >> iStartBit1; + x ^= LoadLittleDWord( (unsigned long*)pData1End, 0 ) << (32 - iStartBit1); + x ^= LoadLittleDWord( (unsigned long*)pData2, 0 ) >> iStartBit2; + x ^= LoadLittleDWord( (unsigned long*)pData2End, 0 ) << (32 - iStartBit2); + return x & g_ExtraMasks[ numbits ]; +} diff --git a/tier1/byteswap.cpp b/tier1/byteswap.cpp new file mode 100644 index 0000000..9f66297 --- /dev/null +++ b/tier1/byteswap.cpp @@ -0,0 +1,90 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Low level byte swapping routines. +// +// $NoKeywords: $ +//============================================================================= + +#include "byteswap.h" + +//----------------------------------------------------------------------------- +// Copy a single field from the input buffer to the output buffer, swapping the bytes if necessary +//----------------------------------------------------------------------------- +void CByteswap::SwapFieldToTargetEndian( void* pOutputBuffer, void *pData, typedescription_t *pField ) +{ + switch ( pField->fieldType ) + { + case FIELD_CHARACTER: + SwapBufferToTargetEndian<char>( (char*)pOutputBuffer, (char*)pData, pField->fieldSize ); + break; + + case FIELD_BOOLEAN: + SwapBufferToTargetEndian<bool>( (bool*)pOutputBuffer, (bool*)pData, pField->fieldSize ); + break; + + case FIELD_SHORT: + SwapBufferToTargetEndian<short>( (short*)pOutputBuffer, (short*)pData, pField->fieldSize ); + break; + + case FIELD_FLOAT: + SwapBufferToTargetEndian<uint>( (uint*)pOutputBuffer, (uint*)pData, pField->fieldSize ); + break; + + case FIELD_INTEGER: + SwapBufferToTargetEndian<int>( (int*)pOutputBuffer, (int*)pData, pField->fieldSize ); + break; + + case FIELD_VECTOR: + SwapBufferToTargetEndian<uint>( (uint*)pOutputBuffer, (uint*)pData, pField->fieldSize * 3 ); + break; + + case FIELD_VECTOR2D: + SwapBufferToTargetEndian<uint>( (uint*)pOutputBuffer, (uint*)pData, pField->fieldSize * 2 ); + break; + + case FIELD_QUATERNION: + SwapBufferToTargetEndian<uint>( (uint*)pOutputBuffer, (uint*)pData, pField->fieldSize * 4 ); + break; + + case FIELD_EMBEDDED: + { + typedescription_t *pEmbed = pField->td->dataDesc; + for ( int i = 0; i < pField->fieldSize; ++i ) + { + SwapFieldsToTargetEndian( (byte*)pOutputBuffer + pEmbed->fieldOffset[ TD_OFFSET_NORMAL ], + (byte*)pData + pEmbed->fieldOffset[ TD_OFFSET_NORMAL ], + pField->td ); + + pOutputBuffer = (byte*)pOutputBuffer + pField->fieldSizeInBytes; + pData = (byte*)pData + pField->fieldSizeInBytes; + } + } + break; + + default: + assert(0); + } +} + +//----------------------------------------------------------------------------- +// Write a block of fields. Works a bit like the saverestore code. +//----------------------------------------------------------------------------- +void CByteswap::SwapFieldsToTargetEndian( void *pOutputBuffer, void *pBaseData, datamap_t *pDataMap ) +{ + // deal with base class first + if ( pDataMap->baseMap ) + { + SwapFieldsToTargetEndian( pOutputBuffer, pBaseData, pDataMap->baseMap ); + } + + typedescription_t *pFields = pDataMap->dataDesc; + int fieldCount = pDataMap->dataNumFields; + for ( int i = 0; i < fieldCount; ++i ) + { + typedescription_t *pField = &pFields[i]; + SwapFieldToTargetEndian( (BYTE*)pOutputBuffer + pField->fieldOffset[ TD_OFFSET_NORMAL ], + (BYTE*)pBaseData + pField->fieldOffset[ TD_OFFSET_NORMAL ], + pField ); + } +} + diff --git a/tier1/characterset.cpp b/tier1/characterset.cpp new file mode 100644 index 0000000..a7c518b --- /dev/null +++ b/tier1/characterset.cpp @@ -0,0 +1,41 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $Workfile: $ +// $Date: $ +// +//----------------------------------------------------------------------------- +// $Log: $ +// +// $NoKeywords: $ +//============================================================================= + +#include <string.h> +#include "characterset.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +//----------------------------------------------------------------------------- +// Purpose: builds a simple lookup table of a group of important characters +// Input : *pParseGroup - pointer to the buffer for the group +// *pGroupString - null terminated list of characters to flag +//----------------------------------------------------------------------------- +void CharacterSetBuild( characterset_t *pSetBuffer, const char *pszSetString ) +{ + int i = 0; + + // Test our pointers + if ( !pSetBuffer || !pszSetString ) + return; + + memset( pSetBuffer->set, 0, sizeof(pSetBuffer->set) ); + + while ( pszSetString[i] ) + { + pSetBuffer->set[ (unsigned)pszSetString[i] ] = 1; + i++; + } + +} diff --git a/tier1/checksum_crc.cpp b/tier1/checksum_crc.cpp new file mode 100644 index 0000000..b9dacbb --- /dev/null +++ b/tier1/checksum_crc.cpp @@ -0,0 +1,180 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Generic CRC functions +// +//=============================================================================// + +#include "basetypes.h" +#include "commonmacros.h" +#include "checksum_crc.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +#define CRC32_INIT_VALUE 0xFFFFFFFFUL +#define CRC32_XOR_VALUE 0xFFFFFFFFUL + +#define NUM_BYTES 256 +static const CRC32_t pulCRCTable[NUM_BYTES] = +{ + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, + 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, + 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, + 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, + 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, + 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, + 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, + 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, + 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, + 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, + 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, + 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, + 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, + 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, + 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, + 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, + 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, + 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, + 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, + 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, + 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, + 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, + 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, + 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, + 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, + 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, + 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, + 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, + 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, + 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, + 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, + 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, + 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, + 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, + 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, + 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, + 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, + 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, + 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d +}; + +void CRC32_Init(CRC32_t *pulCRC) +{ + *pulCRC = CRC32_INIT_VALUE; +} + +void CRC32_Final(CRC32_t *pulCRC) +{ + *pulCRC ^= CRC32_XOR_VALUE; +} + +CRC32_t CRC32_GetTableEntry( unsigned int slot ) +{ + return pulCRCTable[(unsigned char)slot]; +} + +void CRC32_ProcessBuffer(CRC32_t *pulCRC, const void *pBuffer, int nBuffer) +{ + CRC32_t ulCrc = *pulCRC; + unsigned char *pb = (unsigned char *)pBuffer; + unsigned int nFront; + int nMain; + +JustAfew: + + switch (nBuffer) + { + case 7: + ulCrc = pulCRCTable[*pb++ ^ (unsigned char)ulCrc] ^ (ulCrc >> 8); + + case 6: + ulCrc = pulCRCTable[*pb++ ^ (unsigned char)ulCrc] ^ (ulCrc >> 8); + + case 5: + ulCrc = pulCRCTable[*pb++ ^ (unsigned char)ulCrc] ^ (ulCrc >> 8); + + case 4: + ulCrc ^= LittleLong( *(CRC32_t *)pb ); + ulCrc = pulCRCTable[(unsigned char)ulCrc] ^ (ulCrc >> 8); + ulCrc = pulCRCTable[(unsigned char)ulCrc] ^ (ulCrc >> 8); + ulCrc = pulCRCTable[(unsigned char)ulCrc] ^ (ulCrc >> 8); + ulCrc = pulCRCTable[(unsigned char)ulCrc] ^ (ulCrc >> 8); + *pulCRC = ulCrc; + return; + + case 3: + ulCrc = pulCRCTable[*pb++ ^ (unsigned char)ulCrc] ^ (ulCrc >> 8); + + case 2: + ulCrc = pulCRCTable[*pb++ ^ (unsigned char)ulCrc] ^ (ulCrc >> 8); + + case 1: + ulCrc = pulCRCTable[*pb++ ^ (unsigned char)ulCrc] ^ (ulCrc >> 8); + + case 0: + *pulCRC = ulCrc; + return; + } + + // We may need to do some alignment work up front, and at the end, so that + // the main loop is aligned and only has to worry about 8 byte at a time. + // + // The low-order two bits of pb and nBuffer in total control the + // upfront work. + // + nFront = ((unsigned int)pb) & 3; + nBuffer -= nFront; + switch (nFront) + { + case 3: + ulCrc = pulCRCTable[*pb++ ^ (unsigned char)ulCrc] ^ (ulCrc >> 8); + case 2: + ulCrc = pulCRCTable[*pb++ ^ (unsigned char)ulCrc] ^ (ulCrc >> 8); + case 1: + ulCrc = pulCRCTable[*pb++ ^ (unsigned char)ulCrc] ^ (ulCrc >> 8); + } + + nMain = nBuffer >> 3; + while (nMain--) + { + ulCrc ^= LittleLong( *(CRC32_t *)pb ); + ulCrc = pulCRCTable[(unsigned char)ulCrc] ^ (ulCrc >> 8); + ulCrc = pulCRCTable[(unsigned char)ulCrc] ^ (ulCrc >> 8); + ulCrc = pulCRCTable[(unsigned char)ulCrc] ^ (ulCrc >> 8); + ulCrc = pulCRCTable[(unsigned char)ulCrc] ^ (ulCrc >> 8); + ulCrc ^= LittleLong( *(CRC32_t *)(pb + 4) ); + ulCrc = pulCRCTable[(unsigned char)ulCrc] ^ (ulCrc >> 8); + ulCrc = pulCRCTable[(unsigned char)ulCrc] ^ (ulCrc >> 8); + ulCrc = pulCRCTable[(unsigned char)ulCrc] ^ (ulCrc >> 8); + ulCrc = pulCRCTable[(unsigned char)ulCrc] ^ (ulCrc >> 8); + pb += 8; + } + + nBuffer &= 7; + goto JustAfew; +} diff --git a/tier1/checksum_md5.cpp b/tier1/checksum_md5.cpp new file mode 100644 index 0000000..4276d0a --- /dev/null +++ b/tier1/checksum_md5.cpp @@ -0,0 +1,305 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +//===========================================================================// + +#include "basetypes.h" +#include "commonmacros.h" +#include "checksum_md5.h" +#include <string.h> +#include <stdio.h> +#include "tier1/strtools.h" +#include "tier0/dbg.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +// The four core functions - F1 is optimized somewhat +// #define F1(x, y, z) (x & y | ~x & z) +#define F1(x, y, z) (z ^ (x & (y ^ z))) +#define F2(x, y, z) F1(z, x, y) +#define F3(x, y, z) (x ^ y ^ z) +#define F4(x, y, z) (y ^ (x | ~z)) + +// This is the central step in the MD5 algorithm. +#define MD5STEP(f, w, x, y, z, data, s) \ + ( w += f(x, y, z) + data, w = w<<s | w>>(32-s), w += x ) + +//----------------------------------------------------------------------------- +// Purpose: The core of the MD5 algorithm, this alters an existing MD5 hash to +// reflect the addition of 16 longwords of new data. MD5Update blocks +// the data and converts bytes into longwords for this routine. +// Input : buf[4] - +// in[16] - +// Output : static void +//----------------------------------------------------------------------------- +static void MD5Transform(unsigned int buf[4], unsigned int const in[16]) +{ + register unsigned int a, b, c, d; + + a = buf[0]; + b = buf[1]; + c = buf[2]; + d = buf[3]; + + MD5STEP(F1, a, b, c, d, in[0] + 0xd76aa478, 7); + MD5STEP(F1, d, a, b, c, in[1] + 0xe8c7b756, 12); + MD5STEP(F1, c, d, a, b, in[2] + 0x242070db, 17); + MD5STEP(F1, b, c, d, a, in[3] + 0xc1bdceee, 22); + MD5STEP(F1, a, b, c, d, in[4] + 0xf57c0faf, 7); + MD5STEP(F1, d, a, b, c, in[5] + 0x4787c62a, 12); + MD5STEP(F1, c, d, a, b, in[6] + 0xa8304613, 17); + MD5STEP(F1, b, c, d, a, in[7] + 0xfd469501, 22); + MD5STEP(F1, a, b, c, d, in[8] + 0x698098d8, 7); + MD5STEP(F1, d, a, b, c, in[9] + 0x8b44f7af, 12); + MD5STEP(F1, c, d, a, b, in[10] + 0xffff5bb1, 17); + MD5STEP(F1, b, c, d, a, in[11] + 0x895cd7be, 22); + MD5STEP(F1, a, b, c, d, in[12] + 0x6b901122, 7); + MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12); + MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17); + MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22); + + MD5STEP(F2, a, b, c, d, in[1] + 0xf61e2562, 5); + MD5STEP(F2, d, a, b, c, in[6] + 0xc040b340, 9); + MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14); + MD5STEP(F2, b, c, d, a, in[0] + 0xe9b6c7aa, 20); + MD5STEP(F2, a, b, c, d, in[5] + 0xd62f105d, 5); + MD5STEP(F2, d, a, b, c, in[10] + 0x02441453, 9); + MD5STEP(F2, c, d, a, b, in[15] + 0xd8a1e681, 14); + MD5STEP(F2, b, c, d, a, in[4] + 0xe7d3fbc8, 20); + MD5STEP(F2, a, b, c, d, in[9] + 0x21e1cde6, 5); + MD5STEP(F2, d, a, b, c, in[14] + 0xc33707d6, 9); + MD5STEP(F2, c, d, a, b, in[3] + 0xf4d50d87, 14); + MD5STEP(F2, b, c, d, a, in[8] + 0x455a14ed, 20); + MD5STEP(F2, a, b, c, d, in[13] + 0xa9e3e905, 5); + MD5STEP(F2, d, a, b, c, in[2] + 0xfcefa3f8, 9); + MD5STEP(F2, c, d, a, b, in[7] + 0x676f02d9, 14); + MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20); + + MD5STEP(F3, a, b, c, d, in[5] + 0xfffa3942, 4); + MD5STEP(F3, d, a, b, c, in[8] + 0x8771f681, 11); + MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16); + MD5STEP(F3, b, c, d, a, in[14] + 0xfde5380c, 23); + MD5STEP(F3, a, b, c, d, in[1] + 0xa4beea44, 4); + MD5STEP(F3, d, a, b, c, in[4] + 0x4bdecfa9, 11); + MD5STEP(F3, c, d, a, b, in[7] + 0xf6bb4b60, 16); + MD5STEP(F3, b, c, d, a, in[10] + 0xbebfbc70, 23); + MD5STEP(F3, a, b, c, d, in[13] + 0x289b7ec6, 4); + MD5STEP(F3, d, a, b, c, in[0] + 0xeaa127fa, 11); + MD5STEP(F3, c, d, a, b, in[3] + 0xd4ef3085, 16); + MD5STEP(F3, b, c, d, a, in[6] + 0x04881d05, 23); + MD5STEP(F3, a, b, c, d, in[9] + 0xd9d4d039, 4); + MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11); + MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16); + MD5STEP(F3, b, c, d, a, in[2] + 0xc4ac5665, 23); + + MD5STEP(F4, a, b, c, d, in[0] + 0xf4292244, 6); + MD5STEP(F4, d, a, b, c, in[7] + 0x432aff97, 10); + MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15); + MD5STEP(F4, b, c, d, a, in[5] + 0xfc93a039, 21); + MD5STEP(F4, a, b, c, d, in[12] + 0x655b59c3, 6); + MD5STEP(F4, d, a, b, c, in[3] + 0x8f0ccc92, 10); + MD5STEP(F4, c, d, a, b, in[10] + 0xffeff47d, 15); + MD5STEP(F4, b, c, d, a, in[1] + 0x85845dd1, 21); + MD5STEP(F4, a, b, c, d, in[8] + 0x6fa87e4f, 6); + MD5STEP(F4, d, a, b, c, in[15] + 0xfe2ce6e0, 10); + MD5STEP(F4, c, d, a, b, in[6] + 0xa3014314, 15); + MD5STEP(F4, b, c, d, a, in[13] + 0x4e0811a1, 21); + MD5STEP(F4, a, b, c, d, in[4] + 0xf7537e82, 6); + MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10); + MD5STEP(F4, c, d, a, b, in[2] + 0x2ad7d2bb, 15); + MD5STEP(F4, b, c, d, a, in[9] + 0xeb86d391, 21); + + buf[0] += a; + buf[1] += b; + buf[2] += c; + buf[3] += d; +} + +//----------------------------------------------------------------------------- +// Purpose: Start MD5 accumulation. Set bit count to 0 and buffer to mysterious initialization constants. + +// Input : *ctx - +//----------------------------------------------------------------------------- +void MD5Init(MD5Context_t *ctx) +{ + ctx->buf[0] = 0x67452301; + ctx->buf[1] = 0xefcdab89; + ctx->buf[2] = 0x98badcfe; + ctx->buf[3] = 0x10325476; + + ctx->bits[0] = 0; + ctx->bits[1] = 0; +} + +//----------------------------------------------------------------------------- +// Purpose: Update context to reflect the concatenation of another buffer full of bytes. +// Input : *ctx - +// *buf - +// len - +//----------------------------------------------------------------------------- +void MD5Update(MD5Context_t *ctx, unsigned char const *buf, unsigned int len) +{ + unsigned int t; + + /* Update bitcount */ + + t = ctx->bits[0]; + if ((ctx->bits[0] = t + ((unsigned int) len << 3)) < t) + ctx->bits[1]++; /* Carry from low to high */ + ctx->bits[1] += len >> 29; + + t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */ + + /* Handle any leading odd-sized chunks */ + + if (t) + { + unsigned char *p = (unsigned char *) ctx->in + t; + + t = 64 - t; + if (len < t) + { + memcpy(p, buf, len); + return; + } + memcpy(p, buf, t); + //byteReverse(ctx->in, 16); + MD5Transform(ctx->buf, (unsigned int *) ctx->in); + buf += t; + len -= t; + } + /* Process data in 64-byte chunks */ + + while (len >= 64) + { + memcpy(ctx->in, buf, 64); + //byteReverse(ctx->in, 16); + MD5Transform(ctx->buf, (unsigned int *) ctx->in); + buf += 64; + len -= 64; + } + + /* Handle any remaining bytes of data. */ + memcpy(ctx->in, buf, len); +} + +//----------------------------------------------------------------------------- +// Purpose: Final wrapup - pad to 64-byte boundary with the bit pattern +// 1 0* (64-bit count of bits processed, MSB-first) +// Input : digest[MD5_DIGEST_LENGTH] - +// *ctx - +//----------------------------------------------------------------------------- +void MD5Final(unsigned char digest[MD5_DIGEST_LENGTH], MD5Context_t *ctx) +{ + unsigned count; + unsigned char *p; + + /* Compute number of bytes mod 64 */ + count = (ctx->bits[0] >> 3) & 0x3F; + + /* Set the first char of padding to 0x80. This is safe since there is + always at least one byte free */ + p = ctx->in + count; + *p++ = 0x80; + + /* Bytes of padding needed to make 64 bytes */ + count = 64 - 1 - count; + + /* Pad out to 56 mod 64 */ + if (count < 8) + { + /* Two lots of padding: Pad the first block to 64 bytes */ + memset(p, 0, count); + //byteReverse(ctx->in, 16); + MD5Transform(ctx->buf, (unsigned int *) ctx->in); + + /* Now fill the next block with 56 bytes */ + memset(ctx->in, 0, 56); + } + else + { + /* Pad block to 56 bytes */ + memset(p, 0, count - 8); + } + //byteReverse(ctx->in, 14); + + /* Append length in bits and transform */ + ((unsigned int *) ctx->in)[14] = ctx->bits[0]; + ((unsigned int *) ctx->in)[15] = ctx->bits[1]; + + MD5Transform(ctx->buf, (unsigned int *) ctx->in); + //byteReverse((unsigned char *) ctx->buf, 4); + memcpy(digest, ctx->buf, MD5_DIGEST_LENGTH); + memset(ctx, 0, sizeof(*ctx)); /* In case it's sensitive */ +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *hash - +// hashlen - +// Output : char +//----------------------------------------------------------------------------- +char *MD5_Print( unsigned char *hash, int hashlen ) +{ + static char szReturn[64]; + + Assert( hashlen <= 32 ); + + Q_binarytohex( hash, hashlen, szReturn, sizeof( szReturn ) ); + return szReturn; +} + +//----------------------------------------------------------------------------- +// Purpose: generate pseudo random number from a seed number +// Input : seed number +// Output : pseudo random number +//----------------------------------------------------------------------------- +unsigned int MD5_PseudoRandom(unsigned int nSeed) +{ + MD5Context_t ctx; + unsigned char digest[MD5_DIGEST_LENGTH]; // The MD5 Hash + + memset( &ctx, 0, sizeof( ctx ) ); + + MD5Init(&ctx); + MD5Update(&ctx, (unsigned char*)&nSeed, sizeof(nSeed) ); + MD5Final(digest, &ctx); + + return *(unsigned int*)(digest+6); // use 4 middle bytes for random value +} + +//----------------------------------------------------------------------------- +bool MD5_Compare( const MD5Value_t &data, const MD5Value_t &compare ) +{ + return V_memcmp( data.bits, compare.bits, MD5_DIGEST_LENGTH ) == 0; +} + +//----------------------------------------------------------------------------- +void MD5Value_t::Zero() +{ + V_memset( bits, 0, sizeof( bits ) ); +} + +//----------------------------------------------------------------------------- +bool MD5Value_t::IsZero() const +{ + for ( int i = 0 ; i < Q_ARRAYSIZE( bits ) ; ++i ) + { + if ( bits[i] != 0 ) + return false; + } + + return true; +} + +//----------------------------------------------------------------------------- +void MD5_ProcessSingleBuffer( const void *p, int len, MD5Value_t &md5Result ) +{ + Assert( len >= 0 ); + MD5Context_t ctx; + MD5Init( &ctx ); + MD5Update( &ctx, (unsigned char const *)p, len ); + MD5Final( md5Result.bits, &ctx ); +} diff --git a/tier1/checksum_sha1.cpp b/tier1/checksum_sha1.cpp new file mode 100644 index 0000000..ee6e27c --- /dev/null +++ b/tier1/checksum_sha1.cpp @@ -0,0 +1,299 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Implementation of SHA-1 +// +//============================================================================= + +/* + 100% free public domain implementation of the SHA-1 + algorithm by Dominik Reichl <[email protected]> + + + === Test Vectors (from FIPS PUB 180-1) === + + SHA1("abc") = + A9993E36 4706816A BA3E2571 7850C26C 9CD0D89D + + SHA1("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") = + 84983E44 1C3BD26E BAAE4AA1 F95129E5 E54670F1 + + SHA1(A million repetitions of "a") = + 34AA973C D4C4DAA4 F61EEB2B DBAD2731 6534016F +*/ + +#if !defined(_MINIMUM_BUILD_) +#include "checksum_sha1.h" +#else +// +// This path is build in the CEG/DRM projects where we require that no CRT references are made ! +// +#include <intrin.h> // memcpy, memset etc... will be inlined. +#include "tier1/checksum_sha1.h" +#endif + +#define MAX_FILE_READ_BUFFER 8000 + +// Rotate x bits to the left +#ifndef ROL32 +#define ROL32(_val32, _nBits) (((_val32)<<(_nBits))|((_val32)>>(32-(_nBits)))) +#endif + +#ifdef SHA1_LITTLE_ENDIAN + #define SHABLK0(i) (m_block->l[i] = \ + (ROL32(m_block->l[i],24) & 0xFF00FF00) | (ROL32(m_block->l[i],8) & 0x00FF00FF)) +#else + #define SHABLK0(i) (m_block->l[i]) +#endif + +#define SHABLK(i) (m_block->l[i&15] = ROL32(m_block->l[(i+13)&15] ^ m_block->l[(i+8)&15] \ + ^ m_block->l[(i+2)&15] ^ m_block->l[i&15],1)) + +// SHA-1 rounds +#define _R0(v,w,x,y,z,i) { z+=((w&(x^y))^y)+SHABLK0(i)+0x5A827999+ROL32(v,5); w=ROL32(w,30); } +#define _R1(v,w,x,y,z,i) { z+=((w&(x^y))^y)+SHABLK(i)+0x5A827999+ROL32(v,5); w=ROL32(w,30); } +#define _R2(v,w,x,y,z,i) { z+=(w^x^y)+SHABLK(i)+0x6ED9EBA1+ROL32(v,5); w=ROL32(w,30); } +#define _R3(v,w,x,y,z,i) { z+=(((w|x)&y)|(w&x))+SHABLK(i)+0x8F1BBCDC+ROL32(v,5); w=ROL32(w,30); } +#define _R4(v,w,x,y,z,i) { z+=(w^x^y)+SHABLK(i)+0xCA62C1D6+ROL32(v,5); w=ROL32(w,30); } + +#ifdef _MINIMUM_BUILD_ +Minimum_CSHA1::Minimum_CSHA1() +#else +CSHA1::CSHA1() +#endif +{ + m_block = (SHA1_WORKSPACE_BLOCK *)m_workspace; + + Reset(); +} +#ifdef _MINIMUM_BUILD_ +Minimum_CSHA1::~Minimum_CSHA1() +#else +CSHA1::~CSHA1() +#endif +{ + // Reset(); +} +#ifdef _MINIMUM_BUILD_ +void Minimum_CSHA1::Reset() +#else +void CSHA1::Reset() +#endif +{ + // SHA1 initialization constants + m_state[0] = 0x67452301; + m_state[1] = 0xEFCDAB89; + m_state[2] = 0x98BADCFE; + m_state[3] = 0x10325476; + m_state[4] = 0xC3D2E1F0; + + m_count[0] = 0; + m_count[1] = 0; +} + +#ifdef _MINIMUM_BUILD_ +void Minimum_CSHA1::Transform(unsigned long state[5], unsigned char buffer[64]) +#else +void CSHA1::Transform(unsigned long state[5], unsigned char buffer[64]) +#endif +{ + unsigned long a = 0, b = 0, c = 0, d = 0, e = 0; + + memcpy(m_block, buffer, 64); + + // Copy state[] to working vars + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + + // 4 rounds of 20 operations each. Loop unrolled. + _R0(a,b,c,d,e, 0); _R0(e,a,b,c,d, 1); _R0(d,e,a,b,c, 2); _R0(c,d,e,a,b, 3); + _R0(b,c,d,e,a, 4); _R0(a,b,c,d,e, 5); _R0(e,a,b,c,d, 6); _R0(d,e,a,b,c, 7); + _R0(c,d,e,a,b, 8); _R0(b,c,d,e,a, 9); _R0(a,b,c,d,e,10); _R0(e,a,b,c,d,11); + _R0(d,e,a,b,c,12); _R0(c,d,e,a,b,13); _R0(b,c,d,e,a,14); _R0(a,b,c,d,e,15); + _R1(e,a,b,c,d,16); _R1(d,e,a,b,c,17); _R1(c,d,e,a,b,18); _R1(b,c,d,e,a,19); + _R2(a,b,c,d,e,20); _R2(e,a,b,c,d,21); _R2(d,e,a,b,c,22); _R2(c,d,e,a,b,23); + _R2(b,c,d,e,a,24); _R2(a,b,c,d,e,25); _R2(e,a,b,c,d,26); _R2(d,e,a,b,c,27); + _R2(c,d,e,a,b,28); _R2(b,c,d,e,a,29); _R2(a,b,c,d,e,30); _R2(e,a,b,c,d,31); + _R2(d,e,a,b,c,32); _R2(c,d,e,a,b,33); _R2(b,c,d,e,a,34); _R2(a,b,c,d,e,35); + _R2(e,a,b,c,d,36); _R2(d,e,a,b,c,37); _R2(c,d,e,a,b,38); _R2(b,c,d,e,a,39); + _R3(a,b,c,d,e,40); _R3(e,a,b,c,d,41); _R3(d,e,a,b,c,42); _R3(c,d,e,a,b,43); + _R3(b,c,d,e,a,44); _R3(a,b,c,d,e,45); _R3(e,a,b,c,d,46); _R3(d,e,a,b,c,47); + _R3(c,d,e,a,b,48); _R3(b,c,d,e,a,49); _R3(a,b,c,d,e,50); _R3(e,a,b,c,d,51); + _R3(d,e,a,b,c,52); _R3(c,d,e,a,b,53); _R3(b,c,d,e,a,54); _R3(a,b,c,d,e,55); + _R3(e,a,b,c,d,56); _R3(d,e,a,b,c,57); _R3(c,d,e,a,b,58); _R3(b,c,d,e,a,59); + _R4(a,b,c,d,e,60); _R4(e,a,b,c,d,61); _R4(d,e,a,b,c,62); _R4(c,d,e,a,b,63); + _R4(b,c,d,e,a,64); _R4(a,b,c,d,e,65); _R4(e,a,b,c,d,66); _R4(d,e,a,b,c,67); + _R4(c,d,e,a,b,68); _R4(b,c,d,e,a,69); _R4(a,b,c,d,e,70); _R4(e,a,b,c,d,71); + _R4(d,e,a,b,c,72); _R4(c,d,e,a,b,73); _R4(b,c,d,e,a,74); _R4(a,b,c,d,e,75); + _R4(e,a,b,c,d,76); _R4(d,e,a,b,c,77); _R4(c,d,e,a,b,78); _R4(b,c,d,e,a,79); + + // Add the working vars back into state[] + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; + + // Wipe variables + a = b = c = d = e = 0; +} + +// Use this function to hash in binary data and strings +#ifdef _MINIMUM_BUILD_ +void Minimum_CSHA1::Update(unsigned char *data, unsigned int len) +#else +void CSHA1::Update(unsigned char *data, unsigned int len) +#endif +{ + unsigned long i = 0, j; + + j = (m_count[0] >> 3) & 63; + + if((m_count[0] += len << 3) < (len << 3)) m_count[1]++; + + m_count[1] += (len >> 29); + + if((j + len) > 63) + { + memcpy(&m_buffer[j], data, (i = 64 - j)); + Transform(m_state, m_buffer); + + for (; i+63 < len; i += 64) + Transform(m_state, &data[i]); + + j = 0; + } + else i = 0; + + memcpy(&m_buffer[j], &data[i], len - i); +} + +#if !defined(_MINIMUM_BUILD_) +// Hash in file contents +bool CSHA1::HashFile(char *szFileName) +{ + unsigned long ulFileSize = 0, ulRest = 0, ulBlocks = 0; + unsigned long i = 0; + unsigned char uData[MAX_FILE_READ_BUFFER]; + FILE *fIn = NULL; + + if(szFileName == NULL) return(false); + + if((fIn = fopen(szFileName, "rb")) == NULL) return(false); + + fseek(fIn, 0, SEEK_END); + ulFileSize = ftell(fIn); + fseek(fIn, 0, SEEK_SET); + + ulRest = ulFileSize % MAX_FILE_READ_BUFFER; + ulBlocks = ulFileSize / MAX_FILE_READ_BUFFER; + + for(i = 0; i < ulBlocks; i++) + { + fread(uData, 1, MAX_FILE_READ_BUFFER, fIn); + Update(uData, MAX_FILE_READ_BUFFER); + } + + if(ulRest != 0) + { + fread(uData, 1, ulRest, fIn); + Update(uData, ulRest); + } + + fclose(fIn); + fIn = NULL; + + return(true); +} +#endif + +#ifdef _MINIMUM_BUILD_ +void Minimum_CSHA1::Final() +#else +void CSHA1::Final() +#endif +{ + unsigned long i = 0; + unsigned char finalcount[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; + + for (i = 0; i < 8; i++) + finalcount[i] = (unsigned char)((m_count[(i >= 4 ? 0 : 1)] + >> ((3 - (i & 3)) * 8) ) & 255); // Endian independent + + Update((unsigned char *)"\200", 1); + + while ((m_count[0] & 504) != 448) + Update((unsigned char *)"\0", 1); + + Update(finalcount, 8); // Cause a SHA1Transform() + + for (i = 0; i < k_cubHash; i++) + { + m_digest[i] = (unsigned char)((m_state[i >> 2] >> ((3 - (i & 3)) * 8) ) & 255); + } + + // Wipe variables for security reasons + i = 0; + memset(m_buffer, 0, sizeof(m_buffer) ); + memset(m_state, 0, sizeof(m_state) ); + memset(m_count, 0, sizeof(m_count) ); + memset(finalcount, 0, sizeof( finalcount) ); + + Transform(m_state, m_buffer); +} + +#if !defined(_MINIMUM_BUILD_) +// Get the final hash as a pre-formatted string +void CSHA1::ReportHash(char *szReport, unsigned char uReportType) +{ + unsigned char i = 0; + char szTemp[12]; + + if(szReport == NULL) return; + + if(uReportType == REPORT_HEX) + { + sprintf(szTemp, "%02X", m_digest[0]); + strcat(szReport, szTemp); + + for(i = 1; i < k_cubHash; i++) + { + sprintf(szTemp, " %02X", m_digest[i]); + strcat(szReport, szTemp); + } + } + else if(uReportType == REPORT_DIGIT) + { + sprintf(szTemp, "%u", m_digest[0]); + strcat(szReport, szTemp); + + for(i = 1; i < k_cubHash; i++) + { + sprintf(szTemp, " %u", m_digest[i]); + strcat(szReport, szTemp); + } + } + else strcpy(szReport, "Error: Unknown report type!"); +} +#endif // _MINIMUM_BUILD_ + +// Get the raw message digest +#ifdef _MINIMUM_BUILD_ +void Minimum_CSHA1::GetHash(unsigned char *uDest) +#else +void CSHA1::GetHash(unsigned char *uDest) +#endif +{ + memcpy(uDest, m_digest, k_cubHash); +} + +#ifndef _MINIMUM_BUILD_ +// utility hash comparison function +bool HashLessFunc( SHADigest_t const &lhs, SHADigest_t const &rhs ) +{ + int iRes = memcmp( &lhs, &rhs, sizeof( SHADigest_t ) ); + return ( iRes < 0 ); +} +#endif diff --git a/tier1/commandbuffer.cpp b/tier1/commandbuffer.cpp new file mode 100644 index 0000000..2897d03 --- /dev/null +++ b/tier1/commandbuffer.cpp @@ -0,0 +1,636 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $Workfile: $ +// $Date: $ +// $NoKeywords: $ +//===========================================================================// + +#include "tier1/CommandBuffer.h" +#include "tier1/utlbuffer.h" +#include "tier1/strtools.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +#define MAX_ALIAS_NAME 32 +#define MAX_COMMAND_LENGTH 1024 + +struct cmdalias_t +{ + cmdalias_t *next; + char name[ MAX_ALIAS_NAME ]; + char *value; +}; + + +//----------------------------------------------------------------------------- +// Constructor, destructor +//----------------------------------------------------------------------------- +CCommandBuffer::CCommandBuffer( ) : m_Commands( 32, 32 ) +{ + m_hNextCommand = m_Commands.InvalidIndex(); + m_nWaitDelayTicks = 1; + m_nCurrentTick = 0; + m_nLastTickToProcess = -1; + m_nArgSBufferSize = 0; + m_bIsProcessingCommands = false; + m_nMaxArgSBufferLength = ARGS_BUFFER_LENGTH; +} + +CCommandBuffer::~CCommandBuffer() +{ +} + + +//----------------------------------------------------------------------------- +// Indicates how long to delay when encoutering a 'wait' command +//----------------------------------------------------------------------------- +void CCommandBuffer::SetWaitDelayTime( int nTickDelay ) +{ + Assert( nTickDelay >= 0 ); + m_nWaitDelayTicks = nTickDelay; +} + + +//----------------------------------------------------------------------------- +// Specifies a max limit of the args buffer. For unittesting. Size == 0 means use default +//----------------------------------------------------------------------------- +void CCommandBuffer::LimitArgumentBufferSize( int nSize ) +{ + if ( nSize > ARGS_BUFFER_LENGTH ) + { + nSize = ARGS_BUFFER_LENGTH; + } + + m_nMaxArgSBufferLength = ( nSize == 0 ) ? ARGS_BUFFER_LENGTH : nSize; +} + + +//----------------------------------------------------------------------------- +// Parses argv0 out of the buffer +//----------------------------------------------------------------------------- +bool CCommandBuffer::ParseArgV0( CUtlBuffer &buf, char *pArgV0, int nMaxLen, const char **pArgS ) +{ + pArgV0[0] = 0; + *pArgS = NULL; + + if ( !buf.IsValid() ) + return false; + + int nSize = buf.ParseToken( CCommand::DefaultBreakSet(), pArgV0, nMaxLen ); + if ( ( nSize <= 0 ) || ( nMaxLen == nSize ) ) + return false; + + int nArgSLen = buf.TellMaxPut() - buf.TellGet(); + *pArgS = (nArgSLen > 0) ? (const char*)buf.PeekGet() : NULL; + return true; +} + + +//----------------------------------------------------------------------------- +// Insert a command into the command queue +//----------------------------------------------------------------------------- +void CCommandBuffer::InsertCommandAtAppropriateTime( int hCommand ) +{ + int i; + Command_t &command = m_Commands[hCommand]; + for ( i = m_Commands.Head(); i != m_Commands.InvalidIndex(); i = m_Commands.Next(i) ) + { + if ( m_Commands[i].m_nTick > command.m_nTick ) + break; + } + m_Commands.LinkBefore( i, hCommand ); +} + + +//----------------------------------------------------------------------------- +// Insert a command into the command queue at the appropriate time +//----------------------------------------------------------------------------- +void CCommandBuffer::InsertImmediateCommand( int hCommand ) +{ + m_Commands.LinkBefore( m_hNextCommand, hCommand ); +} + + +//----------------------------------------------------------------------------- +// Insert a command into the command queue +//----------------------------------------------------------------------------- +bool CCommandBuffer::InsertCommand( const char *pArgS, int nCommandSize, int nTick ) +{ + if ( nCommandSize >= CCommand::MaxCommandLength() ) + { + Warning( "WARNING: Command too long... ignoring!\n%s\n", pArgS ); + return false; + } + + // Add one for null termination + if ( m_nArgSBufferSize + nCommandSize + 1 > m_nMaxArgSBufferLength ) + { + Compact(); + if ( m_nArgSBufferSize + nCommandSize + 1 > m_nMaxArgSBufferLength ) + return false; + } + + memcpy( &m_pArgSBuffer[m_nArgSBufferSize], pArgS, nCommandSize ); + m_pArgSBuffer[m_nArgSBufferSize + nCommandSize] = 0; + ++nCommandSize; + + int hCommand = m_Commands.Alloc(); + Command_t &command = m_Commands[hCommand]; + command.m_nTick = nTick; + command.m_nFirstArgS = m_nArgSBufferSize; + command.m_nBufferSize = nCommandSize; + + m_nArgSBufferSize += nCommandSize; + + if ( !m_bIsProcessingCommands || ( nTick > m_nCurrentTick ) ) + { + InsertCommandAtAppropriateTime( hCommand ); + } + else + { + InsertImmediateCommand( hCommand ); + } + return true; +} + + +//----------------------------------------------------------------------------- +// Returns the length of the next command +//----------------------------------------------------------------------------- +void CCommandBuffer::GetNextCommandLength( const char *pText, int nMaxLen, int *pCommandLength, int *pNextCommandOffset ) +{ + int nCommandLength = 0; + int nNextCommandOffset; + bool bIsQuoted = false; + bool bIsCommented = false; + for ( nNextCommandOffset=0; nNextCommandOffset < nMaxLen; ++nNextCommandOffset, nCommandLength += bIsCommented ? 0 : 1 ) + { + char c = pText[nNextCommandOffset]; + if ( !bIsCommented ) + { + if ( c == '"' ) + { + bIsQuoted = !bIsQuoted; + continue; + } + + // don't break if inside a C++ style comment + if ( !bIsQuoted && c == '/' ) + { + bIsCommented = ( nNextCommandOffset < nMaxLen-1 ) && pText[nNextCommandOffset+1] == '/'; + if ( bIsCommented ) + { + ++nNextCommandOffset; + continue; + } + } + + // don't break if inside a quoted string + if ( !bIsQuoted && c == ';' ) + break; + } + + // FIXME: This is legacy behavior; should we not break if a \n is inside a quoted string? + if ( c == '\n' ) + break; + } + + *pCommandLength = nCommandLength; + *pNextCommandOffset = nNextCommandOffset; +} + + +//----------------------------------------------------------------------------- +// Add text to command buffer, return false if it couldn't owing to overflow +//----------------------------------------------------------------------------- +bool CCommandBuffer::AddText( const char *pText, int nTickDelay ) +{ + Assert( nTickDelay >= 0 ); + + int nLen = Q_strlen( pText ); + int nTick = m_nCurrentTick + nTickDelay; + + // Parse the text into distinct commands + const char *pCurrentCommand = pText; + int nOffsetToNextCommand; + for( ; nLen > 0; nLen -= nOffsetToNextCommand+1, pCurrentCommand += nOffsetToNextCommand+1 ) + { + // find a \n or ; line break + int nCommandLength; + GetNextCommandLength( pCurrentCommand, nLen, &nCommandLength, &nOffsetToNextCommand ); + if ( nCommandLength <= 0 ) + continue; + + const char *pArgS; + char *pArgV0 = (char*)_alloca( nCommandLength+1 ); + CUtlBuffer bufParse( pCurrentCommand, nCommandLength, CUtlBuffer::TEXT_BUFFER | CUtlBuffer::READ_ONLY ); + ParseArgV0( bufParse, pArgV0, nCommandLength+1, &pArgS ); + if ( pArgV0[0] == 0 ) + continue; + + // Deal with the special 'wait' command + if ( !Q_stricmp( pArgV0, "wait" ) && IsWaitEnabled() ) + { + int nDelay = pArgS ? atoi( pArgS ) : m_nWaitDelayTicks; + nTick += nDelay; + continue; + } + + if ( !InsertCommand( pCurrentCommand, nCommandLength, nTick ) ) + return false; + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Are we in the middle of processing commands? +//----------------------------------------------------------------------------- +bool CCommandBuffer::IsProcessingCommands() +{ + return m_bIsProcessingCommands; +} + + +//----------------------------------------------------------------------------- +// Delays all queued commands to execute at a later time +//----------------------------------------------------------------------------- +void CCommandBuffer::DelayAllQueuedCommands( int nDelay ) +{ + if ( nDelay <= 0 ) + return; + + for ( int i = m_Commands.Head(); i != m_Commands.InvalidIndex(); i = m_Commands.Next(i) ) + { + m_Commands[i].m_nTick += nDelay; + } +} + + +//----------------------------------------------------------------------------- +// Call this to begin iterating over all commands up to flCurrentTime +//----------------------------------------------------------------------------- +void CCommandBuffer::BeginProcessingCommands( int nDeltaTicks ) +{ + if ( nDeltaTicks == 0 ) + return; + + Assert( !m_bIsProcessingCommands ); + m_bIsProcessingCommands = true; + m_nLastTickToProcess = m_nCurrentTick + nDeltaTicks - 1; + + // Necessary to insert commands while commands are being processed + m_hNextCommand = m_Commands.Head(); +} + + +//----------------------------------------------------------------------------- +// Returns the next command +//----------------------------------------------------------------------------- +bool CCommandBuffer::DequeueNextCommand( ) +{ + m_CurrentCommand.Reset(); + + Assert( m_bIsProcessingCommands ); + if ( m_Commands.Count() == 0 ) + return false; + + int nHead = m_Commands.Head(); + Command_t &command = m_Commands[ nHead ]; + if ( command.m_nTick > m_nLastTickToProcess ) + return false; + + m_nCurrentTick = command.m_nTick; + + // Copy the current command into a temp buffer + // NOTE: This is here to avoid the pointers returned by DequeueNextCommand + // to become invalid by calling AddText. Is there a way we can avoid the memcpy? + if ( command.m_nBufferSize > 0 ) + { + m_CurrentCommand.Tokenize( &m_pArgSBuffer[command.m_nFirstArgS] ); + } + + m_Commands.Remove( nHead ); + + // Necessary to insert commands while commands are being processed + m_hNextCommand = m_Commands.Head(); + +// Msg("Dequeue : "); +// for ( int i = 0; i < nArgc; ++i ) +// { +// Msg("%s ", m_pCurrentArgv[i] ); +// } +// Msg("\n"); + return true; +} + + +//----------------------------------------------------------------------------- +// Returns the next command +//----------------------------------------------------------------------------- +int CCommandBuffer::DequeueNextCommand( const char **& ppArgv ) +{ + DequeueNextCommand(); + ppArgv = ArgV(); + return ArgC(); +} + + +//----------------------------------------------------------------------------- +// Compacts the command buffer +//----------------------------------------------------------------------------- +void CCommandBuffer::Compact() +{ + // Compress argvbuffer + argv + // NOTE: I'm using this choice instead of calling malloc + free + // per command to allocate arguments because I expect to post a + // bunch of commands but not have many delayed commands; + // avoiding the allocation cost seems more important that the memcpy + // cost here since I expect to not have much to copy. + m_nArgSBufferSize = 0; + + char pTempBuffer[ ARGS_BUFFER_LENGTH ]; + for ( int i = m_Commands.Head(); i != m_Commands.InvalidIndex(); i = m_Commands.Next(i) ) + { + Command_t &command = m_Commands[ i ]; + + memcpy( &pTempBuffer[m_nArgSBufferSize], &m_pArgSBuffer[command.m_nFirstArgS], command.m_nBufferSize ); + command.m_nFirstArgS = m_nArgSBufferSize; + m_nArgSBufferSize += command.m_nBufferSize; + } + + // NOTE: We could also store 2 buffers in the command buffer and switch + // between the two to avoid the 2nd memcpy; but again I'm guessing the memory + // tradeoff isn't worth it + memcpy( m_pArgSBuffer, pTempBuffer, m_nArgSBufferSize ); +} + + +//----------------------------------------------------------------------------- +// Call this to finish iterating over all commands +//----------------------------------------------------------------------------- +void CCommandBuffer::EndProcessingCommands() +{ + Assert( m_bIsProcessingCommands ); + m_bIsProcessingCommands = false; + m_nCurrentTick = m_nLastTickToProcess + 1; + m_hNextCommand = m_Commands.InvalidIndex(); + + // Extract commands that are before the end time + // NOTE: This is a bug for this to + int i = m_Commands.Head(); + if ( i == m_Commands.InvalidIndex() ) + { + m_nArgSBufferSize = 0; + return; + } + + while ( i != m_Commands.InvalidIndex() ) + { + if ( m_Commands[i].m_nTick >= m_nCurrentTick ) + break; + + AssertMsgOnce( false, "CCommandBuffer::EndProcessingCommands() called before all appropriate commands were dequeued.\n" ); + int nNext = i; + Msg( "Warning: Skipping command %s\n", &m_pArgSBuffer[ m_Commands[i].m_nFirstArgS ] ); + m_Commands.Remove( i ); + i = nNext; + } + + Compact(); +} + + +//----------------------------------------------------------------------------- +// Returns a handle to the next command to process +//----------------------------------------------------------------------------- +CommandHandle_t CCommandBuffer::GetNextCommandHandle() +{ + Assert( m_bIsProcessingCommands ); + return m_Commands.Head(); +} + + +#if 0 +/* +=============== +Cmd_Alias_f + +Creates a new command that executes a command string (possibly ; seperated) +=============== +*/ +void Cmd_Alias_f (void) +{ + cmdalias_t *a; + char cmd[MAX_COMMAND_LENGTH]; + int i, c; + char *s; + + if (Cmd_Argc() == 1) + { + Con_Printf ("Current alias commands:\n"); + for (a = cmd_alias ; a ; a=a->next) + Con_Printf ("%s : %s\n", a->name, a->value); + return; + } + + s = Cmd_Argv(1); + if (strlen(s) >= MAX_ALIAS_NAME) + { + Con_Printf ("Alias name is too long\n"); + return; + } + +// copy the rest of the command line + cmd[0] = 0; // start out with a null string + c = Cmd_Argc(); + for (i=2 ; i< c ; i++) + { + Q_strncat(cmd, Cmd_Argv(i), sizeof( cmd ), COPY_ALL_CHARACTERS); + if (i != c) + { + Q_strncat (cmd, " ", sizeof( cmd ), COPY_ALL_CHARACTERS ); + } + } + Q_strncat (cmd, "\n", sizeof( cmd ), COPY_ALL_CHARACTERS); + + // if the alias already exists, reuse it + for (a = cmd_alias ; a ; a=a->next) + { + if (!strcmp(s, a->name)) + { + if ( !strcmp( a->value, cmd ) ) // Re-alias the same thing + return; + + delete[] a->value; + break; + } + } + + if (!a) + { + a = (cmdalias_t *)new cmdalias_t; + a->next = cmd_alias; + cmd_alias = a; + } + Q_strncpy (a->name, s, sizeof( a->name ) ); + + a->value = COM_StringCopy(cmd); +} + + + +/* +============================================================================= + + COMMAND EXECUTION + +============================================================================= +*/ + +#define MAX_ARGS 80 + +static int cmd_argc; +static char *cmd_argv[MAX_ARGS]; +static char *cmd_null_string = ""; +static const char *cmd_args = NULL; + +cmd_source_t cmd_source; + +//----------------------------------------------------------------------------- +// Purpose: +// Output : void Cmd_Init +//----------------------------------------------------------------------------- + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void Cmd_Shutdown( void ) +{ + // TODO, cleanup + while ( cmd_alias ) + { + cmdalias_t *next = cmd_alias->next; + delete cmd_alias->value; // created by StringCopy() + delete cmd_alias; + cmd_alias = next; + } +} + + + +/* +============ +Cmd_ExecuteString + +A complete command line has been parsed, so try to execute it +FIXME: lookupnoadd the token to speed search? +============ +*/ +const ConCommandBase *Cmd_ExecuteString (const char *text, cmd_source_t src) +{ + cmdalias_t *a; + + cmd_source = src; + Cmd_TokenizeString (text); + +// execute the command line + if (!Cmd_Argc()) + return NULL; // no tokens + +// check alias + for (a=cmd_alias ; a ; a=a->next) + { + if (!Q_strcasecmp (cmd_argv[0], a->name)) + { + Cbuf_InsertText (a->value); + return NULL; + } + } + +// check ConCommands + ConCommandBase const *pCommand = ConCommandBase::FindCommand( cmd_argv[ 0 ] ); + if ( pCommand && pCommand->IsCommand() ) + { + bool isServerCommand = ( pCommand->IsBitSet( FCVAR_GAMEDLL ) && + // Typed at console + cmd_source == src_command && + // Not HLDS + !sv.IsDedicated() ); + + // Hook to allow game .dll to figure out who type the message on a listen server + if ( serverGameClients ) + { + // We're actually the server, so set it up locally + if ( sv.IsActive() ) + { + g_pServerPluginHandler->SetCommandClient( -1 ); + +#ifndef SWDS + // Special processing for listen server player + if ( isServerCommand ) + { + g_pServerPluginHandler->SetCommandClient( cl.m_nPlayerSlot ); + } +#endif + } + // We're not the server, but we've been a listen server (game .dll loaded) + // forward this command tot he server instead of running it locally if we're still + // connected + // Otherwise, things like "say" won't work unless you quit and restart + else if ( isServerCommand ) + { + if ( cl.IsConnected() ) + { + Cmd_ForwardToServer(); + return NULL; + } + else + { + // It's a server command, but we're not connected to a server. Don't try to execute it. + return NULL; + } + } + } + + // Allow cheat commands in singleplayer, debug, or multiplayer with sv_cheats on +#ifndef _DEBUG + if ( pCommand->IsBitSet( FCVAR_CHEAT ) ) + { + if ( !Host_IsSinglePlayerGame() && sv_cheats.GetInt() == 0 ) + { + Msg( "Can't use cheat command %s in multiplayer, unless the server has sv_cheats set to 1.\n", pCommand->GetName() ); + return NULL; + } + } +#endif + + (( ConCommand * )pCommand )->Dispatch(); + return pCommand; + } + + // check cvars + if ( cv->IsCommand() ) + { + return pCommand; + } + + // forward the command line to the server, so the entity DLL can parse it + if ( cmd_source == src_command ) + { + if ( cl.IsConnected() ) + { + Cmd_ForwardToServer(); + return NULL; + } + } + + Msg("Unknown command \"%s\"\n", Cmd_Argv(0)); + + return NULL; +} +#endif diff --git a/tier1/convar.cpp b/tier1/convar.cpp new file mode 100644 index 0000000..ca8e52a --- /dev/null +++ b/tier1/convar.cpp @@ -0,0 +1,1376 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +// +//===========================================================================// + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include "basetypes.h" +#include "tier1/convar.h" +#include "tier1/strtools.h" +#include "tier1/characterset.h" +#include "tier1/utlbuffer.h" +#include "tier1/tier1.h" +#include "tier1/convar_serverbounded.h" +#include "icvar.h" +#include "tier0/dbg.h" +#include "Color.h" +#if defined( _X360 ) +#include "xbox/xbox_console.h" +#endif +#include "tier0/memdbgon.h" + +#ifndef NDEBUG +// Comment this out when we release. +#define ALLOW_DEVELOPMENT_CVARS +#endif + + + +//----------------------------------------------------------------------------- +// Statically constructed list of ConCommandBases, +// used for registering them with the ICVar interface +//----------------------------------------------------------------------------- +ConCommandBase *ConCommandBase::s_pConCommandBases = NULL; +IConCommandBaseAccessor *ConCommandBase::s_pAccessor = NULL; +static int s_nCVarFlag = 0; +static int s_nDLLIdentifier = -1; // A unique identifier indicating which DLL this convar came from +static bool s_bRegistered = false; + +class CDefaultAccessor : public IConCommandBaseAccessor +{ +public: + virtual bool RegisterConCommandBase( ConCommandBase *pVar ) + { + // Link to engine's list instead + g_pCVar->RegisterConCommand( pVar ); + return true; + } +}; + +static CDefaultAccessor s_DefaultAccessor; + +//----------------------------------------------------------------------------- +// Called by the framework to register ConCommandBases with the ICVar +//----------------------------------------------------------------------------- +void ConVar_Register( int nCVarFlag, IConCommandBaseAccessor *pAccessor ) +{ + if ( !g_pCVar || s_bRegistered ) + return; + + Assert( s_nDLLIdentifier < 0 ); + s_bRegistered = true; + s_nCVarFlag = nCVarFlag; + s_nDLLIdentifier = g_pCVar->AllocateDLLIdentifier(); + + ConCommandBase *pCur, *pNext; + + ConCommandBase::s_pAccessor = pAccessor ? pAccessor : &s_DefaultAccessor; + pCur = ConCommandBase::s_pConCommandBases; + while ( pCur ) + { + pNext = pCur->m_pNext; + pCur->AddFlags( s_nCVarFlag ); + pCur->Init(); + pCur = pNext; + } + + g_pCVar->ProcessQueuedMaterialThreadConVarSets(); + ConCommandBase::s_pConCommandBases = NULL; +} + +void ConVar_Unregister( ) +{ + if ( !g_pCVar || !s_bRegistered ) + return; + + Assert( s_nDLLIdentifier >= 0 ); + g_pCVar->UnregisterConCommands( s_nDLLIdentifier ); + s_nDLLIdentifier = -1; + s_bRegistered = false; +} + + +//----------------------------------------------------------------------------- +// Purpose: Default constructor +//----------------------------------------------------------------------------- +ConCommandBase::ConCommandBase( void ) +{ + m_bRegistered = false; + m_pszName = NULL; + m_pszHelpString = NULL; + + m_nFlags = 0; + m_pNext = NULL; +} + +//----------------------------------------------------------------------------- +// Purpose: The base console invoked command/cvar interface +// Input : *pName - name of variable/command +// *pHelpString - help text +// flags - flags +//----------------------------------------------------------------------------- +ConCommandBase::ConCommandBase( const char *pName, const char *pHelpString /*=0*/, int flags /*= 0*/ ) +{ + CreateBase( pName, pHelpString, flags ); +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +ConCommandBase::~ConCommandBase( void ) +{ +} + +//----------------------------------------------------------------------------- +// Purpose: +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool ConCommandBase::IsCommand( void ) const +{ +// Assert( 0 ); This can't assert. . causes a recursive assert in Sys_Printf, etc. + return true; +} + + +//----------------------------------------------------------------------------- +// Returns the DLL identifier +//----------------------------------------------------------------------------- +CVarDLLIdentifier_t ConCommandBase::GetDLLIdentifier() const +{ + return s_nDLLIdentifier; +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *pName - +// callback - +// *pHelpString - +// flags - +//----------------------------------------------------------------------------- +void ConCommandBase::CreateBase( const char *pName, const char *pHelpString /*= 0*/, int flags /*= 0*/ ) +{ + m_bRegistered = false; + + // Name should be static data + Assert( pName ); + m_pszName = pName; + m_pszHelpString = pHelpString ? pHelpString : ""; + + m_nFlags = flags; + +#ifdef ALLOW_DEVELOPMENT_CVARS + m_nFlags &= ~FCVAR_DEVELOPMENTONLY; +#endif + + if ( !( m_nFlags & FCVAR_UNREGISTERED ) ) + { + m_pNext = s_pConCommandBases; + s_pConCommandBases = this; + } + else + { + // It's unregistered + m_pNext = NULL; + } + + // If s_pAccessor is already set (this ConVar is not a global variable), + // register it. + if ( s_pAccessor ) + { + Init(); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Used internally by OneTimeInit to initialize. +//----------------------------------------------------------------------------- +void ConCommandBase::Init() +{ + if ( s_pAccessor ) + { + s_pAccessor->RegisterConCommandBase( this ); + } +} + +void ConCommandBase::Shutdown() +{ + if ( g_pCVar ) + { + g_pCVar->UnregisterConCommand( this ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Return name of the command/var +// Output : const char +//----------------------------------------------------------------------------- +const char *ConCommandBase::GetName( void ) const +{ + return m_pszName; +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : flag - +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool ConCommandBase::IsFlagSet( int flag ) const +{ + return ( flag & m_nFlags ) ? true : false; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : flags - +//----------------------------------------------------------------------------- +void ConCommandBase::AddFlags( int flags ) +{ + m_nFlags |= flags; + +#ifdef ALLOW_DEVELOPMENT_CVARS + m_nFlags &= ~FCVAR_DEVELOPMENTONLY; +#endif +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Output : const ConCommandBase +//----------------------------------------------------------------------------- +const ConCommandBase *ConCommandBase::GetNext( void ) const +{ + return m_pNext; +} + +ConCommandBase *ConCommandBase::GetNext( void ) +{ + return m_pNext; +} + + +//----------------------------------------------------------------------------- +// Purpose: Copies string using local new/delete operators +// Input : *from - +// Output : char +//----------------------------------------------------------------------------- +char *ConCommandBase::CopyString( const char *from ) +{ + int len; + char *to; + + len = V_strlen( from ); + if ( len <= 0 ) + { + to = new char[1]; + to[0] = 0; + } + else + { + to = new char[len+1]; + Q_strncpy( to, from, len+1 ); + } + return to; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Output : const char +//----------------------------------------------------------------------------- +const char *ConCommandBase::GetHelpText( void ) const +{ + return m_pszHelpString; +} + +//----------------------------------------------------------------------------- +// Purpose: Has this cvar been registered +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool ConCommandBase::IsRegistered( void ) const +{ + return m_bRegistered; +} + + +//----------------------------------------------------------------------------- +// +// Con Commands start here +// +//----------------------------------------------------------------------------- + + +//----------------------------------------------------------------------------- +// Global methods +//----------------------------------------------------------------------------- +static characterset_t s_BreakSet; +static bool s_bBuiltBreakSet = false; + + +//----------------------------------------------------------------------------- +// Tokenizer class +//----------------------------------------------------------------------------- +CCommand::CCommand() +{ + if ( !s_bBuiltBreakSet ) + { + s_bBuiltBreakSet = true; + CharacterSetBuild( &s_BreakSet, "{}()':" ); + } + + Reset(); +} + +CCommand::CCommand( int nArgC, const char **ppArgV ) +{ + Assert( nArgC > 0 ); + + if ( !s_bBuiltBreakSet ) + { + s_bBuiltBreakSet = true; + CharacterSetBuild( &s_BreakSet, "{}()':" ); + } + + Reset(); + + char *pBuf = m_pArgvBuffer; + char *pSBuf = m_pArgSBuffer; + m_nArgc = nArgC; + for ( int i = 0; i < nArgC; ++i ) + { + m_ppArgv[i] = pBuf; + int nLen = Q_strlen( ppArgV[i] ); + memcpy( pBuf, ppArgV[i], nLen+1 ); + if ( i == 0 ) + { + m_nArgv0Size = nLen; + } + pBuf += nLen+1; + + bool bContainsSpace = strchr( ppArgV[i], ' ' ) != NULL; + if ( bContainsSpace ) + { + *pSBuf++ = '\"'; + } + memcpy( pSBuf, ppArgV[i], nLen ); + pSBuf += nLen; + if ( bContainsSpace ) + { + *pSBuf++ = '\"'; + } + + if ( i != nArgC - 1 ) + { + *pSBuf++ = ' '; + } + } +} + +void CCommand::Reset() +{ + m_nArgc = 0; + m_nArgv0Size = 0; + m_pArgSBuffer[0] = 0; +} + +characterset_t* CCommand::DefaultBreakSet() +{ + return &s_BreakSet; +} + +bool CCommand::Tokenize( const char *pCommand, characterset_t *pBreakSet ) +{ + Reset(); + if ( !pCommand ) + return false; + + // Use default break set + if ( !pBreakSet ) + { + pBreakSet = &s_BreakSet; + } + + // Copy the current command into a temp buffer + // NOTE: This is here to avoid the pointers returned by DequeueNextCommand + // to become invalid by calling AddText. Is there a way we can avoid the memcpy? + int nLen = Q_strlen( pCommand ); + if ( nLen >= COMMAND_MAX_LENGTH - 1 ) + { + Warning( "CCommand::Tokenize: Encountered command which overflows the tokenizer buffer.. Skipping!\n" ); + return false; + } + + memcpy( m_pArgSBuffer, pCommand, nLen + 1 ); + + // Parse the current command into the current command buffer + CUtlBuffer bufParse( m_pArgSBuffer, nLen, CUtlBuffer::TEXT_BUFFER | CUtlBuffer::READ_ONLY ); + int nArgvBufferSize = 0; + while ( bufParse.IsValid() && ( m_nArgc < COMMAND_MAX_ARGC ) ) + { + char *pArgvBuf = &m_pArgvBuffer[nArgvBufferSize]; + int nMaxLen = COMMAND_MAX_LENGTH - nArgvBufferSize; + int nStartGet = bufParse.TellGet(); + int nSize = bufParse.ParseToken( pBreakSet, pArgvBuf, nMaxLen ); + if ( nSize < 0 ) + break; + + // Check for overflow condition + if ( nMaxLen == nSize ) + { + Reset(); + return false; + } + + if ( m_nArgc == 1 ) + { + // Deal with the case where the arguments were quoted + m_nArgv0Size = bufParse.TellGet(); + bool bFoundEndQuote = m_pArgSBuffer[m_nArgv0Size-1] == '\"'; + if ( bFoundEndQuote ) + { + --m_nArgv0Size; + } + m_nArgv0Size -= nSize; + Assert( m_nArgv0Size != 0 ); + + // The StartGet check is to handle this case: "foo"bar + // which will parse into 2 different args. ArgS should point to bar. + bool bFoundStartQuote = ( m_nArgv0Size > nStartGet ) && ( m_pArgSBuffer[m_nArgv0Size-1] == '\"' ); + Assert( bFoundEndQuote == bFoundStartQuote ); + if ( bFoundStartQuote ) + { + --m_nArgv0Size; + } + } + + m_ppArgv[ m_nArgc++ ] = pArgvBuf; + if( m_nArgc >= COMMAND_MAX_ARGC ) + { + Warning( "CCommand::Tokenize: Encountered command which overflows the argument buffer.. Clamped!\n" ); + } + + nArgvBufferSize += nSize + 1; + Assert( nArgvBufferSize <= COMMAND_MAX_LENGTH ); + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Helper function to parse arguments to commands. +//----------------------------------------------------------------------------- +const char* CCommand::FindArg( const char *pName ) const +{ + int nArgC = ArgC(); + for ( int i = 1; i < nArgC; i++ ) + { + if ( !Q_stricmp( Arg(i), pName ) ) + return (i+1) < nArgC ? Arg( i+1 ) : ""; + } + return 0; +} + +int CCommand::FindArgInt( const char *pName, int nDefaultVal ) const +{ + const char *pVal = FindArg( pName ); + if ( pVal ) + return atoi( pVal ); + else + return nDefaultVal; +} + + +//----------------------------------------------------------------------------- +// Default console command autocompletion function +//----------------------------------------------------------------------------- +int DefaultCompletionFunc( const char *partial, char commands[ COMMAND_COMPLETION_MAXITEMS ][ COMMAND_COMPLETION_ITEM_LENGTH ] ) +{ + return 0; +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructs a console command +//----------------------------------------------------------------------------- +//ConCommand::ConCommand() +//{ +// m_bIsNewConCommand = true; +//} + +ConCommand::ConCommand( const char *pName, FnCommandCallbackVoid_t callback, const char *pHelpString /*= 0*/, int flags /*= 0*/, FnCommandCompletionCallback completionFunc /*= 0*/ ) +{ + // Set the callback + m_fnCommandCallbackV1 = callback; + m_bUsingNewCommandCallback = false; + m_bUsingCommandCallbackInterface = false; + m_fnCompletionCallback = completionFunc ? completionFunc : DefaultCompletionFunc; + m_bHasCompletionCallback = completionFunc != 0 ? true : false; + + // Setup the rest + BaseClass::CreateBase( pName, pHelpString, flags ); +} + +ConCommand::ConCommand( const char *pName, FnCommandCallback_t callback, const char *pHelpString /*= 0*/, int flags /*= 0*/, FnCommandCompletionCallback completionFunc /*= 0*/ ) +{ + // Set the callback + m_fnCommandCallback = callback; + m_bUsingNewCommandCallback = true; + m_fnCompletionCallback = completionFunc ? completionFunc : DefaultCompletionFunc; + m_bHasCompletionCallback = completionFunc != 0 ? true : false; + m_bUsingCommandCallbackInterface = false; + + // Setup the rest + BaseClass::CreateBase( pName, pHelpString, flags ); +} + +ConCommand::ConCommand( const char *pName, ICommandCallback *pCallback, const char *pHelpString /*= 0*/, int flags /*= 0*/, ICommandCompletionCallback *pCompletionCallback /*= 0*/ ) +{ + // Set the callback + m_pCommandCallback = pCallback; + m_bUsingNewCommandCallback = false; + m_pCommandCompletionCallback = pCompletionCallback; + m_bHasCompletionCallback = ( pCompletionCallback != 0 ); + m_bUsingCommandCallbackInterface = true; + + // Setup the rest + BaseClass::CreateBase( pName, pHelpString, flags ); +} + +//----------------------------------------------------------------------------- +// Destructor +//----------------------------------------------------------------------------- +ConCommand::~ConCommand( void ) +{ +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns true if this is a command +//----------------------------------------------------------------------------- +bool ConCommand::IsCommand( void ) const +{ + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Invoke the function if there is one +//----------------------------------------------------------------------------- +void ConCommand::Dispatch( const CCommand &command ) +{ + if ( m_bUsingNewCommandCallback ) + { + if ( m_fnCommandCallback ) + { + ( *m_fnCommandCallback )( command ); + return; + } + } + else if ( m_bUsingCommandCallbackInterface ) + { + if ( m_pCommandCallback ) + { + m_pCommandCallback->CommandCallback( command ); + return; + } + } + else + { + if ( m_fnCommandCallbackV1 ) + { + ( *m_fnCommandCallbackV1 )(); + return; + } + } + + // Command without callback!!! + AssertMsg( 0, "Encountered ConCommand '%s' without a callback!\n", GetName() ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Calls the autocompletion method to get autocompletion suggestions +//----------------------------------------------------------------------------- +int ConCommand::AutoCompleteSuggest( const char *partial, CUtlVector< CUtlString > &commands ) +{ + if ( m_bUsingCommandCallbackInterface ) + { + if ( !m_pCommandCompletionCallback ) + return 0; + return m_pCommandCompletionCallback->CommandCompletionCallback( partial, commands ); + } + + Assert( m_fnCompletionCallback ); + if ( !m_fnCompletionCallback ) + return 0; + + char rgpchCommands[ COMMAND_COMPLETION_MAXITEMS ][ COMMAND_COMPLETION_ITEM_LENGTH ]; + int iret = ( m_fnCompletionCallback )( partial, rgpchCommands ); + for ( int i = 0 ; i < iret; ++i ) + { + CUtlString str = rgpchCommands[ i ]; + commands.AddToTail( str ); + } + return iret; +} + + +//----------------------------------------------------------------------------- +// Returns true if the console command can autocomplete +//----------------------------------------------------------------------------- +bool ConCommand::CanAutoComplete( void ) +{ + return m_bHasCompletionCallback; +} + + + +//----------------------------------------------------------------------------- +// +// Console Variables +// +//----------------------------------------------------------------------------- + +//----------------------------------------------------------------------------- +// Various constructors +//----------------------------------------------------------------------------- +ConVar::ConVar( const char *pName, const char *pDefaultValue, int flags /* = 0 */ ) +{ + Create( pName, pDefaultValue, flags ); +} + +ConVar::ConVar( const char *pName, const char *pDefaultValue, int flags, const char *pHelpString ) +{ + Create( pName, pDefaultValue, flags, pHelpString ); +} + +ConVar::ConVar( const char *pName, const char *pDefaultValue, int flags, const char *pHelpString, bool bMin, float fMin, bool bMax, float fMax ) +{ + Create( pName, pDefaultValue, flags, pHelpString, bMin, fMin, bMax, fMax ); +} + +ConVar::ConVar( const char *pName, const char *pDefaultValue, int flags, const char *pHelpString, FnChangeCallback_t callback ) +{ + Create( pName, pDefaultValue, flags, pHelpString, false, 0.0, false, 0.0, false, 0.0, false, 0.0, callback ); +} + +ConVar::ConVar( const char *pName, const char *pDefaultValue, int flags, const char *pHelpString, bool bMin, float fMin, bool bMax, float fMax, FnChangeCallback_t callback ) +{ + Create( pName, pDefaultValue, flags, pHelpString, bMin, fMin, bMax, fMax, false, 0.0, false, 0.0, callback ); +} + +ConVar::ConVar( const char *pName, const char *pDefaultValue, int flags, const char *pHelpString, bool bMin, float fMin, bool bMax, float fMax, bool bCompMin, float fCompMin, bool bCompMax, float fCompMax, FnChangeCallback_t callback ) +{ + Create( pName, pDefaultValue, flags, pHelpString, bMin, fMin, bMax, fMax, bCompMin, fCompMin, bCompMax, fCompMax, callback ); +} + + +//----------------------------------------------------------------------------- +// Destructor +//----------------------------------------------------------------------------- +ConVar::~ConVar( void ) +{ + if ( m_pszString ) + { + delete[] m_pszString; + m_pszString = NULL; + } +} + + +//----------------------------------------------------------------------------- +// Install a change callback (there shouldn't already be one....) +//----------------------------------------------------------------------------- +void ConVar::InstallChangeCallback( FnChangeCallback_t callback ) +{ + Assert( !m_pParent->m_fnChangeCallback || !callback ); + m_pParent->m_fnChangeCallback = callback; + + if ( m_pParent->m_fnChangeCallback ) + { + // Call it immediately to set the initial value... + m_pParent->m_fnChangeCallback( this, m_pszString, m_fValue ); + } +} + +bool ConVar::IsFlagSet( int flag ) const +{ + return ( flag & m_pParent->m_nFlags ) ? true : false; +} + +const char *ConVar::GetHelpText( void ) const +{ + return m_pParent->m_pszHelpString; +} + +void ConVar::AddFlags( int flags ) +{ + m_pParent->m_nFlags |= flags; + +#ifdef ALLOW_DEVELOPMENT_CVARS + m_pParent->m_nFlags &= ~FCVAR_DEVELOPMENTONLY; +#endif +} + +bool ConVar::IsRegistered( void ) const +{ + return m_pParent->m_bRegistered; +} + +const char *ConVar::GetName( void ) const +{ + return m_pParent->m_pszName; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool ConVar::IsCommand( void ) const +{ + return false; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : +//----------------------------------------------------------------------------- +void ConVar::Init() +{ + BaseClass::Init(); +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *value - +//----------------------------------------------------------------------------- +void ConVar::InternalSetValue( const char *value ) +{ + if ( IsFlagSet( FCVAR_MATERIAL_THREAD_MASK ) ) + { + if ( g_pCVar && !g_pCVar->IsMaterialThreadSetAllowed() ) + { + g_pCVar->QueueMaterialThreadSetValue( this, value ); + return; + } + } + + float fNewValue; + char tempVal[ 32 ]; + char *val; + + Assert(m_pParent == this); // Only valid for root convars. + + float flOldValue = m_fValue; + + val = (char *)value; + if ( !value ) + fNewValue = 0.0f; + else + fNewValue = ( float )atof( value ); + + if ( ClampValue( fNewValue ) ) + { + Q_snprintf( tempVal,sizeof(tempVal), "%f", fNewValue ); + val = tempVal; + } + + // Redetermine value + m_fValue = fNewValue; + m_nValue = ( int )( fNewValue ); + + if ( !( m_nFlags & FCVAR_NEVER_AS_STRING ) ) + { + ChangeStringValue( val, flOldValue ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *tempVal - +//----------------------------------------------------------------------------- +void ConVar::ChangeStringValue( const char *tempVal, float flOldValue ) +{ + Assert( !( m_nFlags & FCVAR_NEVER_AS_STRING ) ); + + char* pszOldValue = (char*)stackalloc( m_StringLength ); + memcpy( pszOldValue, m_pszString, m_StringLength ); + + if ( tempVal ) + { + int len = Q_strlen(tempVal) + 1; + + if ( len > m_StringLength) + { + if (m_pszString) + { + delete[] m_pszString; + } + + m_pszString = new char[len]; + m_StringLength = len; + } + + memcpy( m_pszString, tempVal, len ); + } + else + { + *m_pszString = 0; + } + + // If nothing has changed, don't do the callbacks. + if (V_strcmp(pszOldValue, m_pszString) != 0) + { + // Invoke any necessary callback function + if ( m_fnChangeCallback ) + { + m_fnChangeCallback( this, pszOldValue, flOldValue ); + } + + g_pCVar->CallGlobalChangeCallbacks( this, pszOldValue, flOldValue ); + } + + stackfree( pszOldValue ); +} + +//----------------------------------------------------------------------------- +// Purpose: Check whether to clamp and then perform clamp +// Input : value - +// Output : Returns true if value changed +//----------------------------------------------------------------------------- +bool ConVar::ClampValue( float& value ) +{ + // Competitive /should/ be more restrictive, so do it first. + if ( m_bCompetitiveRestrictions ) + { + if ( m_bHasCompMin && ( value < m_fCompMinVal ) ) + { + value = m_fCompMinVal; + return true; + } + + if ( m_bHasCompMax && ( value > m_fCompMaxVal ) ) + { + value = m_fCompMaxVal; + return true; + } + + if ( !m_bHasCompMin && !m_bHasCompMax ) + { + float fDefaultAsFloat = V_atof( m_pszDefaultValue ); + if ( fabs( value - fDefaultAsFloat ) > 0.0001f ) + { + value = fDefaultAsFloat; + return true; + } + } + } + + if ( m_bHasMin && ( value < m_fMinVal ) ) + { + value = m_fMinVal; + return true; + } + + if ( m_bHasMax && ( value > m_fMaxVal ) ) + { + value = m_fMaxVal; + return true; + } + + return false; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *value - +//----------------------------------------------------------------------------- +void ConVar::InternalSetFloatValue( float fNewValue, bool bForce /*= false */ ) +{ + if ( fNewValue == m_fValue && !bForce ) + return; + + if ( IsFlagSet( FCVAR_MATERIAL_THREAD_MASK ) ) + { + if ( g_pCVar && !g_pCVar->IsMaterialThreadSetAllowed() ) + { + g_pCVar->QueueMaterialThreadSetValue( this, fNewValue ); + return; + } + } + + Assert( m_pParent == this ); // Only valid for root convars. + + // Check bounds + ClampValue( fNewValue ); + + // Redetermine value + float flOldValue = m_fValue; + m_fValue = fNewValue; + m_nValue = ( int )m_fValue; + + if ( !( m_nFlags & FCVAR_NEVER_AS_STRING ) ) + { + char tempVal[ 32 ]; + Q_snprintf( tempVal, sizeof( tempVal), "%f", m_fValue ); + ChangeStringValue( tempVal, flOldValue ); + } + else + { + Assert( !m_fnChangeCallback ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *value - +//----------------------------------------------------------------------------- +void ConVar::InternalSetIntValue( int nValue ) +{ + if ( nValue == m_nValue ) + return; + + if ( IsFlagSet( FCVAR_MATERIAL_THREAD_MASK ) ) + { + if ( g_pCVar && !g_pCVar->IsMaterialThreadSetAllowed() ) + { + g_pCVar->QueueMaterialThreadSetValue( this, nValue ); + return; + } + } + + Assert( m_pParent == this ); // Only valid for root convars. + + float fValue = (float)nValue; + if ( ClampValue( fValue ) ) + { + nValue = ( int )( fValue ); + } + + // Redetermine value + float flOldValue = m_fValue; + m_fValue = fValue; + m_nValue = nValue; + + if ( !( m_nFlags & FCVAR_NEVER_AS_STRING ) ) + { + char tempVal[ 32 ]; + Q_snprintf( tempVal, sizeof( tempVal ), "%d", m_nValue ); + ChangeStringValue( tempVal, flOldValue ); + } + else + { + Assert( !m_fnChangeCallback ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: Private creation +//----------------------------------------------------------------------------- +void ConVar::Create( const char *pName, const char *pDefaultValue, int flags /*= 0*/, + const char *pHelpString /*= NULL*/, bool bMin /*= false*/, float fMin /*= 0.0*/, + bool bMax /*= false*/, float fMax /*= false*/, bool bCompMin /*= false */, + float fCompMin /*= 0.0*/, bool bCompMax /*= false*/, float fCompMax /*= 0.0*/, + FnChangeCallback_t callback /*= NULL*/ ) +{ + m_pParent = this; + + // Name should be static data + SetDefault( pDefaultValue ); + + m_StringLength = V_strlen( m_pszDefaultValue ) + 1; + m_pszString = new char[m_StringLength]; + memcpy( m_pszString, m_pszDefaultValue, m_StringLength ); + + m_bHasMin = bMin; + m_fMinVal = fMin; + m_bHasMax = bMax; + m_fMaxVal = fMax; + + m_bHasCompMin = bCompMin; + m_fCompMinVal = fCompMin; + m_bHasCompMax = bCompMax; + m_fCompMaxVal = fCompMax; + + m_bCompetitiveRestrictions = false; + + m_fnChangeCallback = callback; + + m_fValue = ( float )atof( m_pszString ); + m_nValue = atoi( m_pszString ); // dont convert from float to int and lose bits + + // Bounds Check, should never happen, if it does, no big deal + Assert( !m_bHasMin || m_fValue >= m_fMinVal ); + Assert( !m_bHasMax || m_fValue <= m_fMaxVal ); + + BaseClass::CreateBase( pName, pHelpString, flags ); +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *value - +//----------------------------------------------------------------------------- +void ConVar::SetValue(const char *value) +{ + ConVar *var = ( ConVar * )m_pParent; + var->InternalSetValue( value ); +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : value - +//----------------------------------------------------------------------------- +void ConVar::SetValue( float value ) +{ + ConVar *var = ( ConVar * )m_pParent; + var->InternalSetFloatValue( value ); +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : value - +//----------------------------------------------------------------------------- +void ConVar::SetValue( int value ) +{ + ConVar *var = ( ConVar * )m_pParent; + var->InternalSetIntValue( value ); +} + +//----------------------------------------------------------------------------- +// Purpose: Reset to default value +//----------------------------------------------------------------------------- +void ConVar::Revert( void ) +{ + // Force default value again + ConVar *var = ( ConVar * )m_pParent; + var->SetValue( var->m_pszDefaultValue ); +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : minVal - +// Output : true if there is a min set +//----------------------------------------------------------------------------- +bool ConVar::GetMin( float& minVal ) const +{ + minVal = m_pParent->m_fMinVal; + return m_pParent->m_bHasMin; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : maxVal - +//----------------------------------------------------------------------------- +bool ConVar::GetMax( float& maxVal ) const +{ + maxVal = m_pParent->m_fMaxVal; + return m_pParent->m_bHasMax; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : minVal - +// Output : true if there is a min set +//----------------------------------------------------------------------------- +bool ConVar::GetCompMin( float& minVal ) const +{ + minVal = m_pParent->m_fCompMinVal; + return m_pParent->m_bHasCompMin; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : maxVal - +//----------------------------------------------------------------------------- +bool ConVar::GetCompMax( float& maxVal ) const +{ + maxVal = m_pParent->m_fCompMaxVal; + return m_pParent->m_bHasCompMax; +} + +//----------------------------------------------------------------------------- +// Purpose: Sets that competitive mode is enabled for this var, and then +// attempts to clamp to competitive values. +// Input : maxVal - +// Output : true if the value was successfully updated, otherwise false. +//----------------------------------------------------------------------------- +bool ConVar::SetCompetitiveMode( bool bCompetitive ) +{ + // Should only do this for competitive restricted things. + Assert( IsCompetitiveRestricted() ); + + ConVar* var = m_pParent; + + var->m_bCompetitiveRestrictions = true; + float fDefaultAsFloat = 0.0f; + + bool bRequiresClamp = ( var->m_bHasCompMin && var->m_fCompMinVal > var->m_fValue ) + || ( var->m_bHasCompMax && var->m_fCompMaxVal < var->m_fValue ); + bool bForceToDefault = !var->m_bHasCompMin && !var->m_bHasCompMax + && ( fabs( var->m_fValue - ( fDefaultAsFloat = V_atof( var->m_pszDefaultValue ) ) ) > 0.00001f ); + + if ( bRequiresClamp ) + var->InternalSetFloatValue( var->m_fValue, true ); + else if ( bForceToDefault ) + { + STAGING_ONLY_EXEC( Msg( "Changing Convar: %s ( cur: %.2f ) to %.2f -> ", GetName(), var->m_fValue, fDefaultAsFloat ) ); + var->InternalSetFloatValue( fDefaultAsFloat, true ); + STAGING_ONLY_EXEC( Msg( "%.2f\n", var->m_fValue ) ); + } + + // The clamping should've worked, so if it didn't--need to understand why. + Assert( !bRequiresClamp || IsFlagSet( FCVAR_MATERIAL_THREAD_MASK ) || ( ( !var->m_bHasCompMin || var->m_fCompMinVal <= var->m_fValue ) + && ( !var->m_bHasCompMax || var->m_fCompMaxVal >= var->m_fValue ) ) ); + Assert( !bForceToDefault || IsFlagSet( FCVAR_MATERIAL_THREAD_MASK ) || ( var->m_fValue == fDefaultAsFloat ) ); + return true; +} + + + +//----------------------------------------------------------------------------- +// Purpose: +// Output : const char +//----------------------------------------------------------------------------- +const char *ConVar::GetDefault( void ) const +{ + return m_pParent->m_pszDefaultValue; +} + +void ConVar::SetDefault( const char *pszDefault ) +{ + m_pszDefaultValue = pszDefault ? pszDefault : ""; + Assert( m_pszDefaultValue ); +} + +//----------------------------------------------------------------------------- +// This version is simply used to make reading convars simpler. +// Writing convars isn't allowed in this mode +//----------------------------------------------------------------------------- +class CEmptyConVar : public ConVar +{ +public: + CEmptyConVar() : ConVar( "", "0" ) {} + // Used for optimal read access + virtual void SetValue( const char *pValue ) {} + virtual void SetValue( float flValue ) {} + virtual void SetValue( int nValue ) {} + virtual const char *GetName( void ) const { return ""; } + virtual bool IsFlagSet( int nFlags ) const { return false; } +}; + +static CEmptyConVar s_EmptyConVar; + +ConVarRef::ConVarRef( const char *pName ) +{ + Init( pName, false ); +} + +ConVarRef::ConVarRef( const char *pName, bool bIgnoreMissing ) +{ + Init( pName, bIgnoreMissing ); +} + +void ConVarRef::Init( const char *pName, bool bIgnoreMissing ) +{ + m_pConVar = g_pCVar ? g_pCVar->FindVar( pName ) : &s_EmptyConVar; + if ( !m_pConVar ) + { + m_pConVar = &s_EmptyConVar; + } + m_pConVarState = static_cast< ConVar * >( m_pConVar ); + if( !IsValid() ) + { + static bool bFirst = true; + if ( g_pCVar || bFirst ) + { + if ( !bIgnoreMissing ) + { + Warning( "ConVarRef %s doesn't point to an existing ConVar\n", pName ); + } + bFirst = false; + } + } +} + +ConVarRef::ConVarRef( IConVar *pConVar ) +{ + m_pConVar = pConVar ? pConVar : &s_EmptyConVar; + m_pConVarState = static_cast< ConVar * >( m_pConVar ); +} + +bool ConVarRef::IsValid() const +{ + return m_pConVar != &s_EmptyConVar; +} + + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void ConVar_PrintFlags( const ConCommandBase *var ) +{ + bool any = false; + if ( var->IsFlagSet( FCVAR_GAMEDLL ) ) + { + ConMsg( " game" ); + any = true; + } + + if ( var->IsFlagSet( FCVAR_CLIENTDLL ) ) + { + ConMsg( " client" ); + any = true; + } + + if ( var->IsFlagSet( FCVAR_ARCHIVE ) ) + { + ConMsg( " archive" ); + any = true; + } + + if ( var->IsFlagSet( FCVAR_NOTIFY ) ) + { + ConMsg( " notify" ); + any = true; + } + + if ( var->IsFlagSet( FCVAR_SPONLY ) ) + { + ConMsg( " singleplayer" ); + any = true; + } + + if ( var->IsFlagSet( FCVAR_NOT_CONNECTED ) ) + { + ConMsg( " notconnected" ); + any = true; + } + + if ( var->IsFlagSet( FCVAR_CHEAT ) ) + { + ConMsg( " cheat" ); + any = true; + } + + if ( var->IsFlagSet( FCVAR_REPLICATED ) ) + { + ConMsg( " replicated" ); + any = true; + } + + if ( var->IsFlagSet( FCVAR_SERVER_CAN_EXECUTE ) ) + { + ConMsg( " server_can_execute" ); + any = true; + } + + if ( var->IsFlagSet( FCVAR_CLIENTCMD_CAN_EXECUTE ) ) + { + ConMsg( " clientcmd_can_execute" ); + any = true; + } + + if ( any ) + { + ConMsg( "\n" ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void ConVar_PrintDescription( const ConCommandBase *pVar ) +{ + bool bMin, bMax; + float fMin, fMax; + const char *pStr; + + assert( pVar ); + + Color clr; + clr.SetColor( 255, 100, 100, 255 ); + + if ( !pVar->IsCommand() ) + { + ConVar *var = ( ConVar * )pVar; + const ConVar_ServerBounded *pBounded = dynamic_cast<const ConVar_ServerBounded*>( var ); + + bMin = var->GetMin( fMin ); + bMax = var->GetMax( fMax ); + + const char *value = NULL; + char tempVal[ 32 ]; + + if ( pBounded || var->IsFlagSet( FCVAR_NEVER_AS_STRING ) ) + { + value = tempVal; + + int intVal = pBounded ? pBounded->GetInt() : var->GetInt(); + float floatVal = pBounded ? pBounded->GetFloat() : var->GetFloat(); + + if ( fabs( (float)intVal - floatVal ) < 0.000001 ) + { + Q_snprintf( tempVal, sizeof( tempVal ), "%d", intVal ); + } + else + { + Q_snprintf( tempVal, sizeof( tempVal ), "%f", floatVal ); + } + } + else + { + value = var->GetString(); + } + + if ( value ) + { + ConColorMsg( clr, "\"%s\" = \"%s\"", var->GetName(), value ); + + if ( stricmp( value, var->GetDefault() ) ) + { + ConMsg( " ( def. \"%s\" )", var->GetDefault() ); + } + } + + if ( bMin ) + { + ConMsg( " min. %f", fMin ); + } + if ( bMax ) + { + ConMsg( " max. %f", fMax ); + } + + ConMsg( "\n" ); + + // Handled virtualized cvars. + if ( pBounded && fabs( pBounded->GetFloat() - var->GetFloat() ) > 0.0001f ) + { + ConColorMsg( clr, "** NOTE: The real value is %.3f but the server has temporarily restricted it to %.3f **\n", + var->GetFloat(), pBounded->GetFloat() ); + } + } + else + { + ConCommand *var = ( ConCommand * )pVar; + + ConColorMsg( clr, "\"%s\"\n", var->GetName() ); + } + + ConVar_PrintFlags( pVar ); + + pStr = pVar->GetHelpText(); + if ( pStr && pStr[0] ) + { + ConMsg( " - %s\n", pStr ); + } +} diff --git a/tier1/datamanager.cpp b/tier1/datamanager.cpp new file mode 100644 index 0000000..1fe33f3 --- /dev/null +++ b/tier1/datamanager.cpp @@ -0,0 +1,410 @@ + +// +// Purpose: +// +// $NoKeywords: $ +// +//=============================================================================// + +#include "basetypes.h" +#include "datamanager.h" + +DECLARE_POINTER_HANDLE( memhandle_t ); + +CDataManagerBase::CDataManagerBase( unsigned int maxSize ) +{ + m_targetMemorySize = maxSize; + m_memUsed = 0; + m_lruList = m_memoryLists.CreateList(); + m_lockList = m_memoryLists.CreateList(); + m_freeList = m_memoryLists.CreateList(); + m_listsAreFreed = 0; +} + +CDataManagerBase::~CDataManagerBase() +{ + Assert( m_listsAreFreed ); +} + +void CDataManagerBase::NotifySizeChanged( memhandle_t handle, unsigned int oldSize, unsigned int newSize ) +{ + Lock(); + m_memUsed += (int)newSize - (int)oldSize; + Unlock(); +} + +void CDataManagerBase::SetTargetSize( unsigned int targetSize ) +{ + m_targetMemorySize = targetSize; +} + +unsigned int CDataManagerBase::FlushAllUnlocked() +{ + Lock(); + + int nFlush = m_memoryLists.Count( m_lruList ); + void **pScratch = (void **)_alloca( nFlush * sizeof(void *) ); + CUtlVector<void *> destroyList( pScratch, nFlush ); + + unsigned nBytesInitial = MemUsed_Inline(); + + int node = m_memoryLists.Head(m_lruList); + while ( node != m_memoryLists.InvalidIndex() ) + { + int next = m_memoryLists.Next(node); + m_memoryLists.Unlink( m_lruList, node ); + destroyList.AddToTail( GetForFreeByIndex( node ) ); + node = next; + } + + Unlock(); + + for ( int i = 0; i < nFlush; i++ ) + { + DestroyResourceStorage( destroyList[i] ); + } + + return ( nBytesInitial - MemUsed_Inline() ); +} + +unsigned int CDataManagerBase::FlushToTargetSize() +{ + return EnsureCapacity(0); +} + +// Frees everything! The LRU AND the LOCKED items. This is only used to forcibly free the resources, +// not to make space. + +unsigned int CDataManagerBase::FlushAll() +{ + Lock(); + + int nFlush = m_memoryLists.Count( m_lruList ) + m_memoryLists.Count( m_lockList ); + void **pScratch = (void **)_alloca( nFlush * sizeof(void *) ); + CUtlVector<void *> destroyList( pScratch, nFlush ); + + unsigned result = MemUsed_Inline(); + int node; + int nextNode; + + node = m_memoryLists.Head(m_lruList); + while ( node != m_memoryLists.InvalidIndex() ) + { + nextNode = m_memoryLists.Next(node); + m_memoryLists.Unlink( m_lruList, node ); + destroyList.AddToTail( GetForFreeByIndex( node ) ); + node = nextNode; + } + + node = m_memoryLists.Head(m_lockList); + while ( node != m_memoryLists.InvalidIndex() ) + { + nextNode = m_memoryLists.Next(node); + m_memoryLists.Unlink( m_lockList, node ); + m_memoryLists[node].lockCount = 0; + destroyList.AddToTail( GetForFreeByIndex( node ) ); + node = nextNode; + } + + m_listsAreFreed = false; + Unlock(); + + for ( int i = 0; i < nFlush; i++ ) + { + DestroyResourceStorage( destroyList[i] ); + } + + return result; +} + +unsigned int CDataManagerBase::Purge( unsigned int nBytesToPurge ) +{ + unsigned int nTargetSize = MemUsed_Inline() - nBytesToPurge; + // Check for underflow + if ( MemUsed_Inline() < nBytesToPurge ) + nTargetSize = 0; + unsigned int nImpliedCapacity = MemTotal_Inline() - nTargetSize; + return EnsureCapacity( nImpliedCapacity ); +} + + +void CDataManagerBase::DestroyResource( memhandle_t handle ) +{ + Lock(); + unsigned short index = FromHandle( handle ); + if ( !m_memoryLists.IsValidIndex(index) ) + { + Unlock(); + return; + } + + Assert( m_memoryLists[index].lockCount == 0 ); + if ( m_memoryLists[index].lockCount ) + BreakLock( handle ); + m_memoryLists.Unlink( m_lruList, index ); + void *p = GetForFreeByIndex( index ); + Unlock(); + + DestroyResourceStorage( p ); +} + + +void *CDataManagerBase::LockResource( memhandle_t handle ) +{ + AUTO_LOCK( *this ); + + unsigned short memoryIndex = FromHandle(handle); + if ( memoryIndex != m_memoryLists.InvalidIndex() ) + { + if ( m_memoryLists[memoryIndex].lockCount == 0 ) + { + m_memoryLists.Unlink( m_lruList, memoryIndex ); + m_memoryLists.LinkToTail( m_lockList, memoryIndex ); + } + Assert(m_memoryLists[memoryIndex].lockCount != (unsigned short)-1); + m_memoryLists[memoryIndex].lockCount++; + return m_memoryLists[memoryIndex].pStore; + } + + return NULL; +} + +int CDataManagerBase::UnlockResource( memhandle_t handle ) +{ + AUTO_LOCK( *this ); + unsigned short memoryIndex = FromHandle(handle); + if ( memoryIndex != m_memoryLists.InvalidIndex() ) + { + Assert( m_memoryLists[memoryIndex].lockCount > 0 ); + if ( m_memoryLists[memoryIndex].lockCount > 0 ) + { + m_memoryLists[memoryIndex].lockCount--; + if ( m_memoryLists[memoryIndex].lockCount == 0 ) + { + m_memoryLists.Unlink( m_lockList, memoryIndex ); + m_memoryLists.LinkToTail( m_lruList, memoryIndex ); + } + } + return m_memoryLists[memoryIndex].lockCount; + } + + return 0; +} + +void *CDataManagerBase::GetResource_NoLockNoLRUTouch( memhandle_t handle ) +{ + AUTO_LOCK( *this ); + unsigned short memoryIndex = FromHandle(handle); + if ( memoryIndex != m_memoryLists.InvalidIndex() ) + { + return m_memoryLists[memoryIndex].pStore; + } + return NULL; +} + + +void *CDataManagerBase::GetResource_NoLock( memhandle_t handle ) +{ + AUTO_LOCK( *this ); + unsigned short memoryIndex = FromHandle(handle); + if ( memoryIndex != m_memoryLists.InvalidIndex() ) + { + TouchByIndex( memoryIndex ); + return m_memoryLists[memoryIndex].pStore; + } + return NULL; +} + +void CDataManagerBase::TouchResource( memhandle_t handle ) +{ + AUTO_LOCK( *this ); + TouchByIndex( FromHandle(handle) ); +} + +void CDataManagerBase::MarkAsStale( memhandle_t handle ) +{ + AUTO_LOCK( *this ); + unsigned short memoryIndex = FromHandle(handle); + if ( memoryIndex != m_memoryLists.InvalidIndex() ) + { + if ( m_memoryLists[memoryIndex].lockCount == 0 ) + { + m_memoryLists.Unlink( m_lruList, memoryIndex ); + m_memoryLists.LinkToHead( m_lruList, memoryIndex ); + } + } +} + +int CDataManagerBase::BreakLock( memhandle_t handle ) +{ + AUTO_LOCK( *this ); + unsigned short memoryIndex = FromHandle(handle); + if ( memoryIndex != m_memoryLists.InvalidIndex() && m_memoryLists[memoryIndex].lockCount ) + { + int nBroken = m_memoryLists[memoryIndex].lockCount; + m_memoryLists[memoryIndex].lockCount = 0; + m_memoryLists.Unlink( m_lockList, memoryIndex ); + m_memoryLists.LinkToTail( m_lruList, memoryIndex ); + + return nBroken; + } + return 0; +} + +int CDataManagerBase::BreakAllLocks() +{ + AUTO_LOCK( *this ); + int nBroken = 0; + int node; + int nextNode; + + node = m_memoryLists.Head(m_lockList); + while ( node != m_memoryLists.InvalidIndex() ) + { + nBroken++; + nextNode = m_memoryLists.Next(node); + m_memoryLists[node].lockCount = 0; + m_memoryLists.Unlink( m_lockList, node ); + m_memoryLists.LinkToTail( m_lruList, node ); + node = nextNode; + } + + return nBroken; + +} + +unsigned short CDataManagerBase::CreateHandle( bool bCreateLocked ) +{ + AUTO_LOCK( *this ); + int memoryIndex = m_memoryLists.Head(m_freeList); + unsigned short list = ( bCreateLocked ) ? m_lockList : m_lruList; + if ( memoryIndex != m_memoryLists.InvalidIndex() ) + { + m_memoryLists.Unlink( m_freeList, memoryIndex ); + m_memoryLists.LinkToTail( list, memoryIndex ); + } + else + { + memoryIndex = m_memoryLists.AddToTail( list ); + } + + if ( bCreateLocked ) + { + m_memoryLists[memoryIndex].lockCount++; + } + + return memoryIndex; +} + +memhandle_t CDataManagerBase::StoreResourceInHandle( unsigned short memoryIndex, void *pStore, unsigned int realSize ) +{ + AUTO_LOCK( *this ); + resource_lru_element_t &mem = m_memoryLists[memoryIndex]; + mem.pStore = pStore; + m_memUsed += realSize; + return ToHandle(memoryIndex); +} + +void CDataManagerBase::TouchByIndex( unsigned short memoryIndex ) +{ + if ( memoryIndex != m_memoryLists.InvalidIndex() ) + { + if ( m_memoryLists[memoryIndex].lockCount == 0 ) + { + m_memoryLists.Unlink( m_lruList, memoryIndex ); + m_memoryLists.LinkToTail( m_lruList, memoryIndex ); + } + } +} + +memhandle_t CDataManagerBase::ToHandle( unsigned short index ) +{ + unsigned int hiword = m_memoryLists.Element(index).serial; + hiword <<= 16; + index++; + return (memhandle_t)( hiword|index ); +} + +unsigned int CDataManagerBase::TargetSize() +{ + return MemTotal_Inline(); +} + +unsigned int CDataManagerBase::AvailableSize() +{ + return MemAvailable_Inline(); +} + + +unsigned int CDataManagerBase::UsedSize() +{ + return MemUsed_Inline(); +} + +// free resources until there is enough space to hold "size" +unsigned int CDataManagerBase::EnsureCapacity( unsigned int size ) +{ + unsigned nBytesInitial = MemUsed_Inline(); + while ( MemUsed_Inline() > MemTotal_Inline() || MemAvailable_Inline() < size ) + { + Lock(); + int lruIndex = m_memoryLists.Head( m_lruList ); + if ( lruIndex == m_memoryLists.InvalidIndex() ) + { + Unlock(); + break; + } + m_memoryLists.Unlink( m_lruList, lruIndex ); + void *p = GetForFreeByIndex( lruIndex ); + Unlock(); + DestroyResourceStorage( p ); + } + return ( nBytesInitial - MemUsed_Inline() ); +} + +// free this resource and move the handle to the free list +void *CDataManagerBase::GetForFreeByIndex( unsigned short memoryIndex ) +{ + void *p = NULL; + if ( memoryIndex != m_memoryLists.InvalidIndex() ) + { + Assert( m_memoryLists[memoryIndex].lockCount == 0 ); + + resource_lru_element_t &mem = m_memoryLists[memoryIndex]; + unsigned size = GetRealSize( mem.pStore ); + if ( size > m_memUsed ) + { + ExecuteOnce( Warning( "Data manager 'used' memory incorrect\n" ) ); + size = m_memUsed; + } + m_memUsed -= size; + p = mem.pStore; + mem.pStore = NULL; + mem.serial++; + m_memoryLists.LinkToTail( m_freeList, memoryIndex ); + } + return p; +} + +// get a list of everything in the LRU +void CDataManagerBase::GetLRUHandleList( CUtlVector< memhandle_t >& list ) +{ + for ( int node = m_memoryLists.Tail(m_lruList); + node != m_memoryLists.InvalidIndex(); + node = m_memoryLists.Previous(node) ) + { + list.AddToTail( ToHandle( node ) ); + } +} + +// get a list of everything locked +void CDataManagerBase::GetLockHandleList( CUtlVector< memhandle_t >& list ) +{ + for ( int node = m_memoryLists.Head(m_lockList); + node != m_memoryLists.InvalidIndex(); + node = m_memoryLists.Next(node) ) + { + list.AddToTail( ToHandle( node ) ); + } +} + diff --git a/tier1/diff.cpp b/tier1/diff.cpp new file mode 100644 index 0000000..bad7053 --- /dev/null +++ b/tier1/diff.cpp @@ -0,0 +1,547 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//=============================================================================// + +#include "tier0/platform.h" +#include "tier0/dbg.h" +#include "tier1/diff.h" +#include "mathlib/mathlib.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + + +// format of diff output: +// 0NN (N=1..127) copy next N literaly +// +// 1NN (N=1..127) ofs (-128..127) copy next N bytes from original, changin offset by N bytes from +// last copy end +// 100 N ofs(-32768..32767) copy next N, with larger delta offset +// 00 NNNN(1..65535) ofs(-32768..32767) big copy from old +// 80 00 NN NN NN big raw copy +// +// available codes (could be used for additonal compression ops) +// long offset form whose offset could have fit in short offset + +// note - this algorithm uses storage equal to 8* the old buffer size. 64k=.5mb + + +#define MIN_MATCH_LEN 8 +#define ACCEPTABLE_MATCH_LEN 4096 + +struct BlockPtr +{ + BlockPtr *Next; + uint8 const *dataptr; +}; + +template<class T,class V> static inline void AddToHead(T * & head, V * node) +{ + node->Next=head; + head=node; +} + +void Fail(char const *msg) +{ + Assert(0); +} + +void ApplyDiffs(uint8 const *OldBlock, uint8 const *DiffList, + int OldSize, int DiffListSize, int &ResultListSize,uint8 *Output,uint32 OutSize) +{ + uint8 const *copy_src=OldBlock; + uint8 const *end_of_diff_list=DiffList+DiffListSize; + uint8 const *obuf=Output; + while(DiffList<end_of_diff_list) + { + // printf("dptr=%x ",DiffList-d); + uint8 op=*(DiffList++); + if (op==0) + { + uint16 copy_sz=DiffList[0]+256*DiffList[1]; + int copy_ofs=DiffList[2]+DiffList[3]*256; + if (copy_ofs>32767) + copy_ofs|=0xffff0000; + // printf("long cp from %x to %x len=%d\n", copy_src+copy_ofs-OldBlock,Output-obuf,copy_sz); + + memcpy(Output,copy_src+copy_ofs,copy_sz); + Output+=copy_sz; + copy_src=copy_src+copy_ofs+copy_sz; + DiffList+=4; + } + else + { + if (op & 0x80) + { + int copy_sz=op & 0x7f; + int copy_ofs; + if (copy_sz==0) + { + copy_sz=DiffList[0]; + if (copy_sz==0) + { + // big raw copy + copy_sz=DiffList[1]+256*DiffList[2]+65536*DiffList[3]; + memcpy(Output,DiffList+4,copy_sz); + // printf("big rawcopy to %x len=%d\n", Output-obuf,copy_sz); + + DiffList+=copy_sz+4; + Output+=copy_sz; + } + else + { + copy_ofs=DiffList[1]+(DiffList[2]*256); + if (copy_ofs>32767) + copy_ofs|=0xffff0000; + // printf("long ofs cp from %x to %x len=%d\n", copy_src+copy_ofs-OldBlock,Output-obuf,copy_sz); + + memcpy(Output,copy_src+copy_ofs,copy_sz); + Output+=copy_sz; + copy_src=copy_src+copy_ofs+copy_sz; + DiffList+=3; + } + } + else + { + copy_ofs=DiffList[0]; + if (copy_ofs>127) + copy_ofs|=0xffffff80; + // printf("cp from %x to %x len=%d\n", copy_src+copy_ofs-OldBlock,Output-obuf,copy_sz); + + memcpy(Output,copy_src+copy_ofs,copy_sz); + Output+=copy_sz; + copy_src=copy_src+copy_ofs+copy_sz; + DiffList++; + } + } + else + { + // printf("raw copy %d to %x\n",op & 127,Output-obuf); + memcpy(Output,DiffList,op & 127); + Output+=op & 127; + DiffList+=(op & 127); + } + } + } + ResultListSize=Output-obuf; + +} + +static void CopyPending(int len, uint8 const *rawbytes,uint8 * &outbuf, uint8 const *limit) +{ +// printf("copy raw len=%d\n",len); + if (len<128) + { + if (limit-outbuf < len+1) + Fail("diff buffer overrun"); + *(outbuf++)=len; + memcpy(outbuf,rawbytes,len); + outbuf+=len; + } + else + { + if (limit-outbuf < len+5) + Fail("diff buffer overrun"); + *(outbuf++)=0x80; + *(outbuf++)=0x00; + *(outbuf++)=(len & 255); + *(outbuf++)=((len>>8) & 255); + *(outbuf++)=((len>>16) & 255); + memcpy(outbuf,rawbytes,len); + outbuf+=len; + } +} + +static uint32 hasher(uint8 const *mdata) +{ + // attempt to scramble the bits of h1 and h2 together + uint32 ret=0; + for(int i=0;i<MIN_MATCH_LEN;i++) + { + ret=ret<<4; + ret+=(*mdata++); + } + return ret; +} + +int FindDiffsForLargeFiles(uint8 const *NewBlock, uint8 const *OldBlock, + int NewSize, int OldSize, int &DiffListSize,uint8 *Output, + uint32 OutSize, + int hashsize) +{ + + int ret=0; + if (OldSize!=NewSize) + ret=1; + // first, build the hash table + BlockPtr **HashedMatches=new BlockPtr* [hashsize]; + memset(HashedMatches,0,sizeof(HashedMatches[0])*hashsize); + BlockPtr *Blocks=0; + if (OldSize) + Blocks=new BlockPtr[OldSize]; + BlockPtr *FreeList=Blocks; + // now, build the hash table + uint8 const *walk=OldBlock; + if (OldBlock && OldSize) + while(walk<OldBlock+OldSize-MIN_MATCH_LEN) + { + uint32 hash1=hasher(walk); + hash1 &=(hashsize-1); + BlockPtr *newnode=FreeList; + FreeList++; + newnode->dataptr=walk; + AddToHead(HashedMatches[hash1],newnode); + walk++; + } + else + ret=1; + // now, we have the hash table which may be used to search. begin the output step + int pending_raw_len=0; + walk=NewBlock; + uint8 *outbuf=Output; + uint8 const *lastmatchend=OldBlock; + while(walk<NewBlock+NewSize) + { + int longest=0; + BlockPtr *longest_block=0; + if (walk<NewBlock+NewSize-MIN_MATCH_LEN) + { + // check for a match + uint32 hash1=hasher(walk); + hash1 &= (hashsize-1); + // now, find the longest match in the hash table. If we find one >MIN_MATCH_LEN, take it + for(BlockPtr *b=HashedMatches[hash1];b;b=b->Next) + { + // find the match length + int match_of=b->dataptr-lastmatchend; + if ((match_of>-32768) && (match_of<32767)) + { + int max_mlength=min(65535,(int)((ptrdiff_t)OldBlock+OldSize-(ptrdiff_t)b->dataptr)); + max_mlength=min(max_mlength,(int)((ptrdiff_t)NewBlock+NewSize-(ptrdiff_t)walk)); + int i; + for(i=0;i<max_mlength;i++) + if (walk[i]!=b->dataptr[i]) + break; + if ((i>MIN_MATCH_LEN) && (i>longest)) + { + longest=i; + longest_block=b; + if (longest>ACCEPTABLE_MATCH_LEN) + break; + } + } + } + } + // now, we have a match maybe + if (longest_block) + { + if (pending_raw_len) // must output + { + ret=1; + CopyPending(pending_raw_len,walk-pending_raw_len,outbuf,Output+OutSize); + pending_raw_len=0; + } + // now, output copy block + int match_of=longest_block->dataptr-lastmatchend; + int nremaining=OutSize-(outbuf-Output); + + if (match_of) + ret=1; +// printf("copy from %x to %x len=%d\n", match_of,outbuf-Output,longest); + if (longest>127) + { + // use really long encoding + if (nremaining<5) + Fail("diff buff needs increase"); + *(outbuf++)=00; + *(outbuf++)=(longest & 255); + *(outbuf++)=((longest>>8) & 255); + *(outbuf++)=(match_of & 255); + *(outbuf++)=((match_of>>8) & 255); + + } + else + { + if ((match_of>=-128) && (match_of<128)) + { + if (nremaining<2) + Fail("diff buff needs increase"); + *(outbuf++)=128+longest; + *(outbuf++)=(match_of&255); + } + else + { + // use long encoding + if (nremaining<4) + Fail("diff buff needs increase"); + *(outbuf++)=0x80; + *(outbuf++)=longest; + *(outbuf++)=(match_of & 255); + *(outbuf++)=((match_of>>8) & 255); + } + } + lastmatchend=longest_block->dataptr+longest; + walk+=longest; + } + else + { + walk++; + pending_raw_len++; + } + } + // now, flush pending raw copy + if (pending_raw_len) // must output + { + ret=1; + CopyPending(pending_raw_len,walk-pending_raw_len,outbuf,Output+OutSize); + pending_raw_len=0; + } + delete[] HashedMatches; + if (Blocks) + delete[] Blocks; + DiffListSize=outbuf-Output; + return ret; +} + + +int FindDiffs(uint8 const *NewBlock, uint8 const *OldBlock, + int NewSize, int OldSize, int &DiffListSize,uint8 *Output,uint32 OutSize) +{ + + int ret=0; + if (OldSize!=NewSize) + ret=1; + // first, build the hash table + BlockPtr *HashedMatches[65536]; + memset(HashedMatches,0,sizeof(HashedMatches)); + BlockPtr *Blocks=0; + if (OldSize) + Blocks=new BlockPtr[OldSize]; + BlockPtr *FreeList=Blocks; + // now, build the hash table + uint8 const *walk=OldBlock; + if (OldBlock && OldSize) + while(walk<OldBlock+OldSize-MIN_MATCH_LEN) + { + uint16 hash1=*((uint16 const *) walk)+*((uint16 const *) walk+2); + BlockPtr *newnode=FreeList; + FreeList++; + newnode->dataptr=walk; + AddToHead(HashedMatches[hash1],newnode); + walk++; + } + else + ret=1; + // now, we have the hash table which may be used to search. begin the output step + int pending_raw_len=0; + walk=NewBlock; + uint8 *outbuf=Output; + uint8 const *lastmatchend=OldBlock; + while(walk<NewBlock+NewSize) + { + int longest=0; + BlockPtr *longest_block=0; + if (walk<NewBlock+NewSize-MIN_MATCH_LEN) + { + // check for a match + uint16 hash1=*((uint16 const *) walk)+*((uint16 const *) walk+2); + // now, find the longest match in the hash table. If we find one >MIN_MATCH_LEN, take it + for(BlockPtr *b=HashedMatches[hash1];b;b=b->Next) + { + // find the match length + int match_of=b->dataptr-lastmatchend; + if ((match_of>-32768) && (match_of<32767)) + { + int max_mlength=min(65535,(int)((ptrdiff_t)OldBlock+OldSize-(ptrdiff_t)b->dataptr)); + max_mlength=min(max_mlength,(int)((ptrdiff_t)NewBlock+NewSize-(ptrdiff_t)walk)); + int i; + for(i=0;i<max_mlength;i++) + if (walk[i]!=b->dataptr[i]) + break; + if ((i>MIN_MATCH_LEN) && (i>longest)) + { + longest=i; + longest_block=b; + } + } + } + } + // now, we have a match maybe + if (longest_block) + { + if (pending_raw_len) // must output + { + ret=1; + CopyPending(pending_raw_len,walk-pending_raw_len,outbuf,Output+OutSize); + pending_raw_len=0; + } + // now, output copy block + int match_of=longest_block->dataptr-lastmatchend; + int nremaining=OutSize-(outbuf-Output); + if (match_of) + ret=1; + if (longest>127) + { + // use really long encoding + if (nremaining<5) + Fail("diff buff needs increase"); + *(outbuf++)=00; + *(outbuf++)=(longest & 255); + *(outbuf++)=((longest>>8) & 255); + *(outbuf++)=(match_of & 255); + *(outbuf++)=((match_of>>8) & 255); + } + else + { + if ((match_of>=-128) && (match_of<128)) + { + if (nremaining<2) + Fail("diff buff needs increase"); + *(outbuf++)=128+longest; + *(outbuf++)=(match_of&255); + } + else + { + // use long encoding + if (nremaining<4) + Fail("diff buff needs increase"); + *(outbuf++)=0x80; + *(outbuf++)=longest; + *(outbuf++)=(match_of & 255); + *(outbuf++)=((match_of>>8) & 255); + } + } + lastmatchend=longest_block->dataptr+longest; + walk+=longest; + } + else + { + walk++; + pending_raw_len++; + } + } + // now, flush pending raw copy + if (pending_raw_len) // must output + { + ret=1; + CopyPending(pending_raw_len,walk-pending_raw_len,outbuf,Output+OutSize); + pending_raw_len=0; + } + if (Blocks) + delete[] Blocks; + DiffListSize=outbuf-Output; + return ret; +} + + +int FindDiffsLowMemory(uint8 const *NewBlock, uint8 const *OldBlock, + int NewSize, int OldSize, int &DiffListSize,uint8 *Output,uint32 OutSize) +{ + + int ret=0; + if (OldSize!=NewSize) + ret=1; + uint8 const *old_data_hash[256]; + memset(old_data_hash,0,sizeof(old_data_hash)); + int pending_raw_len=0; + uint8 const *walk=NewBlock; + uint8 const *oldptr=OldBlock; + uint8 *outbuf=Output; + uint8 const *lastmatchend=OldBlock; + while(walk<NewBlock+NewSize) + { + while( (oldptr-OldBlock<walk-NewBlock+40) && (oldptr-OldBlock<OldSize-MIN_MATCH_LEN)) + { + uint16 hash1=(oldptr[0]+oldptr[1]+oldptr[2]+oldptr[3]) & (NELEMS(old_data_hash)-1); + old_data_hash[hash1]=oldptr; + oldptr++; + } + int longest=0; + uint8 const *longest_block=0; + if (walk<NewBlock+NewSize-MIN_MATCH_LEN) + { + // check for a match + uint16 hash1=(walk[0]+walk[1]+walk[2]+walk[3]) & (NELEMS(old_data_hash)-1); + if (old_data_hash[hash1]) + { + int max_bytes_to_compare=min(NewBlock+NewSize-walk,OldBlock+OldSize-old_data_hash[hash1]); + int nmatches; + for(nmatches=0;nmatches<max_bytes_to_compare;nmatches++) + if (walk[nmatches]!=old_data_hash[hash1][nmatches]) + break; + if (nmatches>MIN_MATCH_LEN) + { + longest_block=old_data_hash[hash1]; + longest=nmatches; + } + } + } + // now, we have a match maybe + if (longest_block) + { + if (pending_raw_len) // must output + { + ret=1; + CopyPending(pending_raw_len,walk-pending_raw_len,outbuf,Output+OutSize); + pending_raw_len=0; + } + // now, output copy block + int match_of=longest_block-lastmatchend; + int nremaining=OutSize-(outbuf-Output); + if (match_of) + ret=1; + if (longest>127) + { + // use really long encoding + if (nremaining<5) + Fail("diff buff needs increase"); + *(outbuf++)=00; + *(outbuf++)=(longest & 255); + *(outbuf++)=((longest>>8) & 255); + *(outbuf++)=(match_of & 255); + *(outbuf++)=((match_of>>8) & 255); + } + else + { + if ((match_of>=-128) && (match_of<128)) + { + if (nremaining<2) + Fail("diff buff needs increase"); + *(outbuf++)=128+longest; + *(outbuf++)=(match_of&255); + } + else + { + // use long encoding + if (nremaining<4) + Fail("diff buff needs increase"); + *(outbuf++)=0x80; + *(outbuf++)=longest; + *(outbuf++)=(match_of & 255); + *(outbuf++)=((match_of>>8) & 255); + } + } + lastmatchend=longest_block+longest; + walk+=longest; + } + else + { + walk++; + pending_raw_len++; + } + } + // now, flush pending raw copy + if (pending_raw_len) // must output + { + ret=1; + CopyPending(pending_raw_len,walk-pending_raw_len,outbuf,Output+OutSize); + pending_raw_len=0; + } + DiffListSize=outbuf-Output; + return ret; +} + + diff --git a/tier1/fileio.cpp b/tier1/fileio.cpp new file mode 100644 index 0000000..3a90998 --- /dev/null +++ b/tier1/fileio.cpp @@ -0,0 +1,497 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: A collection of utility classes to simplify file I/O, and +// as much as possible contain portability problems. Here avoiding +// including windows.h. +// +//============================================================================= + +#if defined(_WIN32) +#undef _WIN32_WINNT +#define _WIN32_WINNT 0x0502 // ReadDirectoryChangesW +#endif + +#if defined(OSX) +#include <CoreServices/CoreServices.h> +#include <sys/types.h> +#include <dirent.h> +#include <sys/time.h> +#endif + +#define ASYNC_FILEIO +#if defined( LINUX ) +// Linux hasn't got a good AIO library that we have found yet, so lets punt for now +#undef ASYNC_FILEIO +#endif + +#if defined(_WIN32) +//#include <direct.h> +#include <io.h> +// unset to force to use stdio implementation +#define WIN32_FILEIO + +#if defined(ASYNC_FILEIO) +#if defined(_WIN32) && !defined(WIN32_FILEIO) +#error "trying to use async io without win32 filesystem API usage, that isn't doable" +#endif +#endif + +#else /* not defined (_WIN32) */ +#include <utime.h> +#include <dirent.h> +#include <unistd.h> // for unlink +#include <limits.h> // defines PATH_MAX +#include <alloca.h> // 'cause we like smashing the stack +#if defined( _PS3 ) +#include <fcntl.h> +#else +#include <sys/fcntl.h> +#include <sys/statvfs.h> +#endif +#include <sched.h> +#define int64 int64_t + +#define _A_SUBDIR S_IFDIR + +// FUTURE map _A_HIDDEN via checking filename against .* +#define _A_HIDDEN 0 + +// FUTURE check 'read only' by checking mode against S_IRUSR +#define _A_RDONLY 0 + +// no files under posix are 'system' or 'archive' +#define _A_SYSTEM 0 +#define _A_ARCH 0 + +#endif + +#include "tier1/fileio.h" +#include "tier1/utlbuffer.h" +#include "tier1/strtools.h" +#include <errno.h> + +#if defined( WIN32_FILEIO ) +#include "winlite.h" +#endif + +#if defined( ASYNC_FILEIO ) +#ifdef _WIN32 +#include "winlite.h" +#elif defined(_PS3) +// bugbug ps3 - see some aio files under libfs.. skipping for the moment +#elif defined(POSIX) +#include <aio.h> +#else +#error "aio please" +#endif +#endif + +//----------------------------------------------------------------------------- +// Purpose: Constructor from UTF8 +//----------------------------------------------------------------------------- +CPathString::CPathString( const char *pchUTF8Path ) +{ + // Need to first turn into an absolute path, so \\?\ pre-pended paths will be ok + m_pchUTF8Path = new char[ MAX_UNICODE_PATH_IN_UTF8 ]; + m_pwchWideCharPathPrepended = NULL; + + // First, convert to absolute path, which also does Q_FixSlashes for us. + Q_MakeAbsolutePath( m_pchUTF8Path, MAX_UNICODE_PATH * 4, pchUTF8Path ); + + // Second, fix any double slashes + V_FixDoubleSlashes( m_pchUTF8Path ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Destructor +//----------------------------------------------------------------------------- +CPathString::~CPathString() +{ + if ( m_pwchWideCharPathPrepended ) + { + delete[] m_pwchWideCharPathPrepended; + m_pwchWideCharPathPrepended = NULL; + } + + if ( m_pchUTF8Path ) + { + delete[] m_pchUTF8Path; + m_pchUTF8Path = NULL; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Access UTF8 path +//----------------------------------------------------------------------------- +const char * CPathString::GetUTF8Path() +{ + return m_pchUTF8Path; +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets wchar_t based path, with \\?\ pre-pended (allowing long paths +// on Win32, should only be used with unicode extended path aware filesystem calls) +//----------------------------------------------------------------------------- +const wchar_t *CPathString::GetWCharPathPrePended() +{ + PopulateWCharPath(); + return m_pwchWideCharPathPrepended; +} + + +//----------------------------------------------------------------------------- +// Purpose: Builds wchar path string +//----------------------------------------------------------------------------- +void CPathString::PopulateWCharPath() +{ + if ( m_pwchWideCharPathPrepended ) + return; + + // Check if the UTF8 path starts with \\, which on Win32 means it's a UNC path, and then needs a different prefix + if ( m_pchUTF8Path[0] == '\\' && m_pchUTF8Path[1] == '\\' ) + { + m_pwchWideCharPathPrepended = new wchar_t[MAX_UNICODE_PATH+8]; + Q_memcpy( m_pwchWideCharPathPrepended, L"\\\\?\\UNC\\", 8*sizeof(wchar_t) ); +#ifdef DBGFLAG_ASSERT + int cchResult = +#endif + Q_UTF8ToUnicode( m_pchUTF8Path+2, m_pwchWideCharPathPrepended+8, MAX_UNICODE_PATH*sizeof(wchar_t) ); + Assert( cchResult ); + + // Be sure we NULL terminate within our allocated region incase Q_UTF8ToUnicode failed, though we're already in bad shape then. + m_pwchWideCharPathPrepended[MAX_UNICODE_PATH+7] = 0; + } + else + { + m_pwchWideCharPathPrepended = new wchar_t[MAX_UNICODE_PATH+4]; + Q_memcpy( m_pwchWideCharPathPrepended, L"\\\\?\\", 4*sizeof(wchar_t) ); +#ifdef DBGFLAG_ASSERT + int cchResult = +#endif + Q_UTF8ToUnicode( m_pchUTF8Path, m_pwchWideCharPathPrepended+4, MAX_UNICODE_PATH*sizeof(wchar_t) ); + Assert( cchResult ); + + // Be sure we NULL terminate within our allocated region incase Q_UTF8ToUnicode failed, though we're already in bad shape then. + m_pwchWideCharPathPrepended[MAX_UNICODE_PATH+3] = 0; + } +} + +#ifdef WIN32 +struct DirWatcherOverlapped : public OVERLAPPED +{ + CDirWatcher *m_pDirWatcher; +}; +#endif + +#if !defined(_PS3) && !defined(_X360) +// a buffer full of file names +static const int k_cubDirWatchBufferSize = 8 * 1024; + +//----------------------------------------------------------------------------- +// Purpose: directory watching +//----------------------------------------------------------------------------- +CDirWatcher::CDirWatcher() +{ + m_hFile = NULL; + m_pOverlapped = NULL; + m_pFileInfo = NULL; +#ifdef OSX + m_WatcherStream = 0; +#endif +} + + +//----------------------------------------------------------------------------- +// Purpose: directory watching +//----------------------------------------------------------------------------- +CDirWatcher::~CDirWatcher() +{ +#ifdef WIN32 + if ( m_pOverlapped ) + { + // mark the overlapped structure as gone + DirWatcherOverlapped *pDirWatcherOverlapped = (DirWatcherOverlapped *)m_pOverlapped; + pDirWatcherOverlapped->m_pDirWatcher = NULL; + } + + if ( m_hFile ) + { + // make sure we flush any pending I/O's on the handle + ::CancelIo( m_hFile ); + ::SleepEx( 0, TRUE ); + // close the handle + ::CloseHandle( m_hFile ); + } +#elif defined(OSX) + if ( m_WatcherStream ) + { + FSEventStreamStop( (FSEventStreamRef)m_WatcherStream ); + FSEventStreamInvalidate( (FSEventStreamRef)m_WatcherStream ); + FSEventStreamRelease( (FSEventStreamRef)m_WatcherStream ); + m_WatcherStream = 0; + } +#endif + if ( m_pFileInfo ) + { + free( m_pFileInfo ); + } + if ( m_pOverlapped ) + { + free( m_pOverlapped ); + } +} + + +#ifdef WIN32 +//----------------------------------------------------------------------------- +// Purpose: callback watch +// gets called on the same thread whenever a SleepEx() occurs +//----------------------------------------------------------------------------- +class CDirWatcherFriend +{ +public: + static void WINAPI DirWatchCallback( DWORD dwErrorCode, DWORD dwNumberOfBytesTransfered, OVERLAPPED *pOverlapped ) + { + DirWatcherOverlapped *pDirWatcherOverlapped = (DirWatcherOverlapped *)pOverlapped; + + // see if we've been cancelled + if ( !pDirWatcherOverlapped->m_pDirWatcher ) + return; + + // parse and pass back + if ( dwNumberOfBytesTransfered > sizeof(FILE_NOTIFY_INFORMATION) ) + { + FILE_NOTIFY_INFORMATION *pFileNotifyInformation = (FILE_NOTIFY_INFORMATION *)pDirWatcherOverlapped->m_pDirWatcher->m_pFileInfo; + do + { + // null terminate the string and turn it to UTF-8 + int cNumWChars = pFileNotifyInformation->FileNameLength / sizeof(wchar_t); + wchar_t *pwchT = new wchar_t[cNumWChars + 1]; + memcpy( pwchT, pFileNotifyInformation->FileName, pFileNotifyInformation->FileNameLength ); + pwchT[cNumWChars] = 0; + CStrAutoEncode strAutoEncode( pwchT ); + + // add it to our list + pDirWatcherOverlapped->m_pDirWatcher->AddFileToChangeList( strAutoEncode.ToString() ); + delete[] pwchT; + if ( pFileNotifyInformation->NextEntryOffset == 0 ) + break; + + // move to the next file + pFileNotifyInformation = (FILE_NOTIFY_INFORMATION *)(((byte*)pFileNotifyInformation) + pFileNotifyInformation->NextEntryOffset); + } while ( 1 ); + } + + + // watch again + pDirWatcherOverlapped->m_pDirWatcher->PostDirWatch(); + } +}; +#elif defined(OSX) +void CheckDirectoryForChanges( const char *path_buff, CDirWatcher *pDirWatch, bool bRecurse ) +{ + DIR *dir = opendir(path_buff); + char fullpath[MAX_PATH]; + struct dirent *dirent; + struct timespec ts = { 0, 0 }; + bool bTimeSet = false; + + while ( (dirent = readdir(dir)) != NULL ) + { + if (strcmp(dirent->d_name, ".") == 0 || strcmp(dirent->d_name, "..") == 0) + continue; + + snprintf( fullpath, PATH_MAX, "%s/%s", path_buff, dirent->d_name ); + + struct stat st; + if (lstat(fullpath, &st) != 0) + continue; + + if ( S_ISDIR(st.st_mode) && bRecurse ) + { + CheckDirectoryForChanges( fullpath, pDirWatch, bRecurse ); + } + else if ( st.st_mtimespec.tv_sec > pDirWatch->m_modTime.tv_sec || + ( st.st_mtimespec.tv_sec == pDirWatch->m_modTime.tv_sec && st.st_mtimespec.tv_nsec > pDirWatch->m_modTime.tv_nsec ) ) + { + ts = st.st_mtimespec; + bTimeSet = true; + // the win32 size only sends up the dir relative to the watching dir, so replicate that here + pDirWatch->AddFileToChangeList( fullpath + pDirWatch->m_BaseDir.Length() + 1 ); + } + } + + if ( bTimeSet ) + pDirWatch->m_modTime = ts; + closedir(dir); +} + +static void fsevents_callback( ConstFSEventStreamRef streamRef, void *clientCallBackInfo, size_t numEvents,void *eventPaths, + const FSEventStreamEventFlags eventMasks[], const FSEventStreamEventId eventIDs[] ) +{ + char path_buff[PATH_MAX]; + for (int i=0; i < numEvents; i++) + { + char **paths = (char **)eventPaths; + + strcpy(path_buff, paths[i]); + int len = strlen(path_buff); + if (path_buff[len-1] == '/') + { + // chop off a trailing slash + path_buff[--len] = '\0'; + } + + bool bRecurse = false; + + if (eventMasks[i] & kFSEventStreamEventFlagMustScanSubDirs + || eventMasks[i] & kFSEventStreamEventFlagUserDropped + || eventMasks[i] & kFSEventStreamEventFlagKernelDropped) + { + bRecurse = true; + } + + CDirWatcher *pDirWatch = (CDirWatcher *)clientCallBackInfo; + // make sure its in our subdir + if ( !V_strnicmp( path_buff, pDirWatch->m_BaseDir.String(), pDirWatch->m_BaseDir.Length() ) ) + CheckDirectoryForChanges( path_buff, pDirWatch, bRecurse ); + } +} + + + + +#endif + +//----------------------------------------------------------------------------- +// Purpose: only one directory can be watched at a time +//----------------------------------------------------------------------------- +void CDirWatcher::SetDirToWatch( const char *pchDir ) +{ + if ( !pchDir || !*pchDir ) + return; + + CPathString strPath( pchDir ); +#ifdef WIN32 + // open the directory + m_hFile = ::CreateFileW( strPath.GetWCharPathPrePended(), FILE_LIST_DIRECTORY, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, OPEN_EXISTING, FILE_FLAG_OVERLAPPED | FILE_FLAG_BACKUP_SEMANTICS, NULL ); + + // create our buffers + m_pFileInfo = malloc( k_cubDirWatchBufferSize ); + m_pOverlapped = malloc( sizeof( DirWatcherOverlapped ) ); + + // post a watch + PostDirWatch(); +#elif defined(OSX) + CFStringRef mypath = CFStringCreateWithCString( NULL, strPath.GetUTF8Path(), kCFStringEncodingMacRoman ); + if ( !mypath ) + { + Assert( !"Failed to CFStringCreateWithCString watcher path" ); + return; + } + + CFArrayRef pathsToWatch = CFArrayCreate(NULL, (const void **)&mypath, 1, NULL); + FSEventStreamContext callbackInfo = {0, this, NULL, NULL, NULL}; + CFAbsoluteTime latency = 1.0; // Latency in seconds + + m_WatcherStream = (void *)FSEventStreamCreate(NULL, + &fsevents_callback, + &callbackInfo, + pathsToWatch, + kFSEventStreamEventIdSinceNow, + latency, + kFSEventStreamCreateFlagNoDefer + ); + + FSEventStreamScheduleWithRunLoop( (FSEventStreamRef)m_WatcherStream, CFRunLoopGetCurrent(), kCFRunLoopDefaultMode); + CFRelease(pathsToWatch ); + CFRelease( mypath ); + + FSEventStreamStart( (FSEventStreamRef)m_WatcherStream ); + + char szFullPath[MAX_PATH]; + Q_MakeAbsolutePath( szFullPath, sizeof(szFullPath), pchDir ); + m_BaseDir = szFullPath; + + struct timeval tv; + gettimeofday( &tv, NULL ); + TIMEVAL_TO_TIMESPEC( &tv, &m_modTime ); + +#else + Assert( !"Impl me" ); +#endif +} + + +#ifdef WIN32 +//----------------------------------------------------------------------------- +// Purpose: used by callback functions to push a file onto the list +//----------------------------------------------------------------------------- +void CDirWatcher::PostDirWatch() +{ + memset( m_pOverlapped, 0, sizeof(DirWatcherOverlapped) ); + DirWatcherOverlapped *pDirWatcherOverlapped = (DirWatcherOverlapped *)m_pOverlapped; + pDirWatcherOverlapped->m_pDirWatcher = this; + + DWORD dwBytes; + ::ReadDirectoryChangesW( m_hFile, m_pFileInfo, k_cubDirWatchBufferSize, TRUE, FILE_NOTIFY_CHANGE_LAST_WRITE | FILE_NOTIFY_CHANGE_FILE_NAME, &dwBytes, (OVERLAPPED *)m_pOverlapped, &CDirWatcherFriend::DirWatchCallback ); +} +#endif + + +//----------------------------------------------------------------------------- +// Purpose: used by callback functions to push a file onto the list +//----------------------------------------------------------------------------- +void CDirWatcher::AddFileToChangeList( const char *pchFile ) +{ + // make sure it isn't already in the list + FOR_EACH_LL( m_listChangedFiles, i ) + { + if ( !Q_stricmp( m_listChangedFiles[i], pchFile ) ) + return; + } + + m_listChangedFiles.AddToTail( pchFile ); +} + + +//----------------------------------------------------------------------------- +// Purpose: retrieve any changes +//----------------------------------------------------------------------------- +bool CDirWatcher::GetChangedFile( CUtlString *psFile ) +{ +#ifdef WIN32 + // this will trigger any pending directory reads + // this does get hit other places in the code; so the callback can happen at any time + ::SleepEx( 0, TRUE ); +#endif + + if ( !m_listChangedFiles.Count() ) + return false; + + *psFile = m_listChangedFiles[m_listChangedFiles.Head()]; + m_listChangedFiles.Remove( m_listChangedFiles.Head() ); + return true; +} + + + +#ifdef DBGFLAG_VALIDATE +void CDirWatcher::Validate( CValidator &validator, const char *pchName ) +{ + VALIDATE_SCOPE(); + + validator.ClaimMemory( m_pOverlapped ); + validator.ClaimMemory( m_pFileInfo ); + ValidateObj( m_listChangedFiles ); + FOR_EACH_LL( m_listChangedFiles, i ) + { + ValidateObj( m_listChangedFiles[i] ); + } +} +#endif + +#endif // _PS3 || _X360
\ No newline at end of file diff --git a/tier1/generichash.cpp b/tier1/generichash.cpp new file mode 100644 index 0000000..c5d3d38 --- /dev/null +++ b/tier1/generichash.cpp @@ -0,0 +1,437 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Variant Pearson Hash general purpose hashing algorithm described +// by Cargill in C++ Report 1994. Generates a 16-bit result. +// +//============================================================================= + +#include <stdlib.h> +#include "tier0/basetypes.h" +#include "tier0/platform.h" +#include "generichash.h" +#include <ctype.h> +#include "tier0/dbg.h" + +// NOTE: This has to be the last file included! +#include "tier0/memdbgon.h" + + +//----------------------------------------------------------------------------- +// +// Table of randomly shuffled values from 0-255 generated by: +// +//----------------------------------------------------------------------------- +/* +void MakeRandomValues() +{ + int i, j, r; + unsigned t; + srand( 0xdeadbeef ); + + for ( i = 0; i < 256; i++ ) + { + g_nRandomValues[i] = (unsigned )i; + } + + for (j = 0; j < 8; j++) + { + for (i = 0; i < 256; i++) + { + r = rand() & 0xff; + t = g_nRandomValues[i]; + g_nRandomValues[i] = g_nRandomValues[r]; + g_nRandomValues[r] = t; + } + } + + printf("static unsigned g_nRandomValues[256] =\n{\n"); + + for (i = 0; i < 256; i += 16) + { + printf("\t"); + for (j = 0; j < 16; j++) + printf(" %3d,", g_nRandomValues[i+j]); + printf("\n"); + } + printf("};\n"); +} +*/ + +static unsigned g_nRandomValues[256] = +{ + 238, 164, 191, 168, 115, 16, 142, 11, 213, 214, 57, 151, 248, 252, 26, 198, + 13, 105, 102, 25, 43, 42, 227, 107, 210, 251, 86, 66, 83, 193, 126, 108, + 131, 3, 64, 186, 192, 81, 37, 158, 39, 244, 14, 254, 75, 30, 2, 88, + 172, 176, 255, 69, 0, 45, 116, 139, 23, 65, 183, 148, 33, 46, 203, 20, + 143, 205, 60, 197, 118, 9, 171, 51, 233, 135, 220, 49, 71, 184, 82, 109, + 36, 161, 169, 150, 63, 96, 173, 125, 113, 67, 224, 78, 232, 215, 35, 219, + 79, 181, 41, 229, 149, 153, 111, 217, 21, 72, 120, 163, 133, 40, 122, 140, + 208, 231, 211, 200, 160, 182, 104, 110, 178, 237, 15, 101, 27, 50, 24, 189, + 177, 130, 187, 92, 253, 136, 100, 212, 19, 174, 70, 22, 170, 206, 162, 74, + 247, 5, 47, 32, 179, 117, 132, 195, 124, 123, 245, 128, 236, 223, 12, 84, + 54, 218, 146, 228, 157, 94, 106, 31, 17, 29, 194, 34, 56, 134, 239, 246, + 241, 216, 127, 98, 7, 204, 154, 152, 209, 188, 48, 61, 87, 97, 225, 85, + 90, 167, 155, 112, 145, 114, 141, 93, 250, 4, 201, 156, 38, 89, 226, 196, + 1, 235, 44, 180, 159, 121, 119, 166, 190, 144, 10, 91, 76, 230, 221, 80, + 207, 55, 58, 53, 175, 8, 6, 52, 68, 242, 18, 222, 103, 249, 147, 129, + 138, 243, 28, 185, 62, 59, 240, 202, 234, 99, 77, 73, 199, 137, 95, 165, +}; + +//----------------------------------------------------------------------------- +// String +//----------------------------------------------------------------------------- +unsigned FASTCALL HashString( const char *pszKey ) +{ + const uint8 *k = (const uint8 *)pszKey; + unsigned even = 0, + odd = 0, + n; + + while ((n = *k++) != 0) + { + even = g_nRandomValues[odd ^ n]; + if ((n = *k++) != 0) + odd = g_nRandomValues[even ^ n]; + else + break; + } + + return (even << 8) | odd ; +} + + +//----------------------------------------------------------------------------- +// Case-insensitive string +//----------------------------------------------------------------------------- +unsigned FASTCALL HashStringCaseless( const char *pszKey ) +{ + const uint8 *k = (const uint8 *) pszKey; + unsigned even = 0, + odd = 0, + n; + + while ((n = toupper(*k++)) != 0) + { + even = g_nRandomValues[odd ^ n]; + if ((n = toupper(*k++)) != 0) + odd = g_nRandomValues[even ^ n]; + else + break; + } + + return (even << 8) | odd; +} + +//----------------------------------------------------------------------------- +// 32 bit conventional case-insensitive string +//----------------------------------------------------------------------------- +unsigned FASTCALL HashStringCaselessConventional( const char *pszKey ) +{ + unsigned hash = 0xAAAAAAAA; // Alternating 1's and 0's to maximize the effect of the later multiply and add + + for( ; *pszKey ; pszKey++ ) + { + hash = ( ( hash << 5 ) + hash ) + (uint8)tolower(*pszKey); + } + + return hash; +} + +//----------------------------------------------------------------------------- +// int hash +//----------------------------------------------------------------------------- +unsigned FASTCALL HashInt( const int n ) +{ + unsigned even, odd; + even = g_nRandomValues[n & 0xff]; + odd = g_nRandomValues[((n >> 8) & 0xff)]; + + even = g_nRandomValues[odd ^ (n >> 24)]; + odd = g_nRandomValues[even ^ ((n >> 16) & 0xff)]; + even = g_nRandomValues[odd ^ ((n >> 8) & 0xff)]; + odd = g_nRandomValues[even ^ (n & 0xff)]; + + return (even << 8) | odd; +} + +//----------------------------------------------------------------------------- +// 4-byte hash +//----------------------------------------------------------------------------- +unsigned FASTCALL Hash4( const void *pKey ) +{ + const uint32 * p = (const uint32 *) pKey; + unsigned even, + odd, + n; + n = *p; + even = g_nRandomValues[n & 0xff]; + odd = g_nRandomValues[((n >> 8) & 0xff)]; + + even = g_nRandomValues[odd ^ (n >> 24)]; + odd = g_nRandomValues[even ^ ((n >> 16) & 0xff)]; + even = g_nRandomValues[odd ^ ((n >> 8) & 0xff)]; + odd = g_nRandomValues[even ^ (n & 0xff)]; + + return (even << 8) | odd; +} + + +//----------------------------------------------------------------------------- +// 8-byte hash +//----------------------------------------------------------------------------- +unsigned FASTCALL Hash8( const void *pKey ) +{ + const uint32 * p = (const uint32 *) pKey; + unsigned even, + odd, + n; + n = *p; + even = g_nRandomValues[n & 0xff]; + odd = g_nRandomValues[((n >> 8) & 0xff)]; + + even = g_nRandomValues[odd ^ (n >> 24)]; + odd = g_nRandomValues[even ^ ((n >> 16) & 0xff)]; + even = g_nRandomValues[odd ^ ((n >> 8) & 0xff)]; + odd = g_nRandomValues[even ^ (n & 0xff)]; + + n = *(p+1); + even = g_nRandomValues[odd ^ (n >> 24)]; + odd = g_nRandomValues[even ^ ((n >> 16) & 0xff)]; + even = g_nRandomValues[odd ^ ((n >> 8) & 0xff)]; + odd = g_nRandomValues[even ^ (n & 0xff)]; + + return (even << 8) | odd; +} + + +//----------------------------------------------------------------------------- +// 12-byte hash +//----------------------------------------------------------------------------- +unsigned FASTCALL Hash12( const void *pKey ) +{ + const uint32 * p = (const uint32 *) pKey; + unsigned even, + odd, + n; + n = *p; + even = g_nRandomValues[n & 0xff]; + odd = g_nRandomValues[((n >> 8) & 0xff)]; + + even = g_nRandomValues[odd ^ (n >> 24)]; + odd = g_nRandomValues[even ^ ((n >> 16) & 0xff)]; + even = g_nRandomValues[odd ^ ((n >> 8) & 0xff)]; + odd = g_nRandomValues[even ^ (n & 0xff)]; + + n = *(p+1); + even = g_nRandomValues[odd ^ (n >> 24)]; + odd = g_nRandomValues[even ^ ((n >> 16) & 0xff)]; + even = g_nRandomValues[odd ^ ((n >> 8) & 0xff)]; + odd = g_nRandomValues[even ^ (n & 0xff)]; + + n = *(p+2); + even = g_nRandomValues[odd ^ (n >> 24)]; + odd = g_nRandomValues[even ^ ((n >> 16) & 0xff)]; + even = g_nRandomValues[odd ^ ((n >> 8) & 0xff)]; + odd = g_nRandomValues[even ^ (n & 0xff)]; + + return (even << 8) | odd; +} + + +//----------------------------------------------------------------------------- +// 16-byte hash +//----------------------------------------------------------------------------- +unsigned FASTCALL Hash16( const void *pKey ) +{ + const uint32 * p = (const uint32 *) pKey; + unsigned even, + odd, + n; + n = *p; + even = g_nRandomValues[n & 0xff]; + odd = g_nRandomValues[((n >> 8) & 0xff)]; + + even = g_nRandomValues[odd ^ (n >> 24)]; + odd = g_nRandomValues[even ^ ((n >> 16) & 0xff)]; + even = g_nRandomValues[odd ^ ((n >> 8) & 0xff)]; + odd = g_nRandomValues[even ^ (n & 0xff)]; + + n = *(p+1); + even = g_nRandomValues[odd ^ (n >> 24)]; + odd = g_nRandomValues[even ^ ((n >> 16) & 0xff)]; + even = g_nRandomValues[odd ^ ((n >> 8) & 0xff)]; + odd = g_nRandomValues[even ^ (n & 0xff)]; + + n = *(p+2); + even = g_nRandomValues[odd ^ (n >> 24)]; + odd = g_nRandomValues[even ^ ((n >> 16) & 0xff)]; + even = g_nRandomValues[odd ^ ((n >> 8) & 0xff)]; + odd = g_nRandomValues[even ^ (n & 0xff)]; + + n = *(p+3); + even = g_nRandomValues[odd ^ (n >> 24)]; + odd = g_nRandomValues[even ^ ((n >> 16) & 0xff)]; + even = g_nRandomValues[odd ^ ((n >> 8) & 0xff)]; + odd = g_nRandomValues[even ^ (n & 0xff)]; + + return (even << 8) | odd; +} + + +//----------------------------------------------------------------------------- +// Arbitrary fixed length hash +//----------------------------------------------------------------------------- +unsigned FASTCALL HashBlock( const void *pKey, unsigned size ) +{ + const uint8 * k = (const uint8 *) pKey; + unsigned even = 0, + odd = 0, + n; + + while (size) + { + --size; + n = *k++; + even = g_nRandomValues[odd ^ n]; + if (size) + { + --size; + n = *k++; + odd = g_nRandomValues[even ^ n]; + } + else + break; + } + + return (even << 8) | odd; +} + + +//----------------------------------------------------------------------------- +// Murmur hash +//----------------------------------------------------------------------------- +uint32 MurmurHash2( const void * key, int len, uint32 seed ) +{ + // 'm' and 'r' are mixing constants generated offline. + // They're not really 'magic', they just happen to work well. + + const uint32 m = 0x5bd1e995; + const int r = 24; + + // Initialize the hash to a 'random' value + + uint32 h = seed ^ len; + + // Mix 4 bytes at a time into the hash + + const unsigned char * data = (const unsigned char *)key; + + while(len >= 4) + { + uint32 k = LittleDWord( *(uint32 *)data ); + + k *= m; + k ^= k >> r; + k *= m; + + h *= m; + h ^= k; + + data += 4; + len -= 4; + } + + // Handle the last few bytes of the input array + + switch(len) + { + case 3: h ^= data[2] << 16; + case 2: h ^= data[1] << 8; + case 1: h ^= data[0]; + h *= m; + }; + + // Do a few final mixes of the hash to ensure the last few + // bytes are well-incorporated. + + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; +} + +#define TOLOWERU( c ) ( ( uint32 ) ( ( ( c >= 'A' ) && ( c <= 'Z' ) )? c + 32 : c ) ) +uint32 MurmurHash2LowerCase( char const *pString, uint32 nSeed ) +{ + int nLen = ( int )strlen( pString ); + char *p = ( char * ) stackalloc( nLen + 1 ); + for( int i = 0; i < nLen ; i++ ) + { + p[i] = TOLOWERU( pString[i] ); + } + return MurmurHash2( p, nLen, nSeed ); +} + + +//----------------------------------------------------------------------------- +// Murmur hash, 64 bit- endian neutral +//----------------------------------------------------------------------------- +uint64 MurmurHash64( const void * key, int len, uint32 seed ) +{ + // 'm' and 'r' are mixing constants generated offline. + // They're not really 'magic', they just happen to work well. + + const uint32 m = 0x5bd1e995; + const int r = 24; + + // Initialize the hash to a 'random' value + + uint32 h1 = seed ^ len; + uint32 h2 = 0; + + // Mix 4 bytes at a time into the hash + + const uint32 * data = (const uint32 *)key; + while ( len >= 8 ) + { + uint32 k1 = LittleDWord( *data++ ); + k1 *= m; k1 ^= k1 >> r; k1 *= m; + h1 *= m; h1 ^= k1; + len -= 4; + + uint32 k2 = LittleDWord( *data++ ); + k2 *= m; k2 ^= k2 >> r; k2 *= m; + h2 *= m; h2 ^= k2; + len -= 4; + } + + if(len >= 4) + { + uint32 k1 = LittleDWord( *data++ ); + k1 *= m; k1 ^= k1 >> r; k1 *= m; + h1 *= m; h1 ^= k1; + len -= 4; + } + + // Handle the last few bytes of the input array + switch(len) + { + case 3: h2 ^= ((uint8*)data)[2] << 16; + case 2: h2 ^= ((uint8*)data)[1] << 8; + case 1: h2 ^= ((uint8*)data)[0]; + h2 *= m; + }; + + h1 ^= h2 >> 18; h1 *= m; + h2 ^= h1 >> 22; h2 *= m; + h1 ^= h2 >> 17; h1 *= m; + h2 ^= h1 >> 19; h2 *= m; + + uint64 h = h1; + + h = (h << 32) | h2; + + return h; +} + diff --git a/tier1/ilocalize.cpp b/tier1/ilocalize.cpp new file mode 100644 index 0000000..66673e6 --- /dev/null +++ b/tier1/ilocalize.cpp @@ -0,0 +1,260 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// + +#if defined( WIN32 ) && !defined( _X360 ) + #include <windows.h> +#elif defined( POSIX ) + #include <iconv.h> +#endif + +#include "tier1/ilocalize.h" +#include "utlstring.h" + +#pragma warning( disable: 4018 ) // '<' : signed/unsigned mismatch + +//----------------------------------------------------------------------------- +// Purpose: converts an english string to unicode +//----------------------------------------------------------------------------- +int ILocalize::ConvertANSIToUnicode(const char *ansi, wchar_t *unicode, int unicodeBufferSizeInBytes) +{ +#ifdef POSIX + // Q_UTF8ToUnicode returns the number of bytes. This function is expected to return the number of chars. + return Q_UTF8ToUnicode(ansi, unicode, unicodeBufferSizeInBytes) / sizeof( wchar_t ); +#else + int chars = MultiByteToWideChar(CP_UTF8, 0, ansi, -1, unicode, unicodeBufferSizeInBytes / sizeof(wchar_t)); + unicode[(unicodeBufferSizeInBytes / sizeof(wchar_t)) - 1] = 0; + return chars; +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: converts an unicode string to an english string +//----------------------------------------------------------------------------- +int ILocalize::ConvertUnicodeToANSI(const wchar_t *unicode, char *ansi, int ansiBufferSize) +{ +#ifdef POSIX + return Q_UnicodeToUTF8(unicode, ansi, ansiBufferSize); +#else + int result = WideCharToMultiByte(CP_UTF8, 0, unicode, -1, ansi, ansiBufferSize, NULL, NULL); + ansi[ansiBufferSize - 1] = 0; + return result; +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: construct string helper +//----------------------------------------------------------------------------- +template < typename T > +void ConstructStringVArgsInternal_Impl(T *unicodeOutput, int unicodeBufferSizeInBytes, const T *formatString, int numFormatParameters, va_list argList) +{ + static const int k_cMaxFormatStringArguments = 9; // We only look one character ahead and start at %s1 + Assert( numFormatParameters <= k_cMaxFormatStringArguments ); + + // Safety check + if ( unicodeOutput == NULL || unicodeBufferSizeInBytes < 1 ) + { + return; + } + + if ( !formatString || numFormatParameters > k_cMaxFormatStringArguments ) + { + unicodeOutput[0] = 0; + return; + } + + int unicodeBufferSize = unicodeBufferSizeInBytes / sizeof(T); + const T *searchPos = formatString; + T *outputPos = unicodeOutput; + + T *argParams[k_cMaxFormatStringArguments]; + for ( int i = 0; i < numFormatParameters; i++ ) + { + argParams[i] = va_arg( argList, T* ); + } + + //assumes we can't have %s10 + //assume both are 0 terminated? + int formatLength = StringFuncs<T>::Length( formatString ); + + while ( searchPos[0] != '\0' && unicodeBufferSize > 1 ) + { + if ( formatLength >= 3 && searchPos[0] == '%' && searchPos[1] == 's' ) + { + //this is an escape sequence - %s1, %s2 etc, up to %s9 + + int argindex = ( searchPos[2] ) - '0' - 1; // 0 for %s1, 1 for %s2, etc. + + if ( argindex < 0 || argindex > k_cMaxFormatStringArguments ) + { + Warning( "Bad format string in CLocalizeStringTable::ConstructString\n" ); + *outputPos = '\0'; + return; + } + + if ( argindex < numFormatParameters ) + { + T const *param = argParams[argindex]; + + if ( param == NULL ) + param = StringFuncs<T>::NullDebugString(); + + int paramSize = StringFuncs<T>::Length(param); + if (paramSize >= unicodeBufferSize) + { + paramSize = unicodeBufferSize - 1; + } + + memcpy(outputPos, param, paramSize * sizeof(T)); + + unicodeBufferSize -= paramSize; + outputPos += paramSize; + + searchPos += 3; + formatLength -= 3; + } + else + { + AssertMsg( argindex < numFormatParameters, "ConstructStringVArgsInternal_Impl() - Found a %%s# escape sequence whose index was more than the number of args." ); + + //copy it over, char by char + *outputPos = *searchPos; + + outputPos++; + unicodeBufferSize--; + + searchPos++; + formatLength--; + } + } + else + { + //copy it over, char by char + *outputPos = *searchPos; + + outputPos++; + unicodeBufferSize--; + + searchPos++; + formatLength--; + } + } + + // ensure null termination + Assert( outputPos - unicodeOutput < unicodeBufferSizeInBytes/sizeof(T) ); + *outputPos = L'\0'; +} + +void ILocalize::ConstructStringVArgsInternal(char *unicodeOutput, int unicodeBufferSizeInBytes, const char *formatString, int numFormatParameters, va_list argList) +{ + ConstructStringVArgsInternal_Impl<char>( unicodeOutput, unicodeBufferSizeInBytes, formatString, numFormatParameters, argList ); +} + +void ILocalize::ConstructStringVArgsInternal(wchar_t *unicodeOutput, int unicodeBufferSizeInBytes, const wchar_t *formatString, int numFormatParameters, va_list argList) +{ + ConstructStringVArgsInternal_Impl<wchar_t>( unicodeOutput, unicodeBufferSizeInBytes, formatString, numFormatParameters, argList ); +} + +//----------------------------------------------------------------------------- +// Purpose: construct string helper +//----------------------------------------------------------------------------- +template < typename T > +const T *GetTypedKeyValuesString( KeyValues *pKeyValues, const char *pKeyName ); + +template < > +const char *GetTypedKeyValuesString<char>( KeyValues *pKeyValues, const char *pKeyName ) +{ + return pKeyValues->GetString( pKeyName, "[unknown]" ); +} + +template < > +const wchar_t *GetTypedKeyValuesString<wchar_t>( KeyValues *pKeyValues, const char *pKeyName ) +{ + return pKeyValues->GetWString( pKeyName, L"[unknown]" ); +} + +template < typename T > +void ConstructStringKeyValuesInternal_Impl( T *unicodeOutput, int unicodeBufferSizeInBytes, const T *formatString, KeyValues *localizationVariables ) +{ + T *outputPos = unicodeOutput; + + //assumes we can't have %s10 + //assume both are 0 terminated? + int unicodeBufferSize = unicodeBufferSizeInBytes / sizeof(T); + + while ( *formatString != '\0' && unicodeBufferSize > 1 ) + { + bool shouldAdvance = true; + + if ( *formatString == '%' ) + { + // this is an escape sequence that specifies a variable name + if ( formatString[1] == 's' && formatString[2] >= '0' && formatString[2] <= '9' ) + { + // old style escape sequence, ignore + } + else if ( formatString[1] == '%' ) + { + // just a '%' char, just write the second one + formatString++; + } + else if ( localizationVariables ) + { + // get out the variable name + const T *varStart = formatString + 1; + const T *varEnd = StringFuncs<T>::FindChar( varStart, '%' ); + + if ( varEnd && *varEnd == '%' ) + { + shouldAdvance = false; + + // assume variable names must be ascii, do a quick convert + char variableName[32]; + char *vset = variableName; + for ( const T *pws = varStart; pws < varEnd && (vset < variableName + sizeof(variableName) - 1); ++pws, ++vset ) + { + *vset = (char)*pws; + } + *vset = 0; + + // look up the variable name + const T *value = GetTypedKeyValuesString<T>( localizationVariables, variableName ); + + int paramSize = StringFuncs<T>::Length( value ); + if (paramSize >= unicodeBufferSize) + { + paramSize = MAX( 0, unicodeBufferSize - 1 ); + } + + StringFuncs<T>::Copy( outputPos, value, paramSize ); + + unicodeBufferSize -= paramSize; + outputPos += paramSize; + formatString = varEnd + 1; + } + } + } + + if (shouldAdvance) + { + //copy it over, char by char + *outputPos = *formatString; + + outputPos++; + unicodeBufferSize--; + + formatString++; + } + } + + // ensure null termination + *outputPos = '\0'; +} + +void ILocalize::ConstructStringKeyValuesInternal(char *unicodeOutput, int unicodeBufferSizeInBytes, const char *formatString, KeyValues *localizationVariables) +{ + ConstructStringKeyValuesInternal_Impl<char>( unicodeOutput, unicodeBufferSizeInBytes, formatString, localizationVariables ); +} + +void ILocalize::ConstructStringKeyValuesInternal(wchar_t *unicodeOutput, int unicodeBufferSizeInBytes, const wchar_t *formatString, KeyValues *localizationVariables) +{ + ConstructStringKeyValuesInternal_Impl<wchar_t>( unicodeOutput, unicodeBufferSizeInBytes, formatString, localizationVariables ); +} diff --git a/tier1/interface.cpp b/tier1/interface.cpp new file mode 100644 index 0000000..031a49a --- /dev/null +++ b/tier1/interface.cpp @@ -0,0 +1,567 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +//===========================================================================// +#if defined( _WIN32 ) && !defined( _X360 ) +#include <windows.h> +#endif + +#if !defined( DONT_PROTECT_FILEIO_FUNCTIONS ) +#define DONT_PROTECT_FILEIO_FUNCTIONS // for protected_things.h +#endif + +#if defined( PROTECTED_THINGS_ENABLE ) +#undef PROTECTED_THINGS_ENABLE // from protected_things.h +#endif + +#include <stdio.h> +#include "interface.h" +#include "basetypes.h" +#include "tier0/dbg.h" +#include <string.h> +#include <stdlib.h> +#include "tier1/strtools.h" +#include "tier0/icommandline.h" +#include "tier0/dbg.h" +#include "tier0/threadtools.h" +#ifdef _WIN32 +#include <direct.h> // getcwd +#elif POSIX +#include <dlfcn.h> +#include <unistd.h> +#define _getcwd getcwd +#endif +#if defined( _X360 ) +#include "xbox/xbox_win32stubs.h" +#endif + + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +// ------------------------------------------------------------------------------------ // +// InterfaceReg. +// ------------------------------------------------------------------------------------ // +InterfaceReg *InterfaceReg::s_pInterfaceRegs = NULL; + +InterfaceReg::InterfaceReg( InstantiateInterfaceFn fn, const char *pName ) : + m_pName(pName) +{ + m_CreateFn = fn; + m_pNext = s_pInterfaceRegs; + s_pInterfaceRegs = this; +} + +// ------------------------------------------------------------------------------------ // +// CreateInterface. +// This is the primary exported function by a dll, referenced by name via dynamic binding +// that exposes an opqaue function pointer to the interface. +// +// We have the Internal variant so Sys_GetFactoryThis() returns the correct internal +// symbol under GCC/Linux/Mac as CreateInterface is DLL_EXPORT so its global so the loaders +// on those OS's pick exactly 1 of the CreateInterface symbols to be the one that is process wide and +// all Sys_GetFactoryThis() calls find that one, which doesn't work. Using the internal walkthrough here +// makes sure Sys_GetFactoryThis() has the dll specific symbol and GetProcAddress() returns the module specific +// function for CreateInterface again getting the dll specific symbol we need. +// ------------------------------------------------------------------------------------ // +void* CreateInterfaceInternal( const char *pName, int *pReturnCode ) +{ + InterfaceReg *pCur; + + for (pCur=InterfaceReg::s_pInterfaceRegs; pCur; pCur=pCur->m_pNext) + { + if (strcmp(pCur->m_pName, pName) == 0) + { + if (pReturnCode) + { + *pReturnCode = IFACE_OK; + } + return pCur->m_CreateFn(); + } + } + + if (pReturnCode) + { + *pReturnCode = IFACE_FAILED; + } + return NULL; +} + +void* CreateInterface( const char *pName, int *pReturnCode ) +{ + return CreateInterfaceInternal( pName, pReturnCode ); +} + + + +#ifdef POSIX +// Linux doesn't have this function so this emulates its functionality +void *GetModuleHandle(const char *name) +{ + void *handle; + + if( name == NULL ) + { + // hmm, how can this be handled under linux.... + // is it even needed? + return NULL; + } + + if( (handle=dlopen(name, RTLD_NOW))==NULL) + { + printf("DLOPEN Error:%s\n",dlerror()); + // couldn't open this file + return NULL; + } + + // read "man dlopen" for details + // in short dlopen() inc a ref count + // so dec the ref count by performing the close + dlclose(handle); + return handle; +} +#endif + +#if defined( _WIN32 ) && !defined( _X360 ) +#define WIN32_LEAN_AND_MEAN +#include "windows.h" +#endif + +//----------------------------------------------------------------------------- +// Purpose: returns a pointer to a function, given a module +// Input : pModuleName - module name +// *pName - proc name +//----------------------------------------------------------------------------- +static void *Sys_GetProcAddress( const char *pModuleName, const char *pName ) +{ + HMODULE hModule = (HMODULE)GetModuleHandle( pModuleName ); +#ifdef WIN32 + return (void *)GetProcAddress( hModule, pName ); +#else + return (void *)dlsym( (void *)hModule, pName ); +#endif +} + +#if !defined(LINUX) +static void *Sys_GetProcAddress( HMODULE hModule, const char *pName ) +{ +#ifdef WIN32 + return (void *)GetProcAddress( hModule, pName ); +#else + return (void *)dlsym( (void *)hModule, pName ); +#endif +} +#endif + +bool Sys_IsDebuggerPresent() +{ + return Plat_IsInDebugSession(); +} + +struct ThreadedLoadLibaryContext_t +{ + const char *m_pLibraryName; + HMODULE m_hLibrary; +}; + +#ifdef _WIN32 + +// wraps LoadLibraryEx() since 360 doesn't support that +static HMODULE InternalLoadLibrary( const char *pName, Sys_Flags flags ) +{ +#if defined(_X360) + return LoadLibrary( pName ); +#else + if ( flags & SYS_NOLOAD ) + return GetModuleHandle( pName ); + else + return LoadLibraryEx( pName, NULL, LOAD_WITH_ALTERED_SEARCH_PATH ); +#endif +} +unsigned ThreadedLoadLibraryFunc( void *pParam ) +{ + ThreadedLoadLibaryContext_t *pContext = (ThreadedLoadLibaryContext_t*)pParam; + pContext->m_hLibrary = InternalLoadLibrary( pContext->m_pLibraryName, SYS_NOFLAGS ); + return 0; +} + +#endif // _WIN32 + +HMODULE Sys_LoadLibrary( const char *pLibraryName, Sys_Flags flags ) +{ + char str[ 1024 ]; + // Note: DLL_EXT_STRING can be "_srv.so" or "_360.dll". So be careful + // when using the V_*Extension* routines... + const char *pDllStringExtension = V_GetFileExtension( DLL_EXT_STRING ); + const char *pModuleExtension = pDllStringExtension ? ( pDllStringExtension - 1 ) : DLL_EXT_STRING; + + Q_strncpy( str, pLibraryName, sizeof(str) ); + + if ( IsX360() ) + { + // old, probably busted, behavior for xbox + if ( !Q_stristr( str, pModuleExtension ) ) + { + V_SetExtension( str, pModuleExtension, sizeof(str) ); + } + } + else + { + // always force the final extension to be .dll + V_SetExtension( str, pModuleExtension, sizeof(str) ); + } + + Q_FixSlashes( str ); + +#ifdef _WIN32 + ThreadedLoadLibraryFunc_t threadFunc = GetThreadedLoadLibraryFunc(); + if ( !threadFunc ) + return InternalLoadLibrary( str, flags ); + + // We shouldn't be passing noload while threaded. + Assert( !( flags & SYS_NOLOAD ) ); + + ThreadedLoadLibaryContext_t context; + context.m_pLibraryName = str; + context.m_hLibrary = 0; + + ThreadHandle_t h = CreateSimpleThread( ThreadedLoadLibraryFunc, &context ); + +#ifdef _X360 + ThreadSetAffinity( h, XBOX_PROCESSOR_3 ); +#endif + + unsigned int nTimeout = 0; + while( ThreadWaitForObject( h, true, nTimeout ) == TW_TIMEOUT ) + { + nTimeout = threadFunc(); + } + + ReleaseThreadHandle( h ); + return context.m_hLibrary; + +#elif POSIX + int dlopen_mode = RTLD_NOW; + + if ( flags & SYS_NOLOAD ) + dlopen_mode |= RTLD_NOLOAD; + + HMODULE ret = ( HMODULE )dlopen( str, dlopen_mode ); + if ( !ret && !( flags & SYS_NOLOAD ) ) + { + const char *pError = dlerror(); + if ( pError && ( strstr( pError, "No such file" ) == 0 ) && ( strstr( pError, "image not found" ) == 0 ) ) + { + Msg( " failed to dlopen %s error=%s\n", str, pError ); + } + } + + return ret; +#endif +} +static bool s_bRunningWithDebugModules = false; + +//----------------------------------------------------------------------------- +// Purpose: Loads a DLL/component from disk and returns a handle to it +// Input : *pModuleName - filename of the component +// Output : opaque handle to the module (hides system dependency) +//----------------------------------------------------------------------------- +CSysModule *Sys_LoadModule( const char *pModuleName, Sys_Flags flags /* = SYS_NOFLAGS (0) */ ) +{ + // If using the Steam filesystem, either the DLL must be a minimum footprint + // file in the depot (MFP) or a filesystem GetLocalCopy() call must be made + // prior to the call to this routine. + char szCwd[1024]; + HMODULE hDLL = NULL; + + if ( !Q_IsAbsolutePath( pModuleName ) ) + { + // full path wasn't passed in, using the current working dir + _getcwd( szCwd, sizeof( szCwd ) ); + if ( IsX360() ) + { + int i = CommandLine()->FindParm( "-basedir" ); + if ( i ) + { + V_strcpy_safe( szCwd, CommandLine()->GetParm( i + 1 ) ); + } + } + if (szCwd[strlen(szCwd) - 1] == '/' || szCwd[strlen(szCwd) - 1] == '\\' ) + { + szCwd[strlen(szCwd) - 1] = 0; + } + + char szAbsoluteModuleName[1024]; + size_t cCwd = strlen( szCwd ); + if ( strstr( pModuleName, "bin/") == pModuleName || ( szCwd[ cCwd - 1 ] == 'n' && szCwd[ cCwd - 2 ] == 'i' && szCwd[ cCwd - 3 ] == 'b' ) ) + { + // don't make bin/bin path + Q_snprintf( szAbsoluteModuleName, sizeof(szAbsoluteModuleName), "%s/%s", szCwd, pModuleName ); + } + else + { + Q_snprintf( szAbsoluteModuleName, sizeof(szAbsoluteModuleName), "%s/bin/%s", szCwd, pModuleName ); + } + hDLL = Sys_LoadLibrary( szAbsoluteModuleName, flags ); + } + + if ( !hDLL ) + { + // full path failed, let LoadLibrary() try to search the PATH now + hDLL = Sys_LoadLibrary( pModuleName, flags ); +#if defined( _DEBUG ) + if ( !hDLL ) + { +// So you can see what the error is in the debugger... +#if defined( _WIN32 ) && !defined( _X360 ) + char *lpMsgBuf; + + FormatMessage( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + GetLastError(), + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language + (LPTSTR) &lpMsgBuf, + 0, + NULL + ); + + LocalFree( (HLOCAL)lpMsgBuf ); +#elif defined( _X360 ) + DWORD error = GetLastError(); + Msg( "Error(%d) - Failed to load %s:\n", error, pModuleName ); +#else + Msg( "Failed to load %s: %s\n", pModuleName, dlerror() ); +#endif // _WIN32 + } +#endif // DEBUG + } + +#if !defined(LINUX) + // If running in the debugger, assume debug binaries are okay, otherwise they must run with -allowdebug + if ( Sys_GetProcAddress( hDLL, "BuiltDebug" ) ) + { + if ( !IsX360() && hDLL && + !CommandLine()->FindParm( "-allowdebug" ) && + !Sys_IsDebuggerPresent() ) + { + Error( "Module %s is a debug build\n", pModuleName ); + } + + DevWarning( "Module %s is a debug build\n", pModuleName ); + + if ( !s_bRunningWithDebugModules ) + { + s_bRunningWithDebugModules = true; + +#if 0 //def IS_WINDOWS_PC + char chMemoryName[ MAX_PATH ]; + DebugKernelMemoryObjectName( chMemoryName ); + + (void) CreateFileMapping( INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, 1024, chMemoryName ); + // Created a shared memory kernel object specific to process id + // Existence of this object indicates that we have debug modules loaded +#endif + } + } +#endif + + return reinterpret_cast<CSysModule *>(hDLL); +} + +//----------------------------------------------------------------------------- +// Purpose: Determine if any debug modules were loaded +//----------------------------------------------------------------------------- +bool Sys_RunningWithDebugModules() +{ + if ( !s_bRunningWithDebugModules ) + { +#if 0 //def IS_WINDOWS_PC + char chMemoryName[ MAX_PATH ]; + DebugKernelMemoryObjectName( chMemoryName ); + + HANDLE hObject = OpenFileMapping( FILE_MAP_READ, FALSE, chMemoryName ); + if ( hObject && hObject != INVALID_HANDLE_VALUE ) + { + CloseHandle( hObject ); + s_bRunningWithDebugModules = true; + } +#endif + } + return s_bRunningWithDebugModules; +} + + +//----------------------------------------------------------------------------- +// Purpose: Unloads a DLL/component from +// Input : *pModuleName - filename of the component +// Output : opaque handle to the module (hides system dependency) +//----------------------------------------------------------------------------- +void Sys_UnloadModule( CSysModule *pModule ) +{ + if ( !pModule ) + return; + + HMODULE hDLL = reinterpret_cast<HMODULE>(pModule); + +#ifdef _WIN32 + FreeLibrary( hDLL ); +#elif defined(POSIX) + dlclose((void *)hDLL); +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: returns a pointer to a function, given a module +// Input : module - windows HMODULE from Sys_LoadModule() +// *pName - proc name +// Output : factory for this module +//----------------------------------------------------------------------------- +CreateInterfaceFn Sys_GetFactory( CSysModule *pModule ) +{ + if ( !pModule ) + return NULL; + + HMODULE hDLL = reinterpret_cast<HMODULE>(pModule); +#ifdef _WIN32 + return reinterpret_cast<CreateInterfaceFn>(GetProcAddress( hDLL, CREATEINTERFACE_PROCNAME )); +#elif defined(POSIX) + // Linux gives this error: + //../public/interface.cpp: In function `IBaseInterface *(*Sys_GetFactory + //(CSysModule *)) (const char *, int *)': + //../public/interface.cpp:154: ISO C++ forbids casting between + //pointer-to-function and pointer-to-object + // + // so lets get around it :) + return (CreateInterfaceFn)(GetProcAddress( (void *)hDLL, CREATEINTERFACE_PROCNAME )); +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: returns the instance of this module +// Output : interface_instance_t +//----------------------------------------------------------------------------- +CreateInterfaceFn Sys_GetFactoryThis( void ) +{ + return &CreateInterfaceInternal; +} + +//----------------------------------------------------------------------------- +// Purpose: returns the instance of the named module +// Input : *pModuleName - name of the module +// Output : interface_instance_t - instance of that module +//----------------------------------------------------------------------------- +CreateInterfaceFn Sys_GetFactory( const char *pModuleName ) +{ +#ifdef _WIN32 + return static_cast<CreateInterfaceFn>( Sys_GetProcAddress( pModuleName, CREATEINTERFACE_PROCNAME ) ); +#elif defined(POSIX) + // see Sys_GetFactory( CSysModule *pModule ) for an explanation + return (CreateInterfaceFn)( Sys_GetProcAddress( pModuleName, CREATEINTERFACE_PROCNAME ) ); +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: get the interface for the specified module and version +// Input : +// Output : +//----------------------------------------------------------------------------- +bool Sys_LoadInterface( + const char *pModuleName, + const char *pInterfaceVersionName, + CSysModule **pOutModule, + void **pOutInterface ) +{ + CSysModule *pMod = Sys_LoadModule( pModuleName ); + if ( !pMod ) + return false; + + CreateInterfaceFn fn = Sys_GetFactory( pMod ); + if ( !fn ) + { + Sys_UnloadModule( pMod ); + return false; + } + + *pOutInterface = fn( pInterfaceVersionName, NULL ); + if ( !( *pOutInterface ) ) + { + Sys_UnloadModule( pMod ); + return false; + } + + if ( pOutModule ) + *pOutModule = pMod; + + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Place this as a singleton at module scope (e.g.) and use it to get the factory from the specified module name. +// +// When the singleton goes out of scope (.dll unload if at module scope), +// then it'll call Sys_UnloadModule on the module so that the refcount is decremented +// and the .dll actually can unload from memory. +//----------------------------------------------------------------------------- +CDllDemandLoader::CDllDemandLoader( char const *pchModuleName ) : + m_pchModuleName( pchModuleName ), + m_hModule( 0 ), + m_bLoadAttempted( false ) +{ +} + +CDllDemandLoader::~CDllDemandLoader() +{ + Unload(); +} + +CreateInterfaceFn CDllDemandLoader::GetFactory() +{ + if ( !m_hModule && !m_bLoadAttempted ) + { + m_bLoadAttempted = true; + m_hModule = Sys_LoadModule( m_pchModuleName ); + } + + if ( !m_hModule ) + { + return NULL; + } + + return Sys_GetFactory( m_hModule ); +} + +void CDllDemandLoader::Unload() +{ + if ( m_hModule ) + { + Sys_UnloadModule( m_hModule ); + m_hModule = 0; + } +} + +#if defined( STAGING_ONLY ) && defined( _WIN32 ) + +typedef USHORT( WINAPI RtlCaptureStackBackTrace_FUNC )( + ULONG frames_to_skip, + ULONG frames_to_capture, + PVOID *backtrace, + PULONG backtrace_hash ); + +extern "C" int backtrace( void **buffer, int size ) +{ + HMODULE hNTDll = GetModuleHandleA( "ntdll.dll" ); + static RtlCaptureStackBackTrace_FUNC * const pfnRtlCaptureStackBackTrace = + ( RtlCaptureStackBackTrace_FUNC * )GetProcAddress( hNTDll, "RtlCaptureStackBackTrace" ); + + if ( !pfnRtlCaptureStackBackTrace ) + return 0; + + return (int)pfnRtlCaptureStackBackTrace( 2, size, buffer, 0 ); +} + +#endif // STAGING_ONLY && _WIN32 + diff --git a/tier1/keyvaluesjson.cpp b/tier1/keyvaluesjson.cpp new file mode 100644 index 0000000..7d3dabf --- /dev/null +++ b/tier1/keyvaluesjson.cpp @@ -0,0 +1,714 @@ +//========= Copyright Valve Corporation, All rights reserved. =================// +// +// Read JSON-formatted data into KeyValues +// +//=============================================================================// + +#include "tier1/keyvaluesjson.h" +#include "tier1/utlbuffer.h" +#include "tier1/strtools.h" +#include <stdint.h> // INT32_MIN defn + +KeyValuesJSONParser::KeyValuesJSONParser( const CUtlBuffer &buf ) +{ + Init( (const char *)buf.Base(), buf.TellPut() ); +} + +KeyValuesJSONParser::KeyValuesJSONParser( const char *pszText, int cbSize ) +{ + Init( pszText, cbSize >= 0 ? cbSize : V_strlen(pszText) ); +} + +KeyValuesJSONParser::~KeyValuesJSONParser() {} + +void KeyValuesJSONParser::Init( const char *pszText, int cbSize ) +{ + m_szErrMsg[0] = '\0'; + m_nLine = 1; + m_cur = pszText; + m_end = pszText+cbSize; + + m_eToken = kToken_Null; + NextToken(); +} + +KeyValues *KeyValuesJSONParser::ParseFile() +{ + // A valid JSON object should contain a single object, surrounded by curly braces. + if ( m_eToken == kToken_EOF ) + { + V_sprintf_safe( m_szErrMsg, "Input contains no data" ); + return NULL; + } + if ( m_eToken == kToken_Err ) + return NULL; + if ( m_eToken == '{' ) + { + + // Parse the the entire file as one big object + KeyValues *pResult = new KeyValues(""); + if ( !ParseObject( pResult ) ) + { + pResult->deleteThis(); + return NULL; + } + if ( m_eToken == kToken_EOF ) + return pResult; + pResult->deleteThis(); + } + V_sprintf_safe( m_szErrMsg, "%s not expected here. A valid JSON document should be a single object, which begins with '{' and ends with '}'", GetTokenDebugText() ); + return NULL; +} + +bool KeyValuesJSONParser::ParseObject( KeyValues *pObject ) +{ + Assert( m_eToken == '{' ); + int nOpenDelimLine = m_nLine; + NextToken(); + KeyValues *pLastChild = NULL; + while ( m_eToken != '}' ) + { + // Parse error? + if ( m_eToken == kToken_Err ) + return false; + if ( m_eToken == kToken_EOF ) + { + // Actually report the error at the line of the unmatched delimiter. + // There's no need to report the line number of the end of file, that is always + // useless. + m_nLine = nOpenDelimLine; + V_strcpy_safe( m_szErrMsg, "End of input was reached and '{' was not matched by '}'" ); + return false; + } + + // It must be a string, for the key name + if ( m_eToken != kToken_String ) + { + V_sprintf_safe( m_szErrMsg, "%s not expected here; expected string for key name or '}'", GetTokenDebugText() ); + return false; + } + + KeyValues *pChildValue = new KeyValues( m_vecTokenChars.Base() ); + NextToken(); + + // Expect and eat colon + if ( m_eToken != ':' ) + { + V_sprintf_safe( m_szErrMsg, "%s not expected here. Missing ':'?", GetTokenDebugText() ); + pChildValue->deleteThis(); + return false; + } + NextToken(); + + // Recursively parse the value + if ( !ParseValue( pChildValue ) ) + { + pChildValue->deleteThis(); + return false; + } + + // Add to parent. + pObject->AddSubkeyUsingKnownLastChild( pChildValue, pLastChild ); + pLastChild = pChildValue; + + // Eat the comma, if there is one. If no comma, + // then the other thing that could come next + // is the closing brace to close the object + // NOTE: We are allowing the extra comma after the last item + if ( m_eToken == ',' ) + { + NextToken(); + } + else if ( m_eToken != '}' ) + { + V_sprintf_safe( m_szErrMsg, "%s not expected here. Missing ',' or '}'?", GetTokenDebugText() ); + return false; + } + } + + // Eat closing '}' + NextToken(); + + // Success + return true; +} + +bool KeyValuesJSONParser::ParseArray( KeyValues *pArray ) +{ + Assert( m_eToken == '[' ); + int nOpenDelimLine = m_nLine; + NextToken(); + KeyValues *pLastChild = NULL; + int idx = 0; + while ( m_eToken != ']' ) + { + // Parse error? + if ( m_eToken == kToken_Err ) + return false; + if ( m_eToken == kToken_EOF ) + { + // Actually report the error at the line of the unmatched delimiter. + // There's no need to report the line number of the end of file, that is always + // useless. + m_nLine = nOpenDelimLine; + V_strcpy_safe( m_szErrMsg, "End of input was reached and '[' was not matched by ']'" ); + return false; + } + + // Set a dummy key name based on the index + char szKeyName[ 32 ]; + V_sprintf_safe( szKeyName, "%d", idx ); + ++idx; + KeyValues *pChildValue = new KeyValues( szKeyName ); + + // Recursively parse the value + if ( !ParseValue( pChildValue ) ) + { + pChildValue->deleteThis(); + return false; + } + + // Add to parent. + pArray->AddSubkeyUsingKnownLastChild( pChildValue, pLastChild ); + pLastChild = pChildValue; + + // Handle a colon here specially. If one appears, the odds are they + // are trying to put object-like data inside of an array + if ( m_eToken == ':' ) + { + V_sprintf_safe( m_szErrMsg, "':' not expected inside an array. ('[]' used when '{}' was intended?)" ); + return false; + } + + // Eat the comma, if there is one. If no comma, + // then the other thing that could come next + // is the closing brace to close the object + // NOTE: We are allowing the extra comma after the last item + if ( m_eToken == ',' ) + { + NextToken(); + } + else if ( m_eToken != ']' ) + { + V_sprintf_safe( m_szErrMsg, "%s not expected here. Missing ',' or ']'?", GetTokenDebugText() ); + return false; + } + } + + // Eat closing ']' + NextToken(); + + // Success + return true; +} + +bool KeyValuesJSONParser::ParseValue( KeyValues *pValue ) +{ + switch ( m_eToken ) + { + case '{': return ParseObject( pValue ); + case '[': return ParseArray( pValue ); + case kToken_String: + pValue->SetString( NULL, m_vecTokenChars.Base() ); + NextToken(); + return true; + + case kToken_NumberInt: + { + const char *pszNum = m_vecTokenChars.Base(); + + // Negative? + if ( *pszNum == '-' ) + { + int64 val64 = V_atoi64( pszNum ); + if ( val64 < INT32_MIN ) + { + // !KLUDGE! KeyValues cannot support this! + V_sprintf_safe( m_szErrMsg, "%s is out of range for KeyValues, which doesn't support signed 64-bit numbers", pszNum ); + return false; + } + + pValue->SetInt( NULL, (int)val64 ); + } + else + { + uint64 val64 = V_atoui64( pszNum ); + if ( val64 > 0x7fffffffU ) + { + pValue->SetUint64( NULL, val64 ); + } + else + { + pValue->SetInt( NULL, (int)val64 ); + } + } + NextToken(); + return true; + } + + case kToken_NumberFloat: + { + float f = V_atof( m_vecTokenChars.Base() ); + pValue->SetFloat( NULL, f ); + NextToken(); + return true; + } + + case kToken_True: + pValue->SetBool( NULL, true ); + NextToken(); + return true; + + case kToken_False: + pValue->SetBool( NULL, false ); + NextToken(); + return true; + + case kToken_Null: + pValue->SetPtr( NULL, NULL ); + NextToken(); + return true; + + case kToken_Err: + return false; + } + + V_sprintf_safe( m_szErrMsg, "%s not expected here; missing value?", GetTokenDebugText() ); + return false; +} + +void KeyValuesJSONParser::NextToken() +{ + + // Already in terminal state? + if ( m_eToken < 0 ) + return; + + // Clear token + m_vecTokenChars.SetCount(0); + + // Scan until we hit the end of input + while ( m_cur < m_end ) + { + + // Next character? + char c = *m_cur; + switch (c) + { + // Whitespace? Eat it and keep parsing + case ' ': + case '\t': + ++m_cur; + break; + + // Newline? Eat it and advance line number + case '\n': + case '\r': + ++m_nLine; + ++m_cur; + + // Eat \r\n or \n\r pair as a single character + if ( m_cur < m_end && *m_cur == ( '\n' + '\r' - c ) ) + ++m_cur; + break; + + // Single-character JSON token? + case ':': + case '{': + case '}': + case '[': + case ']': + case ',': + m_eToken = c; + ++m_cur; + return; + + // String? + case '\"': + case '\'': // NOTE: We allow strings to be delimited by single quotes, which is not JSON compliant + ParseStringToken(); + return; + + case '-': + case '.': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + ParseNumberToken(); + return; + + // Literal "true" + case 't': + if ( m_cur + 4 <= m_end && m_cur[1] == 'r' && m_cur[2] == 'u' && m_cur[3] == 'e' ) + { + m_cur += 4; + m_eToken = kToken_True; + return; + } + goto unexpected_char; + + // Literal "false" + case 'f': + if ( m_cur + 5 <= m_end && m_cur[1] == 'a' && m_cur[2] == 'l' && m_cur[3] == 's' && m_cur[4] == 'e' ) + { + m_cur += 5; + m_eToken = kToken_False; + return; + } + goto unexpected_char; + + // Literal "null" + case 'n': + if ( m_cur + 4 <= m_end && m_cur[1] == 'u' && m_cur[2] == 'l' && m_cur[3] == 'l' ) + { + m_cur += 4; + m_eToken = kToken_Null; + return; + } + goto unexpected_char; + + case '/': + // C++-style comment? + if ( m_cur < m_end && m_cur[1] == '/' ) + { + m_cur += 2; + while ( m_cur < m_end && *m_cur != '\n' && *m_cur != '\r' ) + ++m_cur; + // Leave newline as the next character, we'll handle it above + break; + } + // | fall + // | through + // V + + default: + unexpected_char: + if ( V_isprint(c) ) + V_sprintf_safe( m_szErrMsg, "Unexpected character 0x%02x ('%c')", (uint8)c, c ); + else + V_sprintf_safe( m_szErrMsg, "Unexpected character 0x%02x", (uint8)c ); + m_eToken = kToken_Err; + return; + } + } + + m_eToken = kToken_EOF; +} + +void KeyValuesJSONParser::ParseNumberToken() +{ + // Clear token + m_vecTokenChars.SetCount(0); + + // Eat leading minus sign + if ( *m_cur == '-' ) + { + m_vecTokenChars.AddToTail( '-' ); + ++m_cur; + } + + if ( m_cur >= m_end ) + { + V_strcpy_safe( m_szErrMsg, "Unexpected EOF while parsing number" ); + m_eToken = kToken_Err; + return; + } + + char c = *m_cur; + m_vecTokenChars.AddToTail( c ); + bool bHasWholePart = false; + switch ( c ) + { + case '0': + // Leading 0 cannot be followed by any more digits, as per JSON spec (and to make sure nobody tries to parse octal). + ++m_cur; + bHasWholePart = true; + break; + + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + bHasWholePart = true; + ++m_cur; + + // Accumulate digits until we hit a non-digit + while ( m_cur < m_end && *m_cur >= '0' && *m_cur <= '9' ) + m_vecTokenChars.AddToTail( *(m_cur++) ); + break; + + case '.': + // strict JSON doesn't allow a number that starts with a decimal point, but we do + break; + } + + // Assume this is integral, unless we hit a decimal point and/or exponent + m_eToken = kToken_NumberInt; + + // Fractional portion? + if ( m_cur < m_end && *m_cur == '.' ) + { + m_eToken = kToken_NumberFloat; + + // Eat decimal point + m_vecTokenChars.AddToTail( *(m_cur++) ); + + // Accumulate digits until we hit a non-digit + bool bHasFractionPart = false; + while ( m_cur < m_end && *m_cur >= '0' && *m_cur <= '9' ) + { + m_vecTokenChars.AddToTail( *(m_cur++) ); + bHasFractionPart = true; + } + + // Make sure we aren't just a single '.' + if ( !bHasWholePart && !bHasFractionPart ) + { + m_vecTokenChars.AddToTail(0); + V_sprintf_safe( m_szErrMsg, "Invalid number starting with '%s'", m_vecTokenChars.Base() ); + m_eToken = kToken_Err; + return; + } + } + + // Exponent? + if ( m_cur < m_end && ( *m_cur == 'e' || *m_cur == 'E' ) ) + { + m_eToken = kToken_NumberFloat; + + // Eat 'e' + m_vecTokenChars.AddToTail( *(m_cur++) ); + + // Optional sign + if ( m_cur < m_end && ( *m_cur == '-' || *m_cur == '+' ) ) + m_vecTokenChars.AddToTail( *(m_cur++) ); + + // Accumulate digits until we hit a non-digit + bool bHasExponentDigit = false; + while ( m_cur < m_end && *m_cur >= '0' && *m_cur <= '9' ) + { + m_vecTokenChars.AddToTail( *(m_cur++) ); + bHasExponentDigit = true; + } + if ( !bHasExponentDigit ) + { + V_strcpy_safe( m_szErrMsg, "Bad exponent in floating point number" ); + m_eToken = kToken_Err; + return; + } + } + + // OK, We have parsed a valid number. + // Terminate token + m_vecTokenChars.AddToTail( '\0' ); + + // EOF? That's OK for now, at this lexical parsing level. We'll handle the error + // at the higher parse level, when expecting a comma or closing delimiter + if ( m_cur >= m_end ) + return; + + // Is the next thing a valid character? This is the most common case. + c = *m_cur; + if ( V_isspace( c ) || c == ',' || c == '}' || c == ']' || c == '/' ) + return; + + // Handle these guys as "tokens", to provide a slightly more meaningful error message + if ( c == '[' || c == '{' ) + return; + + // Anything else, treat the whole thing as an invalid numerical constant + if ( V_isprint(c) ) + V_sprintf_safe( m_szErrMsg, "Number contains invalid character 0x%02x ('%c')", (uint8)c, c ); + else + V_sprintf_safe( m_szErrMsg, "Number contains invalid character 0x%02x", (uint8)c ); + m_eToken = kToken_Err; +} + +void KeyValuesJSONParser::ParseStringToken() +{ + char cDelim = *(m_cur++); + + while ( m_cur < m_end ) + { + char c = *(m_cur++); + if ( c == '\r' || c == '\n' ) + { + V_sprintf_safe( m_szErrMsg, "Hit end of line before closing quote (%c)", c ); + m_eToken = kToken_Err; + return; + } + if ( c == cDelim ) + { + m_eToken = kToken_String; + m_vecTokenChars.AddToTail( '\0' ); + return; + } + + // Ordinary character? Just append it + if ( c != '\\' ) + { + m_vecTokenChars.AddToTail( c ); + continue; + } + + // Escaped character. + // End of string? We'll handle it above + if ( m_cur >= m_end ) + continue; + + // Check table of allowed escape characters + switch (c) + { + case '\\': + case '/': + case '\'': + case '\"': m_vecTokenChars.AddToTail( c ); break; + case 'b': m_vecTokenChars.AddToTail( '\b' ); break; + case 'f': m_vecTokenChars.AddToTail( '\f' ); break; + case 'n': m_vecTokenChars.AddToTail( '\n' ); break; + case 'r': m_vecTokenChars.AddToTail( '\r' ); break; + case 't': m_vecTokenChars.AddToTail( '\t' ); break; + + case 'u': + { + + // Make sure are followed by exactly 4 hex digits + if ( m_cur + 4 > m_end || !V_isxdigit( m_cur[0] ) || !V_isxdigit( m_cur[1] ) || !V_isxdigit( m_cur[2] ) || !V_isxdigit( m_cur[3] ) ) + { + V_sprintf_safe( m_szErrMsg, "\\u must be followed by exactly 4 hex digits" ); + m_eToken = kToken_Err; + return; + } + + // Parse the codepoint + uchar32 nCodePoint = 0; + for ( int n = 0 ; n < 4 ; ++n ) + { + nCodePoint <<= 4; + char chHex = *(m_cur++); + if ( chHex >= '0' && chHex <= '9' ) + nCodePoint += chHex - '0'; + else if ( chHex >= 'a' && chHex <= 'a' ) + nCodePoint += chHex + 0x0a - 'a'; + else if ( chHex >= 'A' && chHex <= 'A' ) + nCodePoint += chHex + 0x0a - 'A'; + else + Assert( false ); // inconceivable, due to above + } + + // Encode it in UTF-8 + char utf8Encode[8]; + int r = Q_UChar32ToUTF8( nCodePoint, utf8Encode ); + if ( r < 0 || r > 4 ) + { + V_sprintf_safe( m_szErrMsg, "Invalid code point \\u%04x", nCodePoint ); + m_eToken = kToken_Err; + return; + } + for ( int i = 0 ; i < r ; ++i ) + m_vecTokenChars.AddToTail( utf8Encode[i] ); + } break; + + default: + if ( V_isprint(c) ) + V_sprintf_safe( m_szErrMsg, "Invalid escape character 0x%02x ('\\%c')", (uint8)c, c ); + else + V_sprintf_safe( m_szErrMsg, "Invalid escape character 0x%02x", (uint8)c ); + m_eToken = kToken_Err; + return; + } + } + + V_sprintf_safe( m_szErrMsg, "Hit end of input before closing quote (%c)", cDelim ); + m_eToken = kToken_Err; +} + +const char *KeyValuesJSONParser::GetTokenDebugText() +{ + switch ( m_eToken ) + { + case kToken_EOF: return "<EOF>"; + case kToken_String: return "<string>"; + case kToken_NumberInt: + case kToken_NumberFloat: return "<number>"; + case kToken_True: return "'true'"; + case kToken_False: return "'false'"; + case kToken_Null: return "'null'"; + case '{': return "'{'"; + case '}': return "'}'"; + case '[': return "'['"; + case ']': return "']'"; + case ':': return "':'"; + case ',': return "','"; + } + + // We shouldn't ever need to ask for a debug string for the error token, + // and anything else is an error + Assert( false ); + return "<parse error>"; +} + +#ifdef _DEBUG + +static void JSONTest_ParseValid( const char *pszData ) +{ + KeyValuesJSONParser parser( pszData ); + KeyValues *pFile = parser.ParseFile(); + Assert( pFile ); + pFile->deleteThis(); +} + +static void JSONTest_ParseInvalid( const char *pszData, const char *pszExpectedErrMsgSnippet, int nExpectedFailureLine ) +{ + KeyValuesJSONParser parser( pszData ); + KeyValues *pFile = parser.ParseFile(); + Assert( pFile == NULL ); + Assert( V_stristr( parser.m_szErrMsg, pszExpectedErrMsgSnippet ) != NULL ); + Assert( parser.m_nLine == nExpectedFailureLine ); +} + +void TestKeyValuesJSONParser() +{ + JSONTest_ParseValid( "{}" ); + JSONTest_ParseValid( R"JSON({ + "key": "string_value", + "pos_int32": 123, + "pos_int64": 123456789012, + "neg_int32": -456, + "float": -45.23, + "pos_exponent": 1e30, + "neg_exponent": 1e-16, + "decimal_and_exponent": 1.e+30, + "no_leading_zero": .7, // we support this, even though strict JSON says it's no good + "zero": 0, + "true_value": true, + "false_value": false, + "null_value": null, + "with_escaped": "\r \t \n", + "unicode": "\u1234 \\u12f3", + "array_of_ints": [ 1, 2, 3, -45 ], + "empty_array": [], + "array_with_stuff_inside": [ + {}, // this is a comment. + [ 0.45, {}, "hello!" ], + { "id": 0 }, + // Trailing comma above. Comment here + ], + })JSON" ); + JSONTest_ParseInvalid( "{ \"key\": 123", "missing", 1 ); + JSONTest_ParseInvalid( "{ \"key\": 123.4f }", "number", 1 ); +} + +#endif diff --git a/tier1/kvpacker.cpp b/tier1/kvpacker.cpp new file mode 100644 index 0000000..53f7672 --- /dev/null +++ b/tier1/kvpacker.cpp @@ -0,0 +1,285 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Contains a branch-neutral binary packer for KeyValues trees. +// +// $NoKeywords: $ +// +//=============================================================================// + +#include <KeyValues.h> +#include "kvpacker.h" + +#include "tier0/dbg.h" +#include "utlbuffer.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include <tier0/memdbgon.h> + +#define KEYVALUES_TOKEN_SIZE 1024 + +// writes KeyValue as binary data to buffer +bool KVPacker::WriteAsBinary( KeyValues *pNode, CUtlBuffer &buffer ) +{ + if ( buffer.IsText() ) // must be a binary buffer + return false; + + if ( !buffer.IsValid() ) // must be valid, no overflows etc + return false; + + // Write subkeys: + + // loop through all our peers + for ( KeyValues *dat = pNode; dat != NULL; dat = dat->GetNextKey() ) + { + // write type + switch ( dat->GetDataType() ) + { + case KeyValues::TYPE_NONE: + { + buffer.PutUnsignedChar( PACKTYPE_NONE ); + break; + } + case KeyValues::TYPE_STRING: + { + buffer.PutUnsignedChar( PACKTYPE_STRING ); + break; + } + case KeyValues::TYPE_WSTRING: + { + buffer.PutUnsignedChar( PACKTYPE_WSTRING ); + break; + } + + case KeyValues::TYPE_INT: + { + buffer.PutUnsignedChar( PACKTYPE_INT ); + break; + } + + case KeyValues::TYPE_UINT64: + { + buffer.PutUnsignedChar( PACKTYPE_UINT64 ); + break; + } + + case KeyValues::TYPE_FLOAT: + { + buffer.PutUnsignedChar( PACKTYPE_FLOAT ); + break; + } + case KeyValues::TYPE_COLOR: + { + buffer.PutUnsignedChar( PACKTYPE_COLOR ); + break; + } + case KeyValues::TYPE_PTR: + { + buffer.PutUnsignedChar( PACKTYPE_PTR ); + break; + } + + default: + break; + } + + // write name + buffer.PutString( dat->GetName() ); + + // write value + switch ( dat->GetDataType() ) + { + case KeyValues::TYPE_NONE: + { + if( !WriteAsBinary( dat->GetFirstSubKey(), buffer ) ) + return false; + break; + } + case KeyValues::TYPE_STRING: + { + if (dat->GetString() && *(dat->GetString())) + { + buffer.PutString( dat->GetString() ); + } + else + { + buffer.PutString( "" ); + } + break; + } + case KeyValues::TYPE_WSTRING: + { + int nLength = dat->GetWString() ? Q_wcslen( dat->GetWString() ) : 0; + buffer.PutShort( nLength ); + for( int k = 0; k < nLength; ++ k ) + { + buffer.PutShort( ( unsigned short ) dat->GetWString()[k] ); + } + break; + } + + case KeyValues::TYPE_INT: + { + buffer.PutInt( dat->GetInt() ); + break; + } + + case KeyValues::TYPE_UINT64: + { + buffer.PutInt64( dat->GetUint64() ); + break; + } + + case KeyValues::TYPE_FLOAT: + { + buffer.PutFloat( dat->GetFloat() ); + break; + } + case KeyValues::TYPE_COLOR: + { + Color color = dat->GetColor(); + buffer.PutUnsignedChar( color[0] ); + buffer.PutUnsignedChar( color[1] ); + buffer.PutUnsignedChar( color[2] ); + buffer.PutUnsignedChar( color[3] ); + break; + } + case KeyValues::TYPE_PTR: + { + buffer.PutUnsignedInt( (int)dat->GetPtr() ); + break; + } + + default: + break; + } + } + + // write tail, marks end of peers + buffer.PutUnsignedChar( PACKTYPE_NULLMARKER ); + + return buffer.IsValid(); +} + +// read KeyValues from binary buffer, returns true if parsing was successful +bool KVPacker::ReadAsBinary( KeyValues *pNode, CUtlBuffer &buffer ) +{ + if ( buffer.IsText() ) // must be a binary buffer + return false; + + if ( !buffer.IsValid() ) // must be valid, no overflows etc + return false; + + pNode->Clear(); + + char token[KEYVALUES_TOKEN_SIZE]; + KeyValues *dat = pNode; + EPackType ePackType = (EPackType)buffer.GetUnsignedChar(); + + // loop through all our peers + while ( true ) + { + if ( ePackType == PACKTYPE_NULLMARKER ) + break; // no more peers + + buffer.GetString( token ); + token[KEYVALUES_TOKEN_SIZE-1] = 0; + + dat->SetName( token ); + + switch ( ePackType ) + { + case PACKTYPE_NONE: + { + KeyValues *pNewNode = new KeyValues(""); + dat->AddSubKey( pNewNode ); + if( !ReadAsBinary( pNewNode, buffer ) ) + return false; + break; + } + case PACKTYPE_STRING: + { + buffer.GetString( token ); + token[KEYVALUES_TOKEN_SIZE-1] = 0; + dat->SetStringValue( token ); + break; + } + case PACKTYPE_WSTRING: + { + int nLength = buffer.GetShort(); + if ( nLength >= 0 && nLength*sizeof( uint16 ) <= (uint)buffer.GetBytesRemaining() ) + { + if ( nLength > 0 ) + { + wchar_t *pTemp = (wchar_t *)malloc( sizeof( wchar_t ) * (1 + nLength) ); + + for ( int k = 0; k < nLength; ++k ) + { + pTemp[k] = buffer.GetShort(); // ugly, but preserving existing behavior + } + + pTemp[nLength] = 0; + dat->SetWString( NULL, pTemp ); + + free( pTemp ); + } + else + dat->SetWString( NULL, L"" ); + + } + break; + } + + case PACKTYPE_INT: + { + dat->SetInt( NULL, buffer.GetInt() ); + break; + } + + case PACKTYPE_UINT64: + { + dat->SetUint64( NULL, (uint64)buffer.GetInt64() ); + break; + } + + case PACKTYPE_FLOAT: + { + dat->SetFloat( NULL, buffer.GetFloat() ); + break; + } + case PACKTYPE_COLOR: + { + Color color( + buffer.GetUnsignedChar(), + buffer.GetUnsignedChar(), + buffer.GetUnsignedChar(), + buffer.GetUnsignedChar() ); + dat->SetColor( NULL, color ); + break; + } + case PACKTYPE_PTR: + { + dat->SetPtr( NULL, (void*)buffer.GetUnsignedInt() ); + break; + } + + default: + break; + } + + if ( !buffer.IsValid() ) // error occured + return false; + + ePackType = (EPackType)buffer.GetUnsignedChar(); + + if ( ePackType == PACKTYPE_NULLMARKER ) + break; + + // new peer follows + KeyValues *pNewPeer = new KeyValues(""); + dat->SetNextKey( pNewPeer ); + dat = pNewPeer; + } + + return buffer.IsValid(); +} + diff --git a/tier1/lzmaDecoder.cpp b/tier1/lzmaDecoder.cpp new file mode 100644 index 0000000..209c7fc --- /dev/null +++ b/tier1/lzmaDecoder.cpp @@ -0,0 +1,405 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// LZMA Codec interface for engine. +// +// LZMA SDK 9.38 beta +// 2015-01-03 : Igor Pavlov : Public domain +// http://www.7-zip.org/ +// +//========================================================================// + +#define _LZMADECODER_CPP + +#include "tier0/platform.h" +#include "tier0/basetypes.h" +#include "tier0/dbg.h" + +#include "../utils/lzma/C/7zTypes.h" +#include "../utils/lzma/C/LzmaEnc.h" +#include "../utils/lzma/C/LzmaDec.h" + +// Ugly define to let us forward declare the anonymous-struct-typedef that is CLzmaDec in the header. +#define CLzmaDec_t CLzmaDec +#include "tier1/lzmaDecoder.h" +#include "tier1/convar.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +#ifdef OSX +// OS X is having fragmentation issues, and I suspect this 16meg buffer being recreated many times during load is +// hitting a bad case in the default allocator. So this is an experiment to see if it reduces crash rates there. +#define LZMA_DEFAULT_PERSISTENT_BUFFER "1" +#else +#define LZMA_DEFAULT_PERSISTENT_BUFFER "0" +#endif + +ConVar lzma_persistent_buffer( "lzma_persistent_buffer", LZMA_DEFAULT_PERSISTENT_BUFFER, FCVAR_NONE, + "If set, attempt to keep a persistent buffer for the LZMA decoder dictionary. " \ + "This avoids re-allocating a ~16-64meg buffer for each operation, " \ + "at the expensive of keeping extra memory around when it is not in-use." ); + +// Allocator to pass to LZMA functions +static void *g_pStaticLZMABuf = NULL; +static size_t g_unStaticLZMABufSize = 0; +static uint32 g_unStaticLZMABufRef = 0; +static void *SzAlloc(void *p, size_t size) { + // Don't touch static buffer on other threads. + if ( ThreadInMainThread() ) + { + // If nobody is using the persistent buffer and size is above a threshold, use it. + bool bPersistentBuf = (g_pStaticLZMABuf || lzma_persistent_buffer.GetBool()) && size >= (1024 * 1024 * 8) && g_unStaticLZMABufRef == 0; + if ( bPersistentBuf ) + { + if ( g_unStaticLZMABufSize < size ) + { + g_pStaticLZMABuf = g_pStaticLZMABuf ? realloc( g_pStaticLZMABuf, size ) : malloc( size ); + g_unStaticLZMABufSize = size; + } + g_unStaticLZMABufRef++; + return g_pStaticLZMABuf; + } + } + + // Not using the persistent buffer + return malloc(size); +} +static void SzFree(void *p, void *address) { + // Don't touch static buffer on other threads. + if ( ThreadInMainThread() ) + { + if ( address != NULL && g_unStaticLZMABufRef && address == g_pStaticLZMABuf ) + { + g_unStaticLZMABufRef--; + // If the convar was turned off, free the buffer + if ( g_pStaticLZMABuf && g_unStaticLZMABufRef == 0 && !lzma_persistent_buffer.GetBool() ) + { + free( g_pStaticLZMABuf ); + g_pStaticLZMABuf = NULL; + g_unStaticLZMABufSize = 0; + } + return; + } + } + + // Not the static buffer + free(address); +} +static ISzAlloc g_Alloc = { SzAlloc, SzFree }; + +//----------------------------------------------------------------------------- +// Returns true if buffer is compressed. +//----------------------------------------------------------------------------- +/* static */ +bool CLZMA::IsCompressed( unsigned char *pInput ) +{ + lzma_header_t *pHeader = (lzma_header_t *)pInput; + if ( pHeader && pHeader->id == LZMA_ID ) + { + return true; + } + + // unrecognized + return false; +} + +//----------------------------------------------------------------------------- +// Returns uncompressed size of compressed input buffer. Used for allocating output +// buffer for decompression. Returns 0 if input buffer is not compressed. +//----------------------------------------------------------------------------- +/* static */ +unsigned int CLZMA::GetActualSize( unsigned char *pInput ) +{ + lzma_header_t *pHeader = (lzma_header_t *)pInput; + if ( pHeader && pHeader->id == LZMA_ID ) + { + return LittleLong( pHeader->actualSize ); + } + + // unrecognized + return 0; +} + +//----------------------------------------------------------------------------- +// Uncompress a buffer, Returns the uncompressed size. Caller must provide an +// adequate sized output buffer or memory corruption will occur. +//----------------------------------------------------------------------------- +/* static */ +unsigned int CLZMA::Uncompress( unsigned char *pInput, unsigned char *pOutput ) +{ + lzma_header_t *pHeader = (lzma_header_t *)pInput; + if ( pHeader->id != LZMA_ID ) + { + // not ours + return false; + } + + CLzmaDec state; + + LzmaDec_Construct(&state); + + if ( LzmaDec_Allocate(&state, pHeader->properties, LZMA_PROPS_SIZE, &g_Alloc) != SZ_OK ) + { + Assert( false ); + return 0; + } + + // These are in/out variables + SizeT outProcessed = pHeader->actualSize; + SizeT inProcessed = pHeader->lzmaSize; + ELzmaStatus status; + SRes result = LzmaDecode( (Byte *)pOutput, &outProcessed, (Byte *)(pInput + sizeof( lzma_header_t ) ), + &inProcessed, (Byte *)pHeader->properties, LZMA_PROPS_SIZE, LZMA_FINISH_END, &status, &g_Alloc ); + + + LzmaDec_Free(&state, &g_Alloc); + + if ( result != SZ_OK || pHeader->actualSize != outProcessed ) + { + Warning( "LZMA Decompression failed (%i)\n", result ); + return 0; + } + + return outProcessed; +} + +CLZMAStream::CLZMAStream() + : m_pDecoderState( NULL ), + m_nActualSize( 0 ), + m_nActualBytesRead ( 0 ), + m_nCompressedSize( 0 ), + m_nCompressedBytesRead ( 0 ), + m_bParsedHeader( false ), + m_bZIPStyleHeader( false ) +{} + +CLZMAStream::~CLZMAStream() +{ + FreeDecoderState(); +} + +void CLZMAStream::FreeDecoderState() +{ + if ( m_pDecoderState ) + { + LzmaDec_Free( m_pDecoderState, &g_Alloc ); + delete m_pDecoderState; + m_pDecoderState = NULL; + } +} + +bool CLZMAStream::CreateDecoderState( const unsigned char *pProperties ) +{ + CLzmaDec *pDecoderState = new CLzmaDec(); + + LzmaDec_Construct( pDecoderState ); + if ( LzmaDec_Allocate( pDecoderState, pProperties, LZMA_PROPS_SIZE, &g_Alloc) != SZ_OK ) + { + AssertMsg( false, "Failed to allocate lzma decoder state" ); + delete pDecoderState; + return false; + } + + LzmaDec_Init( pDecoderState ); + + // Replace current state + Assert( !m_pDecoderState ); + FreeDecoderState(); + + m_pDecoderState = pDecoderState; + return true; +} + +// Attempt to read up to nMaxInputBytes from the compressed stream, writing up to nMaxOutputBytes to pOutput. +// Returns false if read stops due to an error. +bool CLZMAStream::Read( unsigned char *pInput, unsigned int nMaxInputBytes, + unsigned char *pOutput, unsigned int nMaxOutputBytes, + /* out */ unsigned int &nCompressedBytesRead, + /* out */ unsigned int &nOutputBytesWritten ) +{ + nCompressedBytesRead = 0; + nOutputBytesWritten = 0; + bool bStartedWithHeader = m_bParsedHeader; + + // Check for initial chunk of data + if ( !m_bParsedHeader ) + { + unsigned int nBytesConsumed = 0; + eHeaderParse parseResult = TryParseHeader( pInput, nMaxInputBytes, nBytesConsumed ); + + if ( parseResult == eHeaderParse_NeedMoreBytes ) + { + // Not an error, just need more data to continue + return true; + } + else if ( parseResult != eHeaderParse_OK ) + { + Assert( parseResult == eHeaderParse_Fail ); + // Invalid header + return false; + } + + // Header consumed, fall through to continue read after it + nCompressedBytesRead += nBytesConsumed; + pInput += nBytesConsumed; + nMaxInputBytes -= nBytesConsumed; + } + + // These are input ( available size ) *and* output ( size processed ) vars for lzma + SizeT expectedInputRemaining = m_nCompressedSize - Min( m_nCompressedBytesRead + nCompressedBytesRead, m_nCompressedSize ); + SizeT expectedOutputRemaining = m_nActualSize - m_nActualBytesRead; + SizeT inSize = Min( (SizeT)nMaxInputBytes, expectedInputRemaining ); + SizeT outSize = Min( (SizeT)nMaxOutputBytes, expectedOutputRemaining ); + ELzmaStatus status; + ELzmaFinishMode finishMode = LZMA_FINISH_ANY; + if ( inSize == expectedInputRemaining && outSize == expectedOutputRemaining ) + { + // Expect to finish decoding this call. + finishMode = LZMA_FINISH_END; + } + SRes result = LzmaDec_DecodeToBuf( m_pDecoderState, pOutput, &outSize, + pInput, &inSize, finishMode, &status ); + + // DevMsg("[%p] Running lzmaDecode:\n" + // " pInput: %p\n" + // " nMaxInputBytes: %i\n" + // " pOutput: %p\n" + // " nMaxOutputBytes: %u\n" + // " inSize: %u\n" + // " outSize: %u\n" + // " result: %u\n" + // " status: %i\n" + // " m_nActualSize: %u\n" + // " m_nActualBytesRead: %u\n", + // this, pInput, nMaxInputBytes, pOutput, nMaxOutputBytes, + // inSize, outSize, result, status, m_nActualSize, m_nActualBytesRead); + + if ( result != SZ_OK ) + { + if ( !bStartedWithHeader ) + { + // If we're returning false, we need to pretend we didn't consume anything. + FreeDecoderState(); + m_bParsedHeader = false; + } + return false; + } + + nCompressedBytesRead += inSize; + nOutputBytesWritten += outSize; + + m_nCompressedBytesRead += nCompressedBytesRead; + m_nActualBytesRead += nOutputBytesWritten; + + Assert( m_nCompressedBytesRead <= m_nCompressedSize ); + return true; +} + +bool CLZMAStream::GetExpectedBytesRemaining( /* out */ unsigned int &nBytesRemaining ) +{ + if ( !m_bParsedHeader && !m_bZIPStyleHeader ) { + return false; + } + + nBytesRemaining = m_nActualSize - m_nActualBytesRead; + + return true; +} + +void CLZMAStream::InitZIPHeader( unsigned int nCompressedSize, unsigned int nOriginalSize ) +{ + if ( m_bParsedHeader || m_bZIPStyleHeader ) + { + AssertMsg( !m_bParsedHeader && !m_bZIPStyleHeader, + "LZMA Stream: InitZIPHeader() called on stream past header" ); + return; + } + + m_nCompressedSize = nCompressedSize; + m_nActualSize = nOriginalSize; + // Signal to TryParseHeader to expect a zip-style header (which wont have the size values) + m_bZIPStyleHeader = true; +} + +CLZMAStream::eHeaderParse CLZMAStream::TryParseHeader( unsigned char *pInput, unsigned int nBytesAvailable, /* out */ unsigned int &nBytesConsumed ) +{ + nBytesConsumed = 0; + + if ( m_bParsedHeader ) + { + AssertMsg( !m_bParsedHeader, "CLZMAStream::ReadSourceHeader called on already initialized stream" ); + return eHeaderParse_Fail; + } + + if ( m_bZIPStyleHeader ) + { + // ZIP Spec, 5.8.8 + // LZMA Version Information 2 bytes + // LZMA Properties Size 2 bytes + // LZMA Properties Data variable, defined by "LZMA Properties Size" + + if ( nBytesAvailable < 4 ) + { + // No error, but need more input to continue + return eHeaderParse_NeedMoreBytes; + } + + // Should probably check this + // unsigned char nLZMAVer[2] = { pInput[0], pInput[1] }; + + uint16 nLZMAPropertiesSize = LittleWord( *(uint16 *)(pInput + 2) ); + + nBytesConsumed += 4; + + if ( nLZMAPropertiesSize != LZMA_PROPS_SIZE ) + { + Warning( "LZMA stream: Unexpected LZMA properties size: %hu, expecting %u. Version mismatch?\n", + nLZMAPropertiesSize, LZMA_PROPS_SIZE ); + return eHeaderParse_Fail; + } + + if ( nBytesAvailable < static_cast<unsigned int>(nLZMAPropertiesSize) + 4 ) + { + return eHeaderParse_NeedMoreBytes; + } + + // Looks reasonable, try to parse + if ( !CreateDecoderState( (Byte *)pInput + 4 ) ) + { + AssertMsg( false, "Failed decoding Lzma properties" ); + return eHeaderParse_Fail; + } + + nBytesConsumed += nLZMAPropertiesSize; + } + else + { + // Else native source engine style header + if ( nBytesAvailable < sizeof( lzma_header_t ) ) + { + // need more input to continue + return eHeaderParse_NeedMoreBytes; + } + + m_nActualSize = CLZMA::GetActualSize( pInput ); + + if ( !m_nActualSize ) + { + // unrecognized + Warning( "Unrecognized LZMA data\n" ); + return eHeaderParse_Fail; + } + + if ( !CreateDecoderState( ((lzma_header_t *)pInput)->properties ) ) + { + AssertMsg( false, "Failed decoding Lzma properties" ); + return eHeaderParse_Fail; + } + + m_nCompressedSize = LittleLong( ((lzma_header_t *)pInput)->lzmaSize ) + sizeof( lzma_header_t ); + nBytesConsumed += sizeof( lzma_header_t ); + } + + m_bParsedHeader = true; + return eHeaderParse_OK; +} diff --git a/tier1/lzss.cpp b/tier1/lzss.cpp new file mode 100644 index 0000000..f350cf8 --- /dev/null +++ b/tier1/lzss.cpp @@ -0,0 +1,429 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// LZSS Codec. Designed for fast cheap gametime encoding/decoding. Compression results +// are not aggresive as other alogrithms, but gets 2:1 on most arbitrary uncompressed data. +// +//=====================================================================================// + +#include "tier0/platform.h" +#include "tier0/dbg.h" +#include "tier0/vprof.h" +#include "tier0/etwprof.h" +#include "tier1/lzss.h" +#include "tier1/utlbuffer.h" + +#define LZSS_LOOKSHIFT 4 +#define LZSS_LOOKAHEAD ( 1 << LZSS_LOOKSHIFT ) + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +//----------------------------------------------------------------------------- +// Returns true if buffer is compressed. +//----------------------------------------------------------------------------- +bool CLZSS::IsCompressed( const unsigned char *pInput ) +{ + lzss_header_t *pHeader = (lzss_header_t *)pInput; + if ( pHeader && pHeader->id == LZSS_ID ) + { + return true; + } + + // unrecognized + return false; +} + +//----------------------------------------------------------------------------- +// Returns uncompressed size of compressed input buffer. Used for allocating output +// buffer for decompression. Returns 0 if input buffer is not compressed. +//----------------------------------------------------------------------------- +unsigned int CLZSS::GetActualSize( const unsigned char *pInput ) +{ + lzss_header_t *pHeader = (lzss_header_t *)pInput; + if ( pHeader && pHeader->id == LZSS_ID ) + { + return LittleLong( pHeader->actualSize ); + } + + // unrecognized + return 0; +} + +void CLZSS::BuildHash( const unsigned char *pData ) +{ + lzss_list_t *pList; + lzss_node_t *pTarget; + + int targetindex = (unsigned int)pData & ( m_nWindowSize - 1 ); + pTarget = &m_pHashTarget[targetindex]; + if ( pTarget->pData ) + { + pList = &m_pHashTable[*pTarget->pData]; + if ( pTarget->pPrev ) + { + pList->pEnd = pTarget->pPrev; + pTarget->pPrev->pNext = 0; + } + else + { + pList->pEnd = 0; + pList->pStart = 0; + } + } + + pList = &m_pHashTable[*pData]; + pTarget->pData = pData; + pTarget->pPrev = 0; + pTarget->pNext = pList->pStart; + if ( pList->pStart ) + { + pList->pStart->pPrev = pTarget; + } + else + { + pList->pEnd = pTarget; + } + pList->pStart = pTarget; +} + +unsigned char *CLZSS::CompressNoAlloc( const unsigned char *pInput, int inputLength, unsigned char *pOutputBuf, unsigned int *pOutputSize ) +{ + if ( inputLength <= sizeof( lzss_header_t ) + 8 ) + { + return NULL; + } + VPROF( "CLZSS::CompressNoAlloc" ); + ETWMark1I("CompressNoAlloc", inputLength ); + + // create the compression work buffers, small enough (~64K) for stack + m_pHashTable = (lzss_list_t *)stackalloc( 256 * sizeof( lzss_list_t ) ); + memset( m_pHashTable, 0, 256 * sizeof( lzss_list_t ) ); + m_pHashTarget = (lzss_node_t *)stackalloc( m_nWindowSize * sizeof( lzss_node_t ) ); + memset( m_pHashTarget, 0, m_nWindowSize * sizeof( lzss_node_t ) ); + + // allocate the output buffer, compressed buffer is expected to be less, caller will free + unsigned char *pStart = pOutputBuf; + // prevent compression failure (inflation), leave enough to allow dribble eof bytes + unsigned char *pEnd = pStart + inputLength - sizeof ( lzss_header_t ) - 8; + + // set the header + lzss_header_t *pHeader = (lzss_header_t *)pStart; + pHeader->id = LZSS_ID; + pHeader->actualSize = LittleLong( inputLength ); + + unsigned char *pOutput = pStart + sizeof (lzss_header_t); + const unsigned char *pLookAhead = pInput; + const unsigned char *pWindow = pInput; + const unsigned char *pEncodedPosition = NULL; + unsigned char *pCmdByte = NULL; + int putCmdByte = 0; + + while ( inputLength > 0 ) + { + pWindow = pLookAhead - m_nWindowSize; + if ( pWindow < pInput ) + { + pWindow = pInput; + } + + if ( !putCmdByte ) + { + pCmdByte = pOutput++; + *pCmdByte = 0; + } + putCmdByte = ( putCmdByte + 1 ) & 0x07; + + int encodedLength = 0; + int lookAheadLength = inputLength < LZSS_LOOKAHEAD ? inputLength : LZSS_LOOKAHEAD; + + lzss_node_t *pHash = m_pHashTable[pLookAhead[0]].pStart; + while ( pHash ) + { + int matchLength = 0; + int length = lookAheadLength; + while ( length-- && pHash->pData[matchLength] == pLookAhead[matchLength] ) + { + matchLength++; + } + if ( matchLength > encodedLength ) + { + encodedLength = matchLength; + pEncodedPosition = pHash->pData; + } + if ( matchLength == lookAheadLength ) + { + break; + } + pHash = pHash->pNext; + } + + if ( encodedLength >= 3 ) + { + *pCmdByte = ( *pCmdByte >> 1 ) | 0x80; + *pOutput++ = ( ( pLookAhead-pEncodedPosition-1 ) >> LZSS_LOOKSHIFT ); + *pOutput++ = ( ( pLookAhead-pEncodedPosition-1 ) << LZSS_LOOKSHIFT ) | ( encodedLength-1 ); + } + else + { + encodedLength = 1; + *pCmdByte = ( *pCmdByte >> 1 ); + *pOutput++ = *pLookAhead; + } + + for ( int i=0; i<encodedLength; i++ ) + { + BuildHash( pLookAhead++ ); + } + + inputLength -= encodedLength; + + if ( pOutput >= pEnd ) + { + // compression is worse, abandon + return NULL; + } + } + + if ( inputLength != 0 ) + { + // unexpected failure + Assert( 0 ); + return NULL; + } + + if ( !putCmdByte ) + { + pCmdByte = pOutput++; + *pCmdByte = 0x01; + } + else + { + *pCmdByte = ( ( *pCmdByte >> 1 ) | 0x80 ) >> ( 7 - putCmdByte ); + } + + *pOutput++ = 0; + *pOutput++ = 0; + + if ( pOutputSize ) + { + *pOutputSize = pOutput - pStart; + } + + return pStart; +} + +//----------------------------------------------------------------------------- +// Compress an input buffer. Caller must free output compressed buffer. +// Returns NULL if compression failed (i.e. compression yielded worse results) +//----------------------------------------------------------------------------- +unsigned char* CLZSS::Compress( const unsigned char *pInput, int inputLength, unsigned int *pOutputSize ) +{ + unsigned char *pStart = (unsigned char *)malloc( inputLength ); + unsigned char *pFinal = CompressNoAlloc( pInput, inputLength, pStart, pOutputSize ); + if ( !pFinal ) + { + free( pStart ); + return NULL; + } + + return pStart; +} + +/* +// BUG BUG: This code is flaky, don't use until it's debugged!!! +unsigned int CLZSS::Uncompress( unsigned char *pInput, CUtlBuffer &buf ) +{ + int cmdByte = 0; + int getCmdByte = 0; + + unsigned int actualSize = GetActualSize( pInput ); + if ( !actualSize ) + { + // unrecognized + return 0; + } + + unsigned char *pBase = ( unsigned char * )buf.Base(); + + pInput += sizeof( lzss_header_t ); + + while ( !buf.IsValid() ) + { + if ( !getCmdByte ) + { + cmdByte = *pInput++; + } + getCmdByte = ( getCmdByte + 1 ) & 0x07; + + if ( cmdByte & 0x01 ) + { + int position = *pInput++ << LZSS_LOOKSHIFT; + position |= ( *pInput >> LZSS_LOOKSHIFT ); + int count = ( *pInput++ & 0x0F ) + 1; + if ( count == 1 ) + { + break; + } + unsigned int pos = buf.TellPut(); + unsigned char *pSource = ( pBase + pos ) - position - 1; + + // BUGBUG: + // This is failing!!! + // buf.WriteBytes( pSource, count ); + // So have to iterate them manually + for ( int i =0; i < count; ++i ) + { + buf.PutUnsignedChar( *pSource++ ); + } + } + else + { + buf.PutUnsignedChar( *pInput++ ); + } + cmdByte = cmdByte >> 1; + } + + if ( buf.TellPut() != (int)actualSize ) + { + // unexpected failure + Assert( 0 ); + return 0; + } + + return buf.TellPut(); +} +*/ + +unsigned int CLZSS::SafeUncompress( const unsigned char *pInput, unsigned char *pOutput, unsigned int unBufSize ) +{ + unsigned int totalBytes = 0; + int cmdByte = 0; + int getCmdByte = 0; + + unsigned int actualSize = GetActualSize( pInput ); + if ( !actualSize ) + { + // unrecognized + return 0; + } + + if ( actualSize > unBufSize ) + { + return 0; + } + + pInput += sizeof( lzss_header_t ); + + for ( ;; ) + { + if ( !getCmdByte ) + { + cmdByte = *pInput++; + } + getCmdByte = ( getCmdByte + 1 ) & 0x07; + + if ( cmdByte & 0x01 ) + { + int position = *pInput++ << LZSS_LOOKSHIFT; + position |= ( *pInput >> LZSS_LOOKSHIFT ); + int count = ( *pInput++ & 0x0F ) + 1; + if ( count == 1 ) + { + break; + } + unsigned char *pSource = pOutput - position - 1; + + if ( totalBytes + count > unBufSize ) + { + return 0; + } + + for ( int i=0; i<count; i++ ) + { + *pOutput++ = *pSource++; + } + totalBytes += count; + } + else + { + if ( totalBytes + 1 > unBufSize ) + return 0; + + *pOutput++ = *pInput++; + totalBytes++; + } + cmdByte = cmdByte >> 1; + } + + if ( totalBytes != actualSize ) + { + // unexpected failure + Assert( 0 ); + return 0; + } + + return totalBytes; +} + +//----------------------------------------------------------------------------- +// Uncompress a buffer, Returns the uncompressed size. Caller must provide an +// adequate sized output buffer or memory corruption will occur. +//----------------------------------------------------------------------------- +unsigned int CLZSS::Uncompress( const unsigned char *pInput, unsigned char *pOutput ) +{ + unsigned int totalBytes = 0; + int cmdByte = 0; + int getCmdByte = 0; + + unsigned int actualSize = GetActualSize( pInput ); + if ( !actualSize ) + { + // unrecognized + return 0; + } + + pInput += sizeof( lzss_header_t ); + + for ( ;; ) + { + if ( !getCmdByte ) + { + cmdByte = *pInput++; + } + getCmdByte = ( getCmdByte + 1 ) & 0x07; + + if ( cmdByte & 0x01 ) + { + int position = *pInput++ << LZSS_LOOKSHIFT; + position |= ( *pInput >> LZSS_LOOKSHIFT ); + int count = ( *pInput++ & 0x0F ) + 1; + if ( count == 1 ) + { + break; + } + unsigned char *pSource = pOutput - position - 1; + for ( int i=0; i<count; i++ ) + { + *pOutput++ = *pSource++; + } + totalBytes += count; + } + else + { + *pOutput++ = *pInput++; + totalBytes++; + } + cmdByte = cmdByte >> 1; + } + + if ( totalBytes != actualSize ) + { + // unexpected failure + Assert( 0 ); + return 0; + } + + return totalBytes; +} + + diff --git a/tier1/mempool.cpp b/tier1/mempool.cpp new file mode 100644 index 0000000..fa49edb --- /dev/null +++ b/tier1/mempool.cpp @@ -0,0 +1,312 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +//===========================================================================// + +#include "mempool.h" +#include <stdio.h> +#include <malloc.h> +#include <memory.h> +#include "tier0/dbg.h" +#include <ctype.h> +#include "tier1/strtools.h" + +// Should be last include +#include "tier0/memdbgon.h" + +MemoryPoolReportFunc_t CUtlMemoryPool::g_ReportFunc = 0; + +//----------------------------------------------------------------------------- +// Error reporting... (debug only) +//----------------------------------------------------------------------------- + +void CUtlMemoryPool::SetErrorReportFunc( MemoryPoolReportFunc_t func ) +{ + g_ReportFunc = func; +} + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CUtlMemoryPool::CUtlMemoryPool( int blockSize, int numElements, int growMode, const char *pszAllocOwner, int nAlignment ) +{ +#ifdef _X360 + if( numElements > 0 && growMode != UTLMEMORYPOOL_GROW_NONE ) + { + numElements = 1; + } +#endif + + m_nAlignment = ( nAlignment != 0 ) ? nAlignment : 1; + Assert( IsPowerOfTwo( m_nAlignment ) ); + m_BlockSize = blockSize < sizeof(void*) ? sizeof(void*) : blockSize; + m_BlockSize = AlignValue( m_BlockSize, m_nAlignment ); + m_BlocksPerBlob = numElements; + m_PeakAlloc = 0; + m_GrowMode = growMode; + if ( !pszAllocOwner ) + { + pszAllocOwner = __FILE__; + } + m_pszAllocOwner = pszAllocOwner; + Init(); + AddNewBlob(); +} + +//----------------------------------------------------------------------------- +// Purpose: Frees the memory contained in the mempool, and invalidates it for +// any further use. +// Input : *memPool - the mempool to shutdown +//----------------------------------------------------------------------------- +CUtlMemoryPool::~CUtlMemoryPool() +{ + if (m_BlocksAllocated > 0) + { + ReportLeaks(); + } + Clear(); +} + + +//----------------------------------------------------------------------------- +// Resets the pool +//----------------------------------------------------------------------------- +void CUtlMemoryPool::Init() +{ + m_NumBlobs = 0; + m_BlocksAllocated = 0; + m_pHeadOfFreeList = 0; + m_BlobHead.m_pNext = m_BlobHead.m_pPrev = &m_BlobHead; +} + + +//----------------------------------------------------------------------------- +// Frees everything +//----------------------------------------------------------------------------- +void CUtlMemoryPool::Clear() +{ + // Free everything.. + CBlob *pNext; + for( CBlob *pCur = m_BlobHead.m_pNext; pCur != &m_BlobHead; pCur = pNext ) + { + pNext = pCur->m_pNext; + free( pCur ); + } + Init(); +} + +//----------------------------------------------------------------------------- +// Purpose: Reports memory leaks +//----------------------------------------------------------------------------- + +void CUtlMemoryPool::ReportLeaks() +{ + if (!g_ReportFunc) + return; + + g_ReportFunc("Memory leak: mempool blocks left in memory: %d\n", m_BlocksAllocated); + +#ifdef _DEBUG + // walk and destroy the free list so it doesn't intefere in the scan + while (m_pHeadOfFreeList != NULL) + { + void *next = *((void**)m_pHeadOfFreeList); + memset(m_pHeadOfFreeList, 0, m_BlockSize); + m_pHeadOfFreeList = next; + } + + g_ReportFunc("Dumping memory: \'"); + + for( CBlob *pCur=m_BlobHead.m_pNext; pCur != &m_BlobHead; pCur=pCur->m_pNext ) + { + // scan the memory block and dump the leaks + char *scanPoint = (char *)pCur->m_Data; + char *scanEnd = pCur->m_Data + pCur->m_NumBytes; + bool needSpace = false; + + while (scanPoint < scanEnd) + { + // search for and dump any strings + if ((unsigned)(*scanPoint + 1) <= 256 && isprint(*scanPoint)) + { + g_ReportFunc("%c", *scanPoint); + needSpace = true; + } + else if (needSpace) + { + needSpace = false; + g_ReportFunc(" "); + } + + scanPoint++; + } + } + + g_ReportFunc("\'\n"); +#endif // _DEBUG +} + + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void CUtlMemoryPool::AddNewBlob() +{ + MEM_ALLOC_CREDIT_(m_pszAllocOwner); + + int sizeMultiplier; + + if( m_GrowMode == UTLMEMORYPOOL_GROW_SLOW ) + { + sizeMultiplier = 1; + } + else + { + if ( m_GrowMode == UTLMEMORYPOOL_GROW_NONE ) + { + // Can only have one allocation when we're in this mode + if( m_NumBlobs != 0 ) + { + Assert( !"CUtlMemoryPool::AddNewBlob: mode == UTLMEMORYPOOL_GROW_NONE" ); + return; + } + } + + // GROW_FAST and GROW_NONE use this. + sizeMultiplier = m_NumBlobs + 1; + } + + // maybe use something other than malloc? + int nElements = m_BlocksPerBlob * sizeMultiplier; + int blobSize = m_BlockSize * nElements; + CBlob *pBlob = (CBlob*)malloc( sizeof(CBlob) - 1 + blobSize + ( m_nAlignment - 1 ) ); + Assert( pBlob ); + + // Link it in at the end of the blob list. + pBlob->m_NumBytes = blobSize; + pBlob->m_pNext = &m_BlobHead; + pBlob->m_pPrev = pBlob->m_pNext->m_pPrev; + pBlob->m_pNext->m_pPrev = pBlob->m_pPrev->m_pNext = pBlob; + + // setup the free list + m_pHeadOfFreeList = AlignValue( pBlob->m_Data, m_nAlignment ); + Assert (m_pHeadOfFreeList); + + void **newBlob = (void**)m_pHeadOfFreeList; + for (int j = 0; j < nElements-1; j++) + { + newBlob[0] = (char*)newBlob + m_BlockSize; + newBlob = (void**)newBlob[0]; + } + + // null terminate list + newBlob[0] = NULL; + m_NumBlobs++; +} + + +void* CUtlMemoryPool::Alloc() +{ + return Alloc( m_BlockSize ); +} + + +void* CUtlMemoryPool::AllocZero() +{ + return AllocZero( m_BlockSize ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Allocs a single block of memory from the pool. +// Input : amount - +//----------------------------------------------------------------------------- +void *CUtlMemoryPool::Alloc( size_t amount ) +{ + void *returnBlock; + + if ( amount > (unsigned int)m_BlockSize ) + return NULL; + + if( !m_pHeadOfFreeList ) + { + // returning NULL is fine in UTLMEMORYPOOL_GROW_NONE + if( m_GrowMode == UTLMEMORYPOOL_GROW_NONE ) + { + //Assert( !"CUtlMemoryPool::Alloc: tried to make new blob with UTLMEMORYPOOL_GROW_NONE" ); + return NULL; + } + + // overflow + AddNewBlob(); + + // still failure, error out + if( !m_pHeadOfFreeList ) + { + Assert( !"CUtlMemoryPool::Alloc: ran out of memory" ); + return NULL; + } + } + m_BlocksAllocated++; + m_PeakAlloc = max(m_PeakAlloc, m_BlocksAllocated); + + returnBlock = m_pHeadOfFreeList; + + // move the pointer the next block + m_pHeadOfFreeList = *((void**)m_pHeadOfFreeList); + + return returnBlock; +} + +//----------------------------------------------------------------------------- +// Purpose: Allocs a single block of memory from the pool, zeroes the memory before returning +// Input : amount - +//----------------------------------------------------------------------------- +void *CUtlMemoryPool::AllocZero( size_t amount ) +{ + void *mem = Alloc( amount ); + if ( mem ) + { + V_memset( mem, 0x00, amount ); + } + return mem; +} + +//----------------------------------------------------------------------------- +// Purpose: Frees a block of memory +// Input : *memBlock - the memory to free +//----------------------------------------------------------------------------- +void CUtlMemoryPool::Free( void *memBlock ) +{ + if ( !memBlock ) + return; // trying to delete NULL pointer, ignore + +#ifdef _DEBUG + // check to see if the memory is from the allocated range + bool bOK = false; + for( CBlob *pCur=m_BlobHead.m_pNext; pCur != &m_BlobHead; pCur=pCur->m_pNext ) + { + if (memBlock >= pCur->m_Data && (char*)memBlock < (pCur->m_Data + pCur->m_NumBytes)) + { + bOK = true; + } + } + Assert (bOK); +#endif // _DEBUG + +#ifdef _DEBUG + // invalidate the memory + memset( memBlock, 0xDD, m_BlockSize ); +#endif + + m_BlocksAllocated--; + + // make the block point to the first item in the list + *((void**)memBlock) = m_pHeadOfFreeList; + + // the list head is now the new block + m_pHeadOfFreeList = memBlock; +} + + diff --git a/tier1/memstack.cpp b/tier1/memstack.cpp new file mode 100644 index 0000000..79eb101 --- /dev/null +++ b/tier1/memstack.cpp @@ -0,0 +1,297 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +//=============================================================================// + +#if defined( _WIN32 ) && !defined( _X360 ) +#define WIN_32_LEAN_AND_MEAN +#include <windows.h> +#define VA_COMMIT_FLAGS MEM_COMMIT +#define VA_RESERVE_FLAGS MEM_RESERVE +#elif defined( _X360 ) +#define VA_COMMIT_FLAGS (MEM_COMMIT|MEM_NOZERO|MEM_LARGE_PAGES) +#define VA_RESERVE_FLAGS (MEM_RESERVE|MEM_LARGE_PAGES) +#endif + +#include "tier0/dbg.h" +#include "memstack.h" +#include "utlmap.h" +#include "tier0/memdbgon.h" + +#ifdef _WIN32 +#pragma warning(disable:4073) +#pragma init_seg(lib) +#endif + +//----------------------------------------------------------------------------- + +MEMALLOC_DEFINE_EXTERNAL_TRACKING(CMemoryStack); + +//----------------------------------------------------------------------------- + +CMemoryStack::CMemoryStack() + : m_pBase( NULL ), + m_pNextAlloc( NULL ), + m_pAllocLimit( NULL ), + m_pCommitLimit( NULL ), + m_alignment( 16 ), +#if defined(_WIN32) + m_commitSize( 0 ), + m_minCommit( 0 ), +#endif + m_maxSize( 0 ) +{ +} + +//------------------------------------- + +CMemoryStack::~CMemoryStack() +{ + if ( m_pBase ) + Term(); +} + +//------------------------------------- + +bool CMemoryStack::Init( unsigned maxSize, unsigned commitSize, unsigned initialCommit, unsigned alignment ) +{ + Assert( !m_pBase ); + +#ifdef _X360 + m_bPhysical = false; +#endif + + m_maxSize = maxSize; + m_alignment = AlignValue( alignment, 4 ); + + Assert( m_alignment == alignment ); + Assert( m_maxSize > 0 ); + +#if defined(_WIN32) + if ( commitSize != 0 ) + { + m_commitSize = commitSize; + } + + unsigned pageSize; + +#ifndef _X360 + SYSTEM_INFO sysInfo; + GetSystemInfo( &sysInfo ); + Assert( !( sysInfo.dwPageSize & (sysInfo.dwPageSize-1)) ); + pageSize = sysInfo.dwPageSize; +#else + pageSize = 64*1024; +#endif + + if ( m_commitSize == 0 ) + { + m_commitSize = pageSize; + } + else + { + m_commitSize = AlignValue( m_commitSize, pageSize ); + } + + m_maxSize = AlignValue( m_maxSize, m_commitSize ); + + Assert( m_maxSize % pageSize == 0 && m_commitSize % pageSize == 0 && m_commitSize <= m_maxSize ); + + m_pBase = (unsigned char *)VirtualAlloc( NULL, m_maxSize, VA_RESERVE_FLAGS, PAGE_NOACCESS ); + Assert( m_pBase ); + m_pCommitLimit = m_pNextAlloc = m_pBase; + + if ( initialCommit ) + { + initialCommit = AlignValue( initialCommit, m_commitSize ); + Assert( initialCommit < m_maxSize ); + if ( !VirtualAlloc( m_pCommitLimit, initialCommit, VA_COMMIT_FLAGS, PAGE_READWRITE ) ) + return false; + m_minCommit = initialCommit; + m_pCommitLimit += initialCommit; + MemAlloc_RegisterExternalAllocation( CMemoryStack, GetBase(), GetSize() ); + } + +#else + m_pBase = (byte *)MemAlloc_AllocAligned( m_maxSize, alignment ? alignment : 1 ); + m_pNextAlloc = m_pBase; + m_pCommitLimit = m_pBase + m_maxSize; +#endif + + m_pAllocLimit = m_pBase + m_maxSize; + + return ( m_pBase != NULL ); +} + +//------------------------------------- + +#ifdef _X360 +bool CMemoryStack::InitPhysical( unsigned size, unsigned alignment ) +{ + m_bPhysical = true; + + m_maxSize = m_commitSize = size; + m_alignment = AlignValue( alignment, 4 ); + + int flags = PAGE_READWRITE; + if ( size >= 16*1024*1024 ) + { + flags |= MEM_16MB_PAGES; + } + else + { + flags |= MEM_LARGE_PAGES; + } + m_pBase = (unsigned char *)XPhysicalAlloc( m_maxSize, MAXULONG_PTR, 4096, flags ); + Assert( m_pBase ); + m_pNextAlloc = m_pBase; + m_pCommitLimit = m_pBase + m_maxSize; + m_pAllocLimit = m_pBase + m_maxSize; + + MemAlloc_RegisterExternalAllocation( CMemoryStack, GetBase(), GetSize() ); + return ( m_pBase != NULL ); +} +#endif + +//------------------------------------- + +void CMemoryStack::Term() +{ + FreeAll(); + if ( m_pBase ) + { +#if defined(_WIN32) + VirtualFree( m_pBase, 0, MEM_RELEASE ); +#else + MemAlloc_FreeAligned( m_pBase ); +#endif + m_pBase = NULL; + } +} + +//------------------------------------- + +int CMemoryStack::GetSize() +{ +#ifdef _WIN32 + return m_pCommitLimit - m_pBase; +#else + return m_maxSize; +#endif +} + + +//------------------------------------- + +bool CMemoryStack::CommitTo( byte *pNextAlloc ) RESTRICT +{ +#ifdef _X360 + if ( m_bPhysical ) + { + return NULL; + } +#endif +#if defined(_WIN32) + unsigned char * pNewCommitLimit = AlignValue( pNextAlloc, m_commitSize ); + unsigned commitSize = pNewCommitLimit - m_pCommitLimit; + + if ( GetSize() ) + MemAlloc_RegisterExternalDeallocation( CMemoryStack, GetBase(), GetSize() ); + + if( m_pCommitLimit + commitSize > m_pAllocLimit ) + { + return false; + } + + if ( !VirtualAlloc( m_pCommitLimit, commitSize, VA_COMMIT_FLAGS, PAGE_READWRITE ) ) + { + Assert( 0 ); + return false; + } + m_pCommitLimit = pNewCommitLimit; + + if ( GetSize() ) + MemAlloc_RegisterExternalAllocation( CMemoryStack, GetBase(), GetSize() ); + return true; +#else + Assert( 0 ); + return false; +#endif +} + +//------------------------------------- + +void CMemoryStack::FreeToAllocPoint( MemoryStackMark_t mark, bool bDecommit ) +{ + void *pAllocPoint = m_pBase + mark; + Assert( pAllocPoint >= m_pBase && pAllocPoint <= m_pNextAlloc ); + + if ( pAllocPoint >= m_pBase && pAllocPoint < m_pNextAlloc ) + { + if ( bDecommit ) + { +#if defined(_WIN32) + unsigned char *pDecommitPoint = AlignValue( (unsigned char *)pAllocPoint, m_commitSize ); + + if ( pDecommitPoint < m_pBase + m_minCommit ) + { + pDecommitPoint = m_pBase + m_minCommit; + } + + unsigned decommitSize = m_pCommitLimit - pDecommitPoint; + + if ( decommitSize > 0 ) + { + MemAlloc_RegisterExternalDeallocation( CMemoryStack, GetBase(), GetSize() ); + + VirtualFree( pDecommitPoint, decommitSize, MEM_DECOMMIT ); + m_pCommitLimit = pDecommitPoint; + + if ( mark > 0 ) + { + MemAlloc_RegisterExternalAllocation( CMemoryStack, GetBase(), GetSize() ); + } + } +#endif + } + m_pNextAlloc = (unsigned char *)pAllocPoint; + } +} + +//------------------------------------- + +void CMemoryStack::FreeAll( bool bDecommit ) +{ + if ( m_pBase && m_pCommitLimit - m_pBase > 0 ) + { + if ( bDecommit ) + { +#if defined(_WIN32) + MemAlloc_RegisterExternalDeallocation( CMemoryStack, GetBase(), GetSize() ); + + VirtualFree( m_pBase, m_pCommitLimit - m_pBase, MEM_DECOMMIT ); + m_pCommitLimit = m_pBase; +#endif + } + m_pNextAlloc = m_pBase; + } +} + +//------------------------------------- + +void CMemoryStack::Access( void **ppRegion, unsigned *pBytes ) +{ + *ppRegion = m_pBase; + *pBytes = ( m_pNextAlloc - m_pBase); +} + +//------------------------------------- + +void CMemoryStack::PrintContents() +{ + Msg( "Total used memory: %d\n", GetUsed() ); + Msg( "Total committed memory: %d\n", GetSize() ); +} + +//----------------------------------------------------------------------------- diff --git a/tier1/newbitbuf.cpp b/tier1/newbitbuf.cpp new file mode 100644 index 0000000..ef90f47 --- /dev/null +++ b/tier1/newbitbuf.cpp @@ -0,0 +1,717 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +// +//=============================================================================// + +#include "bitbuf.h" +#include "coordsize.h" +#include "mathlib/vector.h" +#include "mathlib/mathlib.h" +#include "tier1/strtools.h" +#include "bitvec.h" + +// FIXME: Can't use this until we get multithreaded allocations in tier0 working for tools +// This is used by VVIS and fails to link +// NOTE: This must be the last file included!!! +//#include "tier0/memdbgon.h" + +#ifdef _X360 +// mandatory ... wary of above comment and isolating, tier0 is built as MT though +#include "tier0/memdbgon.h" +#endif + +#include "stdio.h" + +#if 0 + +void CBitWrite::StartWriting( void *pData, int nBytes, int iStartBit, int nBits ) +{ + // Make sure it's dword aligned and padded. + Assert( (nBytes % 4) == 0 ); + Assert(((unsigned long)pData & 3) == 0); + Assert( iStartBit == 0 ); + m_pData = (uint32 *) pData; + m_pDataOut = m_pData; + m_nDataBytes = nBytes; + + if ( nBits == -1 ) + { + m_nDataBits = nBytes << 3; + } + else + { + Assert( nBits <= nBytes*8 ); + m_nDataBits = nBits; + } + m_bOverflow = false; + m_nOutBufWord = 0; + m_nOutBitsAvail = 32; + m_pBufferEnd = m_pDataOut + ( nBytes >> 2 ); +} + +const uint32 CBitBuffer::s_nMaskTable[33] = { + 0, + ( 1 << 1 ) - 1, + ( 1 << 2 ) - 1, + ( 1 << 3 ) - 1, + ( 1 << 4 ) - 1, + ( 1 << 5 ) - 1, + ( 1 << 6 ) - 1, + ( 1 << 7 ) - 1, + ( 1 << 8 ) - 1, + ( 1 << 9 ) - 1, + ( 1 << 10 ) - 1, + ( 1 << 11 ) - 1, + ( 1 << 12 ) - 1, + ( 1 << 13 ) - 1, + ( 1 << 14 ) - 1, + ( 1 << 15 ) - 1, + ( 1 << 16 ) - 1, + ( 1 << 17 ) - 1, + ( 1 << 18 ) - 1, + ( 1 << 19 ) - 1, + ( 1 << 20 ) - 1, + ( 1 << 21 ) - 1, + ( 1 << 22 ) - 1, + ( 1 << 23 ) - 1, + ( 1 << 24 ) - 1, + ( 1 << 25 ) - 1, + ( 1 << 26 ) - 1, + ( 1 << 27 ) - 1, + ( 1 << 28 ) - 1, + ( 1 << 29 ) - 1, + ( 1 << 30 ) - 1, + 0x7fffffff, + 0xffffffff, +}; + +bool CBitWrite::WriteString( const char *pStr ) +{ + if(pStr) + { + while( *pStr ) + { + WriteChar( * ( pStr++ ) ); + } + } + WriteChar( 0 ); + return !IsOverflowed(); +} + + +void CBitWrite::WriteLongLong(int64 val) +{ + uint *pLongs = (uint*)&val; + + // Insert the two DWORDS according to network endian + const short endianIndex = 0x0100; + byte *idx = (byte*)&endianIndex; + WriteUBitLong(pLongs[*idx++], sizeof(long) << 3); + WriteUBitLong(pLongs[*idx], sizeof(long) << 3); +} + +bool CBitWrite::WriteBits(const void *pInData, int nBits) +{ + unsigned char *pOut = (unsigned char*)pInData; + int nBitsLeft = nBits; + + // Bounds checking.. + if ( ( GetNumBitsWritten() + nBits) > m_nDataBits ) + { + SetOverflowFlag(); + CallErrorHandler( BITBUFERROR_BUFFER_OVERRUN, m_pDebugName ); + return false; + } + + // !! speed!! need fast paths + // write remaining bytes + while ( nBitsLeft >= 8 ) + { + WriteUBitLong( *pOut, 8, false ); + ++pOut; + nBitsLeft -= 8; + } + + // write remaining bits + if ( nBitsLeft ) + { + WriteUBitLong( *pOut, nBitsLeft, false ); + } + + return !IsOverflowed(); +} + +void CBitWrite::WriteBytes( const void *pBuf, int nBytes ) +{ + WriteBits(pBuf, nBytes << 3); +} + +void CBitWrite::WriteBitCoord (const float f) +{ + int signbit = (f <= -COORD_RESOLUTION); + int intval = (int)abs(f); + int fractval = abs((int)(f*COORD_DENOMINATOR)) & (COORD_DENOMINATOR-1); + + + // Send the bit flags that indicate whether we have an integer part and/or a fraction part. + WriteOneBit( intval ); + WriteOneBit( fractval ); + + if ( intval || fractval ) + { + // Send the sign bit + WriteOneBit( signbit ); + + // Send the integer if we have one. + if ( intval ) + { + // Adjust the integers from [1..MAX_COORD_VALUE] to [0..MAX_COORD_VALUE-1] + intval--; + WriteUBitLong( (unsigned int)intval, COORD_INTEGER_BITS ); + } + + // Send the fraction if we have one + if ( fractval ) + { + WriteUBitLong( (unsigned int)fractval, COORD_FRACTIONAL_BITS ); + } + } +} + +void CBitWrite::WriteBitCoordMP (const float f, bool bIntegral, bool bLowPrecision ) +{ + int signbit = (f <= -( bLowPrecision ? COORD_RESOLUTION_LOWPRECISION : COORD_RESOLUTION )); + int intval = (int)abs(f); + int fractval = bLowPrecision ? + ( abs((int)(f*COORD_DENOMINATOR_LOWPRECISION)) & (COORD_DENOMINATOR_LOWPRECISION-1) ) : + ( abs((int)(f*COORD_DENOMINATOR)) & (COORD_DENOMINATOR-1) ); + + bool bInBounds = intval < (1 << COORD_INTEGER_BITS_MP ); + + WriteOneBit( bInBounds ); + + if ( bIntegral ) + { + // Send the sign bit + WriteOneBit( intval ); + if ( intval ) + { + WriteOneBit( signbit ); + // Send the integer if we have one. + // Adjust the integers from [1..MAX_COORD_VALUE] to [0..MAX_COORD_VALUE-1] + intval--; + if ( bInBounds ) + { + WriteUBitLong( (unsigned int)intval, COORD_INTEGER_BITS_MP ); + } + else + { + WriteUBitLong( (unsigned int)intval, COORD_INTEGER_BITS ); + } + } + } + else + { + // Send the bit flags that indicate whether we have an integer part and/or a fraction part. + WriteOneBit( intval ); + // Send the sign bit + WriteOneBit( signbit ); + + // Send the integer if we have one. + if ( intval ) + { + // Adjust the integers from [1..MAX_COORD_VALUE] to [0..MAX_COORD_VALUE-1] + intval--; + if ( bInBounds ) + { + WriteUBitLong( (unsigned int)intval, COORD_INTEGER_BITS_MP ); + } + else + { + WriteUBitLong( (unsigned int)intval, COORD_INTEGER_BITS ); + } + } + WriteUBitLong( (unsigned int)fractval, bLowPrecision ? COORD_FRACTIONAL_BITS_MP_LOWPRECISION : COORD_FRACTIONAL_BITS ); + } +} + +void CBitWrite::SeekToBit( int nBit ) +{ + TempFlush(); + m_pDataOut = m_pData + ( nBit / 32 ); + m_nOutBufWord = *( m_pDataOut ); + m_nOutBitsAvail = 32 - ( nBit & 31 ); +} + + + +void CBitWrite::WriteBitVec3Coord( const Vector& fa ) +{ + int xflag, yflag, zflag; + + xflag = (fa[0] >= COORD_RESOLUTION) || (fa[0] <= -COORD_RESOLUTION); + yflag = (fa[1] >= COORD_RESOLUTION) || (fa[1] <= -COORD_RESOLUTION); + zflag = (fa[2] >= COORD_RESOLUTION) || (fa[2] <= -COORD_RESOLUTION); + + WriteOneBit( xflag ); + WriteOneBit( yflag ); + WriteOneBit( zflag ); + + if ( xflag ) + WriteBitCoord( fa[0] ); + if ( yflag ) + WriteBitCoord( fa[1] ); + if ( zflag ) + WriteBitCoord( fa[2] ); +} + +void CBitWrite::WriteBitNormal( float f ) +{ + int signbit = (f <= -NORMAL_RESOLUTION); + + // NOTE: Since +/-1 are valid values for a normal, I'm going to encode that as all ones + unsigned int fractval = abs( (int)(f*NORMAL_DENOMINATOR) ); + + // clamp.. + if (fractval > NORMAL_DENOMINATOR) + fractval = NORMAL_DENOMINATOR; + + // Send the sign bit + WriteOneBit( signbit ); + + // Send the fractional component + WriteUBitLong( fractval, NORMAL_FRACTIONAL_BITS ); +} + +void CBitWrite::WriteBitVec3Normal( const Vector& fa ) +{ + int xflag, yflag; + + xflag = (fa[0] >= NORMAL_RESOLUTION) || (fa[0] <= -NORMAL_RESOLUTION); + yflag = (fa[1] >= NORMAL_RESOLUTION) || (fa[1] <= -NORMAL_RESOLUTION); + + WriteOneBit( xflag ); + WriteOneBit( yflag ); + + if ( xflag ) + WriteBitNormal( fa[0] ); + if ( yflag ) + WriteBitNormal( fa[1] ); + + // Write z sign bit + int signbit = (fa[2] <= -NORMAL_RESOLUTION); + WriteOneBit( signbit ); +} + +void CBitWrite::WriteBitAngle( float fAngle, int numbits ) +{ + + unsigned int shift = GetBitForBitnum(numbits); + unsigned int mask = shift - 1; + + int d = (int)( (fAngle / 360.0) * shift ); + d &= mask; + + WriteUBitLong((unsigned int)d, numbits); +} + +bool CBitWrite::WriteBitsFromBuffer( bf_read *pIn, int nBits ) +{ +// This could be optimized a little by + while ( nBits > 32 ) + { + WriteUBitLong( pIn->ReadUBitLong( 32 ), 32 ); + nBits -= 32; + } + + WriteUBitLong( pIn->ReadUBitLong( nBits ), nBits ); + return !IsOverflowed() && !pIn->IsOverflowed(); +} + +void CBitWrite::WriteBitAngles( const QAngle& fa ) +{ + // FIXME: + Vector tmp( fa.x, fa.y, fa.z ); + WriteBitVec3Coord( tmp ); +} + +bool CBitRead::Seek( int nPosition ) +{ + bool bSucc = true; + if ( nPosition < 0 || nPosition > m_nDataBits) + { + SetOverflowFlag(); + bSucc = false; + nPosition = m_nDataBits; + } + int nHead = m_nDataBytes & 3; // non-multiple-of-4 bytes at head of buffer. We put the "round off" + // at the head to make reading and detecting the end efficient. + + int nByteOfs = nPosition / 8; + if ( ( m_nDataBytes < 4 ) || ( nHead && ( nByteOfs < nHead ) ) ) + { + // partial first dword + uint8 const *pPartial = ( uint8 const *) m_pData; + if ( m_pData ) + { + m_nInBufWord = *( pPartial++ ); + if ( nHead > 1 ) + m_nInBufWord |= ( *pPartial++ ) << 8; + if ( nHead > 2 ) + m_nInBufWord |= ( *pPartial++ ) << 16; + } + m_pDataIn = ( uint32 const * ) pPartial; + m_nInBufWord >>= ( nPosition & 31 ); + m_nBitsAvail = ( nHead << 3 ) - ( nPosition & 31 ); + } + else + { + int nAdjPosition = nPosition - ( nHead << 3 ); + m_pDataIn = reinterpret_cast<uint32 const *> ( + reinterpret_cast<uint8 const *>( m_pData ) + ( ( nAdjPosition / 32 ) << 2 ) + nHead ); + if ( m_pData ) + { + m_nBitsAvail = 32; + GrabNextDWord(); + } + else + { + m_nInBufWord = 0; + m_nBitsAvail = 1; + } + m_nInBufWord >>= ( nAdjPosition & 31 ); + m_nBitsAvail = min( m_nBitsAvail, 32 - ( nAdjPosition & 31 ) ); // in case grabnextdword overflowed + } + return bSucc; +} + + +void CBitRead::StartReading( const void *pData, int nBytes, int iStartBit, int nBits ) +{ +// Make sure it's dword aligned and padded. + Assert(((unsigned long)pData & 3) == 0); + m_pData = (uint32 *) pData; + m_pDataIn = m_pData; + m_nDataBytes = nBytes; + + if ( nBits == -1 ) + { + m_nDataBits = nBytes << 3; + } + else + { + Assert( nBits <= nBytes*8 ); + m_nDataBits = nBits; + } + m_bOverflow = false; + m_pBufferEnd = reinterpret_cast<uint32 const *> ( reinterpret_cast< uint8 const *> (m_pData) + nBytes ); + if ( m_pData ) + Seek( iStartBit ); + +} + +bool CBitRead::ReadString( char *pStr, int maxLen, bool bLine, int *pOutNumChars ) +{ + Assert( maxLen != 0 ); + + bool bTooSmall = false; + int iChar = 0; + while(1) + { + char val = ReadChar(); + if ( val == 0 ) + break; + else if ( bLine && val == '\n' ) + break; + + if ( iChar < (maxLen-1) ) + { + pStr[iChar] = val; + ++iChar; + } + else + { + bTooSmall = true; + } + } + + // Make sure it's null-terminated. + Assert( iChar < maxLen ); + pStr[iChar] = 0; + + if ( pOutNumChars ) + *pOutNumChars = iChar; + + return !IsOverflowed() && !bTooSmall; +} + +char* CBitRead::ReadAndAllocateString( bool *pOverflow ) +{ + char str[2048]; + + int nChars; + bool bOverflow = !ReadString( str, sizeof( str ), false, &nChars ); + if ( pOverflow ) + *pOverflow = bOverflow; + + // Now copy into the output and return it; + char *pRet = new char[ nChars + 1 ]; + for ( int i=0; i <= nChars; i++ ) + pRet[i] = str[i]; + + return pRet; +} + +int64 CBitRead::ReadLongLong( void ) +{ + int64 retval; + uint *pLongs = (uint*)&retval; + + // Read the two DWORDs according to network endian + const short endianIndex = 0x0100; + byte *idx = (byte*)&endianIndex; + pLongs[*idx++] = ReadUBitLong(sizeof(long) << 3); + pLongs[*idx] = ReadUBitLong(sizeof(long) << 3); + return retval; +} + +void CBitRead::ReadBits(void *pOutData, int nBits) +{ + unsigned char *pOut = (unsigned char*)pOutData; + int nBitsLeft = nBits; + + + // align output to dword boundary + while( ((unsigned long)pOut & 3) != 0 && nBitsLeft >= 8 ) + { + *pOut = (unsigned char)ReadUBitLong(8); + ++pOut; + nBitsLeft -= 8; + } + + // X360TBD: Can't read dwords in ReadBits because they'll get swapped + if ( IsPC() ) + { + // read dwords + while ( nBitsLeft >= 32 ) + { + *((unsigned long*)pOut) = ReadUBitLong(32); + pOut += sizeof(unsigned long); + nBitsLeft -= 32; + } + } + + // read remaining bytes + while ( nBitsLeft >= 8 ) + { + *pOut = ReadUBitLong(8); + ++pOut; + nBitsLeft -= 8; + } + + // read remaining bits + if ( nBitsLeft ) + { + *pOut = ReadUBitLong(nBitsLeft); + } + +} + +bool CBitRead::ReadBytes(void *pOut, int nBytes) +{ + ReadBits(pOut, nBytes << 3); + return !IsOverflowed(); +} + +float CBitRead::ReadBitAngle( int numbits ) +{ + float shift = (float)( GetBitForBitnum(numbits) ); + + int i = ReadUBitLong( numbits ); + float fReturn = (float)i * (360.0 / shift); + + return fReturn; +} + +// Basic Coordinate Routines (these contain bit-field size AND fixed point scaling constants) +float CBitRead::ReadBitCoord (void) +{ + int intval=0,fractval=0,signbit=0; + float value = 0.0; + + + // Read the required integer and fraction flags + intval = ReadOneBit(); + fractval = ReadOneBit(); + + // If we got either parse them, otherwise it's a zero. + if ( intval || fractval ) + { + // Read the sign bit + signbit = ReadOneBit(); + + // If there's an integer, read it in + if ( intval ) + { + // Adjust the integers from [0..MAX_COORD_VALUE-1] to [1..MAX_COORD_VALUE] + intval = ReadUBitLong( COORD_INTEGER_BITS ) + 1; + } + + // If there's a fraction, read it in + if ( fractval ) + { + fractval = ReadUBitLong( COORD_FRACTIONAL_BITS ); + } + + // Calculate the correct floating point value + value = intval + ((float)fractval * COORD_RESOLUTION); + + // Fixup the sign if negative. + if ( signbit ) + value = -value; + } + + return value; +} + +float CBitRead::ReadBitCoordMP( bool bIntegral, bool bLowPrecision ) +{ + int intval=0,fractval=0,signbit=0; + float value = 0.0; + + bool bInBounds = ReadOneBit() ? true : false; + + if ( bIntegral ) + { + // Read the required integer and fraction flags + intval = ReadOneBit(); + // If we got either parse them, otherwise it's a zero. + if ( intval ) + { + // Read the sign bit + signbit = ReadOneBit(); + + // If there's an integer, read it in + // Adjust the integers from [0..MAX_COORD_VALUE-1] to [1..MAX_COORD_VALUE] + if ( bInBounds ) + { + value = ReadUBitLong( COORD_INTEGER_BITS_MP ) + 1; + } + else + { + value = ReadUBitLong( COORD_INTEGER_BITS ) + 1; + } + } + } + else + { + // Read the required integer and fraction flags + intval = ReadOneBit(); + + // Read the sign bit + signbit = ReadOneBit(); + + // If we got either parse them, otherwise it's a zero. + if ( intval ) + { + if ( bInBounds ) + { + intval = ReadUBitLong( COORD_INTEGER_BITS_MP ) + 1; + } + else + { + intval = ReadUBitLong( COORD_INTEGER_BITS ) + 1; + } + } + + // If there's a fraction, read it in + fractval = ReadUBitLong( bLowPrecision ? COORD_FRACTIONAL_BITS_MP_LOWPRECISION : COORD_FRACTIONAL_BITS ); + + // Calculate the correct floating point value + value = intval + ((float)fractval * ( bLowPrecision ? COORD_RESOLUTION_LOWPRECISION : COORD_RESOLUTION ) ); + } + + // Fixup the sign if negative. + if ( signbit ) + value = -value; + + return value; +} + +void CBitRead::ReadBitVec3Coord( Vector& fa ) +{ + int xflag, yflag, zflag; + + // This vector must be initialized! Otherwise, If any of the flags aren't set, + // the corresponding component will not be read and will be stack garbage. + fa.Init( 0, 0, 0 ); + + xflag = ReadOneBit(); + yflag = ReadOneBit(); + zflag = ReadOneBit(); + + if ( xflag ) + fa[0] = ReadBitCoord(); + if ( yflag ) + fa[1] = ReadBitCoord(); + if ( zflag ) + fa[2] = ReadBitCoord(); +} + +float CBitRead::ReadBitNormal (void) +{ + // Read the sign bit + int signbit = ReadOneBit(); + + // Read the fractional part + unsigned int fractval = ReadUBitLong( NORMAL_FRACTIONAL_BITS ); + + // Calculate the correct floating point value + float value = (float)fractval * NORMAL_RESOLUTION; + + // Fixup the sign if negative. + if ( signbit ) + value = -value; + + return value; +} + +void CBitRead::ReadBitVec3Normal( Vector& fa ) +{ + int xflag = ReadOneBit(); + int yflag = ReadOneBit(); + + if (xflag) + fa[0] = ReadBitNormal(); + else + fa[0] = 0.0f; + + if (yflag) + fa[1] = ReadBitNormal(); + else + fa[1] = 0.0f; + + // The first two imply the third (but not its sign) + int znegative = ReadOneBit(); + + float fafafbfb = fa[0] * fa[0] + fa[1] * fa[1]; + if (fafafbfb < 1.0f) + fa[2] = sqrt( 1.0f - fafafbfb ); + else + fa[2] = 0.0f; + + if (znegative) + fa[2] = -fa[2]; +} + +void CBitRead::ReadBitAngles( QAngle& fa ) +{ + Vector tmp; + ReadBitVec3Coord( tmp ); + fa.Init( tmp.x, tmp.y, tmp.z ); +} + +#endif
\ No newline at end of file diff --git a/tier1/pathmatch.cpp b/tier1/pathmatch.cpp new file mode 100644 index 0000000..f828a9a --- /dev/null +++ b/tier1/pathmatch.cpp @@ -0,0 +1,908 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Utility to interrogate and modify the data in the OSX IPC Server +// +// $NoKeywords: $ +//============================================================================= +// README:README +// +// This file implements the --wrap for ld on linux that lets file i/o api's +// behave as if it were running on a case insensitive file system. Unfortunately, +// this is needed by both steam2 and steam3. It was decided to check the source +// into both locations, otherwise someone would find the .o and have no idea +// where to go for the source if it was in the 'other' tree. Also, because this +// needs to be linked into every elf binary, the .o is checked in for Steam3 so that it is +// always available. In Steam2 it sits with the PosixWin32.cpp implementation and gets +// compiled along side of it through the make system. If you are reading this in Steam3, +// you will probably want to actually make your changes in steam2 and do a baseless merge +// to the steam3 copy. +// +// HOWTO: Add a new function. Add the function with _WRAP to the makefiles as noted below. +// Add the implementation to pathmatch.cpp - probably mimicking the existing functions. +// Build steam2 and copy to matching steam3/client. Take the pathmatch.o from steam 2 +// and check it in to steam3 (in the location noted below). Full rebuild (re-link really) +// of steam3. Test steam and check in. +// +// If you are looking at updating this file, please update the following as needed: +// +// STEAM2.../Projects/GazelleProto/Client/Engine/obj/RELEASE_NORMAL/libsteam_linux/Common/Misc/pathmatch.o +// This is where steam2 builds the pathmatch.o out to. +// +// STEAM2.../Projects/GazelleProto/Makefile.shlib.base - contains _WRAP references +// STEAM2.../Projects/Common/Misc/pathmatch.cpp - Where the source is checked in, keep in sync with: +// STEAM3.../src/common/pathmatch.cpp - should be identical to previous file, but discoverable in steam3. +// STEAM3.../src/lib/linux32/release/pathmatch.o - steam3 checked in version +// STEAM3.../src/devtools/makefile_base_posix.mak - look for the _WRAP references + +#ifdef LINUX + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <stdint.h> +#include <strings.h> +#include <unistd.h> +#include <getopt.h> +#include <errno.h> +#include <signal.h> +#include <ctype.h> +#include <dirent.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <sys/mount.h> +#include <fcntl.h> +#include <utime.h> +#include <map> +#include <string> +#include <time.h> + +// Enable to do pathmatch caching. Beware: this code isn't threadsafe. +// #define DO_PATHMATCH_CACHE + +#ifdef UTF8_PATHMATCH +#define strcasecmp utf8casecmp +#endif + +static bool s_bShowDiag; +#define DEBUG_MSG( ... ) if ( s_bShowDiag ) fprintf( stderr, ##__VA_ARGS__ ) +#define DEBUG_BREAK() __asm__ __volatile__ ( "int $3" ) +#define _COMPILE_TIME_ASSERT(pred) switch(0){case 0:case pred:;} + +#define WRAP( fn, ret, ... ) \ + ret __real_##fn(__VA_ARGS__); \ + ret __wrap_##fn(__VA_ARGS__) + +#define CALL( fn ) __real_##fn + +// Needed by pathmatch code +extern "C" int __real_access(const char *pathname, int mode); +extern "C" DIR *__real_opendir(const char *name); + + +// UTF-8 work from PhysicsFS: http://icculus.org/physfs/ +// Even if it wasn't under the zlib license, Ryan wrote all this code originally. + +#define UNICODE_BOGUS_CHAR_VALUE 0xFFFFFFFF +#define UNICODE_BOGUS_CHAR_CODEPOINT '?' + +inline __attribute__ ((always_inline)) static uint32_t utf8codepoint(const char **_str) +{ + const char *str = *_str; + uint32_t retval = 0; + uint32_t octet = (uint32_t) ((uint8_t) *str); + uint32_t octet2, octet3, octet4; + + if (octet == 0) // null terminator, end of string. + return 0; + + else if (octet < 128) // one octet char: 0 to 127 + { + (*_str)++; // skip to next possible start of codepoint. + return octet; + } + + else if ((octet > 127) && (octet < 192)) // bad (starts with 10xxxxxx). + { + // Apparently each of these is supposed to be flagged as a bogus + // char, instead of just resyncing to the next valid codepoint. + (*_str)++; // skip to next possible start of codepoint. + return UNICODE_BOGUS_CHAR_VALUE; + } + + else if (octet < 224) // two octets + { + octet -= (128+64); + octet2 = (uint32_t) ((uint8_t) *(++str)); + if ((octet2 & (128+64)) != 128) // Format isn't 10xxxxxx? + return UNICODE_BOGUS_CHAR_VALUE; + + *_str += 2; // skip to next possible start of codepoint. + retval = ((octet << 6) | (octet2 - 128)); + if ((retval >= 0x80) && (retval <= 0x7FF)) + return retval; + } + + else if (octet < 240) // three octets + { + octet -= (128+64+32); + octet2 = (uint32_t) ((uint8_t) *(++str)); + if ((octet2 & (128+64)) != 128) // Format isn't 10xxxxxx? + return UNICODE_BOGUS_CHAR_VALUE; + + octet3 = (uint32_t) ((uint8_t) *(++str)); + if ((octet3 & (128+64)) != 128) // Format isn't 10xxxxxx? + return UNICODE_BOGUS_CHAR_VALUE; + + *_str += 3; // skip to next possible start of codepoint. + retval = ( ((octet << 12)) | ((octet2-128) << 6) | ((octet3-128)) ); + + // There are seven "UTF-16 surrogates" that are illegal in UTF-8. + switch (retval) + { + case 0xD800: + case 0xDB7F: + case 0xDB80: + case 0xDBFF: + case 0xDC00: + case 0xDF80: + case 0xDFFF: + return UNICODE_BOGUS_CHAR_VALUE; + } + + // 0xFFFE and 0xFFFF are illegal, too, so we check them at the edge. + if ((retval >= 0x800) && (retval <= 0xFFFD)) + return retval; + } + + else if (octet < 248) // four octets + { + octet -= (128+64+32+16); + octet2 = (uint32_t) ((uint8_t) *(++str)); + if ((octet2 & (128+64)) != 128) // Format isn't 10xxxxxx? + return UNICODE_BOGUS_CHAR_VALUE; + + octet3 = (uint32_t) ((uint8_t) *(++str)); + if ((octet3 & (128+64)) != 128) // Format isn't 10xxxxxx? + return UNICODE_BOGUS_CHAR_VALUE; + + octet4 = (uint32_t) ((uint8_t) *(++str)); + if ((octet4 & (128+64)) != 128) // Format isn't 10xxxxxx? + return UNICODE_BOGUS_CHAR_VALUE; + + *_str += 4; // skip to next possible start of codepoint. + retval = ( ((octet << 18)) | ((octet2 - 128) << 12) | + ((octet3 - 128) << 6) | ((octet4 - 128)) ); + if ((retval >= 0x10000) && (retval <= 0x10FFFF)) + return retval; + } + + // Five and six octet sequences became illegal in rfc3629. + // We throw the codepoint away, but parse them to make sure we move + // ahead the right number of bytes and don't overflow the buffer. + + else if (octet < 252) // five octets + { + octet = (uint32_t) ((uint8_t) *(++str)); + if ((octet & (128+64)) != 128) // Format isn't 10xxxxxx? + return UNICODE_BOGUS_CHAR_VALUE; + + octet = (uint32_t) ((uint8_t) *(++str)); + if ((octet & (128+64)) != 128) // Format isn't 10xxxxxx? + return UNICODE_BOGUS_CHAR_VALUE; + + octet = (uint32_t) ((uint8_t) *(++str)); + if ((octet & (128+64)) != 128) // Format isn't 10xxxxxx? + return UNICODE_BOGUS_CHAR_VALUE; + + octet = (uint32_t) ((uint8_t) *(++str)); + if ((octet & (128+64)) != 128) // Format isn't 10xxxxxx? + return UNICODE_BOGUS_CHAR_VALUE; + + *_str += 5; // skip to next possible start of codepoint. + return UNICODE_BOGUS_CHAR_VALUE; + } + + else // six octets + { + octet = (uint32_t) ((uint8_t) *(++str)); + if ((octet & (128+64)) != 128) // Format isn't 10xxxxxx? + return UNICODE_BOGUS_CHAR_VALUE; + + octet = (uint32_t) ((uint8_t) *(++str)); + if ((octet & (128+64)) != 128) // Format isn't 10xxxxxx? + return UNICODE_BOGUS_CHAR_VALUE; + + octet = (uint32_t) ((uint8_t) *(++str)); + if ((octet & (128+64)) != 128) // Format isn't 10xxxxxx? + return UNICODE_BOGUS_CHAR_VALUE; + + octet = (uint32_t) ((uint8_t) *(++str)); + if ((octet & (128+64)) != 128) // Format isn't 10xxxxxx? + return UNICODE_BOGUS_CHAR_VALUE; + + octet = (uint32_t) ((uint8_t) *(++str)); + if ((octet & (128+64)) != 128) // Format isn't 10xxxxxx? + return UNICODE_BOGUS_CHAR_VALUE; + + *_str += 6; // skip to next possible start of codepoint. + return UNICODE_BOGUS_CHAR_VALUE; + } + + return UNICODE_BOGUS_CHAR_VALUE; +} + +typedef struct CaseFoldMapping +{ + uint32_t from; + uint32_t to0; + uint32_t to1; + uint32_t to2; +} CaseFoldMapping; + +typedef struct CaseFoldHashBucket +{ + const uint8_t count; + const CaseFoldMapping *list; +} CaseFoldHashBucket; + +#include "pathmatch_casefolding.h" + +inline __attribute__ ((always_inline)) static void locate_case_fold_mapping(const uint32_t from, uint32_t *to) +{ + const uint8_t hashed = ((from ^ (from >> 8)) & 0xFF); + const CaseFoldHashBucket *bucket = &case_fold_hash[hashed]; + const CaseFoldMapping *mapping = bucket->list; + uint32_t i; + + for (i = 0; i < bucket->count; i++, mapping++) + { + if (mapping->from == from) + { + to[0] = mapping->to0; + to[1] = mapping->to1; + to[2] = mapping->to2; + return; + } + } + + // Not found...there's no remapping for this codepoint. + to[0] = from; + to[1] = 0; + to[2] = 0; +} + +inline __attribute__ ((always_inline)) static uint32_t *fold_utf8(const char *str) +{ + uint32_t *retval = new uint32_t[(strlen(str) * 3) + 1]; + uint32_t *dst = retval; + while (*str) + { + const char ch = *str; + if (ch & 0x80) // high bit set? UTF-8 sequence! + { + uint32_t fold[3]; + locate_case_fold_mapping(utf8codepoint(&str), fold); + *(dst++) = fold[0]; + if (fold[1]) + { + *(dst++) = fold[1]; + if (fold[2]) + *(dst++) = fold[2]; + } + } + else // simple ASCII test. + { + *(dst++) = (uint32_t) (((ch >= 'A') && (ch <= 'Z')) ? ch + 32 : ch); + str++; + } + } + *dst = 0; + return retval; +} + +inline __attribute__ ((always_inline)) static int utf8casecmp_loop(const uint32_t *folded1, const uint32_t *folded2) +{ + while (true) + { + const uint32_t ch1 = *(folded1++); + const uint32_t ch2 = *(folded2++); + if (ch1 < ch2) + return -1; + else if (ch1 > ch2) + return 1; + else if (ch1 == 0) + return 0; // complete match. + } +} + +#ifdef UTF8_PATHMATCH +static int utf8casecmp(const char *str1, const char *str2) +{ + uint32_t *folded1 = fold_utf8(str1); + uint32_t *folded2 = fold_utf8(str2); + const int retval = utf8casecmp_loop(folded1, folded2); + delete[] folded1; + delete[] folded2; + return retval; +} +#endif + +// Simple object to help make sure a DIR* from opendir +// gets closed when it goes out of scope. +class CDirPtr +{ +public: + CDirPtr() { m_pDir = NULL; } + CDirPtr( DIR *pDir ) : m_pDir(pDir) {} + ~CDirPtr() { Close(); } + + void operator=(DIR *pDir) { Close(); m_pDir = pDir; } + + operator DIR *() { return m_pDir; } + operator bool() { return m_pDir != NULL; } +private: + + void Close() { if ( m_pDir ) closedir( m_pDir ); } + + DIR *m_pDir; +}; + +// Object used to temporarily slice a path into a smaller componentent +// and then repair it when going out of scope. Typically used as an unnamed +// temp object that is a parameter to a function. +class CDirTrimmer +{ +public: + CDirTrimmer( char * pPath, size_t nTrimIdx ) + { + m_pPath = pPath; + m_idx = nTrimIdx; + m_c = m_pPath[nTrimIdx]; + m_pPath[nTrimIdx] = '\0'; + } + ~CDirTrimmer() { m_pPath[m_idx] = m_c; } + + operator const char *() { return m_pPath; } + +private: + size_t m_idx; + char *m_pPath; + char m_c; +}; + + +enum PathMod_t +{ + kPathUnchanged, + kPathLowered, + kPathChanged, + kPathFailed, +}; + +static bool Descend( char *pPath, size_t nStartIdx, bool bAllowBasenameMismatch, size_t nLevel = 0 ) +{ + DEBUG_MSG( "(%zu) Descend: %s, (%s), %s\n", nLevel, pPath, pPath+nStartIdx, bAllowBasenameMismatch ? "true" : "false " ); + // We assume up through nStartIdx is valid and matching + size_t nNextSlash = nStartIdx+1; + + // path might be a dir + if ( pPath[nNextSlash] == '\0' ) + { + return true; + } + + bool bIsDir = false; // is the new component a directory for certain? + while ( pPath[nNextSlash] != '\0' && pPath[nNextSlash] != '/' ) + { + nNextSlash++; + } + + // Modify the pPath string + if ( pPath[nNextSlash] == '/' ) + bIsDir = true; + + // See if we have an immediate match + if ( __real_access( CDirTrimmer(pPath, nNextSlash), F_OK ) == 0 ) + { + if ( !bIsDir ) + return true; + + bool bRet = Descend( pPath, nNextSlash, bAllowBasenameMismatch, nLevel+1 ); + if ( bRet ) + return true; + } + + // Start enumerating dirents + CDirPtr spDir; + if ( nStartIdx ) + { + // we have a path + spDir = __real_opendir( CDirTrimmer( pPath, nStartIdx ) ); + nStartIdx++; + } + else + { + // we either start at root or cwd + const char *pRoot = "."; + if ( *pPath == '/' ) + { + pRoot = "/"; + nStartIdx++; + } + spDir = __real_opendir( pRoot ); + } + + errno = 0; + struct dirent *pEntry = spDir ? readdir( spDir ) : NULL; + char *pszComponent = pPath + nStartIdx; + size_t cbComponent = nNextSlash - nStartIdx; + while ( pEntry ) + { + DEBUG_MSG( "\t(%zu) comparing %s with %s\n", nLevel, pEntry->d_name, (const char *)CDirTrimmer(pszComponent, cbComponent) ); + + // the candidate must match the target, but not be a case-identical match (we would + // have looked there in the short-circuit code above, so don't look again) + bool bMatches = ( strcasecmp( CDirTrimmer(pszComponent, cbComponent), pEntry->d_name ) == 0 && + strcmp( CDirTrimmer(pszComponent, cbComponent), pEntry->d_name ) != 0 ); + + if ( bMatches ) + { + char *pSrc = pEntry->d_name; + char *pDst = &pPath[nStartIdx]; + // found a match; copy it in. + while ( *pSrc && (*pSrc != '/') ) + { + *pDst++ = *pSrc++; + } + + if ( !bIsDir ) + return true; + + if ( Descend( pPath, nNextSlash, bAllowBasenameMismatch, nLevel+1 ) ) + return true; + + // If descend fails, try more directories + } + pEntry = readdir( spDir ); + } + + if ( bIsDir ) + { + DEBUG_MSG( "(%zu) readdir failed to find '%s' in '%s'\n", nLevel, (const char *)CDirTrimmer(pszComponent, cbComponent), (const char *)CDirTrimmer( pPath, nStartIdx ) ); + } + + // Sometimes it's ok for the filename portion to not match + // since we might be opening for write. Note that if + // the filename matches case insensitive, that will be + // preferred over preserving the input name + if ( !bIsDir && bAllowBasenameMismatch ) + return true; + + return false; +} + +#ifdef DO_PATHMATCH_CACHE +typedef std::map<std::string, std::pair<std::string, time_t> > resultCache_t; +typedef std::map<std::string, std::pair<std::string, time_t> >::iterator resultCacheItr_t; +static resultCache_t resultCache; +static const int k_cMaxCacheLifetimeSeconds = 2; +#endif // DO_PATHMATCH_CACHE + +PathMod_t pathmatch( const char *pszIn, char **ppszOut, bool bAllowBasenameMismatch, char *pszOutBuf, size_t OutBufLen ) +{ + // Path matching can be very expensive, and the cost is unpredictable because it + // depends on how many files are in directories on a user's machine. Therefore + // it should be disabled whenever possible, and only enabled in environments (such + // as running with loose files such as out of Perforce) where it is needed. + static const char *s_pszPathMatchEnabled = getenv("ENABLE_PATHMATCH"); + if ( !s_pszPathMatchEnabled ) + return kPathUnchanged; + + static const char *s_pszDbgPathMatch = getenv("DBG_PATHMATCH"); + + s_bShowDiag = ( s_pszDbgPathMatch != NULL ); + + *ppszOut = NULL; + + if ( __real_access( pszIn, F_OK ) == 0 ) + return kPathUnchanged; + +#ifdef DO_PATHMATCH_CACHE + resultCacheItr_t cachedResult = resultCache.find( pszIn ); + if ( cachedResult != resultCache.end() ) + { + unsigned int age = time( NULL ) - cachedResult->second.second; + const char *pszResult = cachedResult->second.first.c_str(); + if ( pszResult[0] != '\0' || age <= k_cMaxCacheLifetimeSeconds ) + { + if ( pszResult[0] != '\0' ) + { + *ppszOut = strdup( pszResult ); + DEBUG_MSG( "Cached '%s' -> '%s'\n", pszIn, *ppszOut ); + return kPathChanged; + } + else + { + DEBUG_MSG( "Cached '%s' -> kPathFailed\n", pszIn ); + return kPathFailed; + } + } + else if ( age <= k_cMaxCacheLifetimeSeconds ) + { + DEBUG_MSG( "Rechecking '%s' - cache is %u seconds old\n", pszIn, age ); + } + } +#endif // DO_PATHMATCH_CACHE + + char *pPath; + if( strlen( pszIn ) >= OutBufLen ) + { + pPath = strdup( pszIn ); + } + else + { + strncpy( pszOutBuf, pszIn, OutBufLen ); + pPath = pszOutBuf; + } + + if ( pPath ) + { + // I believe this code is broken. I'm guessing someone wanted to avoid lowercasing + // the path before the steam directory - but it's actually skipping lowercasing + // whenever steam is found anywhere - including the filename. For example, + // /home/mikesart/valvesrc/console/l4d2/game/left4dead2_dlc1/particles/steam_fx.pcf + // winds up only having the "steam_fx.pcf" portion lowercased. +#ifdef NEVER + // optimization, if the path contained steam somewhere + // assume the path up through the component with 'steam' in + // is valid (because we almost certainly obtained it + // progamatically + char *p = strcasestr( pPath, "steam" ); + if ( p ) + { + while ( p > pPath ) + { + if ( p[-1] == '/' ) + break; + p--; + } + + if ( ( p == pPath+1 ) && ( *pPath != '/' ) ) + p = pPath; + } + else + { + p = pPath; + } +#else + char *p = pPath; +#endif + + // Try the lower casing of the remaining path + char *pBasename = p; + while ( *p ) + { + if ( *p == '/' ) + pBasename = p+1; + + *p = tolower(*p); + p++; + } + if ( __real_access( pPath, F_OK ) == 0 ) + { + *ppszOut = pPath; + DEBUG_MSG( "Lowered '%s' -> '%s'\n", pszIn, pPath ); + return kPathLowered; + } + + // path didn't match lowered successfully, restore the basename + // if bAllowBasenameMismatch was true + if ( bAllowBasenameMismatch ) + { + const char *pSrc = pszIn + (pBasename - pPath); + while ( *pBasename ) + { + *pBasename++ = *pSrc++; + } + } + + if ( s_pszDbgPathMatch && strcasestr( s_pszDbgPathMatch, pszIn ) ) + { + DEBUG_MSG( "Breaking '%s' in '%s'\n", pszIn, s_pszDbgPathMatch ); + DEBUG_BREAK(); + } + + bool bSuccess = Descend( pPath, 0, bAllowBasenameMismatch ); + if ( bSuccess ) + { + *ppszOut = pPath; + DEBUG_MSG( "Matched '%s' -> '%s'\n", pszIn, pPath ); + } + else + { + DEBUG_MSG( "Unmatched %s\n", pszIn ); + } + +#ifndef DO_PATHMATCH_CACHE + return bSuccess ? kPathChanged : kPathFailed; +#else + time_t now = time(NULL); + if ( bSuccess ) + { + resultCache[ pszIn ] = std::make_pair( *ppszOut, now ); + return kPathChanged; + } + else + { + resultCache[ pszIn ] = std::make_pair( "", now ); + return kPathFailed; + } +#endif + } + return kPathFailed; +} + +// Wrapper object that manages the 'typical' usage cases of pathmatch() +class CWrap +{ +public: + CWrap( const char *pSuppliedPath, bool bAllowMismatchedBasename ) + : m_pSuppliedPath( pSuppliedPath ), m_pBestMatch( NULL ) + { + m_eResult = pathmatch( m_pSuppliedPath, &m_pBestMatch, bAllowMismatchedBasename, m_BestMatchBuf, sizeof( m_BestMatchBuf ) ); + if ( m_pBestMatch == NULL ) + { + m_pBestMatch = const_cast<char*>( m_pSuppliedPath ); + } + } + + ~CWrap() + { + if ( ( m_pBestMatch != m_pSuppliedPath ) && ( m_pBestMatch != m_BestMatchBuf ) ) + free( m_pBestMatch ); + } + + const char *GetBest() const { return m_pBestMatch; } + const char *GetOriginal() const { return m_pSuppliedPath; } + PathMod_t GetMatchResult() const { return m_eResult; } + + operator const char*() { return GetBest(); } + +private: + const char *m_pSuppliedPath; + char *m_pBestMatch; + char m_BestMatchBuf[ 512 ]; + PathMod_t m_eResult; +}; + +#ifdef MAIN_TEST +void usage() +{ + puts("pathmatch [options] <path>"); + //puts("options:"); + //puts("\t"); + + exit(-1); +} + +void test( const char *pszFile, bool bAllowBasenameMismatch ) +{ + char *pNewPath; + char NewPathBuf[ 512 ]; + PathMod_t nStat = pathmatch( pszFile, &pNewPath, bAllowBasenameMismatch, NewPathBuf, sizeof( NewPathBuf ) ); + + printf("AllowMismatchedBasename: %s\n", bAllowBasenameMismatch ? "true" : "false" ); + printf("Path Was: "); + switch ( nStat ) + { + case kPathUnchanged: + puts("kPathUnchanged"); + break; + case kPathLowered: + puts("kPathLowered"); + break; + case kPathChanged: + puts("kPathChanged"); + break; + case kPathFailed: + puts("kPathFailed"); + break; + } + + printf(" Path In: %s\n", pszFile ); + printf("Path Out: %s\n", nStat == kPathUnchanged ? pszFile : pNewPath ); + + if ( pNewPath ) + free( pNewPath ); +} + +int +main(int argc, char **argv) +{ + if ( argc <= 1 || argc > 2 ) + usage(); + + test( argv[1], false ); + test( argv[1], true ); + + return 0; +} +#endif + +extern "C" { + + WRAP(freopen, FILE *, const char *path, const char *mode, FILE *stream) + { + // if mode does not have w, a, or +, it's open for read. + bool bAllowBasenameMismatch = strpbrk( mode, "wa+" ) != NULL; + CWrap mpath( path, bAllowBasenameMismatch ); + + return CALL(freopen)( mpath, mode, stream ); + } + + WRAP(fopen, FILE *, const char *path, const char *mode) + { + // if mode does not have w, a, or +, it's open for read. + bool bAllowBasenameMismatch = strpbrk( mode, "wa+" ) != NULL; + CWrap mpath( path, bAllowBasenameMismatch ); + + return CALL(fopen)( mpath, mode ); + } + + + WRAP(fopen64, FILE *, const char *path, const char *mode) + { + // if mode does not have w, a, or +, it's open for read. + bool bAllowBasenameMismatch = strpbrk( mode, "wa+" ) != NULL; + CWrap mpath( path, bAllowBasenameMismatch ); + + return CALL(fopen64)( mpath, mode ); + } + + WRAP(open, int, const char *pathname, int flags, mode_t mode) + { + bool bAllowBasenameMismatch = ((flags & (O_WRONLY | O_RDWR)) != 0); + CWrap mpath( pathname, bAllowBasenameMismatch ); + return CALL(open)( mpath, flags, mode ); + } + + WRAP(open64, int, const char *pathname, int flags, mode_t mode) + { + bool bAllowBasenameMismatch = ((flags & (O_WRONLY | O_RDWR)) != 0); + CWrap mpath( pathname, bAllowBasenameMismatch ); + return CALL(open64)( mpath, flags, mode ); + } + + int __wrap_creat(const char *pathname, mode_t mode) + { + return __wrap_open( pathname, O_CREAT|O_WRONLY|O_TRUNC, mode ); + } + + int __wrap_access(const char *pathname, int mode) + { + return __real_access( CWrap( pathname, false ), mode ); + } + + WRAP(stat, int, const char *path, struct stat *buf) + { + return CALL(stat)( CWrap( path, false ), buf ); + } + + WRAP(lstat, int, const char *path, struct stat *buf) + { + return CALL(lstat)( CWrap( path, false ), buf ); + } + + WRAP(scandir, int, const char *dirp, struct dirent ***namelist, + int (*filter)(const struct dirent *), + int (*compar)(const struct dirent **, const struct dirent **)) + { + return CALL(scandir)( CWrap( dirp, false ), namelist, filter, compar ); + } + + WRAP(opendir, DIR*, const char *name) + { + return CALL(opendir)( CWrap( name, false ) ); + } + + WRAP(__xstat, int, int __ver, __const char *__filename, struct stat *__stat_buf) + { + return CALL(__xstat)( __ver, CWrap( __filename, false), __stat_buf ); + } + + WRAP(__lxstat, int, int __ver, __const char *__filename, struct stat *__stat_buf) + { + return CALL(__lxstat)( __ver, CWrap( __filename, false), __stat_buf ); + } + + WRAP(__xstat64, int, int __ver, __const char *__filename, struct stat *__stat_buf) + { + return CALL(__xstat64)( __ver, CWrap( __filename, false), __stat_buf ); + } + + WRAP(__lxstat64, int, int __ver, __const char *__filename, struct stat *__stat_buf) + { + return CALL(__lxstat64)( __ver, CWrap( __filename, false), __stat_buf ); + } + + WRAP(chmod, int, const char *path, mode_t mode) + { + return CALL(chmod)( CWrap( path, false), mode ); + } + + WRAP(chown, int, const char *path, uid_t owner, gid_t group) + { + return CALL(chown)( CWrap( path, false), owner, group ); + } + + WRAP(lchown, int, const char *path, uid_t owner, gid_t group) + { + return CALL(lchown)( CWrap( path, false), owner, group ); + } + + WRAP(symlink, int, const char *oldpath, const char *newpath) + { + return CALL(symlink)( CWrap( oldpath, false), CWrap( newpath, true ) ); + } + + WRAP(link, int, const char *oldpath, const char *newpath) + { + return CALL(link)( CWrap( oldpath, false), CWrap( newpath, true ) ); + } + + WRAP(mknod, int, const char *pathname, mode_t mode, dev_t dev) + { + return CALL(mknod)( CWrap( pathname, true), mode, dev ); + } + + WRAP(mount, int, const char *source, const char *target, + const char *filesystemtype, unsigned long mountflags, + const void *data) + { + return CALL(mount)( CWrap( source, false ), CWrap( target, false ), filesystemtype, mountflags, data ); + } + + WRAP(unlink, int, const char *pathname) + { + return CALL(unlink)( CWrap( pathname, false ) ); + } + + WRAP(mkfifo, int, const char *pathname, mode_t mode) + { + return CALL(mkfifo)( CWrap( pathname, true ), mode ); + } + + WRAP(rename, int, const char *oldpath, const char *newpath) + { + return CALL(rename)( CWrap( oldpath, false), CWrap( newpath, true ) ); + } + + WRAP(utime, int, const char *filename, const struct utimbuf *times) + { + return CALL(utime)( CWrap( filename, false), times ); + } + + WRAP(utimes, int, const char *filename, const struct timeval times[2]) + { + return CALL(utimes)( CWrap( filename, false), times ); + } + + WRAP(realpath, char *, const char *path, char *resolved_path) + { + return CALL(realpath)( CWrap( path, true ), resolved_path ); + } + + WRAP(mkdir, int, const char *pathname, mode_t mode) + { + return CALL(mkdir)( CWrap( pathname, true ), mode ); + } + + WRAP(rmdir, char *, const char *pathname) + { + return CALL(rmdir)( CWrap( pathname, false ) ); + } + +}; + +#endif diff --git a/tier1/pathmatch_casefolding.h b/tier1/pathmatch_casefolding.h new file mode 100644 index 0000000..2b4381b --- /dev/null +++ b/tier1/pathmatch_casefolding.h @@ -0,0 +1,2008 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// This file was originally part of PhysicsFS (http://icculus.org/physfs/) +// (zlib license, even if Ryan hadn't written it originally.) +// +// This data was originally generated by physfs/extras/makecasefoldhashtable.pl + +#define CASEFOLDING_ARRAYLEN(x) (sizeof (x) / sizeof ((x)[0])) + +static const CaseFoldMapping case_fold_000[] = { + { 0x0202, 0x0203, 0x0000, 0x0000 }, + { 0x0404, 0x0454, 0x0000, 0x0000 }, + { 0x1E1E, 0x1E1F, 0x0000, 0x0000 }, + { 0x2C2C, 0x2C5C, 0x0000, 0x0000 }, + { 0x10404, 0x1042C, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_001[] = { + { 0x0100, 0x0101, 0x0000, 0x0000 }, + { 0x0405, 0x0455, 0x0000, 0x0000 }, + { 0x0504, 0x0505, 0x0000, 0x0000 }, + { 0x2C2D, 0x2C5D, 0x0000, 0x0000 }, + { 0x10405, 0x1042D, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_002[] = { + { 0x0200, 0x0201, 0x0000, 0x0000 }, + { 0x0406, 0x0456, 0x0000, 0x0000 }, + { 0x1E1C, 0x1E1D, 0x0000, 0x0000 }, + { 0x1F1D, 0x1F15, 0x0000, 0x0000 }, + { 0x2C2E, 0x2C5E, 0x0000, 0x0000 }, + { 0x10406, 0x1042E, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_003[] = { + { 0x0102, 0x0103, 0x0000, 0x0000 }, + { 0x0407, 0x0457, 0x0000, 0x0000 }, + { 0x0506, 0x0507, 0x0000, 0x0000 }, + { 0x1F1C, 0x1F14, 0x0000, 0x0000 }, + { 0x10407, 0x1042F, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_004[] = { + { 0x0206, 0x0207, 0x0000, 0x0000 }, + { 0x0400, 0x0450, 0x0000, 0x0000 }, + { 0x1E1A, 0x1E1B, 0x0000, 0x0000 }, + { 0x1F1B, 0x1F13, 0x0000, 0x0000 }, + { 0x2C28, 0x2C58, 0x0000, 0x0000 }, + { 0x10400, 0x10428, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_005[] = { + { 0x0104, 0x0105, 0x0000, 0x0000 }, + { 0x0401, 0x0451, 0x0000, 0x0000 }, + { 0x0500, 0x0501, 0x0000, 0x0000 }, + { 0x1F1A, 0x1F12, 0x0000, 0x0000 }, + { 0x2C29, 0x2C59, 0x0000, 0x0000 }, + { 0x10401, 0x10429, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_006[] = { + { 0x0204, 0x0205, 0x0000, 0x0000 }, + { 0x0402, 0x0452, 0x0000, 0x0000 }, + { 0x1E18, 0x1E19, 0x0000, 0x0000 }, + { 0x1F19, 0x1F11, 0x0000, 0x0000 }, + { 0x2C2A, 0x2C5A, 0x0000, 0x0000 }, + { 0x10402, 0x1042A, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_007[] = { + { 0x0106, 0x0107, 0x0000, 0x0000 }, + { 0x0403, 0x0453, 0x0000, 0x0000 }, + { 0x0502, 0x0503, 0x0000, 0x0000 }, + { 0x1F18, 0x1F10, 0x0000, 0x0000 }, + { 0x2126, 0x03C9, 0x0000, 0x0000 }, + { 0x2C2B, 0x2C5B, 0x0000, 0x0000 }, + { 0x10403, 0x1042B, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_008[] = { + { 0x020A, 0x020B, 0x0000, 0x0000 }, + { 0x040C, 0x045C, 0x0000, 0x0000 }, + { 0x1E16, 0x1E17, 0x0000, 0x0000 }, + { 0x2C24, 0x2C54, 0x0000, 0x0000 }, + { 0x1040C, 0x10434, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_009[] = { + { 0x0108, 0x0109, 0x0000, 0x0000 }, + { 0x040D, 0x045D, 0x0000, 0x0000 }, + { 0x050C, 0x050D, 0x0000, 0x0000 }, + { 0x2C25, 0x2C55, 0x0000, 0x0000 }, + { 0x1040D, 0x10435, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_010[] = { + { 0x0208, 0x0209, 0x0000, 0x0000 }, + { 0x040E, 0x045E, 0x0000, 0x0000 }, + { 0x1E14, 0x1E15, 0x0000, 0x0000 }, + { 0x212B, 0x00E5, 0x0000, 0x0000 }, + { 0x2C26, 0x2C56, 0x0000, 0x0000 }, + { 0x1040E, 0x10436, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_011[] = { + { 0x010A, 0x010B, 0x0000, 0x0000 }, + { 0x040F, 0x045F, 0x0000, 0x0000 }, + { 0x050E, 0x050F, 0x0000, 0x0000 }, + { 0x212A, 0x006B, 0x0000, 0x0000 }, + { 0x2C27, 0x2C57, 0x0000, 0x0000 }, + { 0x1040F, 0x10437, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_012[] = { + { 0x020E, 0x020F, 0x0000, 0x0000 }, + { 0x0408, 0x0458, 0x0000, 0x0000 }, + { 0x1E12, 0x1E13, 0x0000, 0x0000 }, + { 0x2C20, 0x2C50, 0x0000, 0x0000 }, + { 0x10408, 0x10430, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_013[] = { + { 0x010C, 0x010D, 0x0000, 0x0000 }, + { 0x0409, 0x0459, 0x0000, 0x0000 }, + { 0x0508, 0x0509, 0x0000, 0x0000 }, + { 0x2C21, 0x2C51, 0x0000, 0x0000 }, + { 0x10409, 0x10431, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_014[] = { + { 0x020C, 0x020D, 0x0000, 0x0000 }, + { 0x040A, 0x045A, 0x0000, 0x0000 }, + { 0x1E10, 0x1E11, 0x0000, 0x0000 }, + { 0x2C22, 0x2C52, 0x0000, 0x0000 }, + { 0x1040A, 0x10432, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_015[] = { + { 0x010E, 0x010F, 0x0000, 0x0000 }, + { 0x040B, 0x045B, 0x0000, 0x0000 }, + { 0x050A, 0x050B, 0x0000, 0x0000 }, + { 0x2C23, 0x2C53, 0x0000, 0x0000 }, + { 0x1040B, 0x10433, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_016[] = { + { 0x0212, 0x0213, 0x0000, 0x0000 }, + { 0x0414, 0x0434, 0x0000, 0x0000 }, + { 0x1E0E, 0x1E0F, 0x0000, 0x0000 }, + { 0x1F0F, 0x1F07, 0x0000, 0x0000 }, + { 0x10414, 0x1043C, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_017[] = { + { 0x0110, 0x0111, 0x0000, 0x0000 }, + { 0x0415, 0x0435, 0x0000, 0x0000 }, + { 0x1F0E, 0x1F06, 0x0000, 0x0000 }, + { 0x10415, 0x1043D, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_018[] = { + { 0x0210, 0x0211, 0x0000, 0x0000 }, + { 0x0416, 0x0436, 0x0000, 0x0000 }, + { 0x1E0C, 0x1E0D, 0x0000, 0x0000 }, + { 0x1F0D, 0x1F05, 0x0000, 0x0000 }, + { 0x10416, 0x1043E, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_019[] = { + { 0x0112, 0x0113, 0x0000, 0x0000 }, + { 0x0417, 0x0437, 0x0000, 0x0000 }, + { 0x1F0C, 0x1F04, 0x0000, 0x0000 }, + { 0x10417, 0x1043F, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_020[] = { + { 0x0216, 0x0217, 0x0000, 0x0000 }, + { 0x0410, 0x0430, 0x0000, 0x0000 }, + { 0x1E0A, 0x1E0B, 0x0000, 0x0000 }, + { 0x1F0B, 0x1F03, 0x0000, 0x0000 }, + { 0x10410, 0x10438, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_021[] = { + { 0x0114, 0x0115, 0x0000, 0x0000 }, + { 0x0411, 0x0431, 0x0000, 0x0000 }, + { 0x1F0A, 0x1F02, 0x0000, 0x0000 }, + { 0x10411, 0x10439, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_022[] = { + { 0x0214, 0x0215, 0x0000, 0x0000 }, + { 0x0412, 0x0432, 0x0000, 0x0000 }, + { 0x1E08, 0x1E09, 0x0000, 0x0000 }, + { 0x1F09, 0x1F01, 0x0000, 0x0000 }, + { 0x10412, 0x1043A, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_023[] = { + { 0x0116, 0x0117, 0x0000, 0x0000 }, + { 0x0413, 0x0433, 0x0000, 0x0000 }, + { 0x1F08, 0x1F00, 0x0000, 0x0000 }, + { 0x10413, 0x1043B, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_024[] = { + { 0x021A, 0x021B, 0x0000, 0x0000 }, + { 0x041C, 0x043C, 0x0000, 0x0000 }, + { 0x1E06, 0x1E07, 0x0000, 0x0000 }, + { 0x1041C, 0x10444, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_025[] = { + { 0x0118, 0x0119, 0x0000, 0x0000 }, + { 0x041D, 0x043D, 0x0000, 0x0000 }, + { 0x1041D, 0x10445, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_026[] = { + { 0x0218, 0x0219, 0x0000, 0x0000 }, + { 0x041E, 0x043E, 0x0000, 0x0000 }, + { 0x1E04, 0x1E05, 0x0000, 0x0000 }, + { 0x1041E, 0x10446, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_027[] = { + { 0x011A, 0x011B, 0x0000, 0x0000 }, + { 0x041F, 0x043F, 0x0000, 0x0000 }, + { 0x1041F, 0x10447, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_028[] = { + { 0x021E, 0x021F, 0x0000, 0x0000 }, + { 0x0418, 0x0438, 0x0000, 0x0000 }, + { 0x1E02, 0x1E03, 0x0000, 0x0000 }, + { 0x10418, 0x10440, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_029[] = { + { 0x011C, 0x011D, 0x0000, 0x0000 }, + { 0x0419, 0x0439, 0x0000, 0x0000 }, + { 0x10419, 0x10441, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_030[] = { + { 0x021C, 0x021D, 0x0000, 0x0000 }, + { 0x041A, 0x043A, 0x0000, 0x0000 }, + { 0x1E00, 0x1E01, 0x0000, 0x0000 }, + { 0x1041A, 0x10442, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_031[] = { + { 0x011E, 0x011F, 0x0000, 0x0000 }, + { 0x041B, 0x043B, 0x0000, 0x0000 }, + { 0x1041B, 0x10443, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_032[] = { + { 0x0222, 0x0223, 0x0000, 0x0000 }, + { 0x0424, 0x0444, 0x0000, 0x0000 }, + { 0x1E3E, 0x1E3F, 0x0000, 0x0000 }, + { 0x1F3F, 0x1F37, 0x0000, 0x0000 }, + { 0x2C0C, 0x2C3C, 0x0000, 0x0000 }, + { 0x10424, 0x1044C, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_033[] = { + { 0x0120, 0x0121, 0x0000, 0x0000 }, + { 0x0425, 0x0445, 0x0000, 0x0000 }, + { 0x1F3E, 0x1F36, 0x0000, 0x0000 }, + { 0x2C0D, 0x2C3D, 0x0000, 0x0000 }, + { 0x10425, 0x1044D, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_034[] = { + { 0x0220, 0x019E, 0x0000, 0x0000 }, + { 0x0426, 0x0446, 0x0000, 0x0000 }, + { 0x1E3C, 0x1E3D, 0x0000, 0x0000 }, + { 0x1F3D, 0x1F35, 0x0000, 0x0000 }, + { 0x2C0E, 0x2C3E, 0x0000, 0x0000 }, + { 0x10426, 0x1044E, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_035[] = { + { 0x0122, 0x0123, 0x0000, 0x0000 }, + { 0x0427, 0x0447, 0x0000, 0x0000 }, + { 0x1F3C, 0x1F34, 0x0000, 0x0000 }, + { 0x2C0F, 0x2C3F, 0x0000, 0x0000 }, + { 0x10427, 0x1044F, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_036[] = { + { 0x0226, 0x0227, 0x0000, 0x0000 }, + { 0x0420, 0x0440, 0x0000, 0x0000 }, + { 0x1E3A, 0x1E3B, 0x0000, 0x0000 }, + { 0x1F3B, 0x1F33, 0x0000, 0x0000 }, + { 0x2C08, 0x2C38, 0x0000, 0x0000 }, + { 0x10420, 0x10448, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_037[] = { + { 0x0124, 0x0125, 0x0000, 0x0000 }, + { 0x0421, 0x0441, 0x0000, 0x0000 }, + { 0x1F3A, 0x1F32, 0x0000, 0x0000 }, + { 0x2C09, 0x2C39, 0x0000, 0x0000 }, + { 0x10421, 0x10449, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_038[] = { + { 0x0224, 0x0225, 0x0000, 0x0000 }, + { 0x0422, 0x0442, 0x0000, 0x0000 }, + { 0x1E38, 0x1E39, 0x0000, 0x0000 }, + { 0x1F39, 0x1F31, 0x0000, 0x0000 }, + { 0x2C0A, 0x2C3A, 0x0000, 0x0000 }, + { 0x10422, 0x1044A, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_039[] = { + { 0x0126, 0x0127, 0x0000, 0x0000 }, + { 0x0423, 0x0443, 0x0000, 0x0000 }, + { 0x1F38, 0x1F30, 0x0000, 0x0000 }, + { 0x2C0B, 0x2C3B, 0x0000, 0x0000 }, + { 0x10423, 0x1044B, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_040[] = { + { 0x022A, 0x022B, 0x0000, 0x0000 }, + { 0x042C, 0x044C, 0x0000, 0x0000 }, + { 0x1E36, 0x1E37, 0x0000, 0x0000 }, + { 0x2C04, 0x2C34, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_041[] = { + { 0x0128, 0x0129, 0x0000, 0x0000 }, + { 0x042D, 0x044D, 0x0000, 0x0000 }, + { 0x2C05, 0x2C35, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_042[] = { + { 0x0228, 0x0229, 0x0000, 0x0000 }, + { 0x042E, 0x044E, 0x0000, 0x0000 }, + { 0x1E34, 0x1E35, 0x0000, 0x0000 }, + { 0x2C06, 0x2C36, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_043[] = { + { 0x012A, 0x012B, 0x0000, 0x0000 }, + { 0x042F, 0x044F, 0x0000, 0x0000 }, + { 0x2C07, 0x2C37, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_044[] = { + { 0x022E, 0x022F, 0x0000, 0x0000 }, + { 0x0428, 0x0448, 0x0000, 0x0000 }, + { 0x1E32, 0x1E33, 0x0000, 0x0000 }, + { 0x2C00, 0x2C30, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_045[] = { + { 0x012C, 0x012D, 0x0000, 0x0000 }, + { 0x0429, 0x0449, 0x0000, 0x0000 }, + { 0x2C01, 0x2C31, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_046[] = { + { 0x022C, 0x022D, 0x0000, 0x0000 }, + { 0x042A, 0x044A, 0x0000, 0x0000 }, + { 0x1E30, 0x1E31, 0x0000, 0x0000 }, + { 0x2C02, 0x2C32, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_047[] = { + { 0x012E, 0x012F, 0x0000, 0x0000 }, + { 0x042B, 0x044B, 0x0000, 0x0000 }, + { 0x2C03, 0x2C33, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_048[] = { + { 0x0232, 0x0233, 0x0000, 0x0000 }, + { 0x0535, 0x0565, 0x0000, 0x0000 }, + { 0x1E2E, 0x1E2F, 0x0000, 0x0000 }, + { 0x1F2F, 0x1F27, 0x0000, 0x0000 }, + { 0x2C1C, 0x2C4C, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_049[] = { + { 0x0130, 0x0069, 0x0307, 0x0000 }, + { 0x0534, 0x0564, 0x0000, 0x0000 }, + { 0x1F2E, 0x1F26, 0x0000, 0x0000 }, + { 0x2C1D, 0x2C4D, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_050[] = { + { 0x0230, 0x0231, 0x0000, 0x0000 }, + { 0x0537, 0x0567, 0x0000, 0x0000 }, + { 0x1E2C, 0x1E2D, 0x0000, 0x0000 }, + { 0x1F2D, 0x1F25, 0x0000, 0x0000 }, + { 0x2C1E, 0x2C4E, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_051[] = { + { 0x0132, 0x0133, 0x0000, 0x0000 }, + { 0x0536, 0x0566, 0x0000, 0x0000 }, + { 0x1F2C, 0x1F24, 0x0000, 0x0000 }, + { 0x2C1F, 0x2C4F, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_052[] = { + { 0x0531, 0x0561, 0x0000, 0x0000 }, + { 0x1E2A, 0x1E2B, 0x0000, 0x0000 }, + { 0x1F2B, 0x1F23, 0x0000, 0x0000 }, + { 0x2C18, 0x2C48, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_053[] = { + { 0x0134, 0x0135, 0x0000, 0x0000 }, + { 0x1F2A, 0x1F22, 0x0000, 0x0000 }, + { 0x2C19, 0x2C49, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_054[] = { + { 0x0533, 0x0563, 0x0000, 0x0000 }, + { 0x1E28, 0x1E29, 0x0000, 0x0000 }, + { 0x1F29, 0x1F21, 0x0000, 0x0000 }, + { 0x2C1A, 0x2C4A, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_055[] = { + { 0x0136, 0x0137, 0x0000, 0x0000 }, + { 0x0532, 0x0562, 0x0000, 0x0000 }, + { 0x1F28, 0x1F20, 0x0000, 0x0000 }, + { 0x2C1B, 0x2C4B, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_056[] = { + { 0x0139, 0x013A, 0x0000, 0x0000 }, + { 0x053D, 0x056D, 0x0000, 0x0000 }, + { 0x1E26, 0x1E27, 0x0000, 0x0000 }, + { 0x2C14, 0x2C44, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_057[] = { + { 0x023B, 0x023C, 0x0000, 0x0000 }, + { 0x053C, 0x056C, 0x0000, 0x0000 }, + { 0x2C15, 0x2C45, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_058[] = { + { 0x013B, 0x013C, 0x0000, 0x0000 }, + { 0x053F, 0x056F, 0x0000, 0x0000 }, + { 0x1E24, 0x1E25, 0x0000, 0x0000 }, + { 0x2C16, 0x2C46, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_059[] = { + { 0x053E, 0x056E, 0x0000, 0x0000 }, + { 0x2C17, 0x2C47, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_060[] = { + { 0x013D, 0x013E, 0x0000, 0x0000 }, + { 0x0539, 0x0569, 0x0000, 0x0000 }, + { 0x1E22, 0x1E23, 0x0000, 0x0000 }, + { 0x2C10, 0x2C40, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_061[] = { + { 0x0538, 0x0568, 0x0000, 0x0000 }, + { 0x2C11, 0x2C41, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_062[] = { + { 0x013F, 0x0140, 0x0000, 0x0000 }, + { 0x053B, 0x056B, 0x0000, 0x0000 }, + { 0x1E20, 0x1E21, 0x0000, 0x0000 }, + { 0x2C12, 0x2C42, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_063[] = { + { 0x023D, 0x019A, 0x0000, 0x0000 }, + { 0x053A, 0x056A, 0x0000, 0x0000 }, + { 0x2C13, 0x2C43, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_064[] = { + { 0x0141, 0x0142, 0x0000, 0x0000 }, + { 0x0545, 0x0575, 0x0000, 0x0000 }, + { 0x1E5E, 0x1E5F, 0x0000, 0x0000 }, + { 0x1F5F, 0x1F57, 0x0000, 0x0000 }, + { 0x2161, 0x2171, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_065[] = { + { 0x0041, 0x0061, 0x0000, 0x0000 }, + { 0x0544, 0x0574, 0x0000, 0x0000 }, + { 0x2160, 0x2170, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_066[] = { + { 0x0042, 0x0062, 0x0000, 0x0000 }, + { 0x0143, 0x0144, 0x0000, 0x0000 }, + { 0x0547, 0x0577, 0x0000, 0x0000 }, + { 0x1E5C, 0x1E5D, 0x0000, 0x0000 }, + { 0x1F5D, 0x1F55, 0x0000, 0x0000 }, + { 0x2163, 0x2173, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_067[] = { + { 0x0043, 0x0063, 0x0000, 0x0000 }, + { 0x0241, 0x0294, 0x0000, 0x0000 }, + { 0x0546, 0x0576, 0x0000, 0x0000 }, + { 0x2162, 0x2172, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_068[] = { + { 0x0044, 0x0064, 0x0000, 0x0000 }, + { 0x0145, 0x0146, 0x0000, 0x0000 }, + { 0x0541, 0x0571, 0x0000, 0x0000 }, + { 0x1E5A, 0x1E5B, 0x0000, 0x0000 }, + { 0x1F5B, 0x1F53, 0x0000, 0x0000 }, + { 0x2165, 0x2175, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_069[] = { + { 0x0045, 0x0065, 0x0000, 0x0000 }, + { 0x0540, 0x0570, 0x0000, 0x0000 }, + { 0x2164, 0x2174, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_070[] = { + { 0x0046, 0x0066, 0x0000, 0x0000 }, + { 0x0147, 0x0148, 0x0000, 0x0000 }, + { 0x0345, 0x03B9, 0x0000, 0x0000 }, + { 0x0543, 0x0573, 0x0000, 0x0000 }, + { 0x1E58, 0x1E59, 0x0000, 0x0000 }, + { 0x1F59, 0x1F51, 0x0000, 0x0000 }, + { 0x2167, 0x2177, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_071[] = { + { 0x0047, 0x0067, 0x0000, 0x0000 }, + { 0x0542, 0x0572, 0x0000, 0x0000 }, + { 0x2166, 0x2176, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_072[] = { + { 0x0048, 0x0068, 0x0000, 0x0000 }, + { 0x0149, 0x02BC, 0x006E, 0x0000 }, + { 0x054D, 0x057D, 0x0000, 0x0000 }, + { 0x1E56, 0x1E57, 0x0000, 0x0000 }, + { 0x2169, 0x2179, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_073[] = { + { 0x0049, 0x0069, 0x0000, 0x0000 }, + { 0x054C, 0x057C, 0x0000, 0x0000 }, + { 0x1F56, 0x03C5, 0x0313, 0x0342 }, + { 0x2168, 0x2178, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_074[] = { + { 0x004A, 0x006A, 0x0000, 0x0000 }, + { 0x054F, 0x057F, 0x0000, 0x0000 }, + { 0x1E54, 0x1E55, 0x0000, 0x0000 }, + { 0x216B, 0x217B, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_075[] = { + { 0x004B, 0x006B, 0x0000, 0x0000 }, + { 0x014A, 0x014B, 0x0000, 0x0000 }, + { 0x054E, 0x057E, 0x0000, 0x0000 }, + { 0x1F54, 0x03C5, 0x0313, 0x0301 }, + { 0x216A, 0x217A, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_076[] = { + { 0x004C, 0x006C, 0x0000, 0x0000 }, + { 0x0549, 0x0579, 0x0000, 0x0000 }, + { 0x1E52, 0x1E53, 0x0000, 0x0000 }, + { 0x216D, 0x217D, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_077[] = { + { 0x004D, 0x006D, 0x0000, 0x0000 }, + { 0x014C, 0x014D, 0x0000, 0x0000 }, + { 0x0548, 0x0578, 0x0000, 0x0000 }, + { 0x1F52, 0x03C5, 0x0313, 0x0300 }, + { 0x216C, 0x217C, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_078[] = { + { 0x004E, 0x006E, 0x0000, 0x0000 }, + { 0x054B, 0x057B, 0x0000, 0x0000 }, + { 0x1E50, 0x1E51, 0x0000, 0x0000 }, + { 0x216F, 0x217F, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_079[] = { + { 0x004F, 0x006F, 0x0000, 0x0000 }, + { 0x014E, 0x014F, 0x0000, 0x0000 }, + { 0x054A, 0x057A, 0x0000, 0x0000 }, + { 0x1F50, 0x03C5, 0x0313, 0x0000 }, + { 0x216E, 0x217E, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_080[] = { + { 0x0050, 0x0070, 0x0000, 0x0000 }, + { 0x0555, 0x0585, 0x0000, 0x0000 }, + { 0x1E4E, 0x1E4F, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_081[] = { + { 0x0051, 0x0071, 0x0000, 0x0000 }, + { 0x0150, 0x0151, 0x0000, 0x0000 }, + { 0x0554, 0x0584, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_082[] = { + { 0x0052, 0x0072, 0x0000, 0x0000 }, + { 0x1E4C, 0x1E4D, 0x0000, 0x0000 }, + { 0x1F4D, 0x1F45, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_083[] = { + { 0x0053, 0x0073, 0x0000, 0x0000 }, + { 0x0152, 0x0153, 0x0000, 0x0000 }, + { 0x0556, 0x0586, 0x0000, 0x0000 }, + { 0x1F4C, 0x1F44, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_084[] = { + { 0x0054, 0x0074, 0x0000, 0x0000 }, + { 0x0551, 0x0581, 0x0000, 0x0000 }, + { 0x1E4A, 0x1E4B, 0x0000, 0x0000 }, + { 0x1F4B, 0x1F43, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_085[] = { + { 0x0055, 0x0075, 0x0000, 0x0000 }, + { 0x0154, 0x0155, 0x0000, 0x0000 }, + { 0x0550, 0x0580, 0x0000, 0x0000 }, + { 0x1F4A, 0x1F42, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_086[] = { + { 0x0056, 0x0076, 0x0000, 0x0000 }, + { 0x0553, 0x0583, 0x0000, 0x0000 }, + { 0x1E48, 0x1E49, 0x0000, 0x0000 }, + { 0x1F49, 0x1F41, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_087[] = { + { 0x0057, 0x0077, 0x0000, 0x0000 }, + { 0x0156, 0x0157, 0x0000, 0x0000 }, + { 0x0552, 0x0582, 0x0000, 0x0000 }, + { 0x1F48, 0x1F40, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_088[] = { + { 0x0058, 0x0078, 0x0000, 0x0000 }, + { 0x1E46, 0x1E47, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_089[] = { + { 0x0059, 0x0079, 0x0000, 0x0000 }, + { 0x0158, 0x0159, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_090[] = { + { 0x005A, 0x007A, 0x0000, 0x0000 }, + { 0x1E44, 0x1E45, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_091[] = { + { 0x015A, 0x015B, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_092[] = { + { 0x1E42, 0x1E43, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_093[] = { + { 0x015C, 0x015D, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_094[] = { + { 0x1E40, 0x1E41, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_095[] = { + { 0x015E, 0x015F, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_096[] = { + { 0x0464, 0x0465, 0x0000, 0x0000 }, + { 0x1E7E, 0x1E7F, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_097[] = { + { 0x0160, 0x0161, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_098[] = { + { 0x0466, 0x0467, 0x0000, 0x0000 }, + { 0x1E7C, 0x1E7D, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_099[] = { + { 0x0162, 0x0163, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_100[] = { + { 0x0460, 0x0461, 0x0000, 0x0000 }, + { 0x1E7A, 0x1E7B, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_101[] = { + { 0x0164, 0x0165, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_102[] = { + { 0x0462, 0x0463, 0x0000, 0x0000 }, + { 0x1E78, 0x1E79, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_103[] = { + { 0x0166, 0x0167, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_104[] = { + { 0x046C, 0x046D, 0x0000, 0x0000 }, + { 0x1E76, 0x1E77, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_105[] = { + { 0x0168, 0x0169, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_106[] = { + { 0x046E, 0x046F, 0x0000, 0x0000 }, + { 0x1E74, 0x1E75, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_107[] = { + { 0x016A, 0x016B, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_108[] = { + { 0x0468, 0x0469, 0x0000, 0x0000 }, + { 0x1E72, 0x1E73, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_109[] = { + { 0x016C, 0x016D, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_110[] = { + { 0x046A, 0x046B, 0x0000, 0x0000 }, + { 0x1E70, 0x1E71, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_111[] = { + { 0x016E, 0x016F, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_112[] = { + { 0x0474, 0x0475, 0x0000, 0x0000 }, + { 0x1E6E, 0x1E6F, 0x0000, 0x0000 }, + { 0x1F6F, 0x1F67, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_113[] = { + { 0x0170, 0x0171, 0x0000, 0x0000 }, + { 0x1F6E, 0x1F66, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_114[] = { + { 0x0476, 0x0477, 0x0000, 0x0000 }, + { 0x1E6C, 0x1E6D, 0x0000, 0x0000 }, + { 0x1F6D, 0x1F65, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_115[] = { + { 0x0172, 0x0173, 0x0000, 0x0000 }, + { 0x1F6C, 0x1F64, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_116[] = { + { 0x0470, 0x0471, 0x0000, 0x0000 }, + { 0x1E6A, 0x1E6B, 0x0000, 0x0000 }, + { 0x1F6B, 0x1F63, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_117[] = { + { 0x0174, 0x0175, 0x0000, 0x0000 }, + { 0x1F6A, 0x1F62, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_118[] = { + { 0x0472, 0x0473, 0x0000, 0x0000 }, + { 0x1E68, 0x1E69, 0x0000, 0x0000 }, + { 0x1F69, 0x1F61, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_119[] = { + { 0x0176, 0x0177, 0x0000, 0x0000 }, + { 0x1F68, 0x1F60, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_120[] = { + { 0x0179, 0x017A, 0x0000, 0x0000 }, + { 0x047C, 0x047D, 0x0000, 0x0000 }, + { 0x1E66, 0x1E67, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_121[] = { + { 0x0178, 0x00FF, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_122[] = { + { 0x017B, 0x017C, 0x0000, 0x0000 }, + { 0x047E, 0x047F, 0x0000, 0x0000 }, + { 0x1E64, 0x1E65, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_124[] = { + { 0x017D, 0x017E, 0x0000, 0x0000 }, + { 0x0478, 0x0479, 0x0000, 0x0000 }, + { 0x1E62, 0x1E63, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_126[] = { + { 0x017F, 0x0073, 0x0000, 0x0000 }, + { 0x047A, 0x047B, 0x0000, 0x0000 }, + { 0x1E60, 0x1E61, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_128[] = { + { 0x0181, 0x0253, 0x0000, 0x0000 }, + { 0x1F9F, 0x1F27, 0x03B9, 0x0000 }, + { 0x2CAC, 0x2CAD, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_129[] = { + { 0x1F9E, 0x1F26, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_130[] = { + { 0x0587, 0x0565, 0x0582, 0x0000 }, + { 0x1F9D, 0x1F25, 0x03B9, 0x0000 }, + { 0x2CAE, 0x2CAF, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_131[] = { + { 0x0182, 0x0183, 0x0000, 0x0000 }, + { 0x1F9C, 0x1F24, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_132[] = { + { 0x0480, 0x0481, 0x0000, 0x0000 }, + { 0x1E9A, 0x0061, 0x02BE, 0x0000 }, + { 0x1F9B, 0x1F23, 0x03B9, 0x0000 }, + { 0x2CA8, 0x2CA9, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_133[] = { + { 0x0184, 0x0185, 0x0000, 0x0000 }, + { 0x0386, 0x03AC, 0x0000, 0x0000 }, + { 0x1E9B, 0x1E61, 0x0000, 0x0000 }, + { 0x1F9A, 0x1F22, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_134[] = { + { 0x0187, 0x0188, 0x0000, 0x0000 }, + { 0x1E98, 0x0077, 0x030A, 0x0000 }, + { 0x1F99, 0x1F21, 0x03B9, 0x0000 }, + { 0x2CAA, 0x2CAB, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_135[] = { + { 0x0186, 0x0254, 0x0000, 0x0000 }, + { 0x1E99, 0x0079, 0x030A, 0x0000 }, + { 0x1F98, 0x1F20, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_136[] = { + { 0x0189, 0x0256, 0x0000, 0x0000 }, + { 0x048C, 0x048D, 0x0000, 0x0000 }, + { 0x1E96, 0x0068, 0x0331, 0x0000 }, + { 0x1F97, 0x1F27, 0x03B9, 0x0000 }, + { 0x2CA4, 0x2CA5, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_137[] = { + { 0x038A, 0x03AF, 0x0000, 0x0000 }, + { 0x1E97, 0x0074, 0x0308, 0x0000 }, + { 0x1F96, 0x1F26, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_138[] = { + { 0x018B, 0x018C, 0x0000, 0x0000 }, + { 0x0389, 0x03AE, 0x0000, 0x0000 }, + { 0x048E, 0x048F, 0x0000, 0x0000 }, + { 0x1E94, 0x1E95, 0x0000, 0x0000 }, + { 0x1F95, 0x1F25, 0x03B9, 0x0000 }, + { 0x2CA6, 0x2CA7, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_139[] = { + { 0x018A, 0x0257, 0x0000, 0x0000 }, + { 0x0388, 0x03AD, 0x0000, 0x0000 }, + { 0x1F94, 0x1F24, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_140[] = { + { 0x038F, 0x03CE, 0x0000, 0x0000 }, + { 0x1E92, 0x1E93, 0x0000, 0x0000 }, + { 0x1F93, 0x1F23, 0x03B9, 0x0000 }, + { 0x2CA0, 0x2CA1, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_141[] = { + { 0x038E, 0x03CD, 0x0000, 0x0000 }, + { 0x1F92, 0x1F22, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_142[] = { + { 0x018F, 0x0259, 0x0000, 0x0000 }, + { 0x048A, 0x048B, 0x0000, 0x0000 }, + { 0x1E90, 0x1E91, 0x0000, 0x0000 }, + { 0x1F91, 0x1F21, 0x03B9, 0x0000 }, + { 0x2CA2, 0x2CA3, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_143[] = { + { 0x018E, 0x01DD, 0x0000, 0x0000 }, + { 0x038C, 0x03CC, 0x0000, 0x0000 }, + { 0x1F90, 0x1F20, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_144[] = { + { 0x0191, 0x0192, 0x0000, 0x0000 }, + { 0x0393, 0x03B3, 0x0000, 0x0000 }, + { 0x0494, 0x0495, 0x0000, 0x0000 }, + { 0x1E8E, 0x1E8F, 0x0000, 0x0000 }, + { 0x1F8F, 0x1F07, 0x03B9, 0x0000 }, + { 0x2CBC, 0x2CBD, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_145[] = { + { 0x0190, 0x025B, 0x0000, 0x0000 }, + { 0x0392, 0x03B2, 0x0000, 0x0000 }, + { 0x1F8E, 0x1F06, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_146[] = { + { 0x0193, 0x0260, 0x0000, 0x0000 }, + { 0x0391, 0x03B1, 0x0000, 0x0000 }, + { 0x0496, 0x0497, 0x0000, 0x0000 }, + { 0x1E8C, 0x1E8D, 0x0000, 0x0000 }, + { 0x1F8D, 0x1F05, 0x03B9, 0x0000 }, + { 0x24B6, 0x24D0, 0x0000, 0x0000 }, + { 0x2CBE, 0x2CBF, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_147[] = { + { 0x0390, 0x03B9, 0x0308, 0x0301 }, + { 0x1F8C, 0x1F04, 0x03B9, 0x0000 }, + { 0x24B7, 0x24D1, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_148[] = { + { 0x0397, 0x03B7, 0x0000, 0x0000 }, + { 0x0490, 0x0491, 0x0000, 0x0000 }, + { 0x1E8A, 0x1E8B, 0x0000, 0x0000 }, + { 0x1F8B, 0x1F03, 0x03B9, 0x0000 }, + { 0x2CB8, 0x2CB9, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_149[] = { + { 0x0194, 0x0263, 0x0000, 0x0000 }, + { 0x0396, 0x03B6, 0x0000, 0x0000 }, + { 0x1F8A, 0x1F02, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_150[] = { + { 0x0197, 0x0268, 0x0000, 0x0000 }, + { 0x0395, 0x03B5, 0x0000, 0x0000 }, + { 0x0492, 0x0493, 0x0000, 0x0000 }, + { 0x1E88, 0x1E89, 0x0000, 0x0000 }, + { 0x1F89, 0x1F01, 0x03B9, 0x0000 }, + { 0x2CBA, 0x2CBB, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_151[] = { + { 0x0196, 0x0269, 0x0000, 0x0000 }, + { 0x0394, 0x03B4, 0x0000, 0x0000 }, + { 0x1F88, 0x1F00, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_152[] = { + { 0x039B, 0x03BB, 0x0000, 0x0000 }, + { 0x049C, 0x049D, 0x0000, 0x0000 }, + { 0x1E86, 0x1E87, 0x0000, 0x0000 }, + { 0x1F87, 0x1F07, 0x03B9, 0x0000 }, + { 0x24BC, 0x24D6, 0x0000, 0x0000 }, + { 0x2CB4, 0x2CB5, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_153[] = { + { 0x0198, 0x0199, 0x0000, 0x0000 }, + { 0x039A, 0x03BA, 0x0000, 0x0000 }, + { 0x1F86, 0x1F06, 0x03B9, 0x0000 }, + { 0x24BD, 0x24D7, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_154[] = { + { 0x0399, 0x03B9, 0x0000, 0x0000 }, + { 0x049E, 0x049F, 0x0000, 0x0000 }, + { 0x1E84, 0x1E85, 0x0000, 0x0000 }, + { 0x1F85, 0x1F05, 0x03B9, 0x0000 }, + { 0x24BE, 0x24D8, 0x0000, 0x0000 }, + { 0x2CB6, 0x2CB7, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_155[] = { + { 0x0398, 0x03B8, 0x0000, 0x0000 }, + { 0x1F84, 0x1F04, 0x03B9, 0x0000 }, + { 0x24BF, 0x24D9, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_156[] = { + { 0x019D, 0x0272, 0x0000, 0x0000 }, + { 0x039F, 0x03BF, 0x0000, 0x0000 }, + { 0x0498, 0x0499, 0x0000, 0x0000 }, + { 0x1E82, 0x1E83, 0x0000, 0x0000 }, + { 0x1F83, 0x1F03, 0x03B9, 0x0000 }, + { 0x24B8, 0x24D2, 0x0000, 0x0000 }, + { 0x2CB0, 0x2CB1, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_157[] = { + { 0x019C, 0x026F, 0x0000, 0x0000 }, + { 0x039E, 0x03BE, 0x0000, 0x0000 }, + { 0x1F82, 0x1F02, 0x03B9, 0x0000 }, + { 0x24B9, 0x24D3, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_158[] = { + { 0x019F, 0x0275, 0x0000, 0x0000 }, + { 0x039D, 0x03BD, 0x0000, 0x0000 }, + { 0x049A, 0x049B, 0x0000, 0x0000 }, + { 0x1E80, 0x1E81, 0x0000, 0x0000 }, + { 0x1F81, 0x1F01, 0x03B9, 0x0000 }, + { 0x24BA, 0x24D4, 0x0000, 0x0000 }, + { 0x2CB2, 0x2CB3, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_159[] = { + { 0x039C, 0x03BC, 0x0000, 0x0000 }, + { 0x1F80, 0x1F00, 0x03B9, 0x0000 }, + { 0x24BB, 0x24D5, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_160[] = { + { 0x03A3, 0x03C3, 0x0000, 0x0000 }, + { 0x04A4, 0x04A5, 0x0000, 0x0000 }, + { 0x10B0, 0x2D10, 0x0000, 0x0000 }, + { 0x1EBE, 0x1EBF, 0x0000, 0x0000 }, + { 0x2C8C, 0x2C8D, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_161[] = { + { 0x01A0, 0x01A1, 0x0000, 0x0000 }, + { 0x10B1, 0x2D11, 0x0000, 0x0000 }, + { 0x1FBE, 0x03B9, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_162[] = { + { 0x03A1, 0x03C1, 0x0000, 0x0000 }, + { 0x04A6, 0x04A7, 0x0000, 0x0000 }, + { 0x10B2, 0x2D12, 0x0000, 0x0000 }, + { 0x1EBC, 0x1EBD, 0x0000, 0x0000 }, + { 0x2C8E, 0x2C8F, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_163[] = { + { 0x01A2, 0x01A3, 0x0000, 0x0000 }, + { 0x03A0, 0x03C0, 0x0000, 0x0000 }, + { 0x10B3, 0x2D13, 0x0000, 0x0000 }, + { 0x1FBC, 0x03B1, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_164[] = { + { 0x03A7, 0x03C7, 0x0000, 0x0000 }, + { 0x04A0, 0x04A1, 0x0000, 0x0000 }, + { 0x10B4, 0x2D14, 0x0000, 0x0000 }, + { 0x1EBA, 0x1EBB, 0x0000, 0x0000 }, + { 0x1FBB, 0x1F71, 0x0000, 0x0000 }, + { 0x2C88, 0x2C89, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_165[] = { + { 0x01A4, 0x01A5, 0x0000, 0x0000 }, + { 0x03A6, 0x03C6, 0x0000, 0x0000 }, + { 0x10B5, 0x2D15, 0x0000, 0x0000 }, + { 0x1FBA, 0x1F70, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_166[] = { + { 0x01A7, 0x01A8, 0x0000, 0x0000 }, + { 0x03A5, 0x03C5, 0x0000, 0x0000 }, + { 0x04A2, 0x04A3, 0x0000, 0x0000 }, + { 0x10B6, 0x2D16, 0x0000, 0x0000 }, + { 0x1EB8, 0x1EB9, 0x0000, 0x0000 }, + { 0x1FB9, 0x1FB1, 0x0000, 0x0000 }, + { 0x2C8A, 0x2C8B, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_167[] = { + { 0x01A6, 0x0280, 0x0000, 0x0000 }, + { 0x03A4, 0x03C4, 0x0000, 0x0000 }, + { 0x10B7, 0x2D17, 0x0000, 0x0000 }, + { 0x1FB8, 0x1FB0, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_168[] = { + { 0x01A9, 0x0283, 0x0000, 0x0000 }, + { 0x03AB, 0x03CB, 0x0000, 0x0000 }, + { 0x04AC, 0x04AD, 0x0000, 0x0000 }, + { 0x10B8, 0x2D18, 0x0000, 0x0000 }, + { 0x1EB6, 0x1EB7, 0x0000, 0x0000 }, + { 0x1FB7, 0x03B1, 0x0342, 0x03B9 }, + { 0x2C84, 0x2C85, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_169[] = { + { 0x03AA, 0x03CA, 0x0000, 0x0000 }, + { 0x10B9, 0x2D19, 0x0000, 0x0000 }, + { 0x1FB6, 0x03B1, 0x0342, 0x0000 } +}; + +static const CaseFoldMapping case_fold_170[] = { + { 0x03A9, 0x03C9, 0x0000, 0x0000 }, + { 0x04AE, 0x04AF, 0x0000, 0x0000 }, + { 0x10BA, 0x2D1A, 0x0000, 0x0000 }, + { 0x1EB4, 0x1EB5, 0x0000, 0x0000 }, + { 0x2C86, 0x2C87, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_171[] = { + { 0x03A8, 0x03C8, 0x0000, 0x0000 }, + { 0x10BB, 0x2D1B, 0x0000, 0x0000 }, + { 0x1FB4, 0x03AC, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_172[] = { + { 0x04A8, 0x04A9, 0x0000, 0x0000 }, + { 0x10BC, 0x2D1C, 0x0000, 0x0000 }, + { 0x1EB2, 0x1EB3, 0x0000, 0x0000 }, + { 0x1FB3, 0x03B1, 0x03B9, 0x0000 }, + { 0x2C80, 0x2C81, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_173[] = { + { 0x01AC, 0x01AD, 0x0000, 0x0000 }, + { 0x10BD, 0x2D1D, 0x0000, 0x0000 }, + { 0x1FB2, 0x1F70, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_174[] = { + { 0x01AF, 0x01B0, 0x0000, 0x0000 }, + { 0x04AA, 0x04AB, 0x0000, 0x0000 }, + { 0x10BE, 0x2D1E, 0x0000, 0x0000 }, + { 0x1EB0, 0x1EB1, 0x0000, 0x0000 }, + { 0x2C82, 0x2C83, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_175[] = { + { 0x01AE, 0x0288, 0x0000, 0x0000 }, + { 0x10BF, 0x2D1F, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_176[] = { + { 0x01B1, 0x028A, 0x0000, 0x0000 }, + { 0x04B4, 0x04B5, 0x0000, 0x0000 }, + { 0x10A0, 0x2D00, 0x0000, 0x0000 }, + { 0x1EAE, 0x1EAF, 0x0000, 0x0000 }, + { 0x1FAF, 0x1F67, 0x03B9, 0x0000 }, + { 0x2C9C, 0x2C9D, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_177[] = { + { 0x10A1, 0x2D01, 0x0000, 0x0000 }, + { 0x1FAE, 0x1F66, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_178[] = { + { 0x01B3, 0x01B4, 0x0000, 0x0000 }, + { 0x04B6, 0x04B7, 0x0000, 0x0000 }, + { 0x10A2, 0x2D02, 0x0000, 0x0000 }, + { 0x1EAC, 0x1EAD, 0x0000, 0x0000 }, + { 0x1FAD, 0x1F65, 0x03B9, 0x0000 }, + { 0x2C9E, 0x2C9F, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_179[] = { + { 0x01B2, 0x028B, 0x0000, 0x0000 }, + { 0x03B0, 0x03C5, 0x0308, 0x0301 }, + { 0x10A3, 0x2D03, 0x0000, 0x0000 }, + { 0x1FAC, 0x1F64, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_180[] = { + { 0x01B5, 0x01B6, 0x0000, 0x0000 }, + { 0x04B0, 0x04B1, 0x0000, 0x0000 }, + { 0x10A4, 0x2D04, 0x0000, 0x0000 }, + { 0x1EAA, 0x1EAB, 0x0000, 0x0000 }, + { 0x1FAB, 0x1F63, 0x03B9, 0x0000 }, + { 0x2C98, 0x2C99, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_181[] = { + { 0x00B5, 0x03BC, 0x0000, 0x0000 }, + { 0x10A5, 0x2D05, 0x0000, 0x0000 }, + { 0x1FAA, 0x1F62, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_182[] = { + { 0x01B7, 0x0292, 0x0000, 0x0000 }, + { 0x04B2, 0x04B3, 0x0000, 0x0000 }, + { 0x10A6, 0x2D06, 0x0000, 0x0000 }, + { 0x1EA8, 0x1EA9, 0x0000, 0x0000 }, + { 0x1FA9, 0x1F61, 0x03B9, 0x0000 }, + { 0x2C9A, 0x2C9B, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_183[] = { + { 0x10A7, 0x2D07, 0x0000, 0x0000 }, + { 0x1FA8, 0x1F60, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_184[] = { + { 0x04BC, 0x04BD, 0x0000, 0x0000 }, + { 0x10A8, 0x2D08, 0x0000, 0x0000 }, + { 0x1EA6, 0x1EA7, 0x0000, 0x0000 }, + { 0x1FA7, 0x1F67, 0x03B9, 0x0000 }, + { 0x2C94, 0x2C95, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_185[] = { + { 0x01B8, 0x01B9, 0x0000, 0x0000 }, + { 0x10A9, 0x2D09, 0x0000, 0x0000 }, + { 0x1FA6, 0x1F66, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_186[] = { + { 0x04BE, 0x04BF, 0x0000, 0x0000 }, + { 0x10AA, 0x2D0A, 0x0000, 0x0000 }, + { 0x1EA4, 0x1EA5, 0x0000, 0x0000 }, + { 0x1FA5, 0x1F65, 0x03B9, 0x0000 }, + { 0x2C96, 0x2C97, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_187[] = { + { 0x10AB, 0x2D0B, 0x0000, 0x0000 }, + { 0x1FA4, 0x1F64, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_188[] = { + { 0x04B8, 0x04B9, 0x0000, 0x0000 }, + { 0x10AC, 0x2D0C, 0x0000, 0x0000 }, + { 0x1EA2, 0x1EA3, 0x0000, 0x0000 }, + { 0x1FA3, 0x1F63, 0x03B9, 0x0000 }, + { 0x2C90, 0x2C91, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_189[] = { + { 0x01BC, 0x01BD, 0x0000, 0x0000 }, + { 0x10AD, 0x2D0D, 0x0000, 0x0000 }, + { 0x1FA2, 0x1F62, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_190[] = { + { 0x04BA, 0x04BB, 0x0000, 0x0000 }, + { 0x10AE, 0x2D0E, 0x0000, 0x0000 }, + { 0x1EA0, 0x1EA1, 0x0000, 0x0000 }, + { 0x1FA1, 0x1F61, 0x03B9, 0x0000 }, + { 0x2C92, 0x2C93, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_191[] = { + { 0x10AF, 0x2D0F, 0x0000, 0x0000 }, + { 0x1FA0, 0x1F60, 0x03B9, 0x0000 } +}; + +static const CaseFoldMapping case_fold_192[] = { + { 0x00C0, 0x00E0, 0x0000, 0x0000 }, + { 0x1EDE, 0x1EDF, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_193[] = { + { 0x00C1, 0x00E1, 0x0000, 0x0000 }, + { 0x03C2, 0x03C3, 0x0000, 0x0000 }, + { 0x04C5, 0x04C6, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_194[] = { + { 0x00C2, 0x00E2, 0x0000, 0x0000 }, + { 0x1EDC, 0x1EDD, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_195[] = { + { 0x00C3, 0x00E3, 0x0000, 0x0000 }, + { 0x04C7, 0x04C8, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_196[] = { + { 0x00C4, 0x00E4, 0x0000, 0x0000 }, + { 0x01C5, 0x01C6, 0x0000, 0x0000 }, + { 0x1EDA, 0x1EDB, 0x0000, 0x0000 }, + { 0x1FDB, 0x1F77, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_197[] = { + { 0x00C5, 0x00E5, 0x0000, 0x0000 }, + { 0x01C4, 0x01C6, 0x0000, 0x0000 }, + { 0x04C1, 0x04C2, 0x0000, 0x0000 }, + { 0x1FDA, 0x1F76, 0x0000, 0x0000 }, + { 0xFF3A, 0xFF5A, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_198[] = { + { 0x00C6, 0x00E6, 0x0000, 0x0000 }, + { 0x01C7, 0x01C9, 0x0000, 0x0000 }, + { 0x1ED8, 0x1ED9, 0x0000, 0x0000 }, + { 0x1FD9, 0x1FD1, 0x0000, 0x0000 }, + { 0xFF39, 0xFF59, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_199[] = { + { 0x00C7, 0x00E7, 0x0000, 0x0000 }, + { 0x04C3, 0x04C4, 0x0000, 0x0000 }, + { 0x1FD8, 0x1FD0, 0x0000, 0x0000 }, + { 0xFF38, 0xFF58, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_200[] = { + { 0x00C8, 0x00E8, 0x0000, 0x0000 }, + { 0x1ED6, 0x1ED7, 0x0000, 0x0000 }, + { 0x1FD7, 0x03B9, 0x0308, 0x0342 }, + { 0xFF37, 0xFF57, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_201[] = { + { 0x00C9, 0x00E9, 0x0000, 0x0000 }, + { 0x01C8, 0x01C9, 0x0000, 0x0000 }, + { 0x04CD, 0x04CE, 0x0000, 0x0000 }, + { 0x1FD6, 0x03B9, 0x0342, 0x0000 }, + { 0xFF36, 0xFF56, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_202[] = { + { 0x00CA, 0x00EA, 0x0000, 0x0000 }, + { 0x01CB, 0x01CC, 0x0000, 0x0000 }, + { 0x1ED4, 0x1ED5, 0x0000, 0x0000 }, + { 0xFF35, 0xFF55, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_203[] = { + { 0x00CB, 0x00EB, 0x0000, 0x0000 }, + { 0x01CA, 0x01CC, 0x0000, 0x0000 }, + { 0xFF34, 0xFF54, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_204[] = { + { 0x00CC, 0x00EC, 0x0000, 0x0000 }, + { 0x01CD, 0x01CE, 0x0000, 0x0000 }, + { 0x1ED2, 0x1ED3, 0x0000, 0x0000 }, + { 0x1FD3, 0x03B9, 0x0308, 0x0301 }, + { 0x2CE0, 0x2CE1, 0x0000, 0x0000 }, + { 0xFF33, 0xFF53, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_205[] = { + { 0x00CD, 0x00ED, 0x0000, 0x0000 }, + { 0x04C9, 0x04CA, 0x0000, 0x0000 }, + { 0x1FD2, 0x03B9, 0x0308, 0x0300 }, + { 0xFF32, 0xFF52, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_206[] = { + { 0x00CE, 0x00EE, 0x0000, 0x0000 }, + { 0x01CF, 0x01D0, 0x0000, 0x0000 }, + { 0x1ED0, 0x1ED1, 0x0000, 0x0000 }, + { 0x2CE2, 0x2CE3, 0x0000, 0x0000 }, + { 0xFF31, 0xFF51, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_207[] = { + { 0x00CF, 0x00EF, 0x0000, 0x0000 }, + { 0x04CB, 0x04CC, 0x0000, 0x0000 }, + { 0xFF30, 0xFF50, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_208[] = { + { 0x00D0, 0x00F0, 0x0000, 0x0000 }, + { 0x01D1, 0x01D2, 0x0000, 0x0000 }, + { 0x04D4, 0x04D5, 0x0000, 0x0000 }, + { 0x10C0, 0x2D20, 0x0000, 0x0000 }, + { 0x1ECE, 0x1ECF, 0x0000, 0x0000 }, + { 0xFF2F, 0xFF4F, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_209[] = { + { 0x00D1, 0x00F1, 0x0000, 0x0000 }, + { 0x10C1, 0x2D21, 0x0000, 0x0000 }, + { 0xFF2E, 0xFF4E, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_210[] = { + { 0x00D2, 0x00F2, 0x0000, 0x0000 }, + { 0x01D3, 0x01D4, 0x0000, 0x0000 }, + { 0x03D1, 0x03B8, 0x0000, 0x0000 }, + { 0x04D6, 0x04D7, 0x0000, 0x0000 }, + { 0x10C2, 0x2D22, 0x0000, 0x0000 }, + { 0x1ECC, 0x1ECD, 0x0000, 0x0000 }, + { 0xFF2D, 0xFF4D, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_211[] = { + { 0x00D3, 0x00F3, 0x0000, 0x0000 }, + { 0x03D0, 0x03B2, 0x0000, 0x0000 }, + { 0x10C3, 0x2D23, 0x0000, 0x0000 }, + { 0x1FCC, 0x03B7, 0x03B9, 0x0000 }, + { 0xFF2C, 0xFF4C, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_212[] = { + { 0x00D4, 0x00F4, 0x0000, 0x0000 }, + { 0x01D5, 0x01D6, 0x0000, 0x0000 }, + { 0x04D0, 0x04D1, 0x0000, 0x0000 }, + { 0x10C4, 0x2D24, 0x0000, 0x0000 }, + { 0x1ECA, 0x1ECB, 0x0000, 0x0000 }, + { 0x1FCB, 0x1F75, 0x0000, 0x0000 }, + { 0xFF2B, 0xFF4B, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_213[] = { + { 0x00D5, 0x00F5, 0x0000, 0x0000 }, + { 0x03D6, 0x03C0, 0x0000, 0x0000 }, + { 0x10C5, 0x2D25, 0x0000, 0x0000 }, + { 0x1FCA, 0x1F74, 0x0000, 0x0000 }, + { 0xFF2A, 0xFF4A, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_214[] = { + { 0x00D6, 0x00F6, 0x0000, 0x0000 }, + { 0x01D7, 0x01D8, 0x0000, 0x0000 }, + { 0x03D5, 0x03C6, 0x0000, 0x0000 }, + { 0x04D2, 0x04D3, 0x0000, 0x0000 }, + { 0x1EC8, 0x1EC9, 0x0000, 0x0000 }, + { 0x1FC9, 0x1F73, 0x0000, 0x0000 }, + { 0xFF29, 0xFF49, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_215[] = { + { 0x1FC8, 0x1F72, 0x0000, 0x0000 }, + { 0xFF28, 0xFF48, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_216[] = { + { 0x00D8, 0x00F8, 0x0000, 0x0000 }, + { 0x01D9, 0x01DA, 0x0000, 0x0000 }, + { 0x04DC, 0x04DD, 0x0000, 0x0000 }, + { 0x1EC6, 0x1EC7, 0x0000, 0x0000 }, + { 0x1FC7, 0x03B7, 0x0342, 0x03B9 }, + { 0xFF27, 0xFF47, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_217[] = { + { 0x00D9, 0x00F9, 0x0000, 0x0000 }, + { 0x03DA, 0x03DB, 0x0000, 0x0000 }, + { 0x1FC6, 0x03B7, 0x0342, 0x0000 }, + { 0xFF26, 0xFF46, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_218[] = { + { 0x00DA, 0x00FA, 0x0000, 0x0000 }, + { 0x01DB, 0x01DC, 0x0000, 0x0000 }, + { 0x04DE, 0x04DF, 0x0000, 0x0000 }, + { 0x1EC4, 0x1EC5, 0x0000, 0x0000 }, + { 0xFF25, 0xFF45, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_219[] = { + { 0x00DB, 0x00FB, 0x0000, 0x0000 }, + { 0x03D8, 0x03D9, 0x0000, 0x0000 }, + { 0x1FC4, 0x03AE, 0x03B9, 0x0000 }, + { 0xFF24, 0xFF44, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_220[] = { + { 0x00DC, 0x00FC, 0x0000, 0x0000 }, + { 0x04D8, 0x04D9, 0x0000, 0x0000 }, + { 0x1EC2, 0x1EC3, 0x0000, 0x0000 }, + { 0x1FC3, 0x03B7, 0x03B9, 0x0000 }, + { 0xFF23, 0xFF43, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_221[] = { + { 0x00DD, 0x00FD, 0x0000, 0x0000 }, + { 0x03DE, 0x03DF, 0x0000, 0x0000 }, + { 0x1FC2, 0x1F74, 0x03B9, 0x0000 }, + { 0xFF22, 0xFF42, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_222[] = { + { 0x00DE, 0x00FE, 0x0000, 0x0000 }, + { 0x04DA, 0x04DB, 0x0000, 0x0000 }, + { 0x1EC0, 0x1EC1, 0x0000, 0x0000 }, + { 0xFF21, 0xFF41, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_223[] = { + { 0x00DF, 0x0073, 0x0073, 0x0000 }, + { 0x01DE, 0x01DF, 0x0000, 0x0000 }, + { 0x03DC, 0x03DD, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_224[] = { + { 0x04E4, 0x04E5, 0x0000, 0x0000 }, + { 0x24C4, 0x24DE, 0x0000, 0x0000 }, + { 0x2CCC, 0x2CCD, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_225[] = { + { 0x01E0, 0x01E1, 0x0000, 0x0000 }, + { 0x03E2, 0x03E3, 0x0000, 0x0000 }, + { 0x24C5, 0x24DF, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_226[] = { + { 0x04E6, 0x04E7, 0x0000, 0x0000 }, + { 0x24C6, 0x24E0, 0x0000, 0x0000 }, + { 0x2CCE, 0x2CCF, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_227[] = { + { 0x01E2, 0x01E3, 0x0000, 0x0000 }, + { 0x03E0, 0x03E1, 0x0000, 0x0000 }, + { 0x1FFC, 0x03C9, 0x03B9, 0x0000 }, + { 0x24C7, 0x24E1, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_228[] = { + { 0x04E0, 0x04E1, 0x0000, 0x0000 }, + { 0x1FFB, 0x1F7D, 0x0000, 0x0000 }, + { 0x24C0, 0x24DA, 0x0000, 0x0000 }, + { 0x2CC8, 0x2CC9, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_229[] = { + { 0x01E4, 0x01E5, 0x0000, 0x0000 }, + { 0x03E6, 0x03E7, 0x0000, 0x0000 }, + { 0x1FFA, 0x1F7C, 0x0000, 0x0000 }, + { 0x24C1, 0x24DB, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_230[] = { + { 0x04E2, 0x04E3, 0x0000, 0x0000 }, + { 0x1EF8, 0x1EF9, 0x0000, 0x0000 }, + { 0x1FF9, 0x1F79, 0x0000, 0x0000 }, + { 0x24C2, 0x24DC, 0x0000, 0x0000 }, + { 0x2CCA, 0x2CCB, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_231[] = { + { 0x01E6, 0x01E7, 0x0000, 0x0000 }, + { 0x03E4, 0x03E5, 0x0000, 0x0000 }, + { 0x1FF8, 0x1F78, 0x0000, 0x0000 }, + { 0x24C3, 0x24DD, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_232[] = { + { 0x04EC, 0x04ED, 0x0000, 0x0000 }, + { 0x1EF6, 0x1EF7, 0x0000, 0x0000 }, + { 0x1FF7, 0x03C9, 0x0342, 0x03B9 }, + { 0x24CC, 0x24E6, 0x0000, 0x0000 }, + { 0x2CC4, 0x2CC5, 0x0000, 0x0000 }, + { 0xFB13, 0x0574, 0x0576, 0x0000 } +}; + +static const CaseFoldMapping case_fold_233[] = { + { 0x01E8, 0x01E9, 0x0000, 0x0000 }, + { 0x03EA, 0x03EB, 0x0000, 0x0000 }, + { 0x1FF6, 0x03C9, 0x0342, 0x0000 }, + { 0x24CD, 0x24E7, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_234[] = { + { 0x04EE, 0x04EF, 0x0000, 0x0000 }, + { 0x1EF4, 0x1EF5, 0x0000, 0x0000 }, + { 0x24CE, 0x24E8, 0x0000, 0x0000 }, + { 0x2CC6, 0x2CC7, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_235[] = { + { 0x01EA, 0x01EB, 0x0000, 0x0000 }, + { 0x03E8, 0x03E9, 0x0000, 0x0000 }, + { 0x1FF4, 0x03CE, 0x03B9, 0x0000 }, + { 0x24CF, 0x24E9, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_236[] = { + { 0x04E8, 0x04E9, 0x0000, 0x0000 }, + { 0x1EF2, 0x1EF3, 0x0000, 0x0000 }, + { 0x1FF3, 0x03C9, 0x03B9, 0x0000 }, + { 0x24C8, 0x24E2, 0x0000, 0x0000 }, + { 0x2CC0, 0x2CC1, 0x0000, 0x0000 }, + { 0xFB17, 0x0574, 0x056D, 0x0000 } +}; + +static const CaseFoldMapping case_fold_237[] = { + { 0x01EC, 0x01ED, 0x0000, 0x0000 }, + { 0x03EE, 0x03EF, 0x0000, 0x0000 }, + { 0x1FF2, 0x1F7C, 0x03B9, 0x0000 }, + { 0x24C9, 0x24E3, 0x0000, 0x0000 }, + { 0xFB16, 0x057E, 0x0576, 0x0000 } +}; + +static const CaseFoldMapping case_fold_238[] = { + { 0x04EA, 0x04EB, 0x0000, 0x0000 }, + { 0x1EF0, 0x1EF1, 0x0000, 0x0000 }, + { 0x24CA, 0x24E4, 0x0000, 0x0000 }, + { 0x2CC2, 0x2CC3, 0x0000, 0x0000 }, + { 0xFB15, 0x0574, 0x056B, 0x0000 } +}; + +static const CaseFoldMapping case_fold_239[] = { + { 0x01EE, 0x01EF, 0x0000, 0x0000 }, + { 0x03EC, 0x03ED, 0x0000, 0x0000 }, + { 0x24CB, 0x24E5, 0x0000, 0x0000 }, + { 0xFB14, 0x0574, 0x0565, 0x0000 } +}; + +static const CaseFoldMapping case_fold_240[] = { + { 0x01F1, 0x01F3, 0x0000, 0x0000 }, + { 0x04F4, 0x04F5, 0x0000, 0x0000 }, + { 0x1EEE, 0x1EEF, 0x0000, 0x0000 }, + { 0x2CDC, 0x2CDD, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_241[] = { + { 0x01F0, 0x006A, 0x030C, 0x0000 } +}; + +static const CaseFoldMapping case_fold_242[] = { + { 0x03F1, 0x03C1, 0x0000, 0x0000 }, + { 0x04F6, 0x04F7, 0x0000, 0x0000 }, + { 0x1EEC, 0x1EED, 0x0000, 0x0000 }, + { 0x2CDE, 0x2CDF, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_243[] = { + { 0x01F2, 0x01F3, 0x0000, 0x0000 }, + { 0x03F0, 0x03BA, 0x0000, 0x0000 }, + { 0x1FEC, 0x1FE5, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_244[] = { + { 0x03F7, 0x03F8, 0x0000, 0x0000 }, + { 0x04F0, 0x04F1, 0x0000, 0x0000 }, + { 0x1EEA, 0x1EEB, 0x0000, 0x0000 }, + { 0x1FEB, 0x1F7B, 0x0000, 0x0000 }, + { 0x2CD8, 0x2CD9, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_245[] = { + { 0x01F4, 0x01F5, 0x0000, 0x0000 }, + { 0x1FEA, 0x1F7A, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_246[] = { + { 0x01F7, 0x01BF, 0x0000, 0x0000 }, + { 0x03F5, 0x03B5, 0x0000, 0x0000 }, + { 0x04F2, 0x04F3, 0x0000, 0x0000 }, + { 0x1EE8, 0x1EE9, 0x0000, 0x0000 }, + { 0x1FE9, 0x1FE1, 0x0000, 0x0000 }, + { 0x2CDA, 0x2CDB, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_247[] = { + { 0x01F6, 0x0195, 0x0000, 0x0000 }, + { 0x03F4, 0x03B8, 0x0000, 0x0000 }, + { 0x1FE8, 0x1FE0, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_248[] = { + { 0x1EE6, 0x1EE7, 0x0000, 0x0000 }, + { 0x1FE7, 0x03C5, 0x0308, 0x0342 }, + { 0x2CD4, 0x2CD5, 0x0000, 0x0000 }, + { 0xFB03, 0x0066, 0x0066, 0x0069 } +}; + +static const CaseFoldMapping case_fold_249[] = { + { 0x01F8, 0x01F9, 0x0000, 0x0000 }, + { 0x03FA, 0x03FB, 0x0000, 0x0000 }, + { 0x1FE6, 0x03C5, 0x0342, 0x0000 }, + { 0xFB02, 0x0066, 0x006C, 0x0000 } +}; + +static const CaseFoldMapping case_fold_250[] = { + { 0x03F9, 0x03F2, 0x0000, 0x0000 }, + { 0x1EE4, 0x1EE5, 0x0000, 0x0000 }, + { 0x2CD6, 0x2CD7, 0x0000, 0x0000 }, + { 0xFB01, 0x0066, 0x0069, 0x0000 } +}; + +static const CaseFoldMapping case_fold_251[] = { + { 0x01FA, 0x01FB, 0x0000, 0x0000 }, + { 0x1FE4, 0x03C1, 0x0313, 0x0000 }, + { 0xFB00, 0x0066, 0x0066, 0x0000 } +}; + +static const CaseFoldMapping case_fold_252[] = { + { 0x04F8, 0x04F9, 0x0000, 0x0000 }, + { 0x1EE2, 0x1EE3, 0x0000, 0x0000 }, + { 0x1FE3, 0x03C5, 0x0308, 0x0301 }, + { 0x2CD0, 0x2CD1, 0x0000, 0x0000 } +}; + +static const CaseFoldMapping case_fold_253[] = { + { 0x01FC, 0x01FD, 0x0000, 0x0000 }, + { 0x1FE2, 0x03C5, 0x0308, 0x0300 }, + { 0xFB06, 0x0073, 0x0074, 0x0000 } +}; + +static const CaseFoldMapping case_fold_254[] = { + { 0x1EE0, 0x1EE1, 0x0000, 0x0000 }, + { 0x2CD2, 0x2CD3, 0x0000, 0x0000 }, + { 0xFB05, 0x0073, 0x0074, 0x0000 } +}; + +static const CaseFoldMapping case_fold_255[] = { + { 0x01FE, 0x01FF, 0x0000, 0x0000 }, + { 0xFB04, 0x0066, 0x0066, 0x006C } +}; + + +static const CaseFoldHashBucket case_fold_hash[256] = { + { CASEFOLDING_ARRAYLEN(case_fold_000), case_fold_000 }, + { CASEFOLDING_ARRAYLEN(case_fold_001), case_fold_001 }, + { CASEFOLDING_ARRAYLEN(case_fold_002), case_fold_002 }, + { CASEFOLDING_ARRAYLEN(case_fold_003), case_fold_003 }, + { CASEFOLDING_ARRAYLEN(case_fold_004), case_fold_004 }, + { CASEFOLDING_ARRAYLEN(case_fold_005), case_fold_005 }, + { CASEFOLDING_ARRAYLEN(case_fold_006), case_fold_006 }, + { CASEFOLDING_ARRAYLEN(case_fold_007), case_fold_007 }, + { CASEFOLDING_ARRAYLEN(case_fold_008), case_fold_008 }, + { CASEFOLDING_ARRAYLEN(case_fold_009), case_fold_009 }, + { CASEFOLDING_ARRAYLEN(case_fold_010), case_fold_010 }, + { CASEFOLDING_ARRAYLEN(case_fold_011), case_fold_011 }, + { CASEFOLDING_ARRAYLEN(case_fold_012), case_fold_012 }, + { CASEFOLDING_ARRAYLEN(case_fold_013), case_fold_013 }, + { CASEFOLDING_ARRAYLEN(case_fold_014), case_fold_014 }, + { CASEFOLDING_ARRAYLEN(case_fold_015), case_fold_015 }, + { CASEFOLDING_ARRAYLEN(case_fold_016), case_fold_016 }, + { CASEFOLDING_ARRAYLEN(case_fold_017), case_fold_017 }, + { CASEFOLDING_ARRAYLEN(case_fold_018), case_fold_018 }, + { CASEFOLDING_ARRAYLEN(case_fold_019), case_fold_019 }, + { CASEFOLDING_ARRAYLEN(case_fold_020), case_fold_020 }, + { CASEFOLDING_ARRAYLEN(case_fold_021), case_fold_021 }, + { CASEFOLDING_ARRAYLEN(case_fold_022), case_fold_022 }, + { CASEFOLDING_ARRAYLEN(case_fold_023), case_fold_023 }, + { CASEFOLDING_ARRAYLEN(case_fold_024), case_fold_024 }, + { CASEFOLDING_ARRAYLEN(case_fold_025), case_fold_025 }, + { CASEFOLDING_ARRAYLEN(case_fold_026), case_fold_026 }, + { CASEFOLDING_ARRAYLEN(case_fold_027), case_fold_027 }, + { CASEFOLDING_ARRAYLEN(case_fold_028), case_fold_028 }, + { CASEFOLDING_ARRAYLEN(case_fold_029), case_fold_029 }, + { CASEFOLDING_ARRAYLEN(case_fold_030), case_fold_030 }, + { CASEFOLDING_ARRAYLEN(case_fold_031), case_fold_031 }, + { CASEFOLDING_ARRAYLEN(case_fold_032), case_fold_032 }, + { CASEFOLDING_ARRAYLEN(case_fold_033), case_fold_033 }, + { CASEFOLDING_ARRAYLEN(case_fold_034), case_fold_034 }, + { CASEFOLDING_ARRAYLEN(case_fold_035), case_fold_035 }, + { CASEFOLDING_ARRAYLEN(case_fold_036), case_fold_036 }, + { CASEFOLDING_ARRAYLEN(case_fold_037), case_fold_037 }, + { CASEFOLDING_ARRAYLEN(case_fold_038), case_fold_038 }, + { CASEFOLDING_ARRAYLEN(case_fold_039), case_fold_039 }, + { CASEFOLDING_ARRAYLEN(case_fold_040), case_fold_040 }, + { CASEFOLDING_ARRAYLEN(case_fold_041), case_fold_041 }, + { CASEFOLDING_ARRAYLEN(case_fold_042), case_fold_042 }, + { CASEFOLDING_ARRAYLEN(case_fold_043), case_fold_043 }, + { CASEFOLDING_ARRAYLEN(case_fold_044), case_fold_044 }, + { CASEFOLDING_ARRAYLEN(case_fold_045), case_fold_045 }, + { CASEFOLDING_ARRAYLEN(case_fold_046), case_fold_046 }, + { CASEFOLDING_ARRAYLEN(case_fold_047), case_fold_047 }, + { CASEFOLDING_ARRAYLEN(case_fold_048), case_fold_048 }, + { CASEFOLDING_ARRAYLEN(case_fold_049), case_fold_049 }, + { CASEFOLDING_ARRAYLEN(case_fold_050), case_fold_050 }, + { CASEFOLDING_ARRAYLEN(case_fold_051), case_fold_051 }, + { CASEFOLDING_ARRAYLEN(case_fold_052), case_fold_052 }, + { CASEFOLDING_ARRAYLEN(case_fold_053), case_fold_053 }, + { CASEFOLDING_ARRAYLEN(case_fold_054), case_fold_054 }, + { CASEFOLDING_ARRAYLEN(case_fold_055), case_fold_055 }, + { CASEFOLDING_ARRAYLEN(case_fold_056), case_fold_056 }, + { CASEFOLDING_ARRAYLEN(case_fold_057), case_fold_057 }, + { CASEFOLDING_ARRAYLEN(case_fold_058), case_fold_058 }, + { CASEFOLDING_ARRAYLEN(case_fold_059), case_fold_059 }, + { CASEFOLDING_ARRAYLEN(case_fold_060), case_fold_060 }, + { CASEFOLDING_ARRAYLEN(case_fold_061), case_fold_061 }, + { CASEFOLDING_ARRAYLEN(case_fold_062), case_fold_062 }, + { CASEFOLDING_ARRAYLEN(case_fold_063), case_fold_063 }, + { CASEFOLDING_ARRAYLEN(case_fold_064), case_fold_064 }, + { CASEFOLDING_ARRAYLEN(case_fold_065), case_fold_065 }, + { CASEFOLDING_ARRAYLEN(case_fold_066), case_fold_066 }, + { CASEFOLDING_ARRAYLEN(case_fold_067), case_fold_067 }, + { CASEFOLDING_ARRAYLEN(case_fold_068), case_fold_068 }, + { CASEFOLDING_ARRAYLEN(case_fold_069), case_fold_069 }, + { CASEFOLDING_ARRAYLEN(case_fold_070), case_fold_070 }, + { CASEFOLDING_ARRAYLEN(case_fold_071), case_fold_071 }, + { CASEFOLDING_ARRAYLEN(case_fold_072), case_fold_072 }, + { CASEFOLDING_ARRAYLEN(case_fold_073), case_fold_073 }, + { CASEFOLDING_ARRAYLEN(case_fold_074), case_fold_074 }, + { CASEFOLDING_ARRAYLEN(case_fold_075), case_fold_075 }, + { CASEFOLDING_ARRAYLEN(case_fold_076), case_fold_076 }, + { CASEFOLDING_ARRAYLEN(case_fold_077), case_fold_077 }, + { CASEFOLDING_ARRAYLEN(case_fold_078), case_fold_078 }, + { CASEFOLDING_ARRAYLEN(case_fold_079), case_fold_079 }, + { CASEFOLDING_ARRAYLEN(case_fold_080), case_fold_080 }, + { CASEFOLDING_ARRAYLEN(case_fold_081), case_fold_081 }, + { CASEFOLDING_ARRAYLEN(case_fold_082), case_fold_082 }, + { CASEFOLDING_ARRAYLEN(case_fold_083), case_fold_083 }, + { CASEFOLDING_ARRAYLEN(case_fold_084), case_fold_084 }, + { CASEFOLDING_ARRAYLEN(case_fold_085), case_fold_085 }, + { CASEFOLDING_ARRAYLEN(case_fold_086), case_fold_086 }, + { CASEFOLDING_ARRAYLEN(case_fold_087), case_fold_087 }, + { CASEFOLDING_ARRAYLEN(case_fold_088), case_fold_088 }, + { CASEFOLDING_ARRAYLEN(case_fold_089), case_fold_089 }, + { CASEFOLDING_ARRAYLEN(case_fold_090), case_fold_090 }, + { CASEFOLDING_ARRAYLEN(case_fold_091), case_fold_091 }, + { CASEFOLDING_ARRAYLEN(case_fold_092), case_fold_092 }, + { CASEFOLDING_ARRAYLEN(case_fold_093), case_fold_093 }, + { CASEFOLDING_ARRAYLEN(case_fold_094), case_fold_094 }, + { CASEFOLDING_ARRAYLEN(case_fold_095), case_fold_095 }, + { CASEFOLDING_ARRAYLEN(case_fold_096), case_fold_096 }, + { CASEFOLDING_ARRAYLEN(case_fold_097), case_fold_097 }, + { CASEFOLDING_ARRAYLEN(case_fold_098), case_fold_098 }, + { CASEFOLDING_ARRAYLEN(case_fold_099), case_fold_099 }, + { CASEFOLDING_ARRAYLEN(case_fold_100), case_fold_100 }, + { CASEFOLDING_ARRAYLEN(case_fold_101), case_fold_101 }, + { CASEFOLDING_ARRAYLEN(case_fold_102), case_fold_102 }, + { CASEFOLDING_ARRAYLEN(case_fold_103), case_fold_103 }, + { CASEFOLDING_ARRAYLEN(case_fold_104), case_fold_104 }, + { CASEFOLDING_ARRAYLEN(case_fold_105), case_fold_105 }, + { CASEFOLDING_ARRAYLEN(case_fold_106), case_fold_106 }, + { CASEFOLDING_ARRAYLEN(case_fold_107), case_fold_107 }, + { CASEFOLDING_ARRAYLEN(case_fold_108), case_fold_108 }, + { CASEFOLDING_ARRAYLEN(case_fold_109), case_fold_109 }, + { CASEFOLDING_ARRAYLEN(case_fold_110), case_fold_110 }, + { CASEFOLDING_ARRAYLEN(case_fold_111), case_fold_111 }, + { CASEFOLDING_ARRAYLEN(case_fold_112), case_fold_112 }, + { CASEFOLDING_ARRAYLEN(case_fold_113), case_fold_113 }, + { CASEFOLDING_ARRAYLEN(case_fold_114), case_fold_114 }, + { CASEFOLDING_ARRAYLEN(case_fold_115), case_fold_115 }, + { CASEFOLDING_ARRAYLEN(case_fold_116), case_fold_116 }, + { CASEFOLDING_ARRAYLEN(case_fold_117), case_fold_117 }, + { CASEFOLDING_ARRAYLEN(case_fold_118), case_fold_118 }, + { CASEFOLDING_ARRAYLEN(case_fold_119), case_fold_119 }, + { CASEFOLDING_ARRAYLEN(case_fold_120), case_fold_120 }, + { CASEFOLDING_ARRAYLEN(case_fold_121), case_fold_121 }, + { CASEFOLDING_ARRAYLEN(case_fold_122), case_fold_122 }, + { 0, NULL }, + { CASEFOLDING_ARRAYLEN(case_fold_124), case_fold_124 }, + { 0, NULL }, + { CASEFOLDING_ARRAYLEN(case_fold_126), case_fold_126 }, + { 0, NULL }, + { CASEFOLDING_ARRAYLEN(case_fold_128), case_fold_128 }, + { CASEFOLDING_ARRAYLEN(case_fold_129), case_fold_129 }, + { CASEFOLDING_ARRAYLEN(case_fold_130), case_fold_130 }, + { CASEFOLDING_ARRAYLEN(case_fold_131), case_fold_131 }, + { CASEFOLDING_ARRAYLEN(case_fold_132), case_fold_132 }, + { CASEFOLDING_ARRAYLEN(case_fold_133), case_fold_133 }, + { CASEFOLDING_ARRAYLEN(case_fold_134), case_fold_134 }, + { CASEFOLDING_ARRAYLEN(case_fold_135), case_fold_135 }, + { CASEFOLDING_ARRAYLEN(case_fold_136), case_fold_136 }, + { CASEFOLDING_ARRAYLEN(case_fold_137), case_fold_137 }, + { CASEFOLDING_ARRAYLEN(case_fold_138), case_fold_138 }, + { CASEFOLDING_ARRAYLEN(case_fold_139), case_fold_139 }, + { CASEFOLDING_ARRAYLEN(case_fold_140), case_fold_140 }, + { CASEFOLDING_ARRAYLEN(case_fold_141), case_fold_141 }, + { CASEFOLDING_ARRAYLEN(case_fold_142), case_fold_142 }, + { CASEFOLDING_ARRAYLEN(case_fold_143), case_fold_143 }, + { CASEFOLDING_ARRAYLEN(case_fold_144), case_fold_144 }, + { CASEFOLDING_ARRAYLEN(case_fold_145), case_fold_145 }, + { CASEFOLDING_ARRAYLEN(case_fold_146), case_fold_146 }, + { CASEFOLDING_ARRAYLEN(case_fold_147), case_fold_147 }, + { CASEFOLDING_ARRAYLEN(case_fold_148), case_fold_148 }, + { CASEFOLDING_ARRAYLEN(case_fold_149), case_fold_149 }, + { CASEFOLDING_ARRAYLEN(case_fold_150), case_fold_150 }, + { CASEFOLDING_ARRAYLEN(case_fold_151), case_fold_151 }, + { CASEFOLDING_ARRAYLEN(case_fold_152), case_fold_152 }, + { CASEFOLDING_ARRAYLEN(case_fold_153), case_fold_153 }, + { CASEFOLDING_ARRAYLEN(case_fold_154), case_fold_154 }, + { CASEFOLDING_ARRAYLEN(case_fold_155), case_fold_155 }, + { CASEFOLDING_ARRAYLEN(case_fold_156), case_fold_156 }, + { CASEFOLDING_ARRAYLEN(case_fold_157), case_fold_157 }, + { CASEFOLDING_ARRAYLEN(case_fold_158), case_fold_158 }, + { CASEFOLDING_ARRAYLEN(case_fold_159), case_fold_159 }, + { CASEFOLDING_ARRAYLEN(case_fold_160), case_fold_160 }, + { CASEFOLDING_ARRAYLEN(case_fold_161), case_fold_161 }, + { CASEFOLDING_ARRAYLEN(case_fold_162), case_fold_162 }, + { CASEFOLDING_ARRAYLEN(case_fold_163), case_fold_163 }, + { CASEFOLDING_ARRAYLEN(case_fold_164), case_fold_164 }, + { CASEFOLDING_ARRAYLEN(case_fold_165), case_fold_165 }, + { CASEFOLDING_ARRAYLEN(case_fold_166), case_fold_166 }, + { CASEFOLDING_ARRAYLEN(case_fold_167), case_fold_167 }, + { CASEFOLDING_ARRAYLEN(case_fold_168), case_fold_168 }, + { CASEFOLDING_ARRAYLEN(case_fold_169), case_fold_169 }, + { CASEFOLDING_ARRAYLEN(case_fold_170), case_fold_170 }, + { CASEFOLDING_ARRAYLEN(case_fold_171), case_fold_171 }, + { CASEFOLDING_ARRAYLEN(case_fold_172), case_fold_172 }, + { CASEFOLDING_ARRAYLEN(case_fold_173), case_fold_173 }, + { CASEFOLDING_ARRAYLEN(case_fold_174), case_fold_174 }, + { CASEFOLDING_ARRAYLEN(case_fold_175), case_fold_175 }, + { CASEFOLDING_ARRAYLEN(case_fold_176), case_fold_176 }, + { CASEFOLDING_ARRAYLEN(case_fold_177), case_fold_177 }, + { CASEFOLDING_ARRAYLEN(case_fold_178), case_fold_178 }, + { CASEFOLDING_ARRAYLEN(case_fold_179), case_fold_179 }, + { CASEFOLDING_ARRAYLEN(case_fold_180), case_fold_180 }, + { CASEFOLDING_ARRAYLEN(case_fold_181), case_fold_181 }, + { CASEFOLDING_ARRAYLEN(case_fold_182), case_fold_182 }, + { CASEFOLDING_ARRAYLEN(case_fold_183), case_fold_183 }, + { CASEFOLDING_ARRAYLEN(case_fold_184), case_fold_184 }, + { CASEFOLDING_ARRAYLEN(case_fold_185), case_fold_185 }, + { CASEFOLDING_ARRAYLEN(case_fold_186), case_fold_186 }, + { CASEFOLDING_ARRAYLEN(case_fold_187), case_fold_187 }, + { CASEFOLDING_ARRAYLEN(case_fold_188), case_fold_188 }, + { CASEFOLDING_ARRAYLEN(case_fold_189), case_fold_189 }, + { CASEFOLDING_ARRAYLEN(case_fold_190), case_fold_190 }, + { CASEFOLDING_ARRAYLEN(case_fold_191), case_fold_191 }, + { CASEFOLDING_ARRAYLEN(case_fold_192), case_fold_192 }, + { CASEFOLDING_ARRAYLEN(case_fold_193), case_fold_193 }, + { CASEFOLDING_ARRAYLEN(case_fold_194), case_fold_194 }, + { CASEFOLDING_ARRAYLEN(case_fold_195), case_fold_195 }, + { CASEFOLDING_ARRAYLEN(case_fold_196), case_fold_196 }, + { CASEFOLDING_ARRAYLEN(case_fold_197), case_fold_197 }, + { CASEFOLDING_ARRAYLEN(case_fold_198), case_fold_198 }, + { CASEFOLDING_ARRAYLEN(case_fold_199), case_fold_199 }, + { CASEFOLDING_ARRAYLEN(case_fold_200), case_fold_200 }, + { CASEFOLDING_ARRAYLEN(case_fold_201), case_fold_201 }, + { CASEFOLDING_ARRAYLEN(case_fold_202), case_fold_202 }, + { CASEFOLDING_ARRAYLEN(case_fold_203), case_fold_203 }, + { CASEFOLDING_ARRAYLEN(case_fold_204), case_fold_204 }, + { CASEFOLDING_ARRAYLEN(case_fold_205), case_fold_205 }, + { CASEFOLDING_ARRAYLEN(case_fold_206), case_fold_206 }, + { CASEFOLDING_ARRAYLEN(case_fold_207), case_fold_207 }, + { CASEFOLDING_ARRAYLEN(case_fold_208), case_fold_208 }, + { CASEFOLDING_ARRAYLEN(case_fold_209), case_fold_209 }, + { CASEFOLDING_ARRAYLEN(case_fold_210), case_fold_210 }, + { CASEFOLDING_ARRAYLEN(case_fold_211), case_fold_211 }, + { CASEFOLDING_ARRAYLEN(case_fold_212), case_fold_212 }, + { CASEFOLDING_ARRAYLEN(case_fold_213), case_fold_213 }, + { CASEFOLDING_ARRAYLEN(case_fold_214), case_fold_214 }, + { CASEFOLDING_ARRAYLEN(case_fold_215), case_fold_215 }, + { CASEFOLDING_ARRAYLEN(case_fold_216), case_fold_216 }, + { CASEFOLDING_ARRAYLEN(case_fold_217), case_fold_217 }, + { CASEFOLDING_ARRAYLEN(case_fold_218), case_fold_218 }, + { CASEFOLDING_ARRAYLEN(case_fold_219), case_fold_219 }, + { CASEFOLDING_ARRAYLEN(case_fold_220), case_fold_220 }, + { CASEFOLDING_ARRAYLEN(case_fold_221), case_fold_221 }, + { CASEFOLDING_ARRAYLEN(case_fold_222), case_fold_222 }, + { CASEFOLDING_ARRAYLEN(case_fold_223), case_fold_223 }, + { CASEFOLDING_ARRAYLEN(case_fold_224), case_fold_224 }, + { CASEFOLDING_ARRAYLEN(case_fold_225), case_fold_225 }, + { CASEFOLDING_ARRAYLEN(case_fold_226), case_fold_226 }, + { CASEFOLDING_ARRAYLEN(case_fold_227), case_fold_227 }, + { CASEFOLDING_ARRAYLEN(case_fold_228), case_fold_228 }, + { CASEFOLDING_ARRAYLEN(case_fold_229), case_fold_229 }, + { CASEFOLDING_ARRAYLEN(case_fold_230), case_fold_230 }, + { CASEFOLDING_ARRAYLEN(case_fold_231), case_fold_231 }, + { CASEFOLDING_ARRAYLEN(case_fold_232), case_fold_232 }, + { CASEFOLDING_ARRAYLEN(case_fold_233), case_fold_233 }, + { CASEFOLDING_ARRAYLEN(case_fold_234), case_fold_234 }, + { CASEFOLDING_ARRAYLEN(case_fold_235), case_fold_235 }, + { CASEFOLDING_ARRAYLEN(case_fold_236), case_fold_236 }, + { CASEFOLDING_ARRAYLEN(case_fold_237), case_fold_237 }, + { CASEFOLDING_ARRAYLEN(case_fold_238), case_fold_238 }, + { CASEFOLDING_ARRAYLEN(case_fold_239), case_fold_239 }, + { CASEFOLDING_ARRAYLEN(case_fold_240), case_fold_240 }, + { CASEFOLDING_ARRAYLEN(case_fold_241), case_fold_241 }, + { CASEFOLDING_ARRAYLEN(case_fold_242), case_fold_242 }, + { CASEFOLDING_ARRAYLEN(case_fold_243), case_fold_243 }, + { CASEFOLDING_ARRAYLEN(case_fold_244), case_fold_244 }, + { CASEFOLDING_ARRAYLEN(case_fold_245), case_fold_245 }, + { CASEFOLDING_ARRAYLEN(case_fold_246), case_fold_246 }, + { CASEFOLDING_ARRAYLEN(case_fold_247), case_fold_247 }, + { CASEFOLDING_ARRAYLEN(case_fold_248), case_fold_248 }, + { CASEFOLDING_ARRAYLEN(case_fold_249), case_fold_249 }, + { CASEFOLDING_ARRAYLEN(case_fold_250), case_fold_250 }, + { CASEFOLDING_ARRAYLEN(case_fold_251), case_fold_251 }, + { CASEFOLDING_ARRAYLEN(case_fold_252), case_fold_252 }, + { CASEFOLDING_ARRAYLEN(case_fold_253), case_fold_253 }, + { CASEFOLDING_ARRAYLEN(case_fold_254), case_fold_254 }, + { CASEFOLDING_ARRAYLEN(case_fold_255), case_fold_255 }, +}; + diff --git a/tier1/processor_detect.cpp b/tier1/processor_detect.cpp new file mode 100644 index 0000000..0aabeb7 --- /dev/null +++ b/tier1/processor_detect.cpp @@ -0,0 +1,274 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: win32 dependant ASM code for CPU capability detection +// +// $Workfile: $ +// $NoKeywords: $ +//=============================================================================// + +#if defined( _X360 ) || defined( WIN64 ) + +bool CheckMMXTechnology(void) { return false; } +bool CheckSSETechnology(void) { return false; } +bool CheckSSE2Technology(void) { return false; } +bool Check3DNowTechnology(void) { return false; } + +#elif defined( _WIN32 ) && !defined( _X360 ) + +#pragma optimize( "", off ) +#pragma warning( disable: 4800 ) //'int' : forcing value to bool 'true' or 'false' (performance warning) + +// stuff from windows.h +#ifndef EXCEPTION_EXECUTE_HANDLER +#define EXCEPTION_EXECUTE_HANDLER 1 +#endif + +bool CheckMMXTechnology(void) +{ + int retval = true; + unsigned int RegEDX = 0; + +#ifdef CPUID + _asm pushad; +#endif + + __try + { + _asm + { +#ifdef CPUID + xor edx, edx // Clue the compiler that EDX is about to be used. +#endif + mov eax, 1 // set up CPUID to return processor version and features + // 0 = vendor string, 1 = version info, 2 = cache info + CPUID // code bytes = 0fh, 0a2h + mov RegEDX, edx // features returned in edx + } + } + __except(EXCEPTION_EXECUTE_HANDLER) + { + retval = false; + } + + // If CPUID not supported, then certainly no MMX extensions. + if (retval) + { + if (RegEDX & 0x800000) // bit 23 is set for MMX technology + { + __try + { + // try executing the MMX instruction "emms" + _asm EMMS + } + __except(EXCEPTION_EXECUTE_HANDLER) + { + retval = false; + } + } + + else + retval = false; // processor supports CPUID but does not support MMX technology + + // if retval == 0 here, it means the processor has MMX technology but + // floating-point emulation is on; so MMX technology is unavailable + } + +#ifdef CPUID + _asm popad; +#endif + + return retval; +} + +bool CheckSSETechnology(void) +{ + int retval = true; + unsigned int RegEDX = 0; + +#ifdef CPUID + _asm pushad; +#endif + + // Do we have support for the CPUID function? + __try + { + _asm + { +#ifdef CPUID + xor edx, edx // Clue the compiler that EDX is about to be used. +#endif + mov eax, 1 // set up CPUID to return processor version and features + // 0 = vendor string, 1 = version info, 2 = cache info + CPUID // code bytes = 0fh, 0a2h + mov RegEDX, edx // features returned in edx + } + } + __except(EXCEPTION_EXECUTE_HANDLER) + { + retval = false; + } + + // If CPUID not supported, then certainly no SSE extensions. + if (retval) + { + // Do we have support for SSE in this processor? + if ( RegEDX & 0x2000000L ) // bit 25 is set for SSE technology + { + // Make sure that SSE is supported by executing an inline SSE instruction + +// BUGBUG, FIXME - Visual C Version 6.0 does not support SSE inline code YET (No macros from Intel either) +// Fix this if VC7 supports inline SSE instructinons like "xorps" as shown below. +#if 1 + __try + { + _asm + { + // Attempt execution of a SSE instruction to make sure OS supports SSE FPU context switches + xorps xmm0, xmm0 + // This will work on Win2k+ (Including masking SSE FPU exception to "normalized" values) + // This will work on Win98+ (But no "masking" of FPU exceptions provided) + } + } + __except(EXCEPTION_EXECUTE_HANDLER) +#endif + + { + retval = false; + } + } + else + retval = false; + } +#ifdef CPUID + _asm popad; +#endif + + return retval; +} + +bool CheckSSE2Technology(void) +{ + int retval = true; + unsigned int RegEDX = 0; + +#ifdef CPUID + _asm pushad; +#endif + + // Do we have support for the CPUID function? + __try + { + _asm + { +#ifdef CPUID + xor edx, edx // Clue the compiler that EDX is about to be used. +#endif + mov eax, 1 // set up CPUID to return processor version and features + // 0 = vendor string, 1 = version info, 2 = cache info + CPUID // code bytes = 0fh, 0a2h + mov RegEDX, edx // features returned in edx + } + } + __except(EXCEPTION_EXECUTE_HANDLER) + { + retval = false; + } + + // If CPUID not supported, then certainly no SSE extensions. + if (retval) + { + // Do we have support for SSE in this processor? + if ( RegEDX & 0x04000000 ) // bit 26 is set for SSE2 technology + { + // Make sure that SSE is supported by executing an inline SSE instruction + + __try + { + _asm + { + // Attempt execution of a SSE2 instruction to make sure OS supports SSE FPU context switches + xorpd xmm0, xmm0 + } + } + __except(EXCEPTION_EXECUTE_HANDLER) + + { + retval = false; + } + } + else + retval = false; + } +#ifdef CPUID + _asm popad; +#endif + + return retval; +} + +bool Check3DNowTechnology(void) +{ + int retval = true; + unsigned int RegEAX = 0; + +#ifdef CPUID + _asm pushad; +#endif + + // First see if we can execute CPUID at all + __try + { + _asm + { +#ifdef CPUID +// xor edx, edx // Clue the compiler that EDX is about to be used. +#endif + mov eax, 0x80000000 // setup CPUID to return whether AMD >0x80000000 function are supported. + // 0x80000000 = Highest 0x80000000+ function, 0x80000001 = 3DNow support + CPUID // code bytes = 0fh, 0a2h + mov RegEAX, eax // result returned in eax + } + } + __except(EXCEPTION_EXECUTE_HANDLER) + { + retval = false; + } + + // If CPUID not supported, then there is definitely no 3DNow support + if (retval) + { + // Are there any "higher" AMD CPUID functions? + if (RegEAX > 0x80000000L ) + { + __try + { + _asm + { + mov eax, 0x80000001 // setup to test for CPU features + CPUID // code bytes = 0fh, 0a2h + shr edx, 31 // If bit 31 is set, we have 3DNow support! + mov retval, edx // Save the return value for end of function + } + } + __except(EXCEPTION_EXECUTE_HANDLER) + { + retval = false; + } + } + else + { + // processor supports CPUID but does not support AMD CPUID functions + retval = false; + } + } + +#ifdef CPUID + _asm popad; +#endif + + return retval; +} + +#pragma optimize( "", on ) + +#endif // _WIN32 diff --git a/tier1/processor_detect_linux.cpp b/tier1/processor_detect_linux.cpp new file mode 100644 index 0000000..c7c95d0 --- /dev/null +++ b/tier1/processor_detect_linux.cpp @@ -0,0 +1,47 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: linux dependant ASM code for CPU capability detection +// +// $Workfile: $ +// $NoKeywords: $ +//=============================================================================// + +#define cpuid(in,a,b,c,d) \ + asm("pushl %%ebx\n\t" "cpuid\n\t" "movl %%ebx,%%esi\n\t" "pop %%ebx": "=a" (a), "=S" (b), "=c" (c), "=d" (d) : "a" (in)); + +bool CheckMMXTechnology(void) +{ + unsigned long eax,ebx,edx,unused; + cpuid(1,eax,ebx,unused,edx); + + return edx & 0x800000; +} + +bool CheckSSETechnology(void) +{ + unsigned long eax,ebx,edx,unused; + cpuid(1,eax,ebx,unused,edx); + + return edx & 0x2000000L; +} + +bool CheckSSE2Technology(void) +{ + unsigned long eax,ebx,edx,unused; + cpuid(1,eax,ebx,unused,edx); + + return edx & 0x04000000; +} + +bool Check3DNowTechnology(void) +{ + unsigned long eax, unused; + cpuid(0x80000000,eax,unused,unused,unused); + + if ( eax > 0x80000000L ) + { + cpuid(0x80000001,unused,unused,unused,eax); + return ( eax & 1<<31 ); + } + return false; +} diff --git a/tier1/qsort_s.cpp b/tier1/qsort_s.cpp new file mode 100644 index 0000000..820d151 --- /dev/null +++ b/tier1/qsort_s.cpp @@ -0,0 +1,112 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +/******************************************************************/
+/* qsort.c -- Non-Recursive ANSI Quicksort function */
+/* */
+/* Public domain by Raymond Gardner, Englewood CO February 1991 */
+/* */
+/* Usage: */
+/* qsort(base, nbr_elements, width_bytes, compare_function); */
+/* void *base; */
+/* size_t nbr_elements, width_bytes; */
+/* int (*compare_function)(const void *, const void *); */
+/* */
+/* Sorts an array starting at base, of length nbr_elements, each */
+/* element of size width_bytes, ordered via compare_function, */
+/* which is called as (*compare_function)(ptr_to_element1, */
+/* ptr_to_element2) and returns < 0 if element1 < element2, */
+/* 0 if element1 = element2, > 0 if element1 > element2. */
+/* Most refinements are due to R. Sedgewick. See "Implementing */
+/* Quicksort Programs", Comm. ACM, Oct. 1978, and Corrigendum, */
+/* Comm. ACM, June 1979. */
+/******************************************************************/
+
+// modified to take (and use) a context object, ala Microsoft's qsort_s
+// "extension" to the stdlib
+
+#include <stddef.h> /* for size_t definition */
+
+/*
+** swap nbytes between a and b
+*/
+
+static void swap_bytes(char *a, char *b, size_t nbytes)
+{
+ char tmp;
+ do {
+ tmp = *a; *a++ = *b; *b++ = tmp;
+ } while ( --nbytes );
+}
+
+#define SWAP(a, b) (swap_bytes((char *)(a), (char *)(b), size))
+
+#define COMP(ctx, a, b) ((*comp)((void *)ctx, (void *)(a), (void *)(b)))
+
+#define T 7 /* subfiles of T or fewer elements will */
+ /* be sorted by a simple insertion sort */
+ /* Note! T must be at least 3 */
+
+extern "C" void qsort_s(void *basep, size_t nelems, size_t size,
+ int (*comp)(void *, const void *, const void *),
+ void *ctx)
+{
+ char *stack[40], **sp; /* stack and stack pointer */
+ char *i, *j, *limit; /* scan and limit pointers */
+ size_t thresh; /* size of T elements in bytes */
+ char *base; /* base pointer as char * */
+
+ base = (char *)basep; /* set up char * base pointer */
+ thresh = T * size; /* init threshold */
+ sp = stack; /* init stack pointer */
+ limit = base + nelems * size;/* pointer past end of array */
+ for ( ;; ) { /* repeat until break... */
+ if ( limit - base > thresh ) { /* if more than T elements */
+ /* swap base with middle */
+ SWAP((((limit-base)/size)/2)*size+base, base);
+ i = base + size; /* i scans left to right */
+ j = limit - size; /* j scans right to left */
+ if ( COMP(ctx, i, j) > 0 ) /* Sedgewick's */
+ SWAP(i, j); /* three-element sort */
+ if ( COMP(ctx, base, j) > 0 )/* sets things up */
+ SWAP(base, j); /* so that */
+ if ( COMP(ctx, i, base) > 0 )/* *i <= *base <= *j */
+ SWAP(i, base); /* *base is pivot element */
+ for ( ;; ) { /* loop until break */
+ do /* move i right */
+ i += size; /* until *i >= pivot */
+ while ( COMP(ctx, i, base) < 0 );
+ do /* move j left */
+ j -= size; /* until *j <= pivot */
+ while ( COMP(ctx, j, base) > 0 );
+ if ( i > j ) /* if pointers crossed */
+ break; /* break loop */
+ SWAP(i, j); /* else swap elements, keep scanning*/
+ }
+ SWAP(base, j); /* move pivot into correct place */
+ if ( j - base > limit - i ) { /* if left subfile larger */
+ sp[0] = base; /* stack left subfile base */
+ sp[1] = j; /* and limit */
+ base = i; /* sort the right subfile */
+ } else { /* else right subfile larger*/
+ sp[0] = i; /* stack right subfile base */
+ sp[1] = limit; /* and limit */
+ limit = j; /* sort the left subfile */
+ }
+ sp += 2; /* increment stack pointer */
+ } else { /* else subfile is small, use insertion sort */
+ for ( j = base, i = j+size; i < limit; j = i, i += size )
+ for ( ; COMP(ctx, j, j+size) > 0; j -= size ) {
+ SWAP(j, j+size);
+ if ( j == base )
+ break;
+ }
+ if ( sp != stack ) { /* if any entries on stack */
+ sp -= 2; /* pop the base and limit */
+ base = sp[0];
+ limit = sp[1];
+ } else /* else stack empty, done */
+ break;
+ }
+ }
+}
+
+
diff --git a/tier1/rangecheckedvar.cpp b/tier1/rangecheckedvar.cpp new file mode 100644 index 0000000..d37e34a --- /dev/null +++ b/tier1/rangecheckedvar.cpp @@ -0,0 +1,41 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +//=============================================================================// + +#include "rangecheckedvar.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +bool g_bDoRangeChecks = true; + + +static int g_nDisables = 0; + + +CDisableRangeChecks::CDisableRangeChecks() +{ + if ( !ThreadInMainThread() ) + return; + g_nDisables++; + g_bDoRangeChecks = false; +} + + +CDisableRangeChecks::~CDisableRangeChecks() +{ + if ( !ThreadInMainThread() ) + return; + Assert( g_nDisables > 0 ); + --g_nDisables; + if ( g_nDisables == 0 ) + { + g_bDoRangeChecks = true; + } +} + + + + diff --git a/tier1/reliabletimer.cpp b/tier1/reliabletimer.cpp new file mode 100644 index 0000000..af8b842 --- /dev/null +++ b/tier1/reliabletimer.cpp @@ -0,0 +1,93 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +//============================================================================= + +#include "tier1/reliabletimer.h" + +int64 CReliableTimer::sm_nPerformanceFrequency = 0; +bool CReliableTimer::sm_bUseQPC = false; + +#ifdef _WIN32 +#include "winlite.h" +#endif + +//----------------------------------------------------------------------------- +// Purpose: Constructor +//----------------------------------------------------------------------------- +CReliableTimer::CReliableTimer() +{ + m_nPerformanceCounterStart = 0; + m_nPerformanceCounterEnd = 0; + m_nPerformanceCounterLimit = 0; + +#ifdef _WIN32 + // calculate performance frequency the first time we use a timer + if ( 0 == sm_nPerformanceFrequency ) + { + // Are we on a bad CPU? + sm_bUseQPC = false; // todo + const CPUInformation &cpu = *GetCPUInformation(); + sm_bUseQPC = ( ( 0 == Q_stricmp( cpu.m_szProcessorID, "AuthenticAMD" ) ) + && ( cpu.m_nPhysicalProcessors > 1 ) + && !cpu.m_bSSE41 ); + + if ( sm_bUseQPC ) + { + LARGE_INTEGER li; + QueryPerformanceFrequency( &li ); + sm_nPerformanceFrequency = li.QuadPart; + } + else + { + sm_nPerformanceFrequency = g_ClockSpeed; + } + } +#elif defined(_PS3) + // On PowerPC, the time base register increment frequency is implementation dependent, and doesn't have to be constant. + // On PS3, measured it to be just shy of 80Mhz on the PPU and doesn't seem to change + if ( sm_nPerformanceFrequency == 0 ) + sm_nPerformanceFrequency = sys_time_get_timebase_frequency(); +#else + // calculate performance frequency the first time we use a timer + if ( 0 == sm_nPerformanceFrequency ) + { + sm_nPerformanceFrequency = g_ClockSpeed; + } +#endif +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns current QueryPerformanceCounter value +//----------------------------------------------------------------------------- +int64 CReliableTimer::GetPerformanceCountNow() +{ + //VPROF_BUDGET( "CReliableTimer::GetPerformanceCountNow", VPROF_BUDGETGROUP_OTHER_UNACCOUNTED ); +#ifdef _WIN32 + if ( sm_bUseQPC ) + { + LARGE_INTEGER li = {0}; + QueryPerformanceCounter( &li ); + return li.QuadPart; + } + else + { + CCycleCount CycleCount; + CycleCount.Sample(); + return CycleCount.GetLongCycles(); + } +#elif defined( _PS3 ) + // use handy macro to grab tb + uint64 ulNow; + SYS_TIMEBASE_GET( ulNow ); + return ulNow; +#else + uint64 un64; + __asm__ __volatile__ ( + "rdtsc\n\t" + : "=A" (un64) ); + return (int64)un64; +#endif +} diff --git a/tier1/snappy-internal.h b/tier1/snappy-internal.h new file mode 100644 index 0000000..c99d331 --- /dev/null +++ b/tier1/snappy-internal.h @@ -0,0 +1,150 @@ +// Copyright 2008 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Internals shared between the Snappy implementation and its unittest. + +#ifndef UTIL_SNAPPY_SNAPPY_INTERNAL_H_ +#define UTIL_SNAPPY_SNAPPY_INTERNAL_H_ + +#include "snappy-stubs-internal.h" + +namespace snappy { +namespace internal { + +class WorkingMemory { + public: + WorkingMemory() : large_table_(NULL) { } + ~WorkingMemory() { delete[] large_table_; } + + // Allocates and clears a hash table using memory in "*this", + // stores the number of buckets in "*table_size" and returns a pointer to + // the base of the hash table. + uint16* GetHashTable(size_t input_size, int* table_size); + + private: + uint16 small_table_[1<<10]; // 2KB + uint16* large_table_; // Allocated only when needed + + DISALLOW_COPY_AND_ASSIGN(WorkingMemory); +}; + +// Flat array compression that does not emit the "uncompressed length" +// prefix. Compresses "input" string to the "*op" buffer. +// +// REQUIRES: "input_length <= kBlockSize" +// REQUIRES: "op" points to an array of memory that is at least +// "MaxCompressedLength(input_length)" in size. +// REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. +// REQUIRES: "table_size" is a power of two +// +// Returns an "end" pointer into "op" buffer. +// "end - op" is the compressed size of "input". +char* CompressFragment(const char* input, + size_t input_length, + char* op, + uint16* table, + const int table_size); + +// Return the largest n such that +// +// s1[0,n-1] == s2[0,n-1] +// and n <= (s2_limit - s2). +// +// Does not read *s2_limit or beyond. +// Does not read *(s1 + (s2_limit - s2)) or beyond. +// Requires that s2_limit >= s2. +// +// Separate implementation for x86_64, for speed. Uses the fact that +// x86_64 is little endian. +#if defined(ARCH_K8) +static inline int FindMatchLength(const char* s1, + const char* s2, + const char* s2_limit) { + assert(s2_limit >= s2); + int matched = 0; + + // Find out how long the match is. We loop over the data 64 bits at a + // time until we find a 64-bit block that doesn't match; then we find + // the first non-matching bit and use that to calculate the total + // length of the match. + while (PREDICT_TRUE(s2 <= s2_limit - 8)) { + if (PREDICT_FALSE(UNALIGNED_LOAD64(s2) == UNALIGNED_LOAD64(s1 + matched))) { + s2 += 8; + matched += 8; + } else { + // On current (mid-2008) Opteron models there is a 3% more + // efficient code sequence to find the first non-matching byte. + // However, what follows is ~10% better on Intel Core 2 and newer, + // and we expect AMD's bsf instruction to improve. + uint64 x = UNALIGNED_LOAD64(s2) ^ UNALIGNED_LOAD64(s1 + matched); + int matching_bits = Bits::FindLSBSetNonZero64(x); + matched += matching_bits >> 3; + return matched; + } + } + while (PREDICT_TRUE(s2 < s2_limit)) { + if (PREDICT_TRUE(s1[matched] == *s2)) { + ++s2; + ++matched; + } else { + return matched; + } + } + return matched; +} +#else +static inline int FindMatchLength(const char* s1, + const char* s2, + const char* s2_limit) { + // Implementation based on the x86-64 version, above. + assert(s2_limit >= s2); + int matched = 0; + + while (s2 <= s2_limit - 4 && + UNALIGNED_LOAD32(s2) == UNALIGNED_LOAD32(s1 + matched)) { + s2 += 4; + matched += 4; + } + if (LittleEndian::IsLittleEndian() && s2 <= s2_limit - 4) { + uint32 x = UNALIGNED_LOAD32(s2) ^ UNALIGNED_LOAD32(s1 + matched); + int matching_bits = Bits::FindLSBSetNonZero(x); + matched += matching_bits >> 3; + } else { + while ((s2 < s2_limit) && (s1[matched] == *s2)) { + ++s2; + ++matched; + } + } + return matched; +} +#endif + +} // end namespace internal +} // end namespace snappy + +#endif // UTIL_SNAPPY_SNAPPY_INTERNAL_H_ diff --git a/tier1/snappy-sinksource.cpp b/tier1/snappy-sinksource.cpp new file mode 100644 index 0000000..8e868bb --- /dev/null +++ b/tier1/snappy-sinksource.cpp @@ -0,0 +1,74 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <string.h> + +#include "snappy-sinksource.h" + +namespace snappy { + +Source::~Source() { } + +Sink::~Sink() { } + +char* Sink::GetAppendBuffer(size_t length, char* scratch) { + (void)length; + return scratch; +} + +ByteArraySource::~ByteArraySource() { } + +size_t ByteArraySource::Available() const { return left_; } + +const char* ByteArraySource::Peek(size_t* len) { + *len = left_; + return ptr_; +} + +void ByteArraySource::Skip(size_t n) { + left_ -= n; + ptr_ += n; +} + +UncheckedByteArraySink::~UncheckedByteArraySink() { } + +void UncheckedByteArraySink::Append(const char* data, size_t n) { + // Do no copying if the caller filled in the result of GetAppendBuffer() + if (data != dest_) { + memcpy(dest_, data, n); + } + dest_ += n; +} + +char* UncheckedByteArraySink::GetAppendBuffer(size_t len, char* scratch) { + (void)scratch; + (void)len; + return dest_; +} + +} diff --git a/tier1/snappy-stubs-internal.cpp b/tier1/snappy-stubs-internal.cpp new file mode 100644 index 0000000..3b9d79f --- /dev/null +++ b/tier1/snappy-stubs-internal.cpp @@ -0,0 +1,45 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <algorithm> +#ifdef _WIN32 +#pragma warning(disable:4530) // warning C4530: C++ exception handler used, but unwind semantics are not enabled. Specify /EHsc +#endif //_WIN32 +#include <string> + +#include "snappy-stubs-internal.h" + +namespace snappy { + +void Varint::Append32(string* s, uint32 value) { + char buf[Varint::kMax32]; + const char* p = Varint::Encode32(buf, value); + s->append(buf, p - buf); +} + +} // namespace snappy diff --git a/tier1/snappy-stubs-internal.h b/tier1/snappy-stubs-internal.h new file mode 100644 index 0000000..1413825 --- /dev/null +++ b/tier1/snappy-stubs-internal.h @@ -0,0 +1,493 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Various stubs for the open-source version of Snappy. + +#ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_ +#define UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "tier0/platform.h" + +#include <string> + +#include <assert.h> +#include <stdlib.h> +#include <string.h> + +#ifdef HAVE_SYS_MMAN_H +#include <sys/mman.h> +#endif + +#include "snappy-stubs-public.h" + +#if defined(__x86_64__) + +// Enable 64-bit optimized versions of some routines. +#define ARCH_K8 1 + +#endif + +// Needed by OS X, among others. +#ifndef MAP_ANONYMOUS +#define MAP_ANONYMOUS MAP_ANON +#endif + +// Pull in std::min, std::ostream, and the likes. This is safe because this +// header file is never used from any public header files. +using namespace std; + +// The size of an array, if known at compile-time. +// Will give unexpected results if used on a pointer. +// We undefine it first, since some compilers already have a definition. +#ifdef ARRAYSIZE +#undef ARRAYSIZE +#endif +#define ARRAYSIZE(a) (sizeof(a) / sizeof(*(a))) + +// Static prediction hints. +#ifdef HAVE_BUILTIN_EXPECT +#define PREDICT_FALSE(x) (__builtin_expect(x, 0)) +#define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1)) +#else +#define PREDICT_FALSE(x) x +#define PREDICT_TRUE(x) x +#endif + +// This is only used for recomputing the tag byte table used during +// decompression; for simplicity we just remove it from the open-source +// version (anyone who wants to regenerate it can just do the call +// themselves within main()). +#define DEFINE_bool(flag_name, default_value, description) \ + bool FLAGS_ ## flag_name = default_value +#define DECLARE_bool(flag_name) \ + extern bool FLAGS_ ## flag_name + +namespace snappy { + +static const uint32 kuint32max = static_cast<uint32>(0xFFFFFFFF); +static const int64 kint64max = static_cast<int64>(0x7FFFFFFFFFFFFFFFLL); + +// Potentially unaligned loads and stores. + +// x86 and PowerPC can simply do these loads and stores native. + +#if defined(__i386__) || defined(__x86_64__) || defined(__powerpc__) + +#define UNALIGNED_LOAD16(_p) (*reinterpret_cast<const uint16 *>(_p)) +#define UNALIGNED_LOAD32(_p) (*reinterpret_cast<const uint32 *>(_p)) +#define UNALIGNED_LOAD64(_p) (*reinterpret_cast<const uint64 *>(_p)) + +#define UNALIGNED_STORE16(_p, _val) (*reinterpret_cast<uint16 *>(_p) = (_val)) +#define UNALIGNED_STORE32(_p, _val) (*reinterpret_cast<uint32 *>(_p) = (_val)) +#define UNALIGNED_STORE64(_p, _val) (*reinterpret_cast<uint64 *>(_p) = (_val)) + +// ARMv7 and newer support native unaligned accesses, but only of 16-bit +// and 32-bit values (not 64-bit); older versions either raise a fatal signal, +// do an unaligned read and rotate the words around a bit, or do the reads very +// slowly (trip through kernel mode). There's no simple #define that says just +// “ARMv7 or higher”, so we have to filter away all ARMv5 and ARMv6 +// sub-architectures. +// +// This is a mess, but there's not much we can do about it. + +#elif defined(__arm__) && \ + !defined(__ARM_ARCH_4__) && \ + !defined(__ARM_ARCH_4T__) && \ + !defined(__ARM_ARCH_5__) && \ + !defined(__ARM_ARCH_5T__) && \ + !defined(__ARM_ARCH_5TE__) && \ + !defined(__ARM_ARCH_5TEJ__) && \ + !defined(__ARM_ARCH_6__) && \ + !defined(__ARM_ARCH_6J__) && \ + !defined(__ARM_ARCH_6K__) && \ + !defined(__ARM_ARCH_6Z__) && \ + !defined(__ARM_ARCH_6ZK__) && \ + !defined(__ARM_ARCH_6T2__) + +#define UNALIGNED_LOAD16(_p) (*reinterpret_cast<const uint16 *>(_p)) +#define UNALIGNED_LOAD32(_p) (*reinterpret_cast<const uint32 *>(_p)) + +#define UNALIGNED_STORE16(_p, _val) (*reinterpret_cast<uint16 *>(_p) = (_val)) +#define UNALIGNED_STORE32(_p, _val) (*reinterpret_cast<uint32 *>(_p) = (_val)) + +// TODO(user): NEON supports unaligned 64-bit loads and stores. +// See if that would be more efficient on platforms supporting it, +// at least for copies. + +inline uint64 UNALIGNED_LOAD64(const void *p) { + uint64 t; + memcpy(&t, p, sizeof t); + return t; +} + +inline void UNALIGNED_STORE64(void *p, uint64 v) { + memcpy(p, &v, sizeof v); +} + +#else + +// These functions are provided for architectures that don't support +// unaligned loads and stores. + +inline uint16 UNALIGNED_LOAD16(const void *p) { + uint16 t; + memcpy(&t, p, sizeof t); + return t; +} + +inline uint32 UNALIGNED_LOAD32(const void *p) { + uint32 t; + memcpy(&t, p, sizeof t); + return t; +} + +inline uint64 UNALIGNED_LOAD64(const void *p) { + uint64 t; + memcpy(&t, p, sizeof t); + return t; +} + +inline void UNALIGNED_STORE16(void *p, uint16 v) { + memcpy(p, &v, sizeof v); +} + +inline void UNALIGNED_STORE32(void *p, uint32 v) { + memcpy(p, &v, sizeof v); +} + +inline void UNALIGNED_STORE64(void *p, uint64 v) { + memcpy(p, &v, sizeof v); +} + +#endif + +// This can be more efficient than UNALIGNED_LOAD64 + UNALIGNED_STORE64 +// on some platforms, in particular ARM. +inline void UnalignedCopy64(const void *src, void *dst) { + if (sizeof(void *) == 8) { + UNALIGNED_STORE64(dst, UNALIGNED_LOAD64(src)); + } else { + const char *src_char = reinterpret_cast<const char *>(src); + char *dst_char = reinterpret_cast<char *>(dst); + + UNALIGNED_STORE32(dst_char, UNALIGNED_LOAD32(src_char)); + UNALIGNED_STORE32(dst_char + 4, UNALIGNED_LOAD32(src_char + 4)); + } +} + +// The following guarantees declaration of the byte swap functions. +#ifdef WORDS_BIGENDIAN + +#ifdef HAVE_SYS_BYTEORDER_H +#include <sys/byteorder.h> +#endif + +#ifdef HAVE_SYS_ENDIAN_H +#include <sys/endian.h> +#endif + +#ifdef _MSC_VER +#include <stdlib.h> +#define bswap_16(x) _byteswap_ushort(x) +#define bswap_32(x) _byteswap_ulong(x) +#define bswap_64(x) _byteswap_uint64(x) + +#elif defined(__APPLE__) +// Mac OS X / Darwin features +#include <libkern/OSByteOrder.h> +#define bswap_16(x) OSSwapInt16(x) +#define bswap_32(x) OSSwapInt32(x) +#define bswap_64(x) OSSwapInt64(x) + +#elif defined(HAVE_BYTESWAP_H) +#include <byteswap.h> + +#elif defined(bswap32) +// FreeBSD defines bswap{16,32,64} in <sys/endian.h> (already #included). +#define bswap_16(x) bswap16(x) +#define bswap_32(x) bswap32(x) +#define bswap_64(x) bswap64(x) + +#elif defined(BSWAP_64) +// Solaris 10 defines BSWAP_{16,32,64} in <sys/byteorder.h> (already #included). +#define bswap_16(x) BSWAP_16(x) +#define bswap_32(x) BSWAP_32(x) +#define bswap_64(x) BSWAP_64(x) + +#else + +inline uint16 bswap_16(uint16 x) { + return (x << 8) | (x >> 8); +} + +inline uint32 bswap_32(uint32 x) { + x = ((x & 0xff00ff00UL) >> 8) | ((x & 0x00ff00ffUL) << 8); + return (x >> 16) | (x << 16); +} + +inline uint64 bswap_64(uint64 x) { + x = ((x & 0xff00ff00ff00ff00ULL) >> 8) | ((x & 0x00ff00ff00ff00ffULL) << 8); + x = ((x & 0xffff0000ffff0000ULL) >> 16) | ((x & 0x0000ffff0000ffffULL) << 16); + return (x >> 32) | (x << 32); +} + +#endif + +#endif // WORDS_BIGENDIAN + +// Convert to little-endian storage, opposite of network format. +// Convert x from host to little endian: x = LittleEndian.FromHost(x); +// convert x from little endian to host: x = LittleEndian.ToHost(x); +// +// Store values into unaligned memory converting to little endian order: +// LittleEndian.Store16(p, x); +// +// Load unaligned values stored in little endian converting to host order: +// x = LittleEndian.Load16(p); +class LittleEndian { + public: + // Conversion functions. +#ifdef WORDS_BIGENDIAN + + static uint16 FromHost16(uint16 x) { return bswap_16(x); } + static uint16 ToHost16(uint16 x) { return bswap_16(x); } + + static uint32 FromHost32(uint32 x) { return bswap_32(x); } + static uint32 ToHost32(uint32 x) { return bswap_32(x); } + + static bool IsLittleEndian() { return false; } + +#else // !defined(WORDS_BIGENDIAN) + + static uint16 FromHost16(uint16 x) { return x; } + static uint16 ToHost16(uint16 x) { return x; } + + static uint32 FromHost32(uint32 x) { return x; } + static uint32 ToHost32(uint32 x) { return x; } + + static bool IsLittleEndian() { return true; } + +#endif // !defined(WORDS_BIGENDIAN) + + // Functions to do unaligned loads and stores in little-endian order. + static uint16 Load16(const void *p) { + return ToHost16(UNALIGNED_LOAD16(p)); + } + + static void Store16(void *p, uint16 v) { + UNALIGNED_STORE16(p, FromHost16(v)); + } + + static uint32 Load32(const void *p) { + return ToHost32(UNALIGNED_LOAD32(p)); + } + + static void Store32(void *p, uint32 v) { + UNALIGNED_STORE32(p, FromHost32(v)); + } +}; + +// Some bit-manipulation functions. +class Bits { + public: + // Return floor(log2(n)) for positive integer n. Returns -1 iff n == 0. + static int Log2Floor(uint32 n); + + // Return the first set least / most significant bit, 0-indexed. Returns an + // undefined value if n == 0. FindLSBSetNonZero() is similar to ffs() except + // that it's 0-indexed. + static int FindLSBSetNonZero(uint32 n); + static int FindLSBSetNonZero64(uint64 n); + + private: + DISALLOW_COPY_AND_ASSIGN(Bits); +}; + +#ifdef HAVE_BUILTIN_CTZ + +inline int Bits::Log2Floor(uint32 n) { + return n == 0 ? -1 : 31 ^ __builtin_clz(n); +} + +inline int Bits::FindLSBSetNonZero(uint32 n) { + return __builtin_ctz(n); +} + +inline int Bits::FindLSBSetNonZero64(uint64 n) { + return __builtin_ctzll(n); +} + +#else // Portable versions. + +inline int Bits::Log2Floor(uint32 n) { + if (n == 0) + return -1; + int log = 0; + uint32 value = n; + for (int i = 4; i >= 0; --i) { + int shift = (1 << i); + uint32 x = value >> shift; + if (x != 0) { + value = x; + log += shift; + } + } + assert(value == 1); + return log; +} + +inline int Bits::FindLSBSetNonZero(uint32 n) { + int rc = 31; + for (int i = 4, shift = 1 << 4; i >= 0; --i) { + const uint32 x = n << shift; + if (x != 0) { + n = x; + rc -= shift; + } + shift >>= 1; + } + return rc; +} + +// FindLSBSetNonZero64() is defined in terms of FindLSBSetNonZero(). +inline int Bits::FindLSBSetNonZero64(uint64 n) { + const uint32 bottombits = static_cast<uint32>(n); + if (bottombits == 0) { + // Bottom bits are zero, so scan in top bits + return 32 + FindLSBSetNonZero(static_cast<uint32>(n >> 32)); + } else { + return FindLSBSetNonZero(bottombits); + } +} + +#endif // End portable versions. + +// Variable-length integer encoding. +class Varint { + public: + // Maximum lengths of varint encoding of uint32. + static const int kMax32 = 5; + + // Attempts to parse a varint32 from a prefix of the bytes in [ptr,limit-1]. + // Never reads a character at or beyond limit. If a valid/terminated varint32 + // was found in the range, stores it in *OUTPUT and returns a pointer just + // past the last byte of the varint32. Else returns NULL. On success, + // "result <= limit". + static const char* Parse32WithLimit(const char* ptr, const char* limit, + uint32* OUTPUT); + + // REQUIRES "ptr" points to a buffer of length sufficient to hold "v". + // EFFECTS Encodes "v" into "ptr" and returns a pointer to the + // byte just past the last encoded byte. + static char* Encode32(char* ptr, uint32 v); + + // EFFECTS Appends the varint representation of "value" to "*s". + static void Append32(string* s, uint32 value); +}; + +inline const char* Varint::Parse32WithLimit(const char* p, + const char* l, + uint32* OUTPUT) { + const unsigned char* ptr = reinterpret_cast<const unsigned char*>(p); + const unsigned char* limit = reinterpret_cast<const unsigned char*>(l); + uint32 b, result; + if (ptr >= limit) return NULL; + b = *(ptr++); result = b & 127; if (b < 128) goto done; + if (ptr >= limit) return NULL; + b = *(ptr++); result |= (b & 127) << 7; if (b < 128) goto done; + if (ptr >= limit) return NULL; + b = *(ptr++); result |= (b & 127) << 14; if (b < 128) goto done; + if (ptr >= limit) return NULL; + b = *(ptr++); result |= (b & 127) << 21; if (b < 128) goto done; + if (ptr >= limit) return NULL; + b = *(ptr++); result |= (b & 127) << 28; if (b < 16) goto done; + return NULL; // Value is too long to be a varint32 + done: + *OUTPUT = result; + return reinterpret_cast<const char*>(ptr); +} + +inline char* Varint::Encode32(char* sptr, uint32 v) { + // Operate on characters as unsigneds + unsigned char* ptr = reinterpret_cast<unsigned char*>(sptr); + static const int B = 128; + if (v < (1<<7)) { + *(ptr++) = v; + } else if (v < (1<<14)) { + *(ptr++) = v | B; + *(ptr++) = v>>7; + } else if (v < (1<<21)) { + *(ptr++) = v | B; + *(ptr++) = (v>>7) | B; + *(ptr++) = v>>14; + } else if (v < (1<<28)) { + *(ptr++) = v | B; + *(ptr++) = (v>>7) | B; + *(ptr++) = (v>>14) | B; + *(ptr++) = v>>21; + } else { + *(ptr++) = v | B; + *(ptr++) = (v>>7) | B; + *(ptr++) = (v>>14) | B; + *(ptr++) = (v>>21) | B; + *(ptr++) = v>>28; + } + return reinterpret_cast<char*>(ptr); +} + +// If you know the internal layout of the std::string in use, you can +// replace this function with one that resizes the string without +// filling the new space with zeros (if applicable) -- +// it will be non-portable but faster. +inline void STLStringResizeUninitialized(string* s, size_t new_size) { + s->resize(new_size); +} + +// Return a mutable char* pointing to a string's internal buffer, +// which may not be null-terminated. Writing through this pointer will +// modify the string. +// +// string_as_array(&str)[i] is valid for 0 <= i < str.size() until the +// next call to a string method that invalidates iterators. +// +// As of 2006-04, there is no standard-blessed way of getting a +// mutable reference to a string's internal buffer. However, issue 530 +// (http://www.open-std.org/JTC1/SC22/WG21/docs/lwg-defects.html#530) +// proposes this as the method. It will officially be part of the standard +// for C++0x. This should already work on all current implementations. +inline char* string_as_array(string* str) { + return str->empty() ? NULL : &*str->begin(); +} + +} // namespace snappy + +#endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_ diff --git a/tier1/snappy.cpp b/tier1/snappy.cpp new file mode 100644 index 0000000..72e26e0 --- /dev/null +++ b/tier1/snappy.cpp @@ -0,0 +1,1319 @@ +// Copyright 2005 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "snappy.h" +#include "snappy-internal.h" +#include "snappy-sinksource.h" + +#include <stdio.h> + +#include <algorithm> +#include <string> +#include <vector> + +#ifdef _WIN32 + +#pragma warning(disable:4018) // warning C4018: '<' : signed/unsigned mismatch +#pragma warning(disable:4389) // warning C4389: '==' : signed/unsigned mismatch + +/* Define like size_t, omitting the "unsigned" */ +#ifdef _WIN64 +typedef __int64 ssize_t; +#else +typedef int ssize_t; +#endif + +#endif //_WIN32 + +namespace snappy { + +// Any hash function will produce a valid compressed bitstream, but a good +// hash function reduces the number of collisions and thus yields better +// compression for compressible input, and more speed for incompressible +// input. Of course, it doesn't hurt if the hash function is reasonably fast +// either, as it gets called a lot. +static inline uint32 HashBytes(uint32 bytes, int shift) { + uint32 kMul = 0x1e35a7bd; + return (bytes * kMul) >> shift; +} +static inline uint32 Hash(const char* p, int shift) { + return HashBytes(UNALIGNED_LOAD32(p), shift); +} + +size_t MaxCompressedLength(size_t source_len) { + // Compressed data can be defined as: + // compressed := item* literal* + // item := literal* copy + // + // The trailing literal sequence has a space blowup of at most 62/60 + // since a literal of length 60 needs one tag byte + one extra byte + // for length information. + // + // Item blowup is trickier to measure. Suppose the "copy" op copies + // 4 bytes of data. Because of a special check in the encoding code, + // we produce a 4-byte copy only if the offset is < 65536. Therefore + // the copy op takes 3 bytes to encode, and this type of item leads + // to at most the 62/60 blowup for representing literals. + // + // Suppose the "copy" op copies 5 bytes of data. If the offset is big + // enough, it will take 5 bytes to encode the copy op. Therefore the + // worst case here is a one-byte literal followed by a five-byte copy. + // I.e., 6 bytes of input turn into 7 bytes of "compressed" data. + // + // This last factor dominates the blowup, so the final estimate is: + return 32 + source_len + source_len/6; +} + +enum { + LITERAL = 0, + COPY_1_BYTE_OFFSET = 1, // 3 bit length + 3 bits of offset in opcode + COPY_2_BYTE_OFFSET = 2, + COPY_4_BYTE_OFFSET = 3 +}; +static const int kMaximumTagLength = 5; // COPY_4_BYTE_OFFSET plus the actual offset. + +// Copy "len" bytes from "src" to "op", one byte at a time. Used for +// handling COPY operations where the input and output regions may +// overlap. For example, suppose: +// src == "ab" +// op == src + 2 +// len == 20 +// After IncrementalCopy(src, op, len), the result will have +// eleven copies of "ab" +// ababababababababababab +// Note that this does not match the semantics of either memcpy() +// or memmove(). +static inline void IncrementalCopy(const char* src, char* op, ssize_t len) { + assert(len > 0); + do { + *op++ = *src++; + } while (--len > 0); +} + +// Equivalent to IncrementalCopy except that it can write up to ten extra +// bytes after the end of the copy, and that it is faster. +// +// The main part of this loop is a simple copy of eight bytes at a time until +// we've copied (at least) the requested amount of bytes. However, if op and +// src are less than eight bytes apart (indicating a repeating pattern of +// length < 8), we first need to expand the pattern in order to get the correct +// results. For instance, if the buffer looks like this, with the eight-byte +// <src> and <op> patterns marked as intervals: +// +// abxxxxxxxxxxxx +// [------] src +// [------] op +// +// a single eight-byte copy from <src> to <op> will repeat the pattern once, +// after which we can move <op> two bytes without moving <src>: +// +// ababxxxxxxxxxx +// [------] src +// [------] op +// +// and repeat the exercise until the two no longer overlap. +// +// This allows us to do very well in the special case of one single byte +// repeated many times, without taking a big hit for more general cases. +// +// The worst case of extra writing past the end of the match occurs when +// op - src == 1 and len == 1; the last copy will read from byte positions +// [0..7] and write to [4..11], whereas it was only supposed to write to +// position 1. Thus, ten excess bytes. + +namespace { + +const int kMaxIncrementCopyOverflow = 10; + +inline void IncrementalCopyFastPath(const char* src, char* op, ssize_t len) { + while (op - src < 8) { + UnalignedCopy64(src, op); + len -= op - src; + op += op - src; + } + while (len > 0) { + UnalignedCopy64(src, op); + src += 8; + op += 8; + len -= 8; + } +} + +} // namespace + +static inline char* EmitLiteral(char* op, + const char* literal, + int len, + bool allow_fast_path) { + int n = len - 1; // Zero-length literals are disallowed + if (n < 60) { + // Fits in tag byte + *op++ = LITERAL | (n << 2); + + // The vast majority of copies are below 16 bytes, for which a + // call to memcpy is overkill. This fast path can sometimes + // copy up to 15 bytes too much, but that is okay in the + // main loop, since we have a bit to go on for both sides: + // + // - The input will always have kInputMarginBytes = 15 extra + // available bytes, as long as we're in the main loop, and + // if not, allow_fast_path = false. + // - The output will always have 32 spare bytes (see + // MaxCompressedLength). + if (allow_fast_path && len <= 16) { + UnalignedCopy64(literal, op); + UnalignedCopy64(literal + 8, op + 8); + return op + len; + } + } else { + // Encode in upcoming bytes + char* base = op; + int count = 0; + op++; + while (n > 0) { + *op++ = n & 0xff; + n >>= 8; + count++; + } + assert(count >= 1); + assert(count <= 4); + *base = LITERAL | ((59+count) << 2); + } + memcpy(op, literal, len); + return op + len; +} + +static inline char* EmitCopyLessThan64(char* op, size_t offset, int len) { + assert(len <= 64); + assert(len >= 4); + assert(offset < 65536); + + if ((len < 12) && (offset < 2048)) { + size_t len_minus_4 = len - 4; + assert(len_minus_4 < 8); // Must fit in 3 bits + *op++ = (char)(COPY_1_BYTE_OFFSET + ((len_minus_4) << 2) + ((offset >> 8) << 5)); + *op++ = offset & 0xff; + } else { + *op++ = COPY_2_BYTE_OFFSET + ((len-1) << 2); + LittleEndian::Store16(op, (snappy::uint16)offset); + op += 2; + } + return op; +} + +static inline char* EmitCopy(char* op, size_t offset, int len) { + // Emit 64 byte copies but make sure to keep at least four bytes reserved + while (len >= 68) { + op = EmitCopyLessThan64(op, offset, 64); + len -= 64; + } + + // Emit an extra 60 byte copy if have too much data to fit in one copy + if (len > 64) { + op = EmitCopyLessThan64(op, offset, 60); + len -= 60; + } + + // Emit remainder + op = EmitCopyLessThan64(op, offset, len); + return op; +} + + +bool GetUncompressedLength(const char* start, size_t n, size_t* result) { + uint32 v = 0; + const char* limit = start + n; + if (Varint::Parse32WithLimit(start, limit, &v) != NULL) { + *result = v; + return true; + } else { + return false; + } +} + +namespace internal { +uint16* WorkingMemory::GetHashTable(size_t input_size, int* table_size) { + // Use smaller hash table when input.size() is smaller, since we + // fill the table, incurring O(hash table size) overhead for + // compression, and if the input is short, we won't need that + // many hash table entries anyway. + assert(kMaxHashTableSize >= 256); + size_t htsize = 256; + while (htsize < kMaxHashTableSize && htsize < input_size) { + htsize <<= 1; + } + + uint16* table; + if (htsize <= ARRAYSIZE(small_table_)) { + table = small_table_; + } else { + if (large_table_ == NULL) { + large_table_ = new uint16[kMaxHashTableSize]; + } + table = large_table_; + } + + *table_size = (int)htsize; + memset(table, 0, htsize * sizeof(*table)); + return table; +} +} // end namespace internal + +// For 0 <= offset <= 4, GetUint32AtOffset(GetEightBytesAt(p), offset) will +// equal UNALIGNED_LOAD32(p + offset). Motivation: On x86-64 hardware we have +// empirically found that overlapping loads such as +// UNALIGNED_LOAD32(p) ... UNALIGNED_LOAD32(p+1) ... UNALIGNED_LOAD32(p+2) +// are slower than UNALIGNED_LOAD64(p) followed by shifts and casts to uint32. +// +// We have different versions for 64- and 32-bit; ideally we would avoid the +// two functions and just inline the UNALIGNED_LOAD64 call into +// GetUint32AtOffset, but GCC (at least not as of 4.6) is seemingly not clever +// enough to avoid loading the value multiple times then. For 64-bit, the load +// is done when GetEightBytesAt() is called, whereas for 32-bit, the load is +// done at GetUint32AtOffset() time. + +#ifdef ARCH_K8 + +typedef uint64 EightBytesReference; + +static inline EightBytesReference GetEightBytesAt(const char* ptr) { + return UNALIGNED_LOAD64(ptr); +} + +static inline uint32 GetUint32AtOffset(uint64 v, int offset) { + assert(offset >= 0); + assert(offset <= 4); + return v >> (LittleEndian::IsLittleEndian() ? 8 * offset : 32 - 8 * offset); +} + +#else + +typedef const char* EightBytesReference; + +static inline EightBytesReference GetEightBytesAt(const char* ptr) { + return ptr; +} + +static inline uint32 GetUint32AtOffset(const char* v, int offset) { + assert(offset >= 0); + assert(offset <= 4); + return UNALIGNED_LOAD32(v + offset); +} + +#endif + +// Flat array compression that does not emit the "uncompressed length" +// prefix. Compresses "input" string to the "*op" buffer. +// +// REQUIRES: "input" is at most "kBlockSize" bytes long. +// REQUIRES: "op" points to an array of memory that is at least +// "MaxCompressedLength(input.size())" in size. +// REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. +// REQUIRES: "table_size" is a power of two +// +// Returns an "end" pointer into "op" buffer. +// "end - op" is the compressed size of "input". +namespace internal { +char* CompressFragment(const char* input, + size_t input_size, + char* op, + uint16* table, + const int table_size) { + // "ip" is the input pointer, and "op" is the output pointer. + const char* ip = input; + assert(input_size <= kBlockSize); + assert((table_size & (table_size - 1)) == 0); // table must be power of two + const int shift = 32 - Bits::Log2Floor(table_size); + assert(static_cast<int>(kuint32max >> shift) == table_size - 1); + const char* ip_end = input + input_size; + const char* base_ip = ip; + // Bytes in [next_emit, ip) will be emitted as literal bytes. Or + // [next_emit, ip_end) after the main loop. + const char* next_emit = ip; + + const size_t kInputMarginBytes = 15; + if (PREDICT_TRUE(input_size >= kInputMarginBytes)) { + const char* ip_limit = input + input_size - kInputMarginBytes; + + for (uint32 next_hash = Hash(++ip, shift); ; ) { + assert(next_emit < ip); + // The body of this loop calls EmitLiteral once and then EmitCopy one or + // more times. (The exception is that when we're close to exhausting + // the input we goto emit_remainder.) + // + // In the first iteration of this loop we're just starting, so + // there's nothing to copy, so calling EmitLiteral once is + // necessary. And we only start a new iteration when the + // current iteration has determined that a call to EmitLiteral will + // precede the next call to EmitCopy (if any). + // + // Step 1: Scan forward in the input looking for a 4-byte-long match. + // If we get close to exhausting the input then goto emit_remainder. + // + // Heuristic match skipping: If 32 bytes are scanned with no matches + // found, start looking only at every other byte. If 32 more bytes are + // scanned, look at every third byte, etc.. When a match is found, + // immediately go back to looking at every byte. This is a small loss + // (~5% performance, ~0.1% density) for compressible data due to more + // bookkeeping, but for non-compressible data (such as JPEG) it's a huge + // win since the compressor quickly "realizes" the data is incompressible + // and doesn't bother looking for matches everywhere. + // + // The "skip" variable keeps track of how many bytes there are since the + // last match; dividing it by 32 (ie. right-shifting by five) gives the + // number of bytes to move ahead for each iteration. + uint32 skip = 32; + + const char* next_ip = ip; + const char* candidate; + do { + ip = next_ip; + uint32 hash = next_hash; + assert(hash == Hash(ip, shift)); + uint32 bytes_between_hash_lookups = skip++ >> 5; + next_ip = ip + bytes_between_hash_lookups; + if (PREDICT_FALSE(next_ip > ip_limit)) { + goto emit_remainder; + } + next_hash = Hash(next_ip, shift); + candidate = base_ip + table[hash]; + assert(candidate >= base_ip); + assert(candidate < ip); + + table[hash] = ip - base_ip; + } while (PREDICT_TRUE(UNALIGNED_LOAD32(ip) != + UNALIGNED_LOAD32(candidate))); + + // Step 2: A 4-byte match has been found. We'll later see if more + // than 4 bytes match. But, prior to the match, input + // bytes [next_emit, ip) are unmatched. Emit them as "literal bytes." + assert(next_emit + 16 <= ip_end); + op = EmitLiteral(op, next_emit, ip - next_emit, true); + + // Step 3: Call EmitCopy, and then see if another EmitCopy could + // be our next move. Repeat until we find no match for the + // input immediately after what was consumed by the last EmitCopy call. + // + // If we exit this loop normally then we need to call EmitLiteral next, + // though we don't yet know how big the literal will be. We handle that + // by proceeding to the next iteration of the main loop. We also can exit + // this loop via goto if we get close to exhausting the input. + EightBytesReference input_bytes; + uint32 candidate_bytes = 0; + + do { + // We have a 4-byte match at ip, and no need to emit any + // "literal bytes" prior to ip. + const char* base = ip; + int matched = 4 + FindMatchLength(candidate + 4, ip + 4, ip_end); + ip += matched; + size_t offset = base - candidate; + assert(0 == memcmp(base, candidate, matched)); + op = EmitCopy(op, offset, matched); + // We could immediately start working at ip now, but to improve + // compression we first update table[Hash(ip - 1, ...)]. + const char* insert_tail = ip - 1; + next_emit = ip; + if (PREDICT_FALSE(ip >= ip_limit)) { + goto emit_remainder; + } + input_bytes = GetEightBytesAt(insert_tail); + uint32 prev_hash = HashBytes(GetUint32AtOffset(input_bytes, 0), shift); + table[prev_hash] = ip - base_ip - 1; + uint32 cur_hash = HashBytes(GetUint32AtOffset(input_bytes, 1), shift); + candidate = base_ip + table[cur_hash]; + candidate_bytes = UNALIGNED_LOAD32(candidate); + table[cur_hash] = ip - base_ip; + } while (GetUint32AtOffset(input_bytes, 1) == candidate_bytes); + + next_hash = HashBytes(GetUint32AtOffset(input_bytes, 2), shift); + ++ip; + } + } + + emit_remainder: + // Emit the remaining bytes as a literal + if (next_emit < ip_end) { + op = EmitLiteral(op, next_emit, ip_end - next_emit, false); + } + + return op; +} +} // end namespace internal + +// Signature of output types needed by decompression code. +// The decompression code is templatized on a type that obeys this +// signature so that we do not pay virtual function call overhead in +// the middle of a tight decompression loop. +// +// class DecompressionWriter { +// public: +// // Called before decompression +// void SetExpectedLength(size_t length); +// +// // Called after decompression +// bool CheckLength() const; +// +// // Called repeatedly during decompression +// bool Append(const char* ip, size_t length); +// bool AppendFromSelf(uint32 offset, size_t length); +// +// // The rules for how TryFastAppend differs from Append are somewhat +// // convoluted: +// // +// // - TryFastAppend is allowed to decline (return false) at any +// // time, for any reason -- just "return false" would be +// // a perfectly legal implementation of TryFastAppend. +// // The intention is for TryFastAppend to allow a fast path +// // in the common case of a small append. +// // - TryFastAppend is allowed to read up to <available> bytes +// // from the input buffer, whereas Append is allowed to read +// // <length>. However, if it returns true, it must leave +// // at least five (kMaximumTagLength) bytes in the input buffer +// // afterwards, so that there is always enough space to read the +// // next tag without checking for a refill. +// // - TryFastAppend must always return decline (return false) +// // if <length> is 61 or more, as in this case the literal length is not +// // decoded fully. In practice, this should not be a big problem, +// // as it is unlikely that one would implement a fast path accepting +// // this much data. +// // +// bool TryFastAppend(const char* ip, size_t available, size_t length); +// }; + +// ----------------------------------------------------------------------- +// Lookup table for decompression code. Generated by ComputeTable() below. +// ----------------------------------------------------------------------- + +// Mapping from i in range [0,4] to a mask to extract the bottom 8*i bits +static const uint32 wordmask[] = { + 0u, 0xffu, 0xffffu, 0xffffffu, 0xffffffffu +}; + +// Data stored per entry in lookup table: +// Range Bits-used Description +// ------------------------------------ +// 1..64 0..7 Literal/copy length encoded in opcode byte +// 0..7 8..10 Copy offset encoded in opcode byte / 256 +// 0..4 11..13 Extra bytes after opcode +// +// We use eight bits for the length even though 7 would have sufficed +// because of efficiency reasons: +// (1) Extracting a byte is faster than a bit-field +// (2) It properly aligns copy offset so we do not need a <<8 +static const uint16 char_table[256] = { + 0x0001, 0x0804, 0x1001, 0x2001, 0x0002, 0x0805, 0x1002, 0x2002, + 0x0003, 0x0806, 0x1003, 0x2003, 0x0004, 0x0807, 0x1004, 0x2004, + 0x0005, 0x0808, 0x1005, 0x2005, 0x0006, 0x0809, 0x1006, 0x2006, + 0x0007, 0x080a, 0x1007, 0x2007, 0x0008, 0x080b, 0x1008, 0x2008, + 0x0009, 0x0904, 0x1009, 0x2009, 0x000a, 0x0905, 0x100a, 0x200a, + 0x000b, 0x0906, 0x100b, 0x200b, 0x000c, 0x0907, 0x100c, 0x200c, + 0x000d, 0x0908, 0x100d, 0x200d, 0x000e, 0x0909, 0x100e, 0x200e, + 0x000f, 0x090a, 0x100f, 0x200f, 0x0010, 0x090b, 0x1010, 0x2010, + 0x0011, 0x0a04, 0x1011, 0x2011, 0x0012, 0x0a05, 0x1012, 0x2012, + 0x0013, 0x0a06, 0x1013, 0x2013, 0x0014, 0x0a07, 0x1014, 0x2014, + 0x0015, 0x0a08, 0x1015, 0x2015, 0x0016, 0x0a09, 0x1016, 0x2016, + 0x0017, 0x0a0a, 0x1017, 0x2017, 0x0018, 0x0a0b, 0x1018, 0x2018, + 0x0019, 0x0b04, 0x1019, 0x2019, 0x001a, 0x0b05, 0x101a, 0x201a, + 0x001b, 0x0b06, 0x101b, 0x201b, 0x001c, 0x0b07, 0x101c, 0x201c, + 0x001d, 0x0b08, 0x101d, 0x201d, 0x001e, 0x0b09, 0x101e, 0x201e, + 0x001f, 0x0b0a, 0x101f, 0x201f, 0x0020, 0x0b0b, 0x1020, 0x2020, + 0x0021, 0x0c04, 0x1021, 0x2021, 0x0022, 0x0c05, 0x1022, 0x2022, + 0x0023, 0x0c06, 0x1023, 0x2023, 0x0024, 0x0c07, 0x1024, 0x2024, + 0x0025, 0x0c08, 0x1025, 0x2025, 0x0026, 0x0c09, 0x1026, 0x2026, + 0x0027, 0x0c0a, 0x1027, 0x2027, 0x0028, 0x0c0b, 0x1028, 0x2028, + 0x0029, 0x0d04, 0x1029, 0x2029, 0x002a, 0x0d05, 0x102a, 0x202a, + 0x002b, 0x0d06, 0x102b, 0x202b, 0x002c, 0x0d07, 0x102c, 0x202c, + 0x002d, 0x0d08, 0x102d, 0x202d, 0x002e, 0x0d09, 0x102e, 0x202e, + 0x002f, 0x0d0a, 0x102f, 0x202f, 0x0030, 0x0d0b, 0x1030, 0x2030, + 0x0031, 0x0e04, 0x1031, 0x2031, 0x0032, 0x0e05, 0x1032, 0x2032, + 0x0033, 0x0e06, 0x1033, 0x2033, 0x0034, 0x0e07, 0x1034, 0x2034, + 0x0035, 0x0e08, 0x1035, 0x2035, 0x0036, 0x0e09, 0x1036, 0x2036, + 0x0037, 0x0e0a, 0x1037, 0x2037, 0x0038, 0x0e0b, 0x1038, 0x2038, + 0x0039, 0x0f04, 0x1039, 0x2039, 0x003a, 0x0f05, 0x103a, 0x203a, + 0x003b, 0x0f06, 0x103b, 0x203b, 0x003c, 0x0f07, 0x103c, 0x203c, + 0x0801, 0x0f08, 0x103d, 0x203d, 0x1001, 0x0f09, 0x103e, 0x203e, + 0x1801, 0x0f0a, 0x103f, 0x203f, 0x2001, 0x0f0b, 0x1040, 0x2040 +}; + +// In debug mode, allow optional computation of the table at startup. +// Also, check that the decompression table is correct. +#ifndef NDEBUG +DEFINE_bool(snappy_dump_decompression_table, false, + "If true, we print the decompression table at startup."); + +static uint16 MakeEntry(unsigned int extra, + unsigned int len, + unsigned int copy_offset) { + // Check that all of the fields fit within the allocated space + assert(extra == (extra & 0x7)); // At most 3 bits + assert(copy_offset == (copy_offset & 0x7)); // At most 3 bits + assert(len == (len & 0x7f)); // At most 7 bits + return len | (copy_offset << 8) | (extra << 11); +} + +static void ComputeTable() { + uint16 dst[256]; + + // Place invalid entries in all places to detect missing initialization + int assigned = 0; + for (int i = 0; i < 256; i++) { + dst[i] = 0xffff; + } + + // Small LITERAL entries. We store (len-1) in the top 6 bits. + for (unsigned int len = 1; len <= 60; len++) { + dst[LITERAL | ((len-1) << 2)] = MakeEntry(0, len, 0); + assigned++; + } + + // Large LITERAL entries. We use 60..63 in the high 6 bits to + // encode the number of bytes of length info that follow the opcode. + for (unsigned int extra_bytes = 1; extra_bytes <= 4; extra_bytes++) { + // We set the length field in the lookup table to 1 because extra + // bytes encode len-1. + dst[LITERAL | ((extra_bytes+59) << 2)] = MakeEntry(extra_bytes, 1, 0); + assigned++; + } + + // COPY_1_BYTE_OFFSET. + // + // The tag byte in the compressed data stores len-4 in 3 bits, and + // offset/256 in 5 bits. offset%256 is stored in the next byte. + // + // This format is used for length in range [4..11] and offset in + // range [0..2047] + for (unsigned int len = 4; len < 12; len++) { + for (unsigned int offset = 0; offset < 2048; offset += 256) { + dst[COPY_1_BYTE_OFFSET | ((len-4)<<2) | ((offset>>8)<<5)] = + MakeEntry(1, len, offset>>8); + assigned++; + } + } + + // COPY_2_BYTE_OFFSET. + // Tag contains len-1 in top 6 bits, and offset in next two bytes. + for (unsigned int len = 1; len <= 64; len++) { + dst[COPY_2_BYTE_OFFSET | ((len-1)<<2)] = MakeEntry(2, len, 0); + assigned++; + } + + // COPY_4_BYTE_OFFSET. + // Tag contents len-1 in top 6 bits, and offset in next four bytes. + for (unsigned int len = 1; len <= 64; len++) { + dst[COPY_4_BYTE_OFFSET | ((len-1)<<2)] = MakeEntry(4, len, 0); + assigned++; + } + + // Check that each entry was initialized exactly once. + if (assigned != 256) { + fprintf(stderr, "ComputeTable: assigned only %d of 256\n", assigned); + abort(); + } + for (int i = 0; i < 256; i++) { + if (dst[i] == 0xffff) { + fprintf(stderr, "ComputeTable: did not assign byte %d\n", i); + abort(); + } + } + + if (FLAGS_snappy_dump_decompression_table) { + printf("static const uint16 char_table[256] = {\n "); + for (int i = 0; i < 256; i++) { + printf("0x%04x%s", + dst[i], + ((i == 255) ? "\n" : (((i%8) == 7) ? ",\n " : ", "))); + } + printf("};\n"); + } + + // Check that computed table matched recorded table + for (int i = 0; i < 256; i++) { + if (dst[i] != char_table[i]) { + fprintf(stderr, "ComputeTable: byte %d: computed (%x), expect (%x)\n", + i, static_cast<int>(dst[i]), static_cast<int>(char_table[i])); + abort(); + } + } +} +#endif /* !NDEBUG */ + +// Helper class for decompression +class SnappyDecompressor { + private: + Source* reader_; // Underlying source of bytes to decompress + const char* ip_; // Points to next buffered byte + const char* ip_limit_; // Points just past buffered bytes + uint32 peeked_; // Bytes peeked from reader (need to skip) + bool eof_; // Hit end of input without an error? + char scratch_[kMaximumTagLength]; // See RefillTag(). + + // Ensure that all of the tag metadata for the next tag is available + // in [ip_..ip_limit_-1]. Also ensures that [ip,ip+4] is readable even + // if (ip_limit_ - ip_ < 5). + // + // Returns true on success, false on error or end of input. + bool RefillTag(); + + public: + explicit SnappyDecompressor(Source* reader) + : reader_(reader), + ip_(NULL), + ip_limit_(NULL), + peeked_(0), + eof_(false) { + } + + ~SnappyDecompressor() { + // Advance past any bytes we peeked at from the reader + reader_->Skip(peeked_); + } + + // Returns true iff we have hit the end of the input without an error. + bool eof() const { + return eof_; + } + + // Read the uncompressed length stored at the start of the compressed data. + // On succcess, stores the length in *result and returns true. + // On failure, returns false. + bool ReadUncompressedLength(uint32* result) { + assert(ip_ == NULL); // Must not have read anything yet + // Length is encoded in 1..5 bytes + *result = 0; + uint32 shift = 0; + while (true) { + if (shift >= 32) return false; + size_t n; + const char* ip = reader_->Peek(&n); + if (n == 0) return false; + const unsigned char c = *(reinterpret_cast<const unsigned char*>(ip)); + reader_->Skip(1); + *result |= static_cast<uint32>(c & 0x7f) << shift; + if (c < 128) { + break; + } + shift += 7; + } + return true; + } + + // Process the next item found in the input. + // Returns true if successful, false on error or end of input. + template <class Writer> + void DecompressAllTags(Writer* writer) { + const char* ip = ip_; + + // We could have put this refill fragment only at the beginning of the loop. + // However, duplicating it at the end of each branch gives the compiler more + // scope to optimize the <ip_limit_ - ip> expression based on the local + // context, which overall increases speed. + #define MAYBE_REFILL() \ + if (ip_limit_ - ip < kMaximumTagLength) { \ + ip_ = ip; \ + if (!RefillTag()) return; \ + ip = ip_; \ + } + + MAYBE_REFILL(); + for ( ;; ) { + const unsigned char c = *(reinterpret_cast<const unsigned char*>(ip++)); + + if ((c & 0x3) == LITERAL) { + size_t literal_length = (c >> 2) + 1u; + if (writer->TryFastAppend(ip, ip_limit_ - ip, literal_length)) { + assert(literal_length < 61); + ip += literal_length; + // NOTE(user): There is no MAYBE_REFILL() here, as TryFastAppend() + // will not return true unless there's already at least five spare + // bytes in addition to the literal. + continue; + } + if (PREDICT_FALSE(literal_length >= 61)) { + // Long literal. + const size_t literal_length_length = literal_length - 60; + literal_length = + (LittleEndian::Load32(ip) & wordmask[literal_length_length]) + 1; + ip += literal_length_length; + } + + size_t avail = ip_limit_ - ip; + while (avail < literal_length) { + if (!writer->Append(ip, avail)) return; + literal_length -= avail; + reader_->Skip(peeked_); + size_t n; + ip = reader_->Peek(&n); + avail = n; + peeked_ = (snappy::uint32)avail; + if (avail == 0) return; // Premature end of input + ip_limit_ = ip + avail; + } + if (!writer->Append(ip, literal_length)) { + return; + } + ip += literal_length; + MAYBE_REFILL(); + } else { + const uint32 entry = char_table[c]; + const uint32 trailer = LittleEndian::Load32(ip) & wordmask[entry >> 11]; + const uint32 length = entry & 0xff; + ip += entry >> 11; + + // copy_offset/256 is encoded in bits 8..10. By just fetching + // those bits, we get copy_offset (since the bit-field starts at + // bit 8). + const uint32 copy_offset = entry & 0x700; + if (!writer->AppendFromSelf(copy_offset + trailer, length)) { + return; + } + MAYBE_REFILL(); + } + } + +#undef MAYBE_REFILL + } +}; + +bool SnappyDecompressor::RefillTag() { + const char* ip = ip_; + if (ip == ip_limit_) { + // Fetch a new fragment from the reader + reader_->Skip(peeked_); // All peeked bytes are used up + size_t n; + ip = reader_->Peek(&n); + peeked_ = (snappy::uint32)n; + if (n == 0) { + eof_ = true; + return false; + } + ip_limit_ = ip + n; + } + + // Read the tag character + assert(ip < ip_limit_); + const unsigned char c = *(reinterpret_cast<const unsigned char*>(ip)); + const uint32 entry = char_table[c]; + const uint32 needed = (entry >> 11) + 1; // +1 byte for 'c' + assert(needed <= sizeof(scratch_)); + + // Read more bytes from reader if needed + uint32 nbuf = ip_limit_ - ip; + if (nbuf < needed) { + // Stitch together bytes from ip and reader to form the word + // contents. We store the needed bytes in "scratch_". They + // will be consumed immediately by the caller since we do not + // read more than we need. + memmove(scratch_, ip, nbuf); + reader_->Skip(peeked_); // All peeked bytes are used up + peeked_ = 0; + while (nbuf < needed) { + size_t length; + const char* src = reader_->Peek(&length); + if (length == 0) return false; + uint32 to_add = Min(needed - nbuf, (uint32)length); + memcpy(scratch_ + nbuf, src, to_add); + nbuf += to_add; + reader_->Skip(to_add); + } + assert(nbuf == needed); + ip_ = scratch_; + ip_limit_ = scratch_ + needed; + } else if (nbuf < kMaximumTagLength) { + // Have enough bytes, but move into scratch_ so that we do not + // read past end of input + memmove(scratch_, ip, nbuf); + reader_->Skip(peeked_); // All peeked bytes are used up + peeked_ = 0; + ip_ = scratch_; + ip_limit_ = scratch_ + nbuf; + } else { + // Pass pointer to buffer returned by reader_. + ip_ = ip; + } + return true; +} + +template <typename Writer> +static bool InternalUncompress(Source* r, Writer* writer) { + // Read the uncompressed length from the front of the compressed input + SnappyDecompressor decompressor(r); + uint32 uncompressed_len = 0; + if (!decompressor.ReadUncompressedLength(&uncompressed_len)) return false; + return InternalUncompressAllTags(&decompressor, writer, uncompressed_len); +} + +template <typename Writer> +static bool InternalUncompressAllTags(SnappyDecompressor* decompressor, + Writer* writer, + uint32 uncompressed_len) { + writer->SetExpectedLength(uncompressed_len); + + // Process the entire input + decompressor->DecompressAllTags(writer); + return (decompressor->eof() && writer->CheckLength()); +} + +bool GetUncompressedLength(Source* source, uint32* result) { + SnappyDecompressor decompressor(source); + return decompressor.ReadUncompressedLength(result); +} + +size_t Compress(Source* reader, Sink* writer) { + size_t written = 0; + size_t N = reader->Available(); + char ulength[Varint::kMax32]; + char* p = Varint::Encode32(ulength, (snappy::uint32)N); + writer->Append(ulength, p-ulength); + written += (p - ulength); + + internal::WorkingMemory wmem; + char* scratch = NULL; + char* scratch_output = NULL; + + while (N > 0) { + // Get next block to compress (without copying if possible) + size_t fragment_size; + const char* fragment = reader->Peek(&fragment_size); + assert(fragment_size != 0); // premature end of input + const size_t num_to_read = min(N, kBlockSize); + size_t bytes_read = fragment_size; + + size_t pending_advance = 0; + if (bytes_read >= num_to_read) { + // Buffer returned by reader is large enough + pending_advance = num_to_read; + fragment_size = num_to_read; + } else { + // Read into scratch buffer + if (scratch == NULL) { + // If this is the last iteration, we want to allocate N bytes + // of space, otherwise the max possible kBlockSize space. + // num_to_read contains exactly the correct value + scratch = new char[num_to_read]; + } + memcpy(scratch, fragment, bytes_read); + reader->Skip(bytes_read); + + while (bytes_read < num_to_read) { + fragment = reader->Peek(&fragment_size); + size_t n = Min(fragment_size, num_to_read - bytes_read); + memcpy(scratch + bytes_read, fragment, n); + bytes_read += n; + reader->Skip(n); + } + assert(bytes_read == num_to_read); + fragment = scratch; + fragment_size = num_to_read; + } + assert(fragment_size == num_to_read); + + // Get encoding table for compression + int table_size; + uint16* table = wmem.GetHashTable(num_to_read, &table_size); + + // Compress input_fragment and append to dest + const int max_output = (int)MaxCompressedLength(num_to_read); + + // Need a scratch buffer for the output, in case the byte sink doesn't + // have room for us directly. + if (scratch_output == NULL) { + scratch_output = new char[max_output]; + } else { + // Since we encode kBlockSize regions followed by a region + // which is <= kBlockSize in length, a previously allocated + // scratch_output[] region is big enough for this iteration. + } + char* dest = writer->GetAppendBuffer(max_output, scratch_output); + char* end = internal::CompressFragment(fragment, fragment_size, + dest, table, table_size); + writer->Append(dest, end - dest); + written += (end - dest); + + N -= num_to_read; + reader->Skip(pending_advance); + } + + delete[] scratch; + delete[] scratch_output; + + return written; +} + +// ----------------------------------------------------------------------- +// IOVec interfaces +// ----------------------------------------------------------------------- + +// A type that writes to an iovec. +// Note that this is not a "ByteSink", but a type that matches the +// Writer template argument to SnappyDecompressor::DecompressAllTags(). +class SnappyIOVecWriter { + private: + const struct iovec* output_iov_; + const size_t output_iov_count_; + + // We are currently writing into output_iov_[curr_iov_index_]. + int curr_iov_index_; + + // Bytes written to output_iov_[curr_iov_index_] so far. + size_t curr_iov_written_; + + // Total bytes decompressed into output_iov_ so far. + size_t total_written_; + + // Maximum number of bytes that will be decompressed into output_iov_. + size_t output_limit_; + + inline char* GetIOVecPointer(int index, size_t offset) { + return reinterpret_cast<char*>(output_iov_[index].iov_base) + + offset; + } + + public: + // Does not take ownership of iov. iov must be valid during the + // entire lifetime of the SnappyIOVecWriter. + inline SnappyIOVecWriter(const struct iovec* iov, size_t iov_count) + : output_iov_(iov), + output_iov_count_(iov_count), + curr_iov_index_(0), + curr_iov_written_(0), + total_written_(0), + output_limit_((size_t)-1) { + } + + inline void SetExpectedLength(size_t len) { + output_limit_ = len; + } + + inline bool CheckLength() const { + return total_written_ == output_limit_; + } + + inline bool Append(const char* ip, size_t len) { + if (total_written_ + len > output_limit_) { + return false; + } + + while (len > 0) { + assert(curr_iov_written_ <= output_iov_[curr_iov_index_].iov_len); + if (curr_iov_written_ >= output_iov_[curr_iov_index_].iov_len) { + // This iovec is full. Go to the next one. + if (curr_iov_index_ + 1 >= output_iov_count_) { + return false; + } + curr_iov_written_ = 0; + ++curr_iov_index_; + } + + const size_t to_write = Min( + len, output_iov_[curr_iov_index_].iov_len - curr_iov_written_); + memcpy(GetIOVecPointer(curr_iov_index_, curr_iov_written_), + ip, + to_write); + curr_iov_written_ += to_write; + total_written_ += to_write; + ip += to_write; + len -= to_write; + } + + return true; + } + + inline bool TryFastAppend(const char* ip, size_t available, size_t len) { + const size_t space_left = output_limit_ - total_written_; + if (len <= 16 && available >= 16 + kMaximumTagLength && space_left >= 16 && + output_iov_[curr_iov_index_].iov_len - curr_iov_written_ >= 16) { + // Fast path, used for the majority (about 95%) of invocations. + char* ptr = GetIOVecPointer(curr_iov_index_, curr_iov_written_); + UnalignedCopy64(ip, ptr); + UnalignedCopy64(ip + 8, ptr + 8); + curr_iov_written_ += len; + total_written_ += len; + return true; + } + + return false; + } + + inline bool AppendFromSelf(size_t offset, size_t len) { + if (offset > total_written_ || offset == 0) { + return false; + } + const size_t space_left = output_limit_ - total_written_; + if (len > space_left) { + return false; + } + + // Locate the iovec from which we need to start the copy. + int from_iov_index = curr_iov_index_; + size_t from_iov_offset = curr_iov_written_; + while (offset > 0) { + if (from_iov_offset >= offset) { + from_iov_offset -= offset; + break; + } + + offset -= from_iov_offset; + --from_iov_index; + assert(from_iov_index >= 0); + from_iov_offset = output_iov_[from_iov_index].iov_len; + } + + // Copy <len> bytes starting from the iovec pointed to by from_iov_index to + // the current iovec. + while (len > 0) { + assert(from_iov_index <= curr_iov_index_); + if (from_iov_index != curr_iov_index_) { + const size_t to_copy = Min( + output_iov_[from_iov_index].iov_len - from_iov_offset, + len); + Append(GetIOVecPointer(from_iov_index, from_iov_offset), to_copy); + len -= to_copy; + if (len > 0) { + ++from_iov_index; + from_iov_offset = 0; + } + } else { + assert(curr_iov_written_ <= output_iov_[curr_iov_index_].iov_len); + size_t to_copy = Min(output_iov_[curr_iov_index_].iov_len - + curr_iov_written_, + len); + if (to_copy == 0) { + // This iovec is full. Go to the next one. + if (curr_iov_index_ + 1 >= output_iov_count_) { + return false; + } + ++curr_iov_index_; + curr_iov_written_ = 0; + continue; + } + if (to_copy > len) { + to_copy = len; + } + IncrementalCopy(GetIOVecPointer(from_iov_index, from_iov_offset), + GetIOVecPointer(curr_iov_index_, curr_iov_written_), + to_copy); + curr_iov_written_ += to_copy; + from_iov_offset += to_copy; + total_written_ += to_copy; + len -= to_copy; + } + } + + return true; + } + +}; + +bool RawUncompressToIOVec(const char* compressed, size_t compressed_length, + const struct iovec* iov, size_t iov_cnt) { + ByteArraySource reader(compressed, compressed_length); + return RawUncompressToIOVec(&reader, iov, iov_cnt); +} + +bool RawUncompressToIOVec(Source* compressed, const struct iovec* iov, + size_t iov_cnt) { + SnappyIOVecWriter output(iov, iov_cnt); + return InternalUncompress(compressed, &output); +} + +// ----------------------------------------------------------------------- +// Flat array interfaces +// ----------------------------------------------------------------------- + +// A type that writes to a flat array. +// Note that this is not a "ByteSink", but a type that matches the +// Writer template argument to SnappyDecompressor::DecompressAllTags(). +class SnappyArrayWriter { + private: + char* base_; + char* op_; + char* op_limit_; + + public: + inline explicit SnappyArrayWriter(char* dst) + : base_(dst), + op_(dst) { + } + + inline void SetExpectedLength(size_t len) { + op_limit_ = op_ + len; + } + + inline bool CheckLength() const { + return op_ == op_limit_; + } + + inline bool Append(const char* ip, size_t len) { + char* op = op_; + const size_t space_left = op_limit_ - op; + if (space_left < len) { + return false; + } + memcpy(op, ip, len); + op_ = op + len; + return true; + } + + inline bool TryFastAppend(const char* ip, size_t available, size_t len) { + char* op = op_; + const size_t space_left = op_limit_ - op; + if (len <= 16 && available >= 16 + kMaximumTagLength && space_left >= 16) { + // Fast path, used for the majority (about 95%) of invocations. + UnalignedCopy64(ip, op); + UnalignedCopy64(ip + 8, op + 8); + op_ = op + len; + return true; + } else { + return false; + } + } + + inline bool AppendFromSelf(size_t offset, size_t len) { + char* op = op_; + const size_t space_left = op_limit_ - op; + + // Check if we try to append from before the start of the buffer. + // Normally this would just be a check for "produced < offset", + // but "produced <= offset - 1u" is equivalent for every case + // except the one where offset==0, where the right side will wrap around + // to a very big number. This is convenient, as offset==0 is another + // invalid case that we also want to catch, so that we do not go + // into an infinite loop. + assert(op >= base_); + size_t produced = op - base_; + if (produced <= offset - 1u) { + return false; + } + if (len <= 16 && offset >= 8 && space_left >= 16) { + // Fast path, used for the majority (70-80%) of dynamic invocations. + UnalignedCopy64(op - offset, op); + UnalignedCopy64(op - offset + 8, op + 8); + } else { + if (space_left >= len + kMaxIncrementCopyOverflow) { + IncrementalCopyFastPath(op - offset, op, len); + } else { + if (space_left < len) { + return false; + } + IncrementalCopy(op - offset, op, len); + } + } + + op_ = op + len; + return true; + } +}; + +bool RawUncompress(const char* compressed, size_t n, char* uncompressed) { + ByteArraySource reader(compressed, n); + return RawUncompress(&reader, uncompressed); +} + +bool RawUncompress(Source* compressed, char* uncompressed) { + SnappyArrayWriter output(uncompressed); + return InternalUncompress(compressed, &output); +} + +bool Uncompress(const char* compressed, size_t n, string* uncompressed) { + size_t ulength; + if (!GetUncompressedLength(compressed, n, &ulength)) { + return false; + } + // On 32-bit builds: max_size() < kuint32max. Check for that instead + // of crashing (e.g., consider externally specified compressed data). + if (ulength > uncompressed->max_size()) { + return false; + } + STLStringResizeUninitialized(uncompressed, ulength); + return RawUncompress(compressed, n, string_as_array(uncompressed)); +} + + +// A Writer that drops everything on the floor and just does validation +class SnappyDecompressionValidator { + private: + size_t expected_; + size_t produced_; + + public: + inline SnappyDecompressionValidator() : produced_(0) { } + inline void SetExpectedLength(size_t len) { + expected_ = len; + } + inline bool CheckLength() const { + return expected_ == produced_; + } + inline bool Append(const char* ip, size_t len) { + produced_ += len; + return produced_ <= expected_; + } + inline bool TryFastAppend(const char* ip, size_t available, size_t length) { + return false; + } + inline bool AppendFromSelf(size_t offset, size_t len) { + // See SnappyArrayWriter::AppendFromSelf for an explanation of + // the "offset - 1u" trick. + if (produced_ <= offset - 1u) return false; + produced_ += len; + return produced_ <= expected_; + } +}; + +bool IsValidCompressedBuffer(const char* compressed, size_t n) { + ByteArraySource reader(compressed, n); + SnappyDecompressionValidator writer; + return InternalUncompress(&reader, &writer); +} + +void RawCompress(const char* input, + size_t input_length, + char* compressed, + size_t* compressed_length) { + ByteArraySource reader(input, input_length); + UncheckedByteArraySink writer(compressed); + Compress(&reader, &writer); + + // Compute how many bytes were added + *compressed_length = (writer.CurrentDestination() - compressed); +} + +size_t Compress(const char* input, size_t input_length, string* compressed) { + // Pre-grow the buffer to the max length of the compressed output + compressed->resize(MaxCompressedLength(input_length)); + + size_t compressed_length; + RawCompress(input, input_length, string_as_array(compressed), + &compressed_length); + compressed->resize(compressed_length); + return compressed_length; +} + + +} // end namespace snappy + diff --git a/tier1/sparsematrix.cpp b/tier1/sparsematrix.cpp new file mode 100644 index 0000000..3a2df54 --- /dev/null +++ b/tier1/sparsematrix.cpp @@ -0,0 +1,141 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +//===========================================================================// + +#include "tier1/sparsematrix.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + + +void CSparseMatrix::AdjustAllRowIndicesAfter( int nStartRow, int nDelta ) +{ + // now, we need to offset the starting position of all subsequent rows by -1 to compensate for the removal of this element + for( int nOtherRow = nStartRow + 1 ; nOtherRow < Height(); nOtherRow++ ) + { + m_rowDescriptors[nOtherRow].m_nDataIndex += nDelta; + } +} + +void CSparseMatrix::SetDimensions( int nNumRows, int nNumCols ) +{ + m_nNumRows = nNumRows; + m_nNumCols = nNumCols; + m_entries.SetCount( 0 ); + m_rowDescriptors.SetCount( m_nNumRows ); + // and set all rows to be empty + for( int i = 0; i < m_nNumRows; i++ ) + { + m_rowDescriptors[i].m_nNonZeroCount = 0; + m_rowDescriptors[i].m_nDataIndex = 0; + } + m_nHighestRowAppendedTo = -1; + +} + + +void CSparseMatrix::SetElement( int nRow, int nCol, float flValue ) +{ + Assert( nCol < m_nNumCols ); + int nCount = m_rowDescriptors[nRow].m_nNonZeroCount; + bool bValueIsZero = ( flValue == 0.0 ); + int nFirstEntryIndex = m_rowDescriptors[nRow].m_nDataIndex; + if ( nCount ) + { + NonZeroValueDescriptor_t *pValue = &( m_entries[nFirstEntryIndex] ); + int i; + for( i = 0; i < nCount; i++ ) + { + int nIdx = pValue->m_nColumnNumber; + if ( nIdx == nCol ) // we found it! + { + if ( !bValueIsZero ) + { + // we want to overwrite the existing value + pValue->m_flValue = flValue; + } + else + { + // there is a non-zero element currently at this position. We need to remove it + // and we need to remove its storage. + m_rowDescriptors[nRow].m_nNonZeroCount--; + m_entries.Remove( nFirstEntryIndex + i ); + // now, we need to offset the starting position of all subsequent rows by -1 to compensate for the removal of this element + AdjustAllRowIndicesAfter( nRow, -1 ); + } + return; + } + if ( nIdx > nCol ) + { + break; + } + pValue++; + } + // we did not find an entry for this cell. If we were writing zero, fine - we are + // done, otherwise insert + if (! bValueIsZero ) + { + m_rowDescriptors[nRow].m_nNonZeroCount++; + NonZeroValueDescriptor_t newValue; + newValue.m_nColumnNumber = nCol; + newValue.m_flValue = flValue; + if ( i == nCount ) // need to append + { + m_entries.InsertAfter( nFirstEntryIndex + nCount - 1, newValue ); + } + else + { + m_entries.InsertBefore( nFirstEntryIndex + i, newValue ); + } + // now, we need to offset the starting position of all subsequent rows by -1 to compensate for the addition of this element + AdjustAllRowIndicesAfter( nRow, +1 ); + } + } + else + { + // row is empty. We may need to insert + if ( ! bValueIsZero ) + { + m_rowDescriptors[nRow].m_nNonZeroCount++; + NonZeroValueDescriptor_t newValue; + newValue.m_nColumnNumber = nCol; + newValue.m_flValue = flValue; + m_entries.InsertBefore( nFirstEntryIndex, newValue ); + AdjustAllRowIndicesAfter( nRow, +1 ); + } + } +} + +void CSparseMatrix::FinishedAppending( void ) +{ + // set all pointers to space for subsequent rows to the right value + for( int i = m_nHighestRowAppendedTo + 1 ; i < Height(); i++ ) + { + m_rowDescriptors[i].m_nDataIndex = m_entries.Count(); + } +} + +void CSparseMatrix::AppendElement( int nRow, int nColumn, float flValue ) +{ + if ( flValue != 0.0 ) + { + if ( m_nHighestRowAppendedTo != nRow ) + { + Assert( nRow > m_nHighestRowAppendedTo ); + for( int i = m_nHighestRowAppendedTo + 1; i <= nRow; i++ ) + { + m_rowDescriptors[i].m_nDataIndex = m_entries.Count(); + } + } + m_nHighestRowAppendedTo = nRow; + m_rowDescriptors[nRow].m_nNonZeroCount++; + NonZeroValueDescriptor_t newDesc; + newDesc.m_nColumnNumber = nColumn; + newDesc.m_flValue = flValue; + m_entries.AddToTail( newDesc ); + } +} + + + + diff --git a/tier1/splitstring.cpp b/tier1/splitstring.cpp new file mode 100644 index 0000000..448f6d0 --- /dev/null +++ b/tier1/splitstring.cpp @@ -0,0 +1,92 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// +// +//================================================================================================== + +#include "strtools.h" +#include "utlvector.h" + +CSplitString::CSplitString(const char *pString, const char **pSeparators, int nSeparators) +{ + Construct(pString, pSeparators, nSeparators); +}; + +CSplitString::CSplitString( const char *pString, const char *pSeparator) +{ + Construct( pString, &pSeparator, 1 ); +} + +CSplitString::~CSplitString() +{ + if(m_szBuffer) + delete [] m_szBuffer; +} + +void CSplitString::Construct( const char *pString, const char **pSeparators, int nSeparators ) +{ + ////////////////////////////////////////////////////////////////////////// + // make a duplicate of the original string. We'll use pieces of this duplicate to tokenize the string + // and create NULL-terminated tokens of the original string + // + int nOriginalStringLength = V_strlen(pString); + m_szBuffer = new char[nOriginalStringLength + 1]; + memcpy(m_szBuffer, pString, nOriginalStringLength + 1); + + this->Purge(); + const char *pCurPos = pString; + while ( 1 ) + { + int iFirstSeparator = -1; + const char *pFirstSeparator = 0; + for ( int i=0; i < nSeparators; i++ ) + { + const char *pTest = V_stristr( pCurPos, pSeparators[i] ); + if ( pTest && (!pFirstSeparator || pTest < pFirstSeparator) ) + { + iFirstSeparator = i; + pFirstSeparator = pTest; + } + } + + if ( pFirstSeparator ) + { + // Split on this separator and continue on. + int separatorLen = strlen( pSeparators[iFirstSeparator] ); + if ( pFirstSeparator > pCurPos ) + { + ////////////////////////////////////////////////////////////////////////// + /// Cut the token out of the duplicate string + char *pTokenInDuplicate = m_szBuffer + (pCurPos - pString); + int nTokenLength = pFirstSeparator-pCurPos; + Assert(nTokenLength > 0 && !memcmp(pTokenInDuplicate,pCurPos,nTokenLength)); + pTokenInDuplicate[nTokenLength] = '\0'; + + this->AddToTail( pTokenInDuplicate /*AllocString( pCurPos, pFirstSeparator-pCurPos )*/ ); + } + + pCurPos = pFirstSeparator + separatorLen; + } + else + { + // Copy the rest of the string + int nTokenLength = strlen( pCurPos ); + if ( nTokenLength ) + { + ////////////////////////////////////////////////////////////////////////// + // There's no need to cut this token, because there's no separator after it. + // just add its copy in the buffer to the tail + char *pTokenInDuplicate = m_szBuffer + (pCurPos - pString); + Assert(!memcmp(pTokenInDuplicate, pCurPos, nTokenLength)); + + this->AddToTail( pTokenInDuplicate/*AllocString( pCurPos, -1 )*/ ); + } + return; + } + } +} + +void CSplitString::PurgeAndDeleteElements() +{ + Purge(); +} diff --git a/tier1/stringpool.cpp b/tier1/stringpool.cpp new file mode 100644 index 0000000..6b6b0ff --- /dev/null +++ b/tier1/stringpool.cpp @@ -0,0 +1,334 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//===========================================================================// + +#include "convar.h" +#include "tier0/dbg.h" +#include "stringpool.h" +#include "tier1/strtools.h" +#include "generichash.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +//----------------------------------------------------------------------------- +// Purpose: Comparison function for string sorted associative data structures +//----------------------------------------------------------------------------- + +bool StrLess( const char * const &pszLeft, const char * const &pszRight ) +{ + return ( Q_stricmp( pszLeft, pszRight) < 0 ); +} + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- + +CStringPool::CStringPool() + : m_Strings( 32, 256, StrLess ) +{ +} + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- + +CStringPool::~CStringPool() +{ + FreeAll(); +} + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- + +unsigned int CStringPool::Count() const +{ + return m_Strings.Count(); +} + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +const char * CStringPool::Find( const char *pszValue ) +{ + unsigned short i = m_Strings.Find(pszValue); + if ( m_Strings.IsValidIndex(i) ) + return m_Strings[i]; + + return NULL; +} + +const char * CStringPool::Allocate( const char *pszValue ) +{ + char *pszNew; + + unsigned short i = m_Strings.Find(pszValue); + bool bNew = (i == m_Strings.InvalidIndex()); + + if ( !bNew ) + return m_Strings[i]; + + pszNew = strdup( pszValue ); + + if ( bNew ) + m_Strings.Insert( pszNew ); + + return pszNew; +} + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- + +void CStringPool::FreeAll() +{ + unsigned short i = m_Strings.FirstInorder(); + while ( i != m_Strings.InvalidIndex() ) + { + free( (void *)m_Strings[i] ); + i = m_Strings.NextInorder(i); + } + m_Strings.RemoveAll(); +} + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- + + +CCountedStringPool::CCountedStringPool() +{ + MEM_ALLOC_CREDIT(); + m_HashTable.EnsureCount(HASH_TABLE_SIZE); + + for( int i = 0; i < m_HashTable.Count(); i++ ) + { + m_HashTable[i] = INVALID_ELEMENT; + } + + m_FreeListStart = INVALID_ELEMENT; + m_Elements.AddToTail(); + m_Elements[0].pString = NULL; + m_Elements[0].nReferenceCount = 0; + m_Elements[0].nNextElement = INVALID_ELEMENT; +} + +CCountedStringPool::~CCountedStringPool() +{ + FreeAll(); +} + +void CCountedStringPool::FreeAll() +{ + int i; + + // Reset the hash table: + for( i = 0; i < m_HashTable.Count(); i++ ) + { + m_HashTable[i] = INVALID_ELEMENT; + } + + // Blow away the free list: + m_FreeListStart = INVALID_ELEMENT; + + for( i = 0; i < m_Elements.Count(); i++ ) + { + if( m_Elements[i].pString ) + { + delete [] m_Elements[i].pString; + m_Elements[i].pString = NULL; + m_Elements[i].nReferenceCount = 0; + m_Elements[i].nNextElement = INVALID_ELEMENT; + } + } + + // Remove all but the invalid element: + m_Elements.RemoveAll(); + m_Elements.AddToTail(); + m_Elements[0].pString = NULL; + m_Elements[0].nReferenceCount = 0; + m_Elements[0].nNextElement = INVALID_ELEMENT; +} + + +unsigned short CCountedStringPool::FindStringHandle( const char* pIntrinsic ) +{ + if( pIntrinsic == NULL ) + return INVALID_ELEMENT; + + unsigned short nHashBucketIndex = (HashStringCaseless(pIntrinsic ) %HASH_TABLE_SIZE); + unsigned short nCurrentBucket = m_HashTable[ nHashBucketIndex ]; + + // Does the bucket already exist? + if( nCurrentBucket != INVALID_ELEMENT ) + { + for( ; nCurrentBucket != INVALID_ELEMENT ; nCurrentBucket = m_Elements[nCurrentBucket].nNextElement ) + { + if( !Q_stricmp( pIntrinsic, m_Elements[nCurrentBucket].pString ) ) + { + return nCurrentBucket; + } + } + } + + return 0; + +} + +char* CCountedStringPool::FindString( const char* pIntrinsic ) +{ + if( pIntrinsic == NULL ) + return NULL; + + // Yes, this will be NULL on failure. + return m_Elements[FindStringHandle(pIntrinsic)].pString; +} + +unsigned short CCountedStringPool::ReferenceStringHandle( const char* pIntrinsic ) +{ + if( pIntrinsic == NULL ) + return INVALID_ELEMENT; + + unsigned short nHashBucketIndex = (HashStringCaseless( pIntrinsic ) % HASH_TABLE_SIZE); + unsigned short nCurrentBucket = m_HashTable[ nHashBucketIndex ]; + + // Does the bucket already exist? + if( nCurrentBucket != INVALID_ELEMENT ) + { + for( ; nCurrentBucket != INVALID_ELEMENT ; nCurrentBucket = m_Elements[nCurrentBucket].nNextElement ) + { + if( !Q_stricmp( pIntrinsic, m_Elements[nCurrentBucket].pString ) ) + { + // Anyone who hits 65k references is permanant + if( m_Elements[nCurrentBucket].nReferenceCount < MAX_REFERENCE ) + { + m_Elements[nCurrentBucket].nReferenceCount ++ ; + } + return nCurrentBucket; + } + } + } + + if( m_FreeListStart != INVALID_ELEMENT ) + { + nCurrentBucket = m_FreeListStart; + m_FreeListStart = m_Elements[nCurrentBucket].nNextElement; + } + else + { + nCurrentBucket = m_Elements.AddToTail(); + } + + m_Elements[nCurrentBucket].nReferenceCount = 1; + + // Insert at the beginning of the bucket: + m_Elements[nCurrentBucket].nNextElement = m_HashTable[ nHashBucketIndex ]; + m_HashTable[ nHashBucketIndex ] = nCurrentBucket; + + m_Elements[nCurrentBucket].pString = new char[Q_strlen( pIntrinsic ) + 1]; + Q_strcpy( m_Elements[nCurrentBucket].pString, pIntrinsic ); + + return nCurrentBucket; +} + + +char* CCountedStringPool::ReferenceString( const char* pIntrinsic ) +{ + if(!pIntrinsic) + return NULL; + + return m_Elements[ReferenceStringHandle( pIntrinsic)].pString; +} + +void CCountedStringPool::DereferenceString( const char* pIntrinsic ) +{ + // If we get a NULL pointer, just return + if (!pIntrinsic) + return; + + unsigned short nHashBucketIndex = (HashStringCaseless( pIntrinsic ) % m_HashTable.Count()); + unsigned short nCurrentBucket = m_HashTable[ nHashBucketIndex ]; + + // If there isn't anything in the bucket, just return. + if ( nCurrentBucket == INVALID_ELEMENT ) + return; + + for( unsigned short previous = INVALID_ELEMENT; nCurrentBucket != INVALID_ELEMENT ; nCurrentBucket = m_Elements[nCurrentBucket].nNextElement ) + { + if( !Q_stricmp( pIntrinsic, m_Elements[nCurrentBucket].pString ) ) + { + // Anyone who hits 65k references is permanant + if( m_Elements[nCurrentBucket].nReferenceCount < MAX_REFERENCE ) + { + m_Elements[nCurrentBucket].nReferenceCount --; + } + + if( m_Elements[nCurrentBucket].nReferenceCount == 0 ) + { + if( previous == INVALID_ELEMENT ) + { + m_HashTable[nHashBucketIndex] = m_Elements[nCurrentBucket].nNextElement; + } + else + { + m_Elements[previous].nNextElement = m_Elements[nCurrentBucket].nNextElement; + } + + delete [] m_Elements[nCurrentBucket].pString; + m_Elements[nCurrentBucket].pString = NULL; + m_Elements[nCurrentBucket].nReferenceCount = 0; + + m_Elements[nCurrentBucket].nNextElement = m_FreeListStart; + m_FreeListStart = nCurrentBucket; + break; + + } + } + + previous = nCurrentBucket; + } +} + +char* CCountedStringPool::HandleToString( unsigned short handle ) +{ + return m_Elements[handle].pString; +} + +void CCountedStringPool::SpewStrings() +{ + int i; + for ( i = 0; i < m_Elements.Count(); i++ ) + { + char* string = m_Elements[i].pString; + + Msg("String %d: ref:%d %s", i, m_Elements[i].nReferenceCount, string == NULL? "EMPTY - ok for slot zero only!" : string); + } + + Msg("\n%d total counted strings.", m_Elements.Count()); +} + +#ifdef _DEBUG +CON_COMMAND( test_stringpool, "Tests the class CStringPool" ) +{ + CStringPool pool; + + Assert(pool.Count() == 0); + + pool.Allocate("test"); + Assert(pool.Count() == 1); + + pool.Allocate("test"); + Assert(pool.Count() == 1); + + pool.Allocate("test2"); + Assert(pool.Count() == 2); + + Assert( pool.Find("test2") != NULL ); + Assert( pool.Find("TEST") != NULL ); + Assert( pool.Find("Test2") != NULL ); + Assert( pool.Find("test") != NULL ); + + pool.FreeAll(); + Assert(pool.Count() == 0); + + Msg("Pass."); +} +#endif diff --git a/tier1/strtools.cpp b/tier1/strtools.cpp new file mode 100644 index 0000000..a63fa4f --- /dev/null +++ b/tier1/strtools.cpp @@ -0,0 +1,4333 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: String Tools +// +//===========================================================================// + +// These are redefined in the project settings to prevent anyone from using them. +// We in this module are of a higher caste and thus are privileged in their use. +#ifdef strncpy + #undef strncpy +#endif + +#ifdef _snprintf + #undef _snprintf +#endif + +#if defined( sprintf ) + #undef sprintf +#endif + +#if defined( vsprintf ) + #undef vsprintf +#endif + +#ifdef _vsnprintf +#ifdef _WIN32 + #undef _vsnprintf +#endif +#endif + +#ifdef vsnprintf +#ifndef _WIN32 + #undef vsnprintf +#endif +#endif + +#if defined( strcat ) + #undef strcat +#endif + +#ifdef strncat + #undef strncat +#endif + +// NOTE: I have to include stdio + stdarg first so vsnprintf gets compiled in +#include <stdio.h> +#include <stdarg.h> + +#ifdef POSIX +#include <iconv.h> +#include <ctype.h> +#include <unistd.h> +#include <stdlib.h> +#define _getcwd getcwd +#elif _WIN32 +#include <direct.h> +#if !defined( _X360 ) +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#endif +#endif + +#ifdef _WIN32 +#ifndef CP_UTF8 +#define CP_UTF8 65001 +#endif +#endif +#include "tier0/dbg.h" +#include "tier1/strtools.h" +#include <string.h> +#include <stdlib.h> +#include <time.h> +#include "tier0/basetypes.h" +#include "tier1/utldict.h" +#include "tier1/utlbuffer.h" +#include "tier1/utlstring.h" +#include "tier1/fmtstr.h" +#if defined( _X360 ) +#include "xbox/xbox_win32stubs.h" +#endif +#include "tier0/memdbgon.h" + +static int FastToLower( char c ) +{ + int i = (unsigned char) c; + if ( i < 0x80 ) + { + // Brutally fast branchless ASCII tolower(): + i += (((('A'-1) - i) & (i - ('Z'+1))) >> 26) & 0x20; + } + else + { + i += isupper( i ) ? 0x20 : 0; + } + return i; +} + +void _V_memset (const char* file, int line, void *dest, int fill, int count) +{ + Assert( count >= 0 ); + AssertValidWritePtr( dest, count ); + + memset(dest,fill,count); +} + +void _V_memcpy (const char* file, int line, void *dest, const void *src, int count) +{ + Assert( count >= 0 ); + AssertValidReadPtr( src, count ); + AssertValidWritePtr( dest, count ); + + memcpy( dest, src, count ); +} + +void _V_memmove(const char* file, int line, void *dest, const void *src, int count) +{ + Assert( count >= 0 ); + AssertValidReadPtr( src, count ); + AssertValidWritePtr( dest, count ); + + memmove( dest, src, count ); +} + +int _V_memcmp (const char* file, int line, const void *m1, const void *m2, int count) +{ + Assert( count >= 0 ); + AssertValidReadPtr( m1, count ); + AssertValidReadPtr( m2, count ); + + return memcmp( m1, m2, count ); +} + +int _V_strlen(const char* file, int line, const char *str) +{ + AssertValidStringPtr(str); + return strlen( str ); +} + +void _V_strcpy (const char* file, int line, char *dest, const char *src) +{ + AssertValidWritePtr(dest); + AssertValidStringPtr(src); + + strcpy( dest, src ); +} + +int _V_wcslen(const char* file, int line, const wchar_t *pwch) +{ + return wcslen( pwch ); +} + +char *_V_strrchr(const char* file, int line, const char *s, char c) +{ + AssertValidStringPtr( s ); + int len = V_strlen(s); + s += len; + while (len--) + if (*--s == c) return (char *)s; + return 0; +} + +int _V_strcmp (const char* file, int line, const char *s1, const char *s2) +{ + AssertValidStringPtr( s1 ); + AssertValidStringPtr( s2 ); + + return strcmp( s1, s2 ); +} + +int _V_wcscmp (const char* file, int line, const wchar_t *s1, const wchar_t *s2) +{ + AssertValidReadPtr( s1 ); + AssertValidReadPtr( s2 ); + + while ( *s1 == *s2 ) + { + if ( !*s1 ) + return 0; // strings are equal + + s1++; + s2++; + } + + return *s1 > *s2 ? 1 : -1; // strings not equal +} + + +char *_V_strstr(const char* file, int line, const char *s1, const char *search ) +{ + AssertValidStringPtr( s1 ); + AssertValidStringPtr( search ); + +#if defined( _X360 ) + return (char *)strstr( (char *)s1, search ); +#else + return (char *)strstr( s1, search ); +#endif +} + +wchar_t *_V_wcsupr (const char* file, int line, wchar_t *start) +{ + return _wcsupr( start ); +} + + +wchar_t *_V_wcslower (const char* file, int line, wchar_t *start) +{ + return _wcslwr(start); +} + + + +char *V_strupr( char *start ) +{ + unsigned char *str = (unsigned char*)start; + while( *str ) + { + if ( (unsigned char)(*str - 'a') <= ('z' - 'a') ) + *str -= 'a' - 'A'; + else if ( (unsigned char)*str >= 0x80 ) // non-ascii, fall back to CRT + *str = toupper( *str ); + str++; + } + return start; +} + +char *V_strlower( char *start ) +{ + unsigned char *str = (unsigned char*)start; + while( *str ) + { + if ( (unsigned char)(*str - 'A') <= ('Z' - 'A') ) + *str += 'a' - 'A'; + else if ( (unsigned char)*str >= 0x80 ) // non-ascii, fall back to CRT + *str = tolower( *str ); + str++; + } + return start; +} + +char *V_strnlwr(char *s, size_t count) +{ + // Assert( count >= 0 ); tautology since size_t is unsigned + AssertValidStringPtr( s, count ); + + char* pRet = s; + if ( !s || !count ) + return s; + + while ( -- count > 0 ) + { + if ( !*s ) + return pRet; // reached end of string + + *s = tolower( *s ); + ++s; + } + + *s = 0; // null-terminate original string at "count-1" + return pRet; +} + +int V_stricmp( const char *str1, const char *str2 ) +{ + // It is not uncommon to compare a string to itself. See + // VPanelWrapper::GetPanel which does this a lot. Since stricmp + // is expensive and pointer comparison is cheap, this simple test + // can save a lot of cycles, and cache pollution. + if ( str1 == str2 ) + { + return 0; + } + const unsigned char *s1 = (const unsigned char*)str1; + const unsigned char *s2 = (const unsigned char*)str2; + for ( ; *s1; ++s1, ++s2 ) + { + if ( *s1 != *s2 ) + { + // in ascii char set, lowercase = uppercase | 0x20 + unsigned char c1 = *s1 | 0x20; + unsigned char c2 = *s2 | 0x20; + if ( c1 != c2 || (unsigned char)(c1 - 'a') > ('z' - 'a') ) + { + // if non-ascii mismatch, fall back to CRT for locale + if ( (c1 | c2) >= 0x80 ) return stricmp( (const char*)s1, (const char*)s2 ); + // ascii mismatch. only use the | 0x20 value if alphabetic. + if ((unsigned char)(c1 - 'a') > ('z' - 'a')) c1 = *s1; + if ((unsigned char)(c2 - 'a') > ('z' - 'a')) c2 = *s2; + return c1 > c2 ? 1 : -1; + } + } + } + return *s2 ? -1 : 0; +} + +int V_strnicmp( const char *str1, const char *str2, int n ) +{ + const unsigned char *s1 = (const unsigned char*)str1; + const unsigned char *s2 = (const unsigned char*)str2; + for ( ; n > 0 && *s1; --n, ++s1, ++s2 ) + { + if ( *s1 != *s2 ) + { + // in ascii char set, lowercase = uppercase | 0x20 + unsigned char c1 = *s1 | 0x20; + unsigned char c2 = *s2 | 0x20; + if ( c1 != c2 || (unsigned char)(c1 - 'a') > ('z' - 'a') ) + { + // if non-ascii mismatch, fall back to CRT for locale + if ( (c1 | c2) >= 0x80 ) return strnicmp( (const char*)s1, (const char*)s2, n ); + // ascii mismatch. only use the | 0x20 value if alphabetic. + if ((unsigned char)(c1 - 'a') > ('z' - 'a')) c1 = *s1; + if ((unsigned char)(c2 - 'a') > ('z' - 'a')) c2 = *s2; + return c1 > c2 ? 1 : -1; + } + } + } + return (n > 0 && *s2) ? -1 : 0; +} + +int V_strncmp( const char *s1, const char *s2, int count ) +{ + Assert( count >= 0 ); + AssertValidStringPtr( s1, count ); + AssertValidStringPtr( s2, count ); + + while ( count > 0 ) + { + if ( *s1 != *s2 ) + return (unsigned char)*s1 < (unsigned char)*s2 ? -1 : 1; // string different + if ( *s1 == '\0' ) + return 0; // null terminator hit - strings the same + s1++; + s2++; + count--; + } + + return 0; // count characters compared the same +} + + +const char *StringAfterPrefix( const char *str, const char *prefix ) +{ + AssertValidStringPtr( str ); + AssertValidStringPtr( prefix ); + do + { + if ( !*prefix ) + return str; + } + while ( FastToLower( *str++ ) == FastToLower( *prefix++ ) ); + return NULL; +} + +const char *StringAfterPrefixCaseSensitive( const char *str, const char *prefix ) +{ + AssertValidStringPtr( str ); + AssertValidStringPtr( prefix ); + do + { + if ( !*prefix ) + return str; + } + while ( *str++ == *prefix++ ); + return NULL; +} + + +int64 V_atoi64( const char *str ) +{ + AssertValidStringPtr( str ); + + int64 val; + int64 sign; + int64 c; + + Assert( str ); + if (*str == '-') + { + sign = -1; + str++; + } + else if (*str == '+') + { + sign = 1; + str++; + } + else + { + sign = 1; + } + + val = 0; + +// +// check for hex +// + if (str[0] == '0' && (str[1] == 'x' || str[1] == 'X') ) + { + str += 2; + while (1) + { + c = *str++; + if (c >= '0' && c <= '9') + val = (val<<4) + c - '0'; + else if (c >= 'a' && c <= 'f') + val = (val<<4) + c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + val = (val<<4) + c - 'A' + 10; + else + return val*sign; + } + } + +// +// check for character +// + if (str[0] == '\'') + { + return sign * str[1]; + } + +// +// assume decimal +// + while (1) + { + c = *str++; + if (c <'0' || c > '9') + return val*sign; + val = val*10 + c - '0'; + } + + return 0; +} + +uint64 V_atoui64( const char *str ) +{ + AssertValidStringPtr( str ); + + uint64 val; + uint64 c; + + Assert( str ); + + val = 0; + + // + // check for hex + // + if (str[0] == '0' && (str[1] == 'x' || str[1] == 'X') ) + { + str += 2; + while (1) + { + c = *str++; + if (c >= '0' && c <= '9') + val = (val<<4) + c - '0'; + else if (c >= 'a' && c <= 'f') + val = (val<<4) + c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + val = (val<<4) + c - 'A' + 10; + else + return val; + } + } + + // + // check for character + // + if (str[0] == '\'') + { + return str[1]; + } + + // + // assume decimal + // + while (1) + { + c = *str++; + if (c <'0' || c > '9') + return val; + val = val*10 + c - '0'; + } + + return 0; +} + +int V_atoi( const char *str ) +{ + return (int)V_atoi64( str ); +} + +float V_atof (const char *str) +{ + AssertValidStringPtr( str ); + double val; + int sign; + int c; + int decimal, total; + + if (*str == '-') + { + sign = -1; + str++; + } + else if (*str == '+') + { + sign = 1; + str++; + } + else + { + sign = 1; + } + + val = 0; + + // + // check for hex + // + if (str[0] == '0' && (str[1] == 'x' || str[1] == 'X') ) + { + str += 2; + while (1) + { + c = *str++; + if (c >= '0' && c <= '9') + val = (val*16) + c - '0'; + else if (c >= 'a' && c <= 'f') + val = (val*16) + c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + val = (val*16) + c - 'A' + 10; + else + return val*sign; + } + } + + // + // check for character + // + if (str[0] == '\'') + { + return sign * str[1]; + } + + // + // assume decimal + // + decimal = -1; + total = 0; + int exponent = 0; + while (1) + { + c = *str++; + if (c == '.') + { + if ( decimal != -1 ) + { + break; + } + + decimal = total; + continue; + } + if (c <'0' || c > '9') + { + if ( c == 'e' || c == 'E' ) + { + exponent = V_atoi(str); + } + break; + } + val = val*10 + c - '0'; + total++; + } + + if ( exponent != 0 ) + { + val *= pow( 10.0, exponent ); + } + if (decimal == -1) + return val*sign; + while (total > decimal) + { + val /= 10; + total--; + } + + return val*sign; +} + +//----------------------------------------------------------------------------- +// Normalizes a float string in place. +// +// (removes leading zeros, trailing zeros after the decimal point, and the decimal point itself where possible) +//----------------------------------------------------------------------------- +void V_normalizeFloatString( char* pFloat ) +{ + // If we have a decimal point, remove trailing zeroes: + if( strchr( pFloat,'.' ) ) + { + int len = V_strlen(pFloat); + + while( len > 1 && pFloat[len - 1] == '0' ) + { + pFloat[len - 1] = '\0'; + len--; + } + + if( len > 1 && pFloat[ len - 1 ] == '.' ) + { + pFloat[len - 1] = '\0'; + len--; + } + } + + // TODO: Strip leading zeros + +} + + +//----------------------------------------------------------------------------- +// Finds a string in another string with a case insensitive test +//----------------------------------------------------------------------------- +char const* V_stristr( char const* pStr, char const* pSearch ) +{ + AssertValidStringPtr(pStr); + AssertValidStringPtr(pSearch); + + if (!pStr || !pSearch) + return 0; + + char const* pLetter = pStr; + + // Check the entire string + while (*pLetter != 0) + { + // Skip over non-matches + if (FastToLower((unsigned char)*pLetter) == FastToLower((unsigned char)*pSearch)) + { + // Check for match + char const* pMatch = pLetter + 1; + char const* pTest = pSearch + 1; + while (*pTest != 0) + { + // We've run off the end; don't bother. + if (*pMatch == 0) + return 0; + + if (FastToLower((unsigned char)*pMatch) != FastToLower((unsigned char)*pTest)) + break; + + ++pMatch; + ++pTest; + } + + // Found a match! + if (*pTest == 0) + return pLetter; + } + + ++pLetter; + } + + return 0; +} + +char* V_stristr( char* pStr, char const* pSearch ) +{ + AssertValidStringPtr( pStr ); + AssertValidStringPtr( pSearch ); + + return (char*)V_stristr( (char const*)pStr, pSearch ); +} + +//----------------------------------------------------------------------------- +// Finds a string in another string with a case insensitive test w/ length validation +//----------------------------------------------------------------------------- + +char const* V_strnistr( char const* pStr, char const* pSearch, int n ) +{ + AssertValidStringPtr(pStr); + AssertValidStringPtr(pSearch); + + if (!pStr || !pSearch) + return 0; + + char const* pLetter = pStr; + + // Check the entire string + while (*pLetter != 0) + { + if ( n <= 0 ) + return 0; + + // Skip over non-matches + if (FastToLower(*pLetter) == FastToLower(*pSearch)) + { + int n1 = n - 1; + + // Check for match + char const* pMatch = pLetter + 1; + char const* pTest = pSearch + 1; + while (*pTest != 0) + { + if ( n1 <= 0 ) + return 0; + + // We've run off the end; don't bother. + if (*pMatch == 0) + return 0; + + if (FastToLower(*pMatch) != FastToLower(*pTest)) + break; + + ++pMatch; + ++pTest; + --n1; + } + + // Found a match! + if (*pTest == 0) + return pLetter; + } + + ++pLetter; + --n; + } + + return 0; +} + +const char* V_strnchr( const char* pStr, char c, int n ) +{ + char const* pLetter = pStr; + char const* pLast = pStr + n; + + // Check the entire string + while ( (pLetter < pLast) && (*pLetter != 0) ) + { + if (*pLetter == c) + return pLetter; + ++pLetter; + } + return NULL; +} + +void V_strncpy( char *pDest, char const *pSrc, int maxLen ) +{ + Assert( maxLen >= sizeof( *pDest ) ); + AssertValidWritePtr( pDest, maxLen ); + AssertValidStringPtr( pSrc ); + + strncpy( pDest, pSrc, maxLen ); + if ( maxLen > 0 ) + { + pDest[maxLen-1] = 0; + } +} + +// warning C6053: Call to 'wcsncpy' might not zero-terminate string 'pDest' +// warning C6059: Incorrect length parameter in call to 'strncat'. Pass the number of remaining characters, not the buffer size of 'argument 1' +// warning C6386: Buffer overrun: accessing 'argument 1', the writable size is 'destBufferSize' bytes, but '1000' bytes might be written +// These warnings were investigated through code inspection and writing of tests and they are +// believed to all be spurious. +#ifdef _PREFAST_ +#pragma warning( push ) +#pragma warning( disable : 6053 6059 6386 ) +#endif + +void V_wcsncpy( wchar_t *pDest, wchar_t const *pSrc, int maxLenInBytes ) +{ + Assert( maxLenInBytes >= sizeof( *pDest ) ); + AssertValidWritePtr( pDest, maxLenInBytes ); + AssertValidReadPtr( pSrc ); + + int maxLen = maxLenInBytes / sizeof(wchar_t); + + wcsncpy( pDest, pSrc, maxLen ); + if( maxLen ) + { + pDest[maxLen-1] = 0; + } +} + + + +int V_snwprintf( wchar_t *pDest, int maxLen, const wchar_t *pFormat, ... ) +{ + Assert( maxLen > 0 ); + AssertValidWritePtr( pDest, maxLen ); + AssertValidReadPtr( pFormat ); + + va_list marker; + + va_start( marker, pFormat ); +#ifdef _WIN32 + int len = _vsnwprintf( pDest, maxLen, pFormat, marker ); +#elif POSIX + int len = vswprintf( pDest, maxLen, pFormat, marker ); +#else +#error "define vsnwprintf type." +#endif + va_end( marker ); + + // Len > maxLen represents an overflow on POSIX, < 0 is an overflow on windows + if( len < 0 || len >= maxLen ) + { + len = maxLen; + pDest[maxLen-1] = 0; + } + + return len; +} + + +int V_vsnwprintf( wchar_t *pDest, int maxLen, const wchar_t *pFormat, va_list params ) +{ + Assert( maxLen > 0 ); + +#ifdef _WIN32 + int len = _vsnwprintf( pDest, maxLen, pFormat, params ); +#elif POSIX + int len = vswprintf( pDest, maxLen, pFormat, params ); +#else +#error "define vsnwprintf type." +#endif + + // Len < 0 represents an overflow + // Len == maxLen represents exactly fitting with no NULL termination + // Len >= maxLen represents overflow on POSIX + if ( len < 0 || len >= maxLen ) + { + len = maxLen; + pDest[maxLen-1] = 0; + } + + return len; +} + + +int V_snprintf( char *pDest, int maxLen, char const *pFormat, ... ) +{ + Assert( maxLen > 0 ); + AssertValidWritePtr( pDest, maxLen ); + AssertValidStringPtr( pFormat ); + + va_list marker; + + va_start( marker, pFormat ); +#ifdef _WIN32 + int len = _vsnprintf( pDest, maxLen, pFormat, marker ); +#elif POSIX + int len = vsnprintf( pDest, maxLen, pFormat, marker ); +#else + #error "define vsnprintf type." +#endif + va_end( marker ); + + // Len > maxLen represents an overflow on POSIX, < 0 is an overflow on windows + if( len < 0 || len >= maxLen ) + { + len = maxLen; + pDest[maxLen-1] = 0; + } + + return len; +} + + +int V_vsnprintf( char *pDest, int maxLen, char const *pFormat, va_list params ) +{ + Assert( maxLen > 0 ); + AssertValidWritePtr( pDest, maxLen ); + AssertValidStringPtr( pFormat ); + + int len = _vsnprintf( pDest, maxLen, pFormat, params ); + + // Len > maxLen represents an overflow on POSIX, < 0 is an overflow on windows + if( len < 0 || len >= maxLen ) + { + len = maxLen; + pDest[maxLen-1] = 0; + } + + return len; +} + + +int V_vsnprintfRet( char *pDest, int maxLen, const char *pFormat, va_list params, bool *pbTruncated ) +{ + Assert( maxLen > 0 ); + AssertValidWritePtr( pDest, maxLen ); + AssertValidStringPtr( pFormat ); + + int len = _vsnprintf( pDest, maxLen, pFormat, params ); + + if ( pbTruncated ) + { + *pbTruncated = ( len < 0 || len >= maxLen ); + } + + if ( len < 0 || len >= maxLen ) + { + len = maxLen; + pDest[maxLen-1] = 0; + } + + return len; +} + + + +//----------------------------------------------------------------------------- +// Purpose: If COPY_ALL_CHARACTERS == max_chars_to_copy then we try to add the whole pSrc to the end of pDest, otherwise +// we copy only as many characters as are specified in max_chars_to_copy (or the # of characters in pSrc if thats's less). +// Input : *pDest - destination buffer +// *pSrc - string to append +// destBufferSize - sizeof the buffer pointed to by pDest +// max_chars_to_copy - COPY_ALL_CHARACTERS in pSrc or max # to copy +// Output : char * the copied buffer +//----------------------------------------------------------------------------- +char *V_strncat(char *pDest, const char *pSrc, size_t destBufferSize, int max_chars_to_copy ) +{ + size_t charstocopy = (size_t)0; + + Assert( (ptrdiff_t)destBufferSize >= 0 ); + AssertValidStringPtr( pDest); + AssertValidStringPtr( pSrc ); + + size_t len = strlen(pDest); + size_t srclen = strlen( pSrc ); + if ( max_chars_to_copy <= COPY_ALL_CHARACTERS ) + { + charstocopy = srclen; + } + else + { + charstocopy = (size_t)min( max_chars_to_copy, (int)srclen ); + } + + if ( len + charstocopy >= destBufferSize ) + { + charstocopy = destBufferSize - len - 1; + } + + if ( (int)charstocopy <= 0 ) + { + return pDest; + } + + ANALYZE_SUPPRESS( 6059 ); // warning C6059: : Incorrect length parameter in call to 'strncat'. Pass the number of remaining characters, not the buffer size of 'argument 1' + char *pOut = strncat( pDest, pSrc, charstocopy ); + return pOut; +} + +wchar_t *V_wcsncat( INOUT_Z_CAP(cchDest) wchar_t *pDest, const wchar_t *pSrc, size_t cchDest, int max_chars_to_copy ) +{ + size_t charstocopy = (size_t)0; + + Assert( (ptrdiff_t)cchDest >= 0 ); + + size_t len = wcslen(pDest); + size_t srclen = wcslen( pSrc ); + if ( max_chars_to_copy <= COPY_ALL_CHARACTERS ) + { + charstocopy = srclen; + } + else + { + charstocopy = (size_t)min( max_chars_to_copy, (int)srclen ); + } + + if ( len + charstocopy >= cchDest ) + { + charstocopy = cchDest - len - 1; + } + + if ( (int)charstocopy <= 0 ) + { + return pDest; + } + + ANALYZE_SUPPRESS( 6059 ); // warning C6059: : Incorrect length parameter in call to 'strncat'. Pass the number of remaining characters, not the buffer size of 'argument 1' + wchar_t *pOut = wcsncat( pDest, pSrc, charstocopy ); + return pOut; +} + + + +//----------------------------------------------------------------------------- +// Purpose: Converts value into x.xx MB/ x.xx KB, x.xx bytes format, including commas +// Input : value - +// 2 - +// false - +// Output : char +//----------------------------------------------------------------------------- +#define NUM_PRETIFYMEM_BUFFERS 8 +char *V_pretifymem( float value, int digitsafterdecimal /*= 2*/, bool usebinaryonek /*= false*/ ) +{ + static char output[ NUM_PRETIFYMEM_BUFFERS ][ 32 ]; + static int current; + + float onekb = usebinaryonek ? 1024.0f : 1000.0f; + float onemb = onekb * onekb; + + char *out = output[ current ]; + current = ( current + 1 ) & ( NUM_PRETIFYMEM_BUFFERS -1 ); + + char suffix[ 8 ]; + + // First figure out which bin to use + if ( value > onemb ) + { + value /= onemb; + V_snprintf( suffix, sizeof( suffix ), " MB" ); + } + else if ( value > onekb ) + { + value /= onekb; + V_snprintf( suffix, sizeof( suffix ), " KB" ); + } + else + { + V_snprintf( suffix, sizeof( suffix ), " bytes" ); + } + + char val[ 32 ]; + + // Clamp to >= 0 + digitsafterdecimal = max( digitsafterdecimal, 0 ); + + // If it's basically integral, don't do any decimals + if ( FloatMakePositive( value - (int)value ) < 0.00001 ) + { + V_snprintf( val, sizeof( val ), "%i%s", (int)value, suffix ); + } + else + { + char fmt[ 32 ]; + + // Otherwise, create a format string for the decimals + V_snprintf( fmt, sizeof( fmt ), "%%.%if%s", digitsafterdecimal, suffix ); + V_snprintf( val, sizeof( val ), fmt, value ); + } + + // Copy from in to out + char *i = val; + char *o = out; + + // Search for decimal or if it was integral, find the space after the raw number + char *dot = strstr( i, "." ); + if ( !dot ) + { + dot = strstr( i, " " ); + } + + // Compute position of dot + int pos = dot - i; + // Don't put a comma if it's <= 3 long + pos -= 3; + + while ( *i ) + { + // If pos is still valid then insert a comma every third digit, except if we would be + // putting one in the first spot + if ( pos >= 0 && !( pos % 3 ) ) + { + // Never in first spot + if ( o != out ) + { + *o++ = ','; + } + } + + // Count down comma position + pos--; + + // Copy rest of data as normal + *o++ = *i++; + } + + // Terminate + *o = 0; + + return out; +} + +//----------------------------------------------------------------------------- +// Purpose: Returns a string representation of an integer with commas +// separating the 1000s (ie, 37,426,421) +// Input : value - Value to convert +// Output : Pointer to a static buffer containing the output +//----------------------------------------------------------------------------- +#define NUM_PRETIFYNUM_BUFFERS 8 // Must be a power of two +char *V_pretifynum( int64 inputValue ) +{ + static char output[ NUM_PRETIFYMEM_BUFFERS ][ 32 ]; + static int current; + + // Point to the output buffer. + char * const out = output[ current ]; + // Track the output buffer end for easy calculation of bytes-remaining. + const char* const outEnd = out + sizeof( output[ current ] ); + + // Point to the current output location in the output buffer. + char *pchRender = out; + // Move to the next output pointer. + current = ( current + 1 ) & ( NUM_PRETIFYMEM_BUFFERS -1 ); + + *out = 0; + + // In order to handle the most-negative int64 we need to negate it + // into a uint64. + uint64 value; + // Render the leading minus sign, if necessary + if ( inputValue < 0 ) + { + V_snprintf( pchRender, 32, "-" ); + value = (uint64)-inputValue; + // Advance our output pointer. + pchRender += V_strlen( pchRender ); + } + else + { + value = (uint64)inputValue; + } + + // Now let's find out how big our number is. The largest number we can fit + // into 63 bits is about 9.2e18. So, there could potentially be six + // three-digit groups. + + // We need the initial value of 'divisor' to be big enough to divide our + // number down to 1-999 range. + uint64 divisor = 1; + // Loop more than six times to avoid integer overflow. + for ( int i = 0; i < 6; ++i ) + { + // If our divisor is already big enough then stop. + if ( value < divisor * 1000 ) + break; + + divisor *= 1000; + } + + // Print the leading batch of one to three digits. + int toPrint = value / divisor; + V_snprintf( pchRender, outEnd - pchRender, "%d", toPrint ); + + for (;;) + { + // Advance our output pointer. + pchRender += V_strlen( pchRender ); + // Adjust our value to be printed and our divisor. + value -= toPrint * divisor; + divisor /= 1000; + if ( !divisor ) + break; + + // The remaining blocks of digits always include a comma and three digits. + toPrint = value / divisor; + V_snprintf( pchRender, outEnd - pchRender, ",%03d", toPrint ); + } + + return out; +} + + +//----------------------------------------------------------------------------- +// Purpose: returns true if a wide character is a "mean" space; that is, +// if it is technically a space or punctuation, but causes disruptive +// behavior when used in names, web pages, chat windows, etc. +// +// characters in this set are removed from the beginning and/or end of strings +// by Q_AggressiveStripPrecedingAndTrailingWhitespaceW() +//----------------------------------------------------------------------------- +bool Q_IsMeanSpaceW( wchar_t wch ) +{ + bool bIsMean = false; + + switch ( wch ) + { + case L'\x0082': // BREAK PERMITTED HERE + case L'\x0083': // NO BREAK PERMITTED HERE + case L'\x00A0': // NO-BREAK SPACE + case L'\x034F': // COMBINING GRAPHEME JOINER + case L'\x2000': // EN QUAD + case L'\x2001': // EM QUAD + case L'\x2002': // EN SPACE + case L'\x2003': // EM SPACE + case L'\x2004': // THICK SPACE + case L'\x2005': // MID SPACE + case L'\x2006': // SIX SPACE + case L'\x2007': // figure space + case L'\x2008': // PUNCTUATION SPACE + case L'\x2009': // THIN SPACE + case L'\x200A': // HAIR SPACE + case L'\x200B': // ZERO-WIDTH SPACE + case L'\x200C': // ZERO-WIDTH NON-JOINER + case L'\x200D': // ZERO WIDTH JOINER + case L'\x200E': // LEFT-TO-RIGHT MARK + case L'\x2028': // LINE SEPARATOR + case L'\x2029': // PARAGRAPH SEPARATOR + case L'\x202F': // NARROW NO-BREAK SPACE + case L'\x2060': // word joiner + case L'\xFEFF': // ZERO-WIDTH NO BREAK SPACE + case L'\xFFFC': // OBJECT REPLACEMENT CHARACTER + bIsMean = true; + break; + } + + return bIsMean; +} + + +//----------------------------------------------------------------------------- +// Purpose: strips trailing whitespace; returns pointer inside string just past +// any leading whitespace. +// +// bAggresive = true causes this function to also check for "mean" spaces, +// which we don't want in persona names or chat strings as they're disruptive +// to the user experience. +//----------------------------------------------------------------------------- +static wchar_t *StripWhitespaceWorker( int cchLength, wchar_t *pwch, bool *pbStrippedWhitespace, bool bAggressive ) +{ + // walk backwards from the end of the string, killing any whitespace + *pbStrippedWhitespace = false; + + wchar_t *pwchEnd = pwch + cchLength; + while ( --pwchEnd >= pwch ) + { + if ( !iswspace( *pwchEnd ) && ( !bAggressive || !Q_IsMeanSpaceW( *pwchEnd ) ) ) + break; + + *pwchEnd = 0; + *pbStrippedWhitespace = true; + } + + // walk forward in the string + while ( pwch < pwchEnd ) + { + if ( !iswspace( *pwch ) ) + break; + + *pbStrippedWhitespace = true; + pwch++; + } + + return pwch; +} + +//----------------------------------------------------------------------------- +// Purpose: Strips all evil characters (ie. zero-width no-break space) +// from a string. +//----------------------------------------------------------------------------- +bool Q_RemoveAllEvilCharacters( char *pch ) +{ + // convert to unicode + int cch = Q_strlen( pch ); + int cubDest = (cch + 1 ) * sizeof( wchar_t ); + wchar_t *pwch = (wchar_t *)stackalloc( cubDest ); + int cwch = Q_UTF8ToUnicode( pch, pwch, cubDest ) / sizeof( wchar_t ); + + bool bStrippedWhitespace = false; + + // Walk through and skip over evil characters + int nWalk = 0; + for( int i=0; i<cwch; ++i ) + { + if( !Q_IsMeanSpaceW( pwch[i] ) ) + { + pwch[nWalk] = pwch[i]; + ++nWalk; + } + else + { + bStrippedWhitespace = true; + } + } + + // Null terminate + pwch[nWalk-1] = L'\0'; + + + // copy back, if necessary + if ( bStrippedWhitespace ) + { + Q_UnicodeToUTF8( pwch, pch, cch ); + } + + return bStrippedWhitespace; +} + + +//----------------------------------------------------------------------------- +// Purpose: strips leading and trailing whitespace +//----------------------------------------------------------------------------- +bool Q_StripPrecedingAndTrailingWhitespaceW( wchar_t *pwch ) +{ + int cch = Q_wcslen( pwch ); + + // Early out and don't convert if we don't have any chars or leading/trailing ws. + if ( ( cch < 1 ) || ( !iswspace( pwch[ 0 ] ) && !iswspace( pwch[ cch - 1 ] ) ) ) + return false; + + // duplicate on stack + int cubDest = ( cch + 1 ) * sizeof( wchar_t ); + wchar_t *pwchT = (wchar_t *)stackalloc( cubDest ); + Q_wcsncpy( pwchT, pwch, cubDest ); + + bool bStrippedWhitespace = false; + pwchT = StripWhitespaceWorker( cch, pwch, &bStrippedWhitespace, false /* not aggressive */ ); + + // copy back, if necessary + if ( bStrippedWhitespace ) + { + Q_wcsncpy( pwch, pwchT, cubDest ); + } + + return bStrippedWhitespace; +} + + + +//----------------------------------------------------------------------------- +// Purpose: strips leading and trailing whitespace, +// and also strips punctuation and formatting characters with "clear" +// representations. +//----------------------------------------------------------------------------- +bool Q_AggressiveStripPrecedingAndTrailingWhitespaceW( wchar_t *pwch ) +{ + // duplicate on stack + int cch = Q_wcslen( pwch ); + int cubDest = ( cch + 1 ) * sizeof( wchar_t ); + wchar_t *pwchT = (wchar_t *)stackalloc( cubDest ); + Q_wcsncpy( pwchT, pwch, cubDest ); + + bool bStrippedWhitespace = false; + pwchT = StripWhitespaceWorker( cch, pwch, &bStrippedWhitespace, true /* is aggressive */ ); + + // copy back, if necessary + if ( bStrippedWhitespace ) + { + Q_wcsncpy( pwch, pwchT, cubDest ); + } + + return bStrippedWhitespace; +} + + +//----------------------------------------------------------------------------- +// Purpose: strips leading and trailing whitespace +//----------------------------------------------------------------------------- +bool Q_StripPrecedingAndTrailingWhitespace( char *pch ) +{ + int cch = Q_strlen( pch ); + + // Early out and don't convert if we don't have any chars or leading/trailing ws. + if ( ( cch < 1 ) || ( !isspace( (unsigned char)pch[ 0 ] ) && !isspace( (unsigned char)pch[ cch - 1 ] ) ) ) + return false; + + // convert to unicode + int cubDest = (cch + 1 ) * sizeof( wchar_t ); + wchar_t *pwch = (wchar_t *)stackalloc( cubDest ); + int cwch = Q_UTF8ToUnicode( pch, pwch, cubDest ) / sizeof( wchar_t ); + + bool bStrippedWhitespace = false; + pwch = StripWhitespaceWorker( cwch-1, pwch, &bStrippedWhitespace, false /* not aggressive */ ); + + // copy back, if necessary + if ( bStrippedWhitespace ) + { + Q_UnicodeToUTF8( pwch, pch, cch ); + } + + return bStrippedWhitespace; +} + +//----------------------------------------------------------------------------- +// Purpose: strips leading and trailing whitespace +//----------------------------------------------------------------------------- +bool Q_AggressiveStripPrecedingAndTrailingWhitespace( char *pch ) +{ + // convert to unicode + int cch = Q_strlen( pch ); + int cubDest = (cch + 1 ) * sizeof( wchar_t ); + wchar_t *pwch = (wchar_t *)stackalloc( cubDest ); + int cwch = Q_UTF8ToUnicode( pch, pwch, cubDest ) / sizeof( wchar_t ); + + bool bStrippedWhitespace = false; + pwch = StripWhitespaceWorker( cwch-1, pwch, &bStrippedWhitespace, true /* is aggressive */ ); + + // copy back, if necessary + if ( bStrippedWhitespace ) + { + Q_UnicodeToUTF8( pwch, pch, cch ); + } + + return bStrippedWhitespace; +} + +//----------------------------------------------------------------------------- +// Purpose: Converts a ucs2 string to a unicode (wchar_t) one, no-op on win32 +//----------------------------------------------------------------------------- +int _V_UCS2ToUnicode( const ucs2 *pUCS2, wchar_t *pUnicode, int cubDestSizeInBytes ) +{ + Assert( cubDestSizeInBytes >= sizeof( *pUnicode ) ); + AssertValidWritePtr(pUnicode); + AssertValidReadPtr(pUCS2); + + pUnicode[0] = 0; +#ifdef _WIN32 + int cchResult = V_wcslen( pUCS2 ); + V_memcpy( pUnicode, pUCS2, cubDestSizeInBytes ); +#else + iconv_t conv_t = iconv_open( "UCS-4LE", "UCS-2LE" ); + int cchResult = -1; + size_t nLenUnicde = cubDestSizeInBytes; + size_t nMaxUTF8 = cubDestSizeInBytes; + char *pIn = (char *)pUCS2; + char *pOut = (char *)pUnicode; + if ( conv_t > 0 ) + { + cchResult = iconv( conv_t, &pIn, &nLenUnicde, &pOut, &nMaxUTF8 ); + iconv_close( conv_t ); + if ( (int)cchResult < 0 ) + cchResult = 0; + else + cchResult = nMaxUTF8; + } +#endif + pUnicode[(cubDestSizeInBytes / sizeof(wchar_t)) - 1] = 0; + return cchResult; + +} + +#ifdef _PREFAST_ +#pragma warning( pop ) // Restore the /analyze warnings +#endif + + +//----------------------------------------------------------------------------- +// Purpose: Converts a wchar_t string into a UCS2 string -noop on windows +//----------------------------------------------------------------------------- +int _V_UnicodeToUCS2( const wchar_t *pUnicode, int cubSrcInBytes, char *pUCS2, int cubDestSizeInBytes ) +{ +#ifdef _WIN32 + // Figure out which buffer is smaller and convert from bytes to character + // counts. + int cchResult = min( (size_t)cubSrcInBytes/sizeof(wchar_t), cubDestSizeInBytes/sizeof(wchar_t) ); + wchar_t *pDest = (wchar_t*)pUCS2; + wcsncpy( pDest, pUnicode, cchResult ); + // Make sure we NULL-terminate. + pDest[ cchResult - 1 ] = 0; +#elif defined (POSIX) + iconv_t conv_t = iconv_open( "UCS-2LE", "UTF-32LE" ); + size_t cchResult = -1; + size_t nLenUnicde = cubSrcInBytes; + size_t nMaxUCS2 = cubDestSizeInBytes; + char *pIn = (char*)pUnicode; + char *pOut = pUCS2; + if ( conv_t > 0 ) + { + cchResult = iconv( conv_t, &pIn, &nLenUnicde, &pOut, &nMaxUCS2 ); + iconv_close( conv_t ); + if ( (int)cchResult < 0 ) + cchResult = 0; + else + cchResult = cubSrcInBytes / sizeof( wchar_t ); + } +#else + #error Must be implemented for this platform +#endif + return cchResult; +} + + +//----------------------------------------------------------------------------- +// Purpose: Converts a ucs-2 (windows wchar_t) string into a UTF8 (standard) string +//----------------------------------------------------------------------------- +int _V_UCS2ToUTF8( const ucs2 *pUCS2, char *pUTF8, int cubDestSizeInBytes ) +{ + AssertValidStringPtr(pUTF8, cubDestSizeInBytes); + AssertValidReadPtr(pUCS2); + + pUTF8[0] = 0; +#ifdef _WIN32 + // under win32 wchar_t == ucs2, sigh + int cchResult = WideCharToMultiByte( CP_UTF8, 0, pUCS2, -1, pUTF8, cubDestSizeInBytes, NULL, NULL ); +#elif defined(POSIX) + iconv_t conv_t = iconv_open( "UTF-8", "UCS-2LE" ); + size_t cchResult = -1; + + // pUCS2 will be null-terminated so use that to work out the input + // buffer size. Note that we shouldn't assume iconv will stop when it + // finds a zero, and nLenUnicde should be given in bytes, so we multiply + // it by sizeof( ucs2 ) at the end. + size_t nLenUnicde = 0; + while ( pUCS2[nLenUnicde] ) + { + ++nLenUnicde; + } + nLenUnicde *= sizeof( ucs2 ); + + // Calculate number of bytes we want iconv to write, leaving space + // for the null-terminator + size_t nMaxUTF8 = cubDestSizeInBytes - 1; + char *pIn = (char *)pUCS2; + char *pOut = (char *)pUTF8; + if ( conv_t > 0 ) + { + const size_t nBytesToWrite = nMaxUTF8; + cchResult = iconv( conv_t, &pIn, &nLenUnicde, &pOut, &nMaxUTF8 ); + + // Calculate how many bytes were actually written and use that to + // null-terminate our output string. + const size_t nBytesWritten = nBytesToWrite - nMaxUTF8; + pUTF8[nBytesWritten] = 0; + + iconv_close( conv_t ); + if ( (int)cchResult < 0 ) + cchResult = 0; + else + cchResult = nMaxUTF8; + } +#endif + pUTF8[cubDestSizeInBytes - 1] = 0; + return cchResult; +} + + +//----------------------------------------------------------------------------- +// Purpose: Converts a UTF8 to ucs-2 (windows wchar_t) +//----------------------------------------------------------------------------- +int _V_UTF8ToUCS2( const char *pUTF8, int cubSrcInBytes, ucs2 *pUCS2, int cubDestSizeInBytes ) +{ + Assert( cubDestSizeInBytes >= sizeof(pUCS2[0]) ); + AssertValidStringPtr(pUTF8, cubDestSizeInBytes); + AssertValidReadPtr(pUCS2); + + pUCS2[0] = 0; +#ifdef _WIN32 + // under win32 wchar_t == ucs2, sigh + int cchResult = MultiByteToWideChar( CP_UTF8, 0, pUTF8, -1, pUCS2, cubDestSizeInBytes / sizeof(wchar_t) ); +#elif defined( _PS3 ) // bugbug JLB + int cchResult = 0; + Assert( 0 ); +#elif defined(POSIX) + iconv_t conv_t = iconv_open( "UCS-2LE", "UTF-8" ); + size_t cchResult = -1; + size_t nLenUnicde = cubSrcInBytes; + size_t nMaxUTF8 = cubDestSizeInBytes; + char *pIn = (char *)pUTF8; + char *pOut = (char *)pUCS2; + if ( conv_t > 0 ) + { + cchResult = iconv( conv_t, &pIn, &nLenUnicde, &pOut, &nMaxUTF8 ); + iconv_close( conv_t ); + if ( (int)cchResult < 0 ) + cchResult = 0; + else + cchResult = cubSrcInBytes; + + } +#endif + pUCS2[ (cubDestSizeInBytes/sizeof(ucs2)) - 1] = 0; + return cchResult; +} + + + +//----------------------------------------------------------------------------- +// Purpose: Returns the 4 bit nibble for a hex character +// Input : c - +// Output : unsigned char +//----------------------------------------------------------------------------- +unsigned char V_nibble( char c ) +{ + if ( ( c >= '0' ) && + ( c <= '9' ) ) + { + return (unsigned char)(c - '0'); + } + + if ( ( c >= 'A' ) && + ( c <= 'F' ) ) + { + return (unsigned char)(c - 'A' + 0x0a); + } + + if ( ( c >= 'a' ) && + ( c <= 'f' ) ) + { + return (unsigned char)(c - 'a' + 0x0a); + } + + return '0'; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *in - +// numchars - +// *out - +// maxoutputbytes - +//----------------------------------------------------------------------------- +void V_hextobinary( char const *in, int numchars, byte *out, int maxoutputbytes ) +{ + int len = V_strlen( in ); + numchars = min( len, numchars ); + // Make sure it's even + numchars = ( numchars ) & ~0x1; + + // Must be an even # of input characters (two chars per output byte) + Assert( numchars >= 2 ); + + memset( out, 0x00, maxoutputbytes ); + + byte *p; + int i; + + p = out; + for ( i = 0; + ( i < numchars ) && ( ( p - out ) < maxoutputbytes ); + i+=2, p++ ) + { + *p = ( V_nibble( in[i] ) << 4 ) | V_nibble( in[i+1] ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *in - +// inputbytes - +// *out - +// outsize - +//----------------------------------------------------------------------------- +void V_binarytohex( const byte *in, int inputbytes, char *out, int outsize ) +{ + Assert( outsize >= 1 ); + char doublet[10]; + int i; + + out[0]=0; + + for ( i = 0; i < inputbytes; i++ ) + { + unsigned char c = in[i]; + V_snprintf( doublet, sizeof( doublet ), "%02x", c ); + V_strncat( out, doublet, outsize, COPY_ALL_CHARACTERS ); + } +} + +// Even though \ on Posix (Linux&Mac) isn't techincally a path separator we are +// now counting it as one even Posix since so many times our filepaths aren't actual +// paths but rather text strings passed in from data files, treating \ as a pathseparator +// covers the full range of cases +bool PATHSEPARATOR( char c ) +{ + return c == '\\' || c == '/'; +} + + +//----------------------------------------------------------------------------- +// Purpose: Extracts the base name of a file (no path, no extension, assumes '/' or '\' as path separator) +// Input : *in - +// *out - +// maxlen - +//----------------------------------------------------------------------------- +void V_FileBase( const char *in, char *out, int maxlen ) +{ + Assert( maxlen >= 1 ); + Assert( in ); + Assert( out ); + + if ( !in || !in[ 0 ] ) + { + *out = 0; + return; + } + + int len, start, end; + + len = V_strlen( in ); + + // scan backward for '.' + end = len - 1; + while ( end&& in[end] != '.' && !PATHSEPARATOR( in[end] ) ) + { + end--; + } + + if ( in[end] != '.' ) // no '.', copy to end + { + end = len-1; + } + else + { + end--; // Found ',', copy to left of '.' + } + + // Scan backward for '/' + start = len-1; + while ( start >= 0 && !PATHSEPARATOR( in[start] ) ) + { + start--; + } + + if ( start < 0 || !PATHSEPARATOR( in[start] ) ) + { + start = 0; + } + else + { + start++; + } + + // Length of new sting + len = end - start + 1; + + int maxcopy = min( len + 1, maxlen ); + + // Copy partial string + V_strncpy( out, &in[start], maxcopy ); +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *ppath - +//----------------------------------------------------------------------------- +void V_StripTrailingSlash( char *ppath ) +{ + Assert( ppath ); + + int len = V_strlen( ppath ); + if ( len > 0 ) + { + if ( PATHSEPARATOR( ppath[ len - 1 ] ) ) + { + ppath[ len - 1 ] = 0; + } + } +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *ppline - +//----------------------------------------------------------------------------- +void V_StripTrailingWhitespace( char *ppline ) +{ + Assert( ppline ); + + int len = V_strlen( ppline ); + while ( len > 0 ) + { + if ( !V_isspace( ppline[ len - 1 ] ) ) + break; + ppline[ len - 1 ] = 0; + len--; + } +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *ppline - +//----------------------------------------------------------------------------- +void V_StripLeadingWhitespace( char *ppline ) +{ + Assert( ppline ); + + // Skip past initial whitespace + int skip = 0; + while( V_isspace( ppline[ skip ] ) ) + skip++; + // Shuffle the rest of the string back (including the NULL-terminator) + if ( skip ) + { + while( ( ppline[0] = ppline[skip] ) != 0 ) + ppline++; + } +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *ppline - +//----------------------------------------------------------------------------- +void V_StripSurroundingQuotes( char *ppline ) +{ + Assert( ppline ); + + int len = V_strlen( ppline ) - 2; + if ( ( ppline[0] == '"' ) && ( len >= 0 ) && ( ppline[len+1] == '"' ) ) + { + for ( int i = 0; i < len; i++ ) + ppline[i] = ppline[i+1]; + ppline[len] = 0; + } +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *in - +// *out - +// outSize - +//----------------------------------------------------------------------------- +void V_StripExtension( const char *in, char *out, int outSize ) +{ + // Find the last dot. If it's followed by a dot or a slash, then it's part of a + // directory specifier like ../../somedir/./blah. + + // scan backward for '.' + int end = V_strlen( in ) - 1; + while ( end > 0 && in[end] != '.' && !PATHSEPARATOR( in[end] ) ) + { + --end; + } + + if (end > 0 && !PATHSEPARATOR( in[end] ) && end < outSize) + { + int nChars = min( end, outSize-1 ); + if ( out != in ) + { + memcpy( out, in, nChars ); + } + out[nChars] = 0; + } + else + { + // nothing found + if ( out != in ) + { + V_strncpy( out, in, outSize ); + } + } +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *path - +// *extension - +// pathStringLength - +//----------------------------------------------------------------------------- +void V_DefaultExtension( char *path, const char *extension, int pathStringLength ) +{ + Assert( path ); + Assert( pathStringLength >= 1 ); + Assert( extension ); + Assert( extension[0] == '.' ); + + char *src; + + // if path doesn't have a .EXT, append extension + // (extension should include the .) + src = path + V_strlen(path) - 1; + + while ( !PATHSEPARATOR( *src ) && ( src > path ) ) + { + if (*src == '.') + { + // it has an extension + return; + } + src--; + } + + // Concatenate the desired extension + V_strncat( path, extension, pathStringLength, COPY_ALL_CHARACTERS ); +} + +//----------------------------------------------------------------------------- +// Purpose: Force extension... +// Input : *path - +// *extension - +// pathStringLength - +//----------------------------------------------------------------------------- +void V_SetExtension( char *path, const char *extension, int pathStringLength ) +{ + V_StripExtension( path, path, pathStringLength ); + + // We either had an extension and stripped it, or didn't have an extension + // at all. Either way, we need to concatenate our extension now. + + // extension is not required to start with '.', so if it's not there, + // then append that first. + if ( extension[0] != '.' ) + { + V_strncat( path, ".", pathStringLength, COPY_ALL_CHARACTERS ); + } + + V_strncat( path, extension, pathStringLength, COPY_ALL_CHARACTERS ); +} + +//----------------------------------------------------------------------------- +// Purpose: Remove final filename from string +// Input : *path - +// Output : void V_StripFilename +//----------------------------------------------------------------------------- +void V_StripFilename (char *path) +{ + int length; + + length = V_strlen( path )-1; + if ( length <= 0 ) + return; + + while ( length > 0 && + !PATHSEPARATOR( path[length] ) ) + { + length--; + } + + path[ length ] = 0; +} + +#ifdef _WIN32 +#define CORRECT_PATH_SEPARATOR '\\' +#define INCORRECT_PATH_SEPARATOR '/' +#elif POSIX +#define CORRECT_PATH_SEPARATOR '/' +#define INCORRECT_PATH_SEPARATOR '\\' +#endif + +//----------------------------------------------------------------------------- +// Purpose: Changes all '/' or '\' characters into separator +// Input : *pname - +// separator - +//----------------------------------------------------------------------------- +void V_FixSlashes( char *pname, char separator /* = CORRECT_PATH_SEPARATOR */ ) +{ + while ( *pname ) + { + if ( *pname == INCORRECT_PATH_SEPARATOR || *pname == CORRECT_PATH_SEPARATOR ) + { + *pname = separator; + } + pname++; + } +} + + +//----------------------------------------------------------------------------- +// Purpose: This function fixes cases of filenames like materials\\blah.vmt or somepath\otherpath\\ and removes the extra double slash. +//----------------------------------------------------------------------------- +void V_FixDoubleSlashes( char *pStr ) +{ + int len = V_strlen( pStr ); + + for ( int i=1; i < len-1; i++ ) + { + if ( (pStr[i] == '/' || pStr[i] == '\\') && (pStr[i+1] == '/' || pStr[i+1] == '\\') ) + { + // This means there's a double slash somewhere past the start of the filename. That + // can happen in Hammer if they use a material in the root directory. You'll get a filename + // that looks like 'materials\\blah.vmt' + V_memmove( &pStr[i], &pStr[i+1], len - i ); + --len; + } + } +} + +//----------------------------------------------------------------------------- +// Purpose: Strip off the last directory from dirName +// Input : *dirName - +// maxlen - +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool V_StripLastDir( char *dirName, int maxlen ) +{ + if( dirName[0] == 0 || + !V_stricmp( dirName, "./" ) || + !V_stricmp( dirName, ".\\" ) ) + return false; + + int len = V_strlen( dirName ); + + Assert( len < maxlen ); + + // skip trailing slash + if ( PATHSEPARATOR( dirName[len-1] ) ) + { + len--; + } + + while ( len > 0 ) + { + if ( PATHSEPARATOR( dirName[len-1] ) ) + { + dirName[len] = 0; + V_FixSlashes( dirName, CORRECT_PATH_SEPARATOR ); + return true; + } + len--; + } + + // Allow it to return an empty string and true. This can happen if something like "tf2/" is passed in. + // The correct behavior is to strip off the last directory ("tf2") and return true. + if( len == 0 ) + { + V_snprintf( dirName, maxlen, ".%c", CORRECT_PATH_SEPARATOR ); + return true; + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns a pointer to the beginning of the unqualified file name +// (no path information) +// Input: in - file name (may be unqualified, relative or absolute path) +// Output: pointer to unqualified file name +//----------------------------------------------------------------------------- +const char * V_UnqualifiedFileName( const char * in ) +{ + // back up until the character after the first path separator we find, + // or the beginning of the string + const char * out = in + strlen( in ) - 1; + while ( ( out > in ) && ( !PATHSEPARATOR( *( out-1 ) ) ) ) + out--; + return out; +} + + +//----------------------------------------------------------------------------- +// Purpose: Composes a path and filename together, inserting a path separator +// if need be +// Input: path - path to use +// filename - filename to use +// dest - buffer to compose result in +// destSize - size of destination buffer +//----------------------------------------------------------------------------- +void V_ComposeFileName( const char *path, const char *filename, char *dest, int destSize ) +{ + V_strncpy( dest, path, destSize ); + V_FixSlashes( dest ); + V_AppendSlash( dest, destSize ); + V_strncat( dest, filename, destSize, COPY_ALL_CHARACTERS ); + V_FixSlashes( dest ); +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *path - +// *dest - +// destSize - +// Output : void V_ExtractFilePath +//----------------------------------------------------------------------------- +bool V_ExtractFilePath (const char *path, char *dest, int destSize ) +{ + Assert( destSize >= 1 ); + if ( destSize < 1 ) + { + return false; + } + + // Last char + int len = V_strlen(path); + const char *src = path + (len ? len-1 : 0); + + // back up until a \ or the start + while ( src != path && !PATHSEPARATOR( *(src-1) ) ) + { + src--; + } + + int copysize = min( (int)((ptrdiff_t)src - (ptrdiff_t)path), destSize - 1 ); + memcpy( dest, path, copysize ); + dest[copysize] = 0; + + return copysize != 0 ? true : false; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *path - +// *dest - +// destSize - +// Output : void V_ExtractFileExtension +//----------------------------------------------------------------------------- +void V_ExtractFileExtension( const char *path, char *dest, int destSize ) +{ + *dest = NULL; + const char * extension = V_GetFileExtension( path ); + if ( NULL != extension ) + V_strncpy( dest, extension, destSize ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns a pointer to the file extension within a file name string +// Input: in - file name +// Output: pointer to beginning of extension (after the "."), or NULL +// if there is no extension +//----------------------------------------------------------------------------- +const char * V_GetFileExtension( const char * path ) +{ + const char *src; + + src = path + strlen(path) - 1; + +// +// back up until a . or the start +// + while (src != path && *(src-1) != '.' ) + src--; + + // check to see if the '.' is part of a pathname + if (src == path || PATHSEPARATOR( *src ) ) + { + return NULL; // no extension + } + + return src; +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns a pointer to the filename part of a path string +// Input: in - file name +// Output: pointer to beginning of filename (after the "/"). If there were no /, +// output is identical to input +//----------------------------------------------------------------------------- +const char * V_GetFileName( const char * path ) +{ + return V_UnqualifiedFileName( path ); +} + + +bool V_RemoveDotSlashes( char *pFilename, char separator, bool bRemoveDoubleSlashes /* = true */ ) +{ + char *pIn = pFilename; + char *pOut = pFilename; + bool bRetVal = true; + + bool bBoundary = true; + while ( *pIn ) + { + if ( bBoundary && pIn[0] == '.' && pIn[1] == '.' && ( PATHSEPARATOR( pIn[2] ) || !pIn[2] ) ) + { + // Get rid of /../ or trailing /.. by backing pOut up to previous separator + + // Eat the last separator (or repeated separators) we wrote out + while ( pOut != pFilename && pOut[-1] == separator ) + { + --pOut; + } + + while ( true ) + { + if ( pOut == pFilename ) + { + bRetVal = false; // backwards compat. return value, even though we continue handling + break; + } + --pOut; + if ( *pOut == separator ) + { + break; + } + } + + // Skip the '..' but not the slash, next loop iteration will handle separator + pIn += 2; + bBoundary = ( pOut == pFilename ); + } + else if ( bBoundary && pIn[0] == '.' && ( PATHSEPARATOR( pIn[1] ) || !pIn[1] ) ) + { + // Handle "./" by simply skipping this sequence. bBoundary is unchanged. + if ( PATHSEPARATOR( pIn[1] ) ) + { + pIn += 2; + } + else + { + // Special case: if trailing "." is preceded by separator, eg "path/.", + // then the final separator should also be stripped. bBoundary may then + // be in an incorrect state, but we are at the end of processing anyway + // so we don't really care (the processing loop is about to terminate). + if ( pOut != pFilename && pOut[-1] == separator ) + { + --pOut; + } + pIn += 1; + } + } + else if ( PATHSEPARATOR( pIn[0] ) ) + { + *pOut = separator; + pOut += 1 - (bBoundary & bRemoveDoubleSlashes & (pOut != pFilename)); + pIn += 1; + bBoundary = true; + } + else + { + if ( pOut != pIn ) + { + *pOut = *pIn; + } + pOut += 1; + pIn += 1; + bBoundary = false; + } + } + *pOut = 0; + + return bRetVal; +} + + +void V_AppendSlash( char *pStr, int strSize ) +{ + int len = V_strlen( pStr ); + if ( len > 0 && !PATHSEPARATOR(pStr[len-1]) ) + { + if ( len+1 >= strSize ) + Error( "V_AppendSlash: ran out of space on %s.", pStr ); + + pStr[len] = CORRECT_PATH_SEPARATOR; + pStr[len+1] = 0; + } +} + + +void V_MakeAbsolutePath( char *pOut, int outLen, const char *pPath, const char *pStartingDir ) +{ + if ( V_IsAbsolutePath( pPath ) ) + { + // pPath is not relative.. just copy it. + V_strncpy( pOut, pPath, outLen ); + } + else + { + // Make sure the starting directory is absolute.. + if ( pStartingDir && V_IsAbsolutePath( pStartingDir ) ) + { + V_strncpy( pOut, pStartingDir, outLen ); + } + else + { + if ( !_getcwd( pOut, outLen ) ) + Error( "V_MakeAbsolutePath: _getcwd failed." ); + + if ( pStartingDir ) + { + V_AppendSlash( pOut, outLen ); + V_strncat( pOut, pStartingDir, outLen, COPY_ALL_CHARACTERS ); + } + } + + // Concatenate the paths. + V_AppendSlash( pOut, outLen ); + V_strncat( pOut, pPath, outLen, COPY_ALL_CHARACTERS ); + } + + if ( !V_RemoveDotSlashes( pOut ) ) + Error( "V_MakeAbsolutePath: tried to \"..\" past the root." ); + + //V_FixSlashes( pOut ); - handled by V_RemoveDotSlashes +} + + +//----------------------------------------------------------------------------- +// Makes a relative path +//----------------------------------------------------------------------------- +bool V_MakeRelativePath( const char *pFullPath, const char *pDirectory, char *pRelativePath, int nBufLen ) +{ + pRelativePath[0] = 0; + + const char *pPath = pFullPath; + const char *pDir = pDirectory; + + // Strip out common parts of the path + const char *pLastCommonPath = NULL; + const char *pLastCommonDir = NULL; + while ( *pPath && ( FastToLower( *pPath ) == FastToLower( *pDir ) || + ( PATHSEPARATOR( *pPath ) && ( PATHSEPARATOR( *pDir ) || (*pDir == 0) ) ) ) ) + { + if ( PATHSEPARATOR( *pPath ) ) + { + pLastCommonPath = pPath + 1; + pLastCommonDir = pDir + 1; + } + if ( *pDir == 0 ) + { + --pLastCommonDir; + break; + } + ++pDir; ++pPath; + } + + // Nothing in common + if ( !pLastCommonPath ) + return false; + + // For each path separator remaining in the dir, need a ../ + int nOutLen = 0; + bool bLastCharWasSeparator = true; + for ( ; *pLastCommonDir; ++pLastCommonDir ) + { + if ( PATHSEPARATOR( *pLastCommonDir ) ) + { + pRelativePath[nOutLen++] = '.'; + pRelativePath[nOutLen++] = '.'; + pRelativePath[nOutLen++] = CORRECT_PATH_SEPARATOR; + bLastCharWasSeparator = true; + } + else + { + bLastCharWasSeparator = false; + } + } + + // Deal with relative paths not specified with a trailing slash + if ( !bLastCharWasSeparator ) + { + pRelativePath[nOutLen++] = '.'; + pRelativePath[nOutLen++] = '.'; + pRelativePath[nOutLen++] = CORRECT_PATH_SEPARATOR; + } + + // Copy the remaining part of the relative path over, fixing the path separators + for ( ; *pLastCommonPath; ++pLastCommonPath ) + { + if ( PATHSEPARATOR( *pLastCommonPath ) ) + { + pRelativePath[nOutLen++] = CORRECT_PATH_SEPARATOR; + } + else + { + pRelativePath[nOutLen++] = *pLastCommonPath; + } + + // Check for overflow + if ( nOutLen == nBufLen - 1 ) + break; + } + + pRelativePath[nOutLen] = 0; + return true; +} + + +//----------------------------------------------------------------------------- +// small helper function shared by lots of modules +//----------------------------------------------------------------------------- +bool V_IsAbsolutePath( const char *pStr ) +{ + bool bIsAbsolute = ( pStr[0] && pStr[1] == ':' ) || pStr[0] == '/' || pStr[0] == '\\'; + if ( IsX360() && !bIsAbsolute ) + { + bIsAbsolute = ( V_stristr( pStr, ":" ) != NULL ); + } + return bIsAbsolute; +} + + +// Copies at most nCharsToCopy bytes from pIn into pOut. +// Returns false if it would have overflowed pOut's buffer. +static bool CopyToMaxChars( char *pOut, int outSize, const char *pIn, int nCharsToCopy ) +{ + if ( outSize == 0 ) + return false; + + int iOut = 0; + while ( *pIn && nCharsToCopy > 0 ) + { + if ( iOut == (outSize-1) ) + { + pOut[iOut] = 0; + return false; + } + pOut[iOut] = *pIn; + ++iOut; + ++pIn; + --nCharsToCopy; + } + + pOut[iOut] = 0; + return true; +} + + +//----------------------------------------------------------------------------- +// Fixes up a file name, removing dot slashes, fixing slashes, converting to lowercase, etc. +//----------------------------------------------------------------------------- +void V_FixupPathName( char *pOut, size_t nOutLen, const char *pPath ) +{ + V_strncpy( pOut, pPath, nOutLen ); + V_RemoveDotSlashes( pOut, CORRECT_PATH_SEPARATOR, true ); +#ifdef WIN32 + V_strlower( pOut ); +#endif +} + + +// Returns true if it completed successfully. +// If it would overflow pOut, it fills as much as it can and returns false. +bool V_StrSubst( + const char *pIn, + const char *pMatch, + const char *pReplaceWith, + char *pOut, + int outLen, + bool bCaseSensitive + ) +{ + int replaceFromLen = strlen( pMatch ); + int replaceToLen = strlen( pReplaceWith ); + + const char *pInStart = pIn; + char *pOutPos = pOut; + pOutPos[0] = 0; + + while ( 1 ) + { + int nRemainingOut = outLen - (pOutPos - pOut); + + const char *pTestPos = ( bCaseSensitive ? strstr( pInStart, pMatch ) : V_stristr( pInStart, pMatch ) ); + if ( pTestPos ) + { + // Found an occurence of pMatch. First, copy whatever leads up to the string. + int copyLen = pTestPos - pInStart; + if ( !CopyToMaxChars( pOutPos, nRemainingOut, pInStart, copyLen ) ) + return false; + + // Did we hit the end of the output string? + if ( copyLen > nRemainingOut-1 ) + return false; + + pOutPos += strlen( pOutPos ); + nRemainingOut = outLen - (pOutPos - pOut); + + // Now add the replacement string. + if ( !CopyToMaxChars( pOutPos, nRemainingOut, pReplaceWith, replaceToLen ) ) + return false; + + pInStart += copyLen + replaceFromLen; + pOutPos += replaceToLen; + } + else + { + // We're at the end of pIn. Copy whatever remains and get out. + int copyLen = strlen( pInStart ); + V_strncpy( pOutPos, pInStart, nRemainingOut ); + return ( copyLen <= nRemainingOut-1 ); + } + } +} + + +char* AllocString( const char *pStr, int nMaxChars ) +{ + int allocLen; + if ( nMaxChars == -1 ) + allocLen = strlen( pStr ) + 1; + else + allocLen = min( (int)strlen(pStr), nMaxChars ) + 1; + + char *pOut = new char[allocLen]; + V_strncpy( pOut, pStr, allocLen ); + return pOut; +} + + +void V_SplitString2( const char *pString, const char **pSeparators, int nSeparators, CUtlVector<char*> &outStrings ) +{ + outStrings.Purge(); + const char *pCurPos = pString; + while ( 1 ) + { + int iFirstSeparator = -1; + const char *pFirstSeparator = 0; + for ( int i=0; i < nSeparators; i++ ) + { + const char *pTest = V_stristr( pCurPos, pSeparators[i] ); + if ( pTest && (!pFirstSeparator || pTest < pFirstSeparator) ) + { + iFirstSeparator = i; + pFirstSeparator = pTest; + } + } + + if ( pFirstSeparator ) + { + // Split on this separator and continue on. + int separatorLen = strlen( pSeparators[iFirstSeparator] ); + if ( pFirstSeparator > pCurPos ) + { + outStrings.AddToTail( AllocString( pCurPos, pFirstSeparator-pCurPos ) ); + } + + pCurPos = pFirstSeparator + separatorLen; + } + else + { + // Copy the rest of the string + if ( strlen( pCurPos ) ) + { + outStrings.AddToTail( AllocString( pCurPos, -1 ) ); + } + return; + } + } +} + + +void V_SplitString( const char *pString, const char *pSeparator, CUtlVector<char*> &outStrings ) +{ + V_SplitString2( pString, &pSeparator, 1, outStrings ); +} + + +bool V_GetCurrentDirectory( char *pOut, int maxLen ) +{ + return _getcwd( pOut, maxLen ) == pOut; +} + + +bool V_SetCurrentDirectory( const char *pDirName ) +{ + return _chdir( pDirName ) == 0; +} + + +// This function takes a slice out of pStr and stores it in pOut. +// It follows the Python slice convention: +// Negative numbers wrap around the string (-1 references the last character). +// Numbers are clamped to the end of the string. +void V_StrSlice( const char *pStr, int firstChar, int lastCharNonInclusive, char *pOut, int outSize ) +{ + if ( outSize == 0 ) + return; + + int length = strlen( pStr ); + + // Fixup the string indices. + if ( firstChar < 0 ) + { + firstChar = length - (-firstChar % length); + } + else if ( firstChar >= length ) + { + pOut[0] = 0; + return; + } + + if ( lastCharNonInclusive < 0 ) + { + lastCharNonInclusive = length - (-lastCharNonInclusive % length); + } + else if ( lastCharNonInclusive > length ) + { + lastCharNonInclusive %= length; + } + + if ( lastCharNonInclusive <= firstChar ) + { + pOut[0] = 0; + return; + } + + int copyLen = lastCharNonInclusive - firstChar; + if ( copyLen <= (outSize-1) ) + { + memcpy( pOut, &pStr[firstChar], copyLen ); + pOut[copyLen] = 0; + } + else + { + memcpy( pOut, &pStr[firstChar], outSize-1 ); + pOut[outSize-1] = 0; + } +} + + +void V_StrLeft( const char *pStr, int nChars, char *pOut, int outSize ) +{ + if ( nChars == 0 ) + { + if ( outSize != 0 ) + pOut[0] = 0; + + return; + } + + V_StrSlice( pStr, 0, nChars, pOut, outSize ); +} + + +void V_StrRight( const char *pStr, int nChars, char *pOut, int outSize ) +{ + int len = strlen( pStr ); + if ( nChars >= len ) + { + V_strncpy( pOut, pStr, outSize ); + } + else + { + V_StrSlice( pStr, -nChars, strlen( pStr ), pOut, outSize ); + } +} + +//----------------------------------------------------------------------------- +// Convert multibyte to wchar + back +//----------------------------------------------------------------------------- +void V_strtowcs( const char *pString, int nInSize, wchar_t *pWString, int nOutSizeInBytes ) +{ + Assert( nOutSizeInBytes >= sizeof(pWString[0]) ); +#ifdef _WIN32 + int nOutSizeInChars = nOutSizeInBytes / sizeof(pWString[0]); + int result = MultiByteToWideChar( CP_UTF8, 0, pString, nInSize, pWString, nOutSizeInChars ); + // If the string completely fails to fit then MultiByteToWideChar will return 0. + // If the string exactly fits but with no room for a null-terminator then MultiByteToWideChar + // will happily fill the buffer and omit the null-terminator, returning nOutSizeInChars. + // Either way we need to return an empty string rather than a bogus and possibly not + // null-terminated result. + if ( result <= 0 || result >= nOutSizeInChars ) + { + // If nInSize includes the null-terminator then a result of nOutSizeInChars is + // legal. We check this by seeing if the last character in the output buffer is + // a zero. + if ( result == nOutSizeInChars && pWString[ nOutSizeInChars - 1 ] == 0) + { + // We're okay! Do nothing. + } + else + { + // The string completely to fit. Null-terminate the buffer. + *pWString = L'\0'; + } + } + else + { + // We have successfully converted our string. Now we need to null-terminate it, because + // MultiByteToWideChar will only do that if nInSize includes the source null-terminator! + pWString[ result ] = 0; + } +#elif POSIX + if ( mbstowcs( pWString, pString, nOutSizeInBytes / sizeof(pWString[0]) ) <= 0 ) + { + *pWString = 0; + } +#endif +} + +void V_wcstostr( const wchar_t *pWString, int nInSize, char *pString, int nOutSizeInChars ) +{ +#ifdef _WIN32 + int result = WideCharToMultiByte( CP_UTF8, 0, pWString, nInSize, pString, nOutSizeInChars, NULL, NULL ); + // If the string completely fails to fit then MultiByteToWideChar will return 0. + // If the string exactly fits but with no room for a null-terminator then MultiByteToWideChar + // will happily fill the buffer and omit the null-terminator, returning nOutSizeInChars. + // Either way we need to return an empty string rather than a bogus and possibly not + // null-terminated result. + if ( result <= 0 || result >= nOutSizeInChars ) + { + // If nInSize includes the null-terminator then a result of nOutSizeInChars is + // legal. We check this by seeing if the last character in the output buffer is + // a zero. + if ( result == nOutSizeInChars && pWString[ nOutSizeInChars - 1 ] == 0) + { + // We're okay! Do nothing. + } + else + { + *pString = '\0'; + } + } + else + { + // We have successfully converted our string. Now we need to null-terminate it, because + // MultiByteToWideChar will only do that if nInSize includes the source null-terminator! + pString[ result ] = '\0'; + } +#elif POSIX + if ( wcstombs( pString, pWString, nOutSizeInChars ) <= 0 ) + { + *pString = '\0'; + } +#endif +} + + + +//-------------------------------------------------------------------------------- +// backslashification +//-------------------------------------------------------------------------------- + +static char s_BackSlashMap[]="\tt\nn\rr\"\"\\\\"; + +char *V_AddBackSlashesToSpecialChars( char const *pSrc ) +{ + // first, count how much space we are going to need + int nSpaceNeeded = 0; + for( char const *pScan = pSrc; *pScan; pScan++ ) + { + nSpaceNeeded++; + for(char const *pCharSet=s_BackSlashMap; *pCharSet; pCharSet += 2 ) + { + if ( *pCharSet == *pScan ) + nSpaceNeeded++; // we need to store a bakslash + } + } + char *pRet = new char[ nSpaceNeeded + 1 ]; // +1 for null + char *pOut = pRet; + + for( char const *pScan = pSrc; *pScan; pScan++ ) + { + bool bIsSpecial = false; + for(char const *pCharSet=s_BackSlashMap; *pCharSet; pCharSet += 2 ) + { + if ( *pCharSet == *pScan ) + { + *( pOut++ ) = '\\'; + *( pOut++ ) = pCharSet[1]; + bIsSpecial = true; + break; + } + } + if (! bIsSpecial ) + { + *( pOut++ ) = *pScan; + } + } + *( pOut++ ) = 0; + return pRet; +} + +//----------------------------------------------------------------------------- +// Purpose: Helper for converting a numeric value to a hex digit, value should be 0-15. +//----------------------------------------------------------------------------- +char cIntToHexDigit( int nValue ) +{ + Assert( nValue >= 0 && nValue <= 15 ); + return "0123456789ABCDEF"[ nValue & 15 ]; +} + +//----------------------------------------------------------------------------- +// Purpose: Helper for converting a hex char value to numeric, return -1 if the char +// is not a valid hex digit. +//----------------------------------------------------------------------------- +int iHexCharToInt( char cValue ) +{ + int32 iValue = cValue; + if ( (uint32)( iValue - '0' ) < 10 ) + return iValue - '0'; + + iValue |= 0x20; + if ( (uint32)( iValue - 'a' ) < 6 ) + return iValue - 'a' + 10; + + return -1; +} + +//----------------------------------------------------------------------------- +// Purpose: Internal implementation of encode, works in the strict RFC manner, or +// with spaces turned to + like HTML form encoding. +//----------------------------------------------------------------------------- +void Q_URLEncodeInternal( char *pchDest, int nDestLen, const char *pchSource, int nSourceLen, bool bUsePlusForSpace ) +{ + if ( nDestLen < 3*nSourceLen ) + { + pchDest[0] = '\0'; + AssertMsg( false, "Target buffer for Q_URLEncode needs to be 3 times larger than source to guarantee enough space\n" ); + return; + } + + int iDestPos = 0; + for ( int i=0; i < nSourceLen; ++i ) + { + // We allow only a-z, A-Z, 0-9, period, underscore, and hyphen to pass through unescaped. + // These are the characters allowed by both the original RFC 1738 and the latest RFC 3986. + // Current specs also allow '~', but that is forbidden under original RFC 1738. + if ( !( pchSource[i] >= 'a' && pchSource[i] <= 'z' ) && !( pchSource[i] >= 'A' && pchSource[i] <= 'Z' ) && !(pchSource[i] >= '0' && pchSource[i] <= '9' ) + && pchSource[i] != '-' && pchSource[i] != '_' && pchSource[i] != '.' + ) + { + if ( bUsePlusForSpace && pchSource[i] == ' ' ) + { + pchDest[iDestPos++] = '+'; + } + else + { + pchDest[iDestPos++] = '%'; + uint8 iValue = pchSource[i]; + if ( iValue == 0 ) + { + pchDest[iDestPos++] = '0'; + pchDest[iDestPos++] = '0'; + } + else + { + char cHexDigit1 = cIntToHexDigit( iValue % 16 ); + iValue /= 16; + char cHexDigit2 = cIntToHexDigit( iValue ); + pchDest[iDestPos++] = cHexDigit2; + pchDest[iDestPos++] = cHexDigit1; + } + } + } + else + { + pchDest[iDestPos++] = pchSource[i]; + } + } + + // Null terminate + pchDest[iDestPos++] = 0; +} + + +//----------------------------------------------------------------------------- +// Purpose: Internal implementation of decode, works in the strict RFC manner, or +// with spaces turned to + like HTML form encoding. +// +// Returns the amount of space used in the output buffer. +//----------------------------------------------------------------------------- +size_t Q_URLDecodeInternal( char *pchDecodeDest, int nDecodeDestLen, const char *pchEncodedSource, int nEncodedSourceLen, bool bUsePlusForSpace ) +{ + if ( nDecodeDestLen < nEncodedSourceLen ) + { + AssertMsg( false, "Q_URLDecode needs a dest buffer at least as large as the source" ); + return 0; + } + + int iDestPos = 0; + for( int i=0; i < nEncodedSourceLen; ++i ) + { + if ( bUsePlusForSpace && pchEncodedSource[i] == '+' ) + { + pchDecodeDest[ iDestPos++ ] = ' '; + } + else if ( pchEncodedSource[i] == '%' ) + { + // Percent signifies an encoded value, look ahead for the hex code, convert to numeric, and use that + + // First make sure we have 2 more chars + if ( i < nEncodedSourceLen - 2 ) + { + char cHexDigit1 = pchEncodedSource[i+1]; + char cHexDigit2 = pchEncodedSource[i+2]; + + // Turn the chars into a hex value, if they are not valid, then we'll + // just place the % and the following two chars direct into the string, + // even though this really shouldn't happen, who knows what bad clients + // may do with encoding. + bool bValid = false; + int iValue = iHexCharToInt( cHexDigit1 ); + if ( iValue != -1 ) + { + iValue *= 16; + int iValue2 = iHexCharToInt( cHexDigit2 ); + if ( iValue2 != -1 ) + { + iValue += iValue2; + pchDecodeDest[ iDestPos++ ] = iValue; + bValid = true; + } + } + + if ( !bValid ) + { + pchDecodeDest[ iDestPos++ ] = '%'; + pchDecodeDest[ iDestPos++ ] = cHexDigit1; + pchDecodeDest[ iDestPos++ ] = cHexDigit2; + } + } + + // Skip ahead + i += 2; + } + else + { + pchDecodeDest[ iDestPos++ ] = pchEncodedSource[i]; + } + } + + // We may not have extra room to NULL terminate, since this can be used on raw data, but if we do + // go ahead and do it as this can avoid bugs. + if ( iDestPos < nDecodeDestLen ) + { + pchDecodeDest[iDestPos] = 0; + } + + return (size_t)iDestPos; +} + +//----------------------------------------------------------------------------- +// Purpose: Encodes a string (or binary data) from URL encoding format, see rfc1738 section 2.2. +// This version of the call isn't a strict RFC implementation, but uses + for space as is +// the standard in HTML form encoding, despite it not being part of the RFC. +// +// Dest buffer should be at least as large as source buffer to guarantee room for decode. +//----------------------------------------------------------------------------- +void Q_URLEncode( char *pchDest, int nDestLen, const char *pchSource, int nSourceLen ) +{ + return Q_URLEncodeInternal( pchDest, nDestLen, pchSource, nSourceLen, true ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Decodes a string (or binary data) from URL encoding format, see rfc1738 section 2.2. +// This version of the call isn't a strict RFC implementation, but uses + for space as is +// the standard in HTML form encoding, despite it not being part of the RFC. +// +// Dest buffer should be at least as large as source buffer to guarantee room for decode. +// Dest buffer being the same as the source buffer (decode in-place) is explicitly allowed. +//----------------------------------------------------------------------------- +size_t Q_URLDecode( char *pchDecodeDest, int nDecodeDestLen, const char *pchEncodedSource, int nEncodedSourceLen ) +{ + return Q_URLDecodeInternal( pchDecodeDest, nDecodeDestLen, pchEncodedSource, nEncodedSourceLen, true ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Encodes a string (or binary data) from URL encoding format, see rfc1738 section 2.2. +// This version will not encode space as + (which HTML form encoding uses despite not being part of the RFC) +// +// Dest buffer should be at least as large as source buffer to guarantee room for decode. +//----------------------------------------------------------------------------- +void Q_URLEncodeRaw( char *pchDest, int nDestLen, const char *pchSource, int nSourceLen ) +{ + return Q_URLEncodeInternal( pchDest, nDestLen, pchSource, nSourceLen, false ); +} + + +//----------------------------------------------------------------------------- +// Purpose: Decodes a string (or binary data) from URL encoding format, see rfc1738 section 2.2. +// This version will not recognize + as a space (which HTML form encoding uses despite not being part of the RFC) +// +// Dest buffer should be at least as large as source buffer to guarantee room for decode. +// Dest buffer being the same as the source buffer (decode in-place) is explicitly allowed. +//----------------------------------------------------------------------------- +size_t Q_URLDecodeRaw( char *pchDecodeDest, int nDecodeDestLen, const char *pchEncodedSource, int nEncodedSourceLen ) +{ + return Q_URLDecodeInternal( pchDecodeDest, nDecodeDestLen, pchEncodedSource, nEncodedSourceLen, false ); +} + +#if defined( LINUX ) || defined( _PS3 ) +extern "C" void qsort_s( void *base, size_t num, size_t width, int (*compare )(void *, const void *, const void *), void * context ); +#endif + +void V_qsort_s( void *base, size_t num, size_t width, int ( __cdecl *compare )(void *, const void *, const void *), void * context ) +{ +#if defined OSX + // the arguments are swapped 'round on the mac - awesome, huh? + return qsort_r( base, num, width, context, compare ); +#else + return qsort_s( base, num, width, compare, context ); +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: format the time and/or date with the user's current locale +// If timeVal is 0, gets the current time +// +// This is generally for use with chatroom dialogs, etc. which need to be +// able to say "Last message received: %date% at %time%" +// +// Note that this uses time_t because RTime32 is not hooked-up on the client +//----------------------------------------------------------------------------- +bool BGetLocalFormattedDateAndTime( time_t timeVal, char *pchDate, int cubDate, char *pchTime, int cubTime ) +{ + if ( 0 == timeVal || timeVal < 0 ) + { + // get the current time + time( &timeVal ); + } + + if ( timeVal ) + { + // Convert it to our local time + struct tm tmStruct; + struct tm tmToDisplay = *( Plat_localtime( ( const time_t* )&timeVal, &tmStruct ) ); +#ifdef POSIX + if ( pchDate != NULL ) + { + pchDate[ 0 ] = 0; + if ( 0 == strftime( pchDate, cubDate, "%A %b %d", &tmToDisplay ) ) + return false; + } + + if ( pchTime != NULL ) + { + pchTime[ 0 ] = 0; + if ( 0 == strftime( pchTime, cubTime - 6, "%I:%M ", &tmToDisplay ) ) + return false; + + // append am/pm in lower case (since strftime doesn't have a lowercase formatting option) + if (tmToDisplay.tm_hour >= 12) + { + Q_strcat( pchTime, "p.m.", cubTime ); + } + else + { + Q_strcat( pchTime, "a.m.", cubTime ); + } + } +#else // WINDOWS + // convert time_t to a SYSTEMTIME + SYSTEMTIME st; + st.wHour = tmToDisplay.tm_hour; + st.wMinute = tmToDisplay.tm_min; + st.wSecond = tmToDisplay.tm_sec; + st.wDay = tmToDisplay.tm_mday; + st.wMonth = tmToDisplay.tm_mon + 1; + st.wYear = tmToDisplay.tm_year + 1900; + st.wDayOfWeek = tmToDisplay.tm_wday; + st.wMilliseconds = 0; + + WCHAR rgwch[ MAX_PATH ]; + + if ( pchDate != NULL ) + { + pchDate[ 0 ] = 0; + if ( !GetDateFormatW( LOCALE_USER_DEFAULT, DATE_LONGDATE, &st, NULL, rgwch, MAX_PATH ) ) + return false; + Q_strncpy( pchDate, CStrAutoEncode( rgwch ).ToString(), cubDate ); + } + + if ( pchTime != NULL ) + { + pchTime[ 0 ] = 0; + if ( !GetTimeFormatW( LOCALE_USER_DEFAULT, TIME_NOSECONDS, &st, NULL, rgwch, MAX_PATH ) ) + return false; + Q_strncpy( pchTime, CStrAutoEncode( rgwch ).ToString(), cubTime ); + } +#endif + return true; + } + + return false; +} + + +// And a couple of helpers so people don't have to remember the order of the parameters in the above function +bool BGetLocalFormattedDate( time_t timeVal, char *pchDate, int cubDate ) +{ + return BGetLocalFormattedDateAndTime( timeVal, pchDate, cubDate, NULL, 0 ); +} +bool BGetLocalFormattedTime( time_t timeVal, char *pchTime, int cubTime ) +{ + return BGetLocalFormattedDateAndTime( timeVal, NULL, 0, pchTime, cubTime ); +} + +// Prints out a memory dump where stuff that's ascii is human readable, etc. +void V_LogMultiline( bool input, char const *label, const char *data, size_t len, CUtlString &output ) +{ + static const char HEX[] = "0123456789abcdef"; + const char * direction = (input ? " << " : " >> "); + const size_t LINE_SIZE = 24; + char hex_line[LINE_SIZE * 9 / 4 + 2], asc_line[LINE_SIZE + 1]; + while (len > 0) + { + V_memset(asc_line, ' ', sizeof(asc_line)); + V_memset(hex_line, ' ', sizeof(hex_line)); + size_t line_len = MIN(len, LINE_SIZE); + for (size_t i=0; i<line_len; ++i) { + unsigned char ch = static_cast<unsigned char>(data[i]); + asc_line[i] = ( V_isprint(ch) && !V_iscntrl(ch) ) ? data[i] : '.'; + hex_line[i*2 + i/4] = HEX[ch >> 4]; + hex_line[i*2 + i/4 + 1] = HEX[ch & 0xf]; + } + asc_line[sizeof(asc_line)-1] = 0; + hex_line[sizeof(hex_line)-1] = 0; + output += CFmtStr( "%s %s %s %s\n", label, direction, asc_line, hex_line ); + data += line_len; + len -= line_len; + } +} + + +#ifdef WIN32 +// Win32 CRT doesn't support the full range of UChar32, has no extended planes +inline int V_iswspace( int c ) { return ( c <= 0xFFFF ) ? iswspace( (wint_t)c ) : 0; } +#else +#define V_iswspace(x) iswspace(x) +#endif + + +//----------------------------------------------------------------------------- +// Purpose: Slightly modified strtok. Does not modify the input string. Does +// not skip over more than one separator at a time. This allows parsing +// strings where tokens between separators may or may not be present: +// +// Door01,,,0 would be parsed as "Door01" "" "" "0" +// Door01,Open,,0 would be parsed as "Door01" "Open" "" "0" +// +// Input : token - Returns with a token, or zero length if the token was missing. +// str - String to parse. +// sep - Character to use as separator. UNDONE: allow multiple separator chars +// Output : Returns a pointer to the next token to be parsed. +//----------------------------------------------------------------------------- +const char *nexttoken(char *token, size_t nMaxTokenLen, const char *str, char sep) +{ + if (nMaxTokenLen < 1) + { + Assert(nMaxTokenLen > 0); + return NULL; + } + + if ((str == NULL) || (*str == '\0')) + { + *token = '\0'; + return(NULL); + } + + char *pTokenLast = token + nMaxTokenLen - 1; + + // + // Copy everything up to the first separator into the return buffer. + // Do not include separators in the return buffer. + // + while ((*str != sep) && (*str != '\0') && (token < pTokenLast)) + { + *token++ = *str++; + } + *token = '\0'; + + // + // Advance the pointer unless we hit the end of the input string. + // + if (*str == '\0') + { + return(str); + } + + return(++str); +} + +int V_StrTrim( char *pStr ) +{ + char *pSource = pStr; + char *pDest = pStr; + + // skip white space at the beginning + while ( *pSource != 0 && V_isspace( *pSource ) ) + { + pSource++; + } + + // copy everything else + char *pLastWhiteBlock = NULL; + char *pStart = pDest; + while ( *pSource != 0 ) + { + *pDest = *pSource++; + if ( V_isspace( *pDest ) ) + { + if ( pLastWhiteBlock == NULL ) + pLastWhiteBlock = pDest; + } + else + { + pLastWhiteBlock = NULL; + } + pDest++; + } + *pDest = 0; + + // did we end in a whitespace block? + if ( pLastWhiteBlock != NULL ) + { + // yep; shorten the string + pDest = pLastWhiteBlock; + *pLastWhiteBlock = 0; + } + + return pDest - pStart; +} + +#ifdef _WIN32 +int64 V_strtoi64( const char *nptr, char **endptr, int base ) +{ + return _strtoi64( nptr, endptr, base ); +} + +uint64 V_strtoui64( const char *nptr, char **endptr, int base ) +{ + return _strtoui64( nptr, endptr, base ); +} +#elif POSIX +int64 V_strtoi64( const char *nptr, char **endptr, int base ) +{ + return strtoll( nptr, endptr, base ); +} + +uint64 V_strtoui64( const char *nptr, char **endptr, int base ) +{ + return strtoull( nptr, endptr, base ); +} +#endif + + +struct HtmlEntity_t +{ + unsigned short uCharCode; + const char *pchEntity; + int nEntityLength; +}; + +const static HtmlEntity_t g_BasicHTMLEntities[] = { + { '"', """, 6 }, + { '\'', "'", 6 }, + { '<', "<", 4 }, + { '>', ">", 4 }, + { '&', "&", 5 }, + { 0, NULL, 0 } // sentinel for end of array +}; + +const static HtmlEntity_t g_WhitespaceEntities[] = { + { ' ', " ", 6 }, + { '\n', "<br>", 4 }, + { 0, NULL, 0 } // sentinel for end of array +}; + + +struct Tier1FullHTMLEntity_t +{ + uchar32 uCharCode; + const char *pchEntity; + int nEntityLength; +}; + + +#pragma warning( push ) +#pragma warning( disable : 4428 ) // universal-character-name encountered in source +const Tier1FullHTMLEntity_t g_Tier1_FullHTMLEntities[] = +{ + { L'"', """, 6 }, + { L'\'', "'", 6 }, + { L'&', "&", 5 }, + { L'<', "<", 4 }, + { L'>', ">", 4 }, + { L' ', " ", 6 }, + { L'\u2122', "™", 7 }, + { L'\u00A9', "©", 6 }, + { L'\u00AE', "®", 5 }, + { L'\u2013', "–", 7 }, + { L'\u2014', "—", 7 }, + { L'\u20AC', "€", 6 }, + { L'\u00A1', "¡", 7 }, + { L'\u00A2', "¢", 6 }, + { L'\u00A3', "£", 7 }, + { L'\u00A4', "¤", 8 }, + { L'\u00A5', "¥", 5 }, + { L'\u00A6', "¦", 8 }, + { L'\u00A7', "§", 6 }, + { L'\u00A8', "¨", 5 }, + { L'\u00AA', "ª", 6 }, + { L'\u00AB', "«", 7 }, + { L'\u00AC', "¬", 8 }, + { L'\u00AD', "­", 5 }, + { L'\u00AF', "¯", 6 }, + { L'\u00B0', "°", 5 }, + { L'\u00B1', "±", 8 }, + { L'\u00B2', "²", 6 }, + { L'\u00B3', "³", 6 }, + { L'\u00B4', "´", 7 }, + { L'\u00B5', "µ", 7 }, + { L'\u00B6', "¶", 6 }, + { L'\u00B7', "·", 8 }, + { L'\u00B8', "¸", 7 }, + { L'\u00B9', "¹", 6 }, + { L'\u00BA', "º", 6 }, + { L'\u00BB', "»", 7 }, + { L'\u00BC', "¼", 8 }, + { L'\u00BD', "½", 8 }, + { L'\u00BE', "¾", 8 }, + { L'\u00BF', "¿", 8 }, + { L'\u00D7', "×", 7 }, + { L'\u00F7', "÷", 8 }, + { L'\u00C0', "À", 8 }, + { L'\u00C1', "Á", 8 }, + { L'\u00C2', "Â", 7 }, + { L'\u00C3', "Ã", 8 }, + { L'\u00C4', "Ä", 6 }, + { L'\u00C5', "Å", 7 }, + { L'\u00C6', "Æ", 7 }, + { L'\u00C7', "Ç", 8 }, + { L'\u00C8', "È", 8 }, + { L'\u00C9', "É", 8 }, + { L'\u00CA', "Ê", 7 }, + { L'\u00CB', "Ë", 6 }, + { L'\u00CC', "Ì", 8 }, + { L'\u00CD', "Í", 8 }, + { L'\u00CE', "Î", 7 }, + { L'\u00CF', "Ï", 6 }, + { L'\u00D0', "Ð", 5 }, + { L'\u00D1', "Ñ", 8 }, + { L'\u00D2', "Ò", 8 }, + { L'\u00D3', "Ó", 8 }, + { L'\u00D4', "Ô", 7 }, + { L'\u00D5', "Õ", 8 }, + { L'\u00D6', "Ö", 6 }, + { L'\u00D8', "Ø", 8 }, + { L'\u00D9', "Ù", 8 }, + { L'\u00DA', "Ú", 8 }, + { L'\u00DB', "Û", 7 }, + { L'\u00DC', "Ü", 6 }, + { L'\u00DD', "Ý", 8 }, + { L'\u00DE', "Þ", 7 }, + { L'\u00DF', "ß", 7 }, + { L'\u00E0', "à", 8 }, + { L'\u00E1', "á", 8 }, + { L'\u00E2', "â", 7 }, + { L'\u00E3', "ã", 8 }, + { L'\u00E4', "ä", 6 }, + { L'\u00E5', "å", 7 }, + { L'\u00E6', "æ", 7 }, + { L'\u00E7', "ç", 8 }, + { L'\u00E8', "è", 8 }, + { L'\u00E9', "é", 8 }, + { L'\u00EA', "ê", 7 }, + { L'\u00EB', "ë", 6 }, + { L'\u00EC', "ì", 8 }, + { L'\u00ED', "í", 8 }, + { L'\u00EE', "î", 7 }, + { L'\u00EF', "ï", 6 }, + { L'\u00F0', "ð", 5 }, + { L'\u00F1', "ñ", 8 }, + { L'\u00F2', "ò", 8 }, + { L'\u00F3', "ó", 8 }, + { L'\u00F4', "ô", 7 }, + { L'\u00F5', "õ", 8 }, + { L'\u00F6', "ö", 6 }, + { L'\u00F8', "ø", 8 }, + { L'\u00F9', "ù", 8 }, + { L'\u00FA', "ú", 8 }, + { L'\u00FB', "û", 7 }, + { L'\u00FC', "ü", 6 }, + { L'\u00FD', "ý", 8 }, + { L'\u00FE', "þ", 7 }, + { L'\u00FF', "ÿ", 6 }, + { 0, NULL, 0 } // sentinel for end of array +}; +#pragma warning( pop ) + + + +bool V_BasicHtmlEntityEncode( char *pDest, const int nDestSize, char const *pIn, const int nInSize, bool bPreserveWhitespace /*= false*/ ) +{ + Assert( nDestSize == 0 || pDest != NULL ); + int iOutput = 0; + for ( int iInput = 0; iInput < nInSize; ++iInput ) + { + bool bReplacementDone = false; + // See if the current char matches any of the basic entities + for ( int i = 0; g_BasicHTMLEntities[ i ].uCharCode != 0; ++i ) + { + if ( pIn[ iInput ] == g_BasicHTMLEntities[ i ].uCharCode ) + { + bReplacementDone = true; + for ( int j = 0; j < g_BasicHTMLEntities[ i ].nEntityLength; ++j ) + { + if ( iOutput >= nDestSize - 1 ) + { + pDest[ nDestSize - 1 ] = 0; + return false; + } + pDest[ iOutput++ ] = g_BasicHTMLEntities[ i ].pchEntity[ j ]; + } + } + } + + if ( bPreserveWhitespace && !bReplacementDone ) + { + // See if the current char matches any of the basic entities + for ( int i = 0; g_WhitespaceEntities[ i ].uCharCode != 0; ++i ) + { + if ( pIn[ iInput ] == g_WhitespaceEntities[ i ].uCharCode ) + { + bReplacementDone = true; + for ( int j = 0; j < g_WhitespaceEntities[ i ].nEntityLength; ++j ) + { + if ( iOutput >= nDestSize - 1 ) + { + pDest[ nDestSize - 1 ] = 0; + return false; + } + pDest[ iOutput++ ] = g_WhitespaceEntities[ i ].pchEntity[ j ]; + } + } + } + } + + if ( !bReplacementDone ) + { + pDest[ iOutput++ ] = pIn[ iInput ]; + } + } + + // Null terminate the output + pDest[ iOutput ] = 0; + return true; +} + + +bool V_HtmlEntityDecodeToUTF8( char *pDest, const int nDestSize, char const *pIn, const int nInSize ) +{ + Assert( nDestSize == 0 || pDest != NULL ); + int iOutput = 0; + for ( int iInput = 0; iInput < nInSize && iOutput < nDestSize; ++iInput ) + { + bool bReplacementDone = false; + if ( pIn[ iInput ] == '&' ) + { + bReplacementDone = true; + + uchar32 wrgchReplacement[ 2 ] = { 0, 0 }; + char rgchReplacement[ 8 ]; + rgchReplacement[ 0 ] = 0; + + const char *pchEnd = Q_strstr( pIn + iInput + 1, ";" ); + if ( pchEnd ) + { + if ( iInput + 1 < nInSize && pIn[ iInput + 1 ] == '#' ) + { + // Numeric + int iBase = 10; + int iOffset = 2; + if ( iInput + 3 < nInSize && pIn[ iInput + 2 ] == 'x' ) + { + iBase = 16; + iOffset = 3; + } + + wrgchReplacement[ 0 ] = (uchar32)V_strtoi64( pIn + iInput + iOffset, NULL, iBase ); + if ( !Q_UTF32ToUTF8( wrgchReplacement, rgchReplacement, sizeof( rgchReplacement ) ) ) + { + rgchReplacement[ 0 ] = 0; + } + } + else + { + // Lookup in map + const Tier1FullHTMLEntity_t *pFullEntities = g_Tier1_FullHTMLEntities; + for ( int i = 0; pFullEntities[ i ].uCharCode != 0; ++i ) + { + if ( nInSize - iInput - 1 >= pFullEntities[ i ].nEntityLength ) + { + if ( Q_memcmp( pIn + iInput, pFullEntities[ i ].pchEntity, pFullEntities[ i ].nEntityLength ) == 0 ) + { + wrgchReplacement[ 0 ] = pFullEntities[ i ].uCharCode; + if ( !Q_UTF32ToUTF8( wrgchReplacement, rgchReplacement, sizeof( rgchReplacement ) ) ) + { + rgchReplacement[ 0 ] = 0; + } + break; + } + } + } + } + + // make sure we found a replacement. If not, skip + int cchReplacement = V_strlen( rgchReplacement ); + if ( cchReplacement > 0 ) + { + if ( (int)cchReplacement + iOutput < nDestSize ) + { + for ( int i = 0; rgchReplacement[ i ] != 0; ++i ) + { + pDest[ iOutput++ ] = rgchReplacement[ i ]; + } + } + + // Skip extra space that we passed + iInput += pchEnd - ( pIn + iInput ); + } + else + { + bReplacementDone = false; + } + } + } + + if ( !bReplacementDone ) + { + pDest[ iOutput++ ] = pIn[ iInput ]; + } + } + + // Null terminate the output + if ( iOutput < nDestSize ) + { + pDest[ iOutput ] = 0; + } + else + { + pDest[ nDestSize - 1 ] = 0; + } + + return true; +} + +static const char *g_pszSimpleBBCodeReplacements[] = { + "[b]", "<b>", + "[/b]", "</b>", + "[i]", "<i>", + "[/i]", "</i>", + "[u]", "<u>", + "[/u]", "</u>", + "[s]", "<s>", + "[/s]", "</s>", + "[code]", "<pre>", + "[/code]", "</pre>", + "[h1]", "<h1>", + "[/h1]", "</h1>", + "[list]", "<ul>", + "[/list]", "</ul>", + "[*]", "<li>", + "[/url]", "</a>", + "[img]", "<img src=\"", + "[/img]", "\"></img>", +}; + +// Converts BBCode tags to HTML tags +bool V_BBCodeToHTML( OUT_Z_CAP( nDestSize ) char *pDest, const int nDestSize, char const *pIn, const int nInSize ) +{ + Assert( nDestSize == 0 || pDest != NULL ); + int iOutput = 0; + + for ( int iInput = 0; iInput < nInSize && iOutput < nDestSize && pIn[ iInput ]; ++iInput ) + { + if ( pIn[ iInput ] == '[' ) + { + // check simple replacements + bool bFoundReplacement = false; + for ( int r = 0; r < ARRAYSIZE( g_pszSimpleBBCodeReplacements ); r += 2 ) + { + int nBBCodeLength = V_strlen( g_pszSimpleBBCodeReplacements[ r ] ); + if ( !V_strnicmp( &pIn[ iInput ], g_pszSimpleBBCodeReplacements[ r ], nBBCodeLength ) ) + { + int nHTMLReplacementLength = V_strlen( g_pszSimpleBBCodeReplacements[ r + 1 ] ); + for ( int c = 0; c < nHTMLReplacementLength && iOutput < nDestSize; c++ ) + { + pDest[ iOutput ] = g_pszSimpleBBCodeReplacements[ r + 1 ][ c ]; + iOutput++; + } + iInput += nBBCodeLength - 1; + bFoundReplacement = true; + break; + } + } + // check URL replacement + if ( !bFoundReplacement && !V_strnicmp( &pIn[ iInput ], "[url=", 5 ) && nDestSize - iOutput > 9 ) + { + iInput += 5; + pDest[ iOutput++ ] = '<'; + pDest[ iOutput++ ] = 'a'; + pDest[ iOutput++ ] = ' '; + pDest[ iOutput++ ] = 'h'; + pDest[ iOutput++ ] = 'r'; + pDest[ iOutput++ ] = 'e'; + pDest[ iOutput++ ] = 'f'; + pDest[ iOutput++ ] = '='; + pDest[ iOutput++ ] = '\"'; + + // copy all characters up to the closing square bracket + while ( pIn[ iInput ] != ']' && iInput < nInSize && iOutput < nDestSize ) + { + pDest[ iOutput++ ] = pIn[ iInput++ ]; + } + if ( pIn[ iInput ] == ']' && nDestSize - iOutput > 2 ) + { + pDest[ iOutput++ ] = '\"'; + pDest[ iOutput++ ] = '>'; + } + bFoundReplacement = true; + } + // otherwise, skip over everything up to the closing square bracket + if ( !bFoundReplacement ) + { + while ( pIn[ iInput ] != ']' && iInput < nInSize ) + { + iInput++; + } + } + } + else if ( pIn[ iInput ] == '\r' && pIn[ iInput + 1 ] == '\n' ) + { + // convert carriage return and newline to a <br> + if ( nDestSize - iOutput > 4 ) + { + pDest[ iOutput++ ] = '<'; + pDest[ iOutput++ ] = 'b'; + pDest[ iOutput++ ] = 'r'; + pDest[ iOutput++ ] = '>'; + } + iInput++; + } + else if ( pIn[ iInput ] == '\n' ) + { + // convert newline to a <br> + if ( nDestSize - iOutput > 4 ) + { + pDest[ iOutput++ ] = '<'; + pDest[ iOutput++ ] = 'b'; + pDest[ iOutput++ ] = 'r'; + pDest[ iOutput++ ] = '>'; + } + } + else + { + // copy character to destination + pDest[ iOutput++ ] = pIn[ iInput ]; + } + } + // always terminate string + if ( iOutput >= nDestSize ) + { + iOutput = nDestSize - 1; + } + pDest[ iOutput ] = 0; + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: returns true if a wide character is a "mean" space; that is, +// if it is technically a space or punctuation, but causes disruptive +// behavior when used in names, web pages, chat windows, etc. +// +// characters in this set are removed from the beginning and/or end of strings +// by Q_AggressiveStripPrecedingAndTrailingWhitespaceW() +//----------------------------------------------------------------------------- +bool V_IsMeanUnderscoreW( wchar_t wch ) +{ + bool bIsMean = false; + + switch ( wch ) + { + case L'\x005f': // low line (normal underscore) + case L'\xff3f': // fullwidth low line + case L'\x0332': // combining low line + bIsMean = true; + break; + default: + break; + } + + return bIsMean; +} + + +//----------------------------------------------------------------------------- +// Purpose: returns true if a wide character is a "mean" space; that is, +// if it is technically a space or punctuation, but causes disruptive +// behavior when used in names, web pages, chat windows, etc. +// +// characters in this set are removed from the beginning and/or end of strings +// by Q_AggressiveStripPrecedingAndTrailingWhitespaceW() +//----------------------------------------------------------------------------- +bool V_IsMeanSpaceW( wchar_t wch ) +{ + bool bIsMean = false; + + switch ( wch ) + { + case L'\x0080': // PADDING CHARACTER + case L'\x0081': // HIGH OCTET PRESET + case L'\x0082': // BREAK PERMITTED HERE + case L'\x0083': // NO BREAK PERMITTED HERE + case L'\x0084': // INDEX + case L'\x0085': // NEXT LINE + case L'\x0086': // START OF SELECTED AREA + case L'\x0087': // END OF SELECTED AREA + case L'\x0088': // CHARACTER TABULATION SET + case L'\x0089': // CHARACTER TABULATION WITH JUSTIFICATION + case L'\x008A': // LINE TABULATION SET + case L'\x008B': // PARTIAL LINE FORWARD + case L'\x008C': // PARTIAL LINE BACKWARD + case L'\x008D': // REVERSE LINE FEED + case L'\x008E': // SINGLE SHIFT 2 + case L'\x008F': // SINGLE SHIFT 3 + case L'\x0090': // DEVICE CONTROL STRING + case L'\x0091': // PRIVATE USE + case L'\x0092': // PRIVATE USE + case L'\x0093': // SET TRANSMIT STATE + case L'\x0094': // CANCEL CHARACTER + case L'\x0095': // MESSAGE WAITING + case L'\x0096': // START OF PROTECTED AREA + case L'\x0097': // END OF PROTECED AREA + case L'\x0098': // START OF STRING + case L'\x0099': // SINGLE GRAPHIC CHARACTER INTRODUCER + case L'\x009A': // SINGLE CHARACTER INTRODUCER + case L'\x009B': // CONTROL SEQUENCE INTRODUCER + case L'\x009C': // STRING TERMINATOR + case L'\x009D': // OPERATING SYSTEM COMMAND + case L'\x009E': // PRIVACY MESSAGE + case L'\x009F': // APPLICATION PROGRAM COMMAND + case L'\x00A0': // NO-BREAK SPACE + case L'\x034F': // COMBINING GRAPHEME JOINER + case L'\x2000': // EN QUAD + case L'\x2001': // EM QUAD + case L'\x2002': // EN SPACE + case L'\x2003': // EM SPACE + case L'\x2004': // THICK SPACE + case L'\x2005': // MID SPACE + case L'\x2006': // SIX SPACE + case L'\x2007': // figure space + case L'\x2008': // PUNCTUATION SPACE + case L'\x2009': // THIN SPACE + case L'\x200A': // HAIR SPACE + case L'\x200B': // ZERO-WIDTH SPACE + case L'\x200C': // ZERO-WIDTH NON-JOINER + case L'\x200D': // ZERO WIDTH JOINER + case L'\x2028': // LINE SEPARATOR + case L'\x2029': // PARAGRAPH SEPARATOR + case L'\x202F': // NARROW NO-BREAK SPACE + case L'\x2060': // word joiner + case L'\xFEFF': // ZERO-WIDTH NO BREAK SPACE + case L'\xFFFC': // OBJECT REPLACEMENT CHARACTER + bIsMean = true; + break; + } + + return bIsMean; +} + + +//----------------------------------------------------------------------------- +// Purpose: tell us if a Unicode character is deprecated +// +// See Unicode Technical Report #20: http://www.unicode.org/reports/tr20/ +// +// Some characters are difficult or unreliably rendered. These characters eventually +// fell out of the Unicode standard, but are abusable by users. For example, +// setting "RIGHT-TO-LEFT OVERRIDE" without popping or undoing the action causes +// the layout instruction to bleed into following characters in HTML renderings, +// or upset layout calculations in vgui panels. +// +// Many games don't cope with these characters well, and end up providing opportunities +// for griefing others. For example, a user might join a game with a malformed player +// name and it turns out that player name can't be selected or typed into the admin +// console or UI to mute, kick, or ban the disruptive player. +// +// Ideally, we'd perfectly support these end-to-end but we never realistically will. +// The benefit of doing so far outweighs the cost, anyway. +//----------------------------------------------------------------------------- +bool V_IsDeprecatedW( wchar_t wch ) +{ + bool bIsDeprecated = false; + + switch ( wch ) + { + case L'\x202A': // LEFT-TO-RIGHT EMBEDDING + case L'\x202B': // RIGHT-TO-LEFT EMBEDDING + case L'\x202C': // POP DIRECTIONAL FORMATTING + case L'\x202D': // LEFT-TO-RIGHT OVERRIDE + case L'\x202E': // RIGHT-TO-LEFT OVERRIDE + + case L'\x206A': // INHIBIT SYMMETRIC SWAPPING + case L'\x206B': // ACTIVATE SYMMETRIC SWAPPING + case L'\x206C': // INHIBIT ARABIC FORM SHAPING + case L'\x206D': // ACTIVATE ARABIC FORM SHAPING + case L'\x206E': // NATIONAL DIGIT SHAPES + case L'\x206F': // NOMINAL DIGIT SHAPES + bIsDeprecated = true; + } + + return bIsDeprecated; +} + + +//----------------------------------------------------------------------------- +// returns true if the character is allowed in a DNS doman name, false otherwise +//----------------------------------------------------------------------------- +bool V_IsValidDomainNameCharacter( const char *pch, int *pAdvanceBytes ) +{ + if ( pAdvanceBytes ) + *pAdvanceBytes = 0; + + + // We allow unicode in Domain Names without the an encoding unless it corresponds to + // a whitespace or control sequence or something we think is an underscore looking thing. + // If this character is the start of a UTF-8 sequence, try decoding it. + unsigned char ch = (unsigned char)*pch; + if ( ( ch & 0xC0 ) == 0xC0 ) + { + uchar32 rgch32Buf; + bool bError = false; + int iAdvance = Q_UTF8ToUChar32( pch, rgch32Buf, bError ); + if ( bError || iAdvance == 0 ) + { + // Invalid UTF8 sequence, lets consider that invalid + return false; + } + + if ( pAdvanceBytes ) + *pAdvanceBytes = iAdvance; + + if ( iAdvance ) + { + // Ick. Want uchar32 versions of unicode character classification functions. + // Really would like Q_IsWhitespace32 and Q_IsNonPrintable32, but this is OK. + if ( rgch32Buf < 0x10000 && ( V_IsMeanSpaceW( (wchar_t)rgch32Buf ) || V_IsDeprecatedW( (wchar_t)rgch32Buf ) || V_IsMeanUnderscoreW( (wchar_t)rgch32Buf ) ) ) + { + return false; + } + + return true; + } + else + { + // Unreachable but would be invalid utf8 + return false; + } + } + else + { + // Was not unicode + if ( pAdvanceBytes ) + *pAdvanceBytes = 1; + + // The only allowable non-unicode chars are a-z A-Z 0-9 and - + if ( ( ch >= 'a' && ch <= 'z' ) || ( ch >= 'A' && ch <= 'Z' ) || ( ch >= '0' && ch <= '9' ) || ch == '-' || ch == '.' ) + return true; + + return false; + } +} + + +//----------------------------------------------------------------------------- +// returns true if the character is allowed in a URL, false otherwise +//----------------------------------------------------------------------------- +bool V_IsValidURLCharacter( const char *pch, int *pAdvanceBytes ) +{ + if ( pAdvanceBytes ) + *pAdvanceBytes = 0; + + + // We allow unicode in URLs unless it corresponds to a whitespace or control sequence. + // If this character is the start of a UTF-8 sequence, try decoding it. + unsigned char ch = (unsigned char)*pch; + if ( ( ch & 0xC0 ) == 0xC0 ) + { + uchar32 rgch32Buf; + bool bError = false; + int iAdvance = Q_UTF8ToUChar32( pch, rgch32Buf, bError ); + if ( bError || iAdvance == 0 ) + { + // Invalid UTF8 sequence, lets consider that invalid + return false; + } + + if ( pAdvanceBytes ) + *pAdvanceBytes = iAdvance; + + if ( iAdvance ) + { + // Ick. Want uchar32 versions of unicode character classification functions. + // Really would like Q_IsWhitespace32 and Q_IsNonPrintable32, but this is OK. + if ( rgch32Buf < 0x10000 && ( V_IsMeanSpaceW( (wchar_t)rgch32Buf ) || V_IsDeprecatedW( (wchar_t)rgch32Buf ) ) ) + { + return false; + } + + return true; + } + else + { + // Unreachable but would be invalid utf8 + return false; + } + } + else + { + // Was not unicode + if ( pAdvanceBytes ) + *pAdvanceBytes = 1; + + // Spaces, control characters, quotes, and angle brackets are not legal URL characters. + if ( ch <= 32 || ch == 127 || ch == '"' || ch == '<' || ch == '>' ) + return false; + + return true; + } + +} + + +//----------------------------------------------------------------------------- +// Purpose: helper function to get a domain from a url +// Checks both standard url and steam://openurl/<url> +//----------------------------------------------------------------------------- +bool V_ExtractDomainFromURL( const char *pchURL, char *pchDomain, int cchDomain ) +{ + pchDomain[ 0 ] = 0; + + static const char *k_pchSteamOpenUrl = "steam://openurl/"; + static const char *k_pchSteamOpenUrlExt = "steam://openurl_external/"; + + const char *pchOpenUrlSuffix = StringAfterPrefix( pchURL, k_pchSteamOpenUrl ); + if ( pchOpenUrlSuffix == NULL ) + pchOpenUrlSuffix = StringAfterPrefix( pchURL, k_pchSteamOpenUrlExt ); + + if ( pchOpenUrlSuffix ) + pchURL = pchOpenUrlSuffix; + + if ( !pchURL || pchURL[ 0 ] == '\0' ) + return false; + + const char *pchDoubleSlash = strstr( pchURL, "//" ); + + // Put the domain and everything after into pchDomain. + // We'll find where to terminate it later. + if ( pchDoubleSlash ) + { + // Skip the slashes + pchDoubleSlash += 2; + + // If that's all there was, then there's no domain here. Bail. + if ( *pchDoubleSlash == '\0' ) + { + return false; + } + + // Skip any extra slashes + // ex: http:///steamcommunity.com/ + while ( *pchDoubleSlash == '/' ) + { + pchDoubleSlash++; + } + + Q_strncpy( pchDomain, pchDoubleSlash, cchDomain ); + } + else + { + // No double slash, so pchURL has no protocol. + Q_strncpy( pchDomain, pchURL, cchDomain ); + } + + // First character has to be valid + if ( *pchDomain == '?' || *pchDomain == '\0' ) + { + return false; + } + + // terminate the domain after the first non domain char + int iAdvance = 0; + int iStrLen = 0; + char cLast = 0; + while ( pchDomain[ iStrLen ] ) + { + if ( !V_IsValidDomainNameCharacter( pchDomain + iStrLen, &iAdvance ) || ( pchDomain[ iStrLen ] == '.' && cLast == '.' ) ) + { + pchDomain[ iStrLen ] = 0; + break; + } + + cLast = pchDomain[ iStrLen ]; + iStrLen += iAdvance; + } + + return ( pchDomain[ 0 ] != 0 ); +} + + +//----------------------------------------------------------------------------- +// Purpose: helper function to get a domain from a url +//----------------------------------------------------------------------------- +bool V_URLContainsDomain( const char *pchURL, const char *pchDomain ) +{ + char rgchExtractedDomain[ 2048 ]; + if ( V_ExtractDomainFromURL( pchURL, rgchExtractedDomain, sizeof( rgchExtractedDomain ) ) ) + { + // see if the last part of the domain matches what we extracted + int cchExtractedDomain = V_strlen( rgchExtractedDomain ); + if ( pchDomain[ 0 ] == '.' ) + { + ++pchDomain; // If the domain has a leading '.', skip it. The test below assumes there is none. + } + int cchDomain = V_strlen( pchDomain ); + + if ( cchDomain > cchExtractedDomain ) + { + return false; + } + else if ( cchExtractedDomain >= cchDomain ) + { + // If the actual domain is longer than what we're searching for, the character previous + // to the domain we're searching for must be a period + if ( cchExtractedDomain > cchDomain && rgchExtractedDomain[ cchExtractedDomain - cchDomain - 1 ] != '.' ) + return false; + + if ( 0 == V_stricmp( rgchExtractedDomain + cchExtractedDomain - cchDomain, pchDomain ) ) + return true; + } + } + return false; +} + +//----------------------------------------------------------------------------- +// Purpose: Strips all HTML tags not specified in rgszPreserveTags +// Does some additional formatting, like turning <li> into * when not preserving that tag, +// and auto-closing unclosed tags if they aren't specified in rgszNoCloseTags +//----------------------------------------------------------------------------- +void V_StripAndPreserveHTMLCore( CUtlBuffer *pbuffer, const char *pchHTML, const char **rgszPreserveTags, uint cPreserveTags, const char **rgszNoCloseTags, uint cNoCloseTags, uint cMaxResultSize ) +{ + uint cHTMLCur = 0; + + bool bStripNewLines = true; + if ( cPreserveTags > 0 ) + { + for ( uint i = 0; i < cPreserveTags; ++i ) + { + if ( !Q_stricmp( rgszPreserveTags[ i ], "\n" ) ) + bStripNewLines = false; + } + } + + //state- + bool bInStrippedTag = false; + bool bInStrippedContentTag = false; + bool bInPreservedTag = false; + bool bInListItemTag = false; + bool bLastCharWasWhitespace = true; //set to true to strip leading whitespace + bool bInComment = false; + bool bInDoubleQuote = false; + bool bInSingleQuote = false; + int nPreTagDepth = 0; + CUtlVector< const char* > vecTagStack; + + for ( int iContents = 0; pchHTML[ iContents ] != '\0' && cHTMLCur < cMaxResultSize; iContents++ ) + { + char c = pchHTML[ iContents ]; + + // If we are entering a comment, flag as such and skip past the begin comment tag + const char *pchCur = &pchHTML[ iContents ]; + if ( !Q_strnicmp( pchCur, "<!--", 4 ) ) + { + bInComment = true; + iContents += 3; + continue; + } + + // If we are in a comment, check if we are exiting + if ( bInComment ) + { + if ( !Q_strnicmp( pchCur, "-->", 3 ) ) + { + bInComment = false; + iContents += 2; + continue; + } + else + { + continue; + } + } + + if ( bInStrippedTag || bInPreservedTag ) + { + // we're inside a tag, keep stripping/preserving until we get to a > + if ( bInPreservedTag ) + pbuffer->PutChar( c ); + + // While inside a tag, ignore ending > properties if they are inside a property value in "" or '' + if ( c == '"' ) + { + if ( bInDoubleQuote ) + bInDoubleQuote = false; + else + bInDoubleQuote = true; + } + + if ( c == '\'' ) + { + if ( bInSingleQuote ) + bInSingleQuote = false; + else + bInSingleQuote = true; + } + + if ( !bInDoubleQuote && !bInSingleQuote && c == '>' ) + { + if ( bInPreservedTag ) + bLastCharWasWhitespace = false; + + bInPreservedTag = false; + bInStrippedTag = false; + } + } + else if ( bInStrippedContentTag ) + { + if ( c == '<' && !Q_strnicmp( pchCur, "</script>", 9 ) ) + { + bInStrippedContentTag = false; + iContents += 8; + continue; + } + else + { + continue; + } + } + else if ( c & 0x80 && !bInStrippedContentTag ) + { + // start/continuation of a multibyte sequence, copy to output. + int nMultibyteRemaining = 0; + if ( ( c & 0xF8 ) == 0xF0 ) // first 5 bits are 11110 + nMultibyteRemaining = 3; + else if ( ( c & 0xF0 ) == 0xE0 ) // first 4 bits are 1110 + nMultibyteRemaining = 2; + else if ( ( c & 0xE0 ) == 0xC0 ) // first 3 bits are 110 + nMultibyteRemaining = 1; + + // cHTMLCur is in characters, so just +1 + cHTMLCur++; + pbuffer->Put( pchCur, 1 + nMultibyteRemaining ); + + iContents += nMultibyteRemaining; + + // Need to determine if we just added whitespace or not + wchar_t rgwch[ 3 ] = { 0 }; + Q_UTF8CharsToWString( pchCur, 1, rgwch, sizeof( rgwch ) ); + if ( !V_iswspace( rgwch[ 0 ] ) ) + bLastCharWasWhitespace = false; + else + bLastCharWasWhitespace = true; + } + else + { + //not in a multibyte sequence- do our parsing/stripping + if ( c == '<' ) + { + if ( !rgszPreserveTags || cPreserveTags == 0 ) + { + //not preserving any tags, just strip it + bInStrippedTag = true; + } + else + { + //look ahead, is this our kind of tag? + bool bPreserve = false; + bool bEndTag = false; + const char *szTagStart = &pchHTML[ iContents + 1 ]; + // if it's a close tag, skip the / + if ( *szTagStart == '/' ) + { + bEndTag = true; + szTagStart++; + } + if ( Q_strnicmp( "script", szTagStart, 6 ) == 0 ) + { + bInStrippedTag = true; + bInStrippedContentTag = true; + } + else + { + //see if this tag is one we want to preserve + for ( uint iTag = 0; iTag < cPreserveTags; iTag++ ) + { + const char *szTag = rgszPreserveTags[ iTag ]; + int cchTag = Q_strlen( szTag ); + + //make sure characters match, and are followed by some non-alnum char + // so "i" can match <i> or <i class=...>, but not <img> + if ( Q_strnicmp( szTag, szTagStart, cchTag ) == 0 && !V_isalnum( szTagStart[ cchTag ] ) ) + { + bPreserve = true; + if ( bEndTag ) + { + // ending a paragraph tag is optional. If we were expecting to find one, and didn't, skip + if ( Q_stricmp( szTag, "p" ) != 0 ) + { + while ( vecTagStack.Count() > 0 && Q_stricmp( vecTagStack[ vecTagStack.Count() - 1 ], "p" ) == 0 ) + { + vecTagStack.Remove( vecTagStack.Count() - 1 ); + } + } + + if ( vecTagStack.Count() > 0 && vecTagStack[ vecTagStack.Count() - 1 ] == szTag ) + { + vecTagStack.Remove( vecTagStack.Count() - 1 ); + + if ( Q_stricmp( szTag, "pre" ) == 0 ) + { + nPreTagDepth--; + if ( nPreTagDepth < 0 ) + { + nPreTagDepth = 0; + } + } + } + else + { + // don't preserve this unbalanced tag. All open tags will be closed at the end of the blurb + bPreserve = false; + } + } + else + { + bool bNoCloseTag = false; + for ( uint iNoClose = 0; iNoClose < cNoCloseTags; iNoClose++ ) + { + if ( Q_stricmp( szTag, rgszNoCloseTags[ iNoClose ] ) == 0 ) + { + bNoCloseTag = true; + break; + } + } + + if ( !bNoCloseTag ) + { + vecTagStack.AddToTail( szTag ); + if ( Q_stricmp( szTag, "pre" ) == 0 ) + { + nPreTagDepth++; + } + } + } + break; + } + } + if ( !bPreserve ) + { + bInStrippedTag = true; + } + else + { + bInPreservedTag = true; + pbuffer->PutChar( c ); + } + + } + } + if ( bInStrippedTag ) + { + const char *szTagStart = &pchHTML[ iContents ]; + if ( Q_strnicmp( szTagStart, "<li>", Q_strlen( "<li>" ) ) == 0 ) + { + if ( bInListItemTag ) + { + pbuffer->PutChar( ';' ); + cHTMLCur++; + bInListItemTag = false; + } + + if ( !bLastCharWasWhitespace ) + { + pbuffer->PutChar( ' ' ); + cHTMLCur++; + } + + pbuffer->PutChar( '*' ); + pbuffer->PutChar( ' ' ); + cHTMLCur += 2; + bInListItemTag = true; + } + else if ( !bLastCharWasWhitespace ) + { + + if ( bInListItemTag ) + { + char cLastChar = ' '; + + if ( pbuffer->TellPut() > 0 ) + { + cLastChar = ( ( (char*)pbuffer->Base() ) + pbuffer->TellPut() - 1 )[ 0 ]; + } + if ( cLastChar != '.' && cLastChar != '?' && cLastChar != '!' ) + { + pbuffer->PutChar( ';' ); + cHTMLCur++; + } + bInListItemTag = false; + } + + //we're decided to remove a tag, simulate a space in the original text + pbuffer->PutChar( ' ' ); + cHTMLCur++; + } + bLastCharWasWhitespace = true; + } + } + else + { + //just a normal character, nothin' special. + if ( nPreTagDepth == 0 && V_isspace( c ) && ( bStripNewLines || c != '\n' ) ) + { + if ( !bLastCharWasWhitespace ) + { + //replace any block of whitespace with a single space + cHTMLCur++; + pbuffer->PutChar( ' ' ); + bLastCharWasWhitespace = true; + } + // don't put anything for whitespace if the previous character was whitespace + // (effectively trimming all blocks of whitespace down to a single ' ') + } + else + { + cHTMLCur++; + pbuffer->PutChar( c ); + bLastCharWasWhitespace = false; + } + } + } + } + if ( cHTMLCur >= cMaxResultSize ) + { + // we terminated because the blurb was full. Add a '...' to the end + pbuffer->Put( "...", 3 ); + } + //close any preserved tags that were open at the end. + FOR_EACH_VEC_BACK( vecTagStack, iTagStack ) + { + pbuffer->PutChar( '<' ); + pbuffer->PutChar( '/' ); + pbuffer->Put( vecTagStack[ iTagStack ], Q_strlen( vecTagStack[ iTagStack ] ) ); + pbuffer->PutChar( '>' ); + } + + // Null terminate + pbuffer->PutChar( '\0' ); +} + +//----------------------------------------------------------------------------- +// Purpose: Strips all HTML tags not specified in rgszPreserveTags +// Does some additional formatting, like turning <li> into * when not preserving that tag +//----------------------------------------------------------------------------- +void V_StripAndPreserveHTML( CUtlBuffer *pbuffer, const char *pchHTML, const char **rgszPreserveTags, uint cPreserveTags, uint cMaxResultSize ) +{ + const char *rgszNoCloseTags[] = { "br", "img" }; + V_StripAndPreserveHTMLCore( pbuffer, pchHTML, rgszPreserveTags, cPreserveTags, rgszNoCloseTags, V_ARRAYSIZE( rgszNoCloseTags ), cMaxResultSize ); +} + + diff --git a/tier1/strtools_unicode.cpp b/tier1/strtools_unicode.cpp new file mode 100644 index 0000000..77685ec --- /dev/null +++ b/tier1/strtools_unicode.cpp @@ -0,0 +1,572 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +// +//=============================================================================// + +#include <limits.h> +#include "tier0/dbg.h" +#include "tier1/strtools.h" + +// This code was copied from steam +#define DbgAssert Assert + +//----------------------------------------------------------------------------- +// Purpose: determine if a uchar32 represents a valid Unicode code point +//----------------------------------------------------------------------------- +bool Q_IsValidUChar32( uchar32 uVal ) +{ + // Values > 0x10FFFF are explicitly invalid; ditto for UTF-16 surrogate halves, + // values ending in FFFE or FFFF, or values in the 0x00FDD0-0x00FDEF reserved range + return ( uVal < 0x110000u ) && ( (uVal - 0x00D800u) > 0x7FFu ) && ( (uVal & 0xFFFFu) < 0xFFFEu ) && ( ( uVal - 0x00FDD0u ) > 0x1Fu ); +} + +//----------------------------------------------------------------------------- +// Purpose: return number of UTF-8 bytes required to encode a Unicode code point +//----------------------------------------------------------------------------- +int Q_UChar32ToUTF8Len( uchar32 uVal ) +{ + DbgAssert( Q_IsValidUChar32( uVal ) ); + if ( uVal <= 0x7F ) + return 1; + if ( uVal <= 0x7FF ) + return 2; + if ( uVal <= 0xFFFF ) + return 3; + return 4; +} + + +//----------------------------------------------------------------------------- +// Purpose: return number of UTF-16 elements required to encode a Unicode code point +//----------------------------------------------------------------------------- +int Q_UChar32ToUTF16Len( uchar32 uVal ) +{ + DbgAssert( Q_IsValidUChar32( uVal ) ); + if ( uVal <= 0xFFFF ) + return 1; + return 2; +} + + +//----------------------------------------------------------------------------- +// Purpose: encode Unicode code point as UTF-8, returns number of bytes written +//----------------------------------------------------------------------------- +int Q_UChar32ToUTF8( uchar32 uVal, char *pUTF8Out ) +{ + DbgAssert( Q_IsValidUChar32( uVal ) ); + if ( uVal <= 0x7F ) + { + pUTF8Out[0] = (unsigned char) uVal; + return 1; + } + if ( uVal <= 0x7FF ) + { + pUTF8Out[0] = (unsigned char)(uVal >> 6) | 0xC0; + pUTF8Out[1] = (unsigned char)(uVal & 0x3F) | 0x80; + return 2; + } + if ( uVal <= 0xFFFF ) + { + pUTF8Out[0] = (unsigned char)(uVal >> 12) | 0xE0; + pUTF8Out[1] = (unsigned char)((uVal >> 6) & 0x3F) | 0x80; + pUTF8Out[2] = (unsigned char)(uVal & 0x3F) | 0x80; + return 3; + } + pUTF8Out[0] = (unsigned char)((uVal >> 18) & 0x07) | 0xF0; + pUTF8Out[1] = (unsigned char)((uVal >> 12) & 0x3F) | 0x80; + pUTF8Out[2] = (unsigned char)((uVal >> 6) & 0x3F) | 0x80; + pUTF8Out[3] = (unsigned char)(uVal & 0x3F) | 0x80; + return 4; +} + +//----------------------------------------------------------------------------- +// Purpose: encode Unicode code point as UTF-16, returns number of elements written +//----------------------------------------------------------------------------- +int Q_UChar32ToUTF16( uchar32 uVal, uchar16 *pUTF16Out ) +{ + DbgAssert( Q_IsValidUChar32( uVal ) ); + if ( uVal <= 0xFFFF ) + { + pUTF16Out[0] = (uchar16) uVal; + return 1; + } + uVal -= 0x010000; + pUTF16Out[0] = (uchar16)(uVal >> 10) | 0xD800; + pUTF16Out[1] = (uchar16)(uVal & 0x3FF) | 0xDC00; + return 2; +} + + +// Decode one character from a UTF-8 encoded string. Treats 6-byte CESU-8 sequences +// as a single character, as if they were a correctly-encoded 4-byte UTF-8 sequence. +int Q_UTF8ToUChar32( const char *pUTF8_, uchar32 &uValueOut, bool &bErrorOut ) +{ + const uint8 *pUTF8 = (const uint8 *)pUTF8_; + + int nBytes = 1; + uint32 uValue = pUTF8[0]; + uint32 uMinValue = 0; + + // 0....... single byte + if ( uValue < 0x80 ) + goto decodeFinishedNoCheck; + + // Expecting at least a two-byte sequence with 0xC0 <= first <= 0xF7 (110...... and 11110...) + if ( (uValue - 0xC0u) > 0x37u || ( pUTF8[1] & 0xC0 ) != 0x80 ) + goto decodeError; + + uValue = (uValue << 6) - (0xC0 << 6) + pUTF8[1] - 0x80; + nBytes = 2; + uMinValue = 0x80; + + // 110..... two-byte lead byte + if ( !( uValue & (0x20 << 6) ) ) + goto decodeFinished; + + // Expecting at least a three-byte sequence + if ( ( pUTF8[2] & 0xC0 ) != 0x80 ) + goto decodeError; + + uValue = (uValue << 6) - (0x20 << 12) + pUTF8[2] - 0x80; + nBytes = 3; + uMinValue = 0x800; + + // 1110.... three-byte lead byte + if ( !( uValue & (0x10 << 12) ) ) + goto decodeFinishedMaybeCESU8; + + // Expecting a four-byte sequence, longest permissible in UTF-8 + if ( ( pUTF8[3] & 0xC0 ) != 0x80 ) + goto decodeError; + + uValue = (uValue << 6) - (0x10 << 18) + pUTF8[3] - 0x80; + nBytes = 4; + uMinValue = 0x10000; + + // 11110... four-byte lead byte. fall through to finished. + +decodeFinished: + if ( uValue >= uMinValue && Q_IsValidUChar32( uValue ) ) + { +decodeFinishedNoCheck: + uValueOut = uValue; + bErrorOut = false; + return nBytes; + } +decodeError: + uValueOut = '?'; + bErrorOut = true; + return nBytes; + +decodeFinishedMaybeCESU8: + // Do we have a full UTF-16 surrogate pair that's been UTF-8 encoded afterwards? + // That is, do we have 0xD800-0xDBFF followed by 0xDC00-0xDFFF? If so, decode it all. + if ( ( uValue - 0xD800u ) < 0x400u && pUTF8[3] == 0xED && (uint8)( pUTF8[4] - 0xB0 ) < 0x10 && ( pUTF8[5] & 0xC0 ) == 0x80 ) + { + uValue = 0x10000 + ( ( uValue - 0xD800u ) << 10 ) + ( (uint8)( pUTF8[4] - 0xB0 ) << 6 ) + pUTF8[5] - 0x80; + nBytes = 6; + uMinValue = 0x10000; + } + goto decodeFinished; +} + +// Decode one character from a UTF-16 encoded string. +int Q_UTF16ToUChar32( const uchar16 *pUTF16, uchar32 &uValueOut, bool &bErrorOut ) +{ + if ( Q_IsValidUChar32( pUTF16[0] ) ) + { + uValueOut = pUTF16[0]; + bErrorOut = false; + return 1; + } + else if ( (pUTF16[0] - 0xD800u) < 0x400u && (pUTF16[1] - 0xDC00u) < 0x400u ) + { + // Valid surrogate pair, but maybe not encoding a valid Unicode code point... + uchar32 uVal = 0x010000 + ((pUTF16[0] - 0xD800u) << 10) + (pUTF16[1] - 0xDC00); + if ( Q_IsValidUChar32( uVal ) ) + { + uValueOut = uVal; + bErrorOut = false; + return 2; + } + else + { + uValueOut = '?'; + bErrorOut = true; + return 2; + } + } + else + { + uValueOut = '?'; + bErrorOut = true; + return 1; + } +} + +namespace // internal use only +{ + // Identity transformations and validity tests for use with Q_UnicodeConvertT + int Q_UTF32ToUChar32( const uchar32 *pUTF32, uchar32 &uVal, bool &bErr ) + { + bErr = !Q_IsValidUChar32( *pUTF32 ); + uVal = bErr ? '?' : *pUTF32; + return 1; + } + + int Q_UChar32ToUTF32Len( uchar32 uVal ) + { + return 1; + } + + int Q_UChar32ToUTF32( uchar32 uVal, uchar32 *pUTF32 ) + { + *pUTF32 = uVal; + return 1; + } + + // A generic Unicode processing loop: decode one character from input to uchar32, handle errors, encode uchar32 to output + template < typename SrcType, typename DstType, bool bStopAtNull, int (&DecodeSrc)( const SrcType*, uchar32&, bool& ), int (&EncodeDstLen)( uchar32 ), int (&EncodeDst)( uchar32, DstType* ) > + int Q_UnicodeConvertT( const SrcType *pIn, int nInChars, DstType *pOut, int nOutBytes, EStringConvertErrorPolicy ePolicy ) + { + if ( !pIn ) + { + // For now, assert and return 0. Once these are cleaned out a bit + // we should remove this return and just leave in the assert... + AssertMsg( pIn, "We shouldn't be passing in NULL!" ); + return 0; + } + + int nOut = 0; + + if ( !pOut ) + { + while ( bStopAtNull ? ( *pIn ) : ( nInChars-- > 0 ) ) + { + uchar32 uVal; + // Initialize in order to avoid /analyze warnings. + bool bErr = false; + pIn += DecodeSrc( pIn, uVal, bErr ); + nOut += EncodeDstLen( uVal ); + if ( bErr ) + { +#ifdef _DEBUG + AssertMsg( !(ePolicy & _STRINGCONVERTFLAG_ASSERT), "invalid Unicode byte sequence" ); +#endif + if ( ePolicy & _STRINGCONVERTFLAG_SKIP ) + { + nOut -= EncodeDstLen( uVal ); + } + else if ( ePolicy & _STRINGCONVERTFLAG_FAIL ) + { + pOut[0] = 0; + return 0; + } + } + } + } + else + { + int nOutElems = nOutBytes / sizeof( DstType ); + if ( nOutElems <= 0 ) + return 0; + + int nMaxOut = nOutElems - 1; + while ( bStopAtNull ? ( *pIn ) : ( nInChars-- > 0 ) ) + { + uchar32 uVal; + // Initialize in order to avoid /analyze warnings. + bool bErr = false; + pIn += DecodeSrc( pIn, uVal, bErr ); + if ( nOut + EncodeDstLen( uVal ) > nMaxOut ) + break; + nOut += EncodeDst( uVal, pOut + nOut ); + if ( bErr ) + { +#ifdef _DEBUG + AssertMsg( !(ePolicy & _STRINGCONVERTFLAG_ASSERT), "invalid Unicode byte sequence" ); +#endif + if ( ePolicy & _STRINGCONVERTFLAG_SKIP ) + { + nOut -= EncodeDstLen( uVal ); + } + else if ( ePolicy & _STRINGCONVERTFLAG_FAIL ) + { + pOut[0] = 0; + return 0; + } + } + } + pOut[nOut] = 0; + } + + return (nOut + 1) * sizeof( DstType ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: Returns true if UTF-8 string contains invalid sequences. +//----------------------------------------------------------------------------- +bool Q_UnicodeValidate( const char *pUTF8 ) +{ + bool bError = false; + while ( *pUTF8 ) + { + uchar32 uVal; + // Our UTF-8 decoder silently fixes up 6-byte CESU-8 (improperly re-encoded UTF-16) sequences. + // However, these are technically not valid UTF-8. So if we eat 6 bytes at once, it's an error. + int nCharSize = Q_UTF8ToUChar32( pUTF8, uVal, bError ); + if ( bError || nCharSize == 6 ) + return false; + pUTF8 += nCharSize; + } + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Returns true if UTF-16 string contains invalid sequences. +//----------------------------------------------------------------------------- +bool Q_UnicodeValidate( const uchar16 *pUTF16 ) +{ + bool bError = false; + while ( *pUTF16 ) + { + uchar32 uVal; + pUTF16 += Q_UTF16ToUChar32( pUTF16, uVal, bError ); + if ( bError ) + return false; + } + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Returns true if UTF-32 string contains invalid sequences. +//----------------------------------------------------------------------------- +bool Q_UnicodeValidate( const uchar32 *pUTF32 ) +{ + while ( *pUTF32 ) + { + if ( !Q_IsValidUChar32( *pUTF32++ ) ) + return false; + ++pUTF32; + } + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Returns number of Unicode code points (aka glyphs / characters) encoded in the UTF-8 string +//----------------------------------------------------------------------------- +int Q_UnicodeLength( const char *pUTF8 ) +{ + int nChars = 0; + while ( *pUTF8 ) + { + bool bError; + uchar32 uVal; + pUTF8 += Q_UTF8ToUChar32( pUTF8, uVal, bError ); + ++nChars; + } + return nChars; +} + +//----------------------------------------------------------------------------- +// Purpose: Returns number of Unicode code points (aka glyphs / characters) encoded in the UTF-16 string +//----------------------------------------------------------------------------- +int Q_UnicodeLength( const uchar16 *pUTF16 ) +{ + int nChars = 0; + while ( *pUTF16 ) + { + bool bError; + uchar32 uVal; + pUTF16 += Q_UTF16ToUChar32( pUTF16, uVal, bError ); + ++nChars; + } + return nChars; +} + +//----------------------------------------------------------------------------- +// Purpose: Returns number of Unicode code points (aka glyphs / characters) encoded in the UTF-32 string +//----------------------------------------------------------------------------- +int Q_UnicodeLength( const uchar32 *pUTF32 ) +{ + int nChars = 0; + while ( *pUTF32++ ) + ++nChars; + return nChars; +} + +//----------------------------------------------------------------------------- +// Purpose: Advance a UTF-8 string pointer by a certain number of Unicode code points, stopping at end of string +//----------------------------------------------------------------------------- +char *Q_UnicodeAdvance( char *pUTF8, int nChars ) +{ + while ( nChars > 0 && *pUTF8 ) + { + uchar32 uVal; + bool bError; + pUTF8 += Q_UTF8ToUChar32( pUTF8, uVal, bError ); + --nChars; + } + return pUTF8; +} + +//----------------------------------------------------------------------------- +// Purpose: Advance a UTF-16 string pointer by a certain number of Unicode code points, stopping at end of string +//----------------------------------------------------------------------------- +uchar16 *Q_UnicodeAdvance( uchar16 *pUTF16, int nChars ) +{ + while ( nChars > 0 && *pUTF16 ) + { + uchar32 uVal; + bool bError; + pUTF16 += Q_UTF16ToUChar32( pUTF16, uVal, bError ); + --nChars; + } + return pUTF16; +} + +//----------------------------------------------------------------------------- +// Purpose: Advance a UTF-32 string pointer by a certain number of Unicode code points, stopping at end of string +//----------------------------------------------------------------------------- +uchar32 *Q_UnicodeAdvance( uchar32 *pUTF32, int nChars ) +{ + while ( nChars > 0 && *pUTF32 ) + { + ++pUTF32; + --nChars; + } + return pUTF32; +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF8ToUTF16( const char *pUTF8, uchar16 *pUTF16, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< char, uchar16, true, Q_UTF8ToUChar32, Q_UChar32ToUTF16Len, Q_UChar32ToUTF16 >( pUTF8, 0, pUTF16, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF8ToUTF32( const char *pUTF8, uchar32 *pUTF32, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< char, uchar32, true, Q_UTF8ToUChar32, Q_UChar32ToUTF32Len, Q_UChar32ToUTF32 >( pUTF8, 0, pUTF32, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF16ToUTF8( const uchar16 *pUTF16, char *pUTF8, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar16, char, true, Q_UTF16ToUChar32, Q_UChar32ToUTF8Len, Q_UChar32ToUTF8 >( pUTF16, 0, pUTF8, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF16ToUTF32( const uchar16 *pUTF16, uchar32 *pUTF32, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar16, uchar32, true, Q_UTF16ToUChar32, Q_UChar32ToUTF32Len, Q_UChar32ToUTF32 >( pUTF16, 0, pUTF32, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF32ToUTF8( const uchar32 *pUTF32, char *pUTF8, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar32, char, true, Q_UTF32ToUChar32, Q_UChar32ToUTF8Len, Q_UChar32ToUTF8 >( pUTF32, 0, pUTF8, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF32ToUTF16( const uchar32 *pUTF32, uchar16 *pUTF16, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar32, uchar16, true, Q_UTF32ToUChar32, Q_UChar32ToUTF16Len, Q_UChar32ToUTF16 >( pUTF32, 0, pUTF16, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF32ToUTF32( const uchar32 *pUTF32Source, uchar32 *pUTF32Dest, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar32, uchar32, true, Q_UTF32ToUChar32, Q_UChar32ToUTF32Len, Q_UChar32ToUTF32 >( pUTF32Source, 0, pUTF32Dest, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF8CharsToUTF16( const char *pUTF8, int nElements, uchar16 *pUTF16, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< char, uchar16, false, Q_UTF8ToUChar32, Q_UChar32ToUTF16Len, Q_UChar32ToUTF16 >( pUTF8, nElements, pUTF16, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF8CharsToUTF32( const char *pUTF8, int nElements, uchar32 *pUTF32, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< char, uchar32, false, Q_UTF8ToUChar32, Q_UChar32ToUTF32Len, Q_UChar32ToUTF32 >( pUTF8, nElements, pUTF32, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF16CharsToUTF8( const uchar16 *pUTF16, int nElements, char *pUTF8, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar16, char, false, Q_UTF16ToUChar32, Q_UChar32ToUTF8Len, Q_UChar32ToUTF8 >( pUTF16, nElements, pUTF8, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF16CharsToUTF32( const uchar16 *pUTF16, int nElements, uchar32 *pUTF32, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar16, uchar32, false, Q_UTF16ToUChar32, Q_UChar32ToUTF32Len, Q_UChar32ToUTF32 >( pUTF16, nElements, pUTF32, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF32CharsToUTF8( const uchar32 *pUTF32, int nElements, char *pUTF8, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar32, char, false, Q_UTF32ToUChar32, Q_UChar32ToUTF8Len, Q_UChar32ToUTF8 >( pUTF32, nElements, pUTF8, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF32CharsToUTF16( const uchar32 *pUTF32, int nElements, uchar16 *pUTF16, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar32, uchar16, false, Q_UTF32ToUChar32, Q_UChar32ToUTF16Len, Q_UChar32ToUTF16 >( pUTF32, nElements, pUTF16, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Repair a UTF-8 string by removing or replacing invalid seqeuences. Returns non-zero on success. +//----------------------------------------------------------------------------- +int Q_UnicodeRepair( char *pUTF8, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< char, char, true, Q_UTF8ToUChar32, Q_UChar32ToUTF8Len, Q_UChar32ToUTF8 >( pUTF8, 0, pUTF8, INT_MAX, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Repair a UTF-16 string by removing or replacing invalid seqeuences. Returns non-zero on success. +//----------------------------------------------------------------------------- +int Q_UnicodeRepair( uchar16 *pUTF16, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar16, uchar16, true, Q_UTF16ToUChar32, Q_UChar32ToUTF16Len, Q_UChar32ToUTF16 >( pUTF16, 0, pUTF16, INT_MAX/sizeof(uchar16), ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Repair a UTF-32 string by removing or replacing invalid seqeuences. Returns non-zero on success. +//----------------------------------------------------------------------------- +int Q_UnicodeRepair( uchar32 *pUTF32, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar32, uchar32, true, Q_UTF32ToUChar32, Q_UChar32ToUTF32Len, Q_UChar32ToUTF32 >( pUTF32, 0, pUTF32, INT_MAX/sizeof(uchar32), ePolicy ); +} + diff --git a/tier1/tier1.cpp b/tier1/tier1.cpp new file mode 100644 index 0000000..d683b87 --- /dev/null +++ b/tier1/tier1.cpp @@ -0,0 +1,63 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: A higher level link library for general use in the game and tools. +// +//===========================================================================// + +#include <tier1/tier1.h> +#include "tier0/dbg.h" +#include "vstdlib/iprocessutils.h" +#include "icvar.h" + + +//----------------------------------------------------------------------------- +// These tier1 libraries must be set by any users of this library. +// They can be set by calling ConnectTier1Libraries or InitDefaultFileSystem. +// It is hoped that setting this, and using this library will be the common mechanism for +// allowing link libraries to access tier1 library interfaces +//----------------------------------------------------------------------------- +ICvar *cvar = 0; +ICvar *g_pCVar = 0; +IProcessUtils *g_pProcessUtils = 0; +static bool s_bConnected = false; + +// for utlsortvector.h +#ifndef _WIN32 + void *g_pUtlSortVectorQSortContext = NULL; +#endif + + +//----------------------------------------------------------------------------- +// Call this to connect to all tier 1 libraries. +// It's up to the caller to check the globals it cares about to see if ones are missing +//----------------------------------------------------------------------------- +void ConnectTier1Libraries( CreateInterfaceFn *pFactoryList, int nFactoryCount ) +{ + // Don't connect twice.. + if ( s_bConnected ) + return; + + s_bConnected = true; + + for ( int i = 0; i < nFactoryCount; ++i ) + { + if ( !g_pCVar ) + { + cvar = g_pCVar = ( ICvar * )pFactoryList[i]( CVAR_INTERFACE_VERSION, NULL ); + } + if ( !g_pProcessUtils ) + { + g_pProcessUtils = ( IProcessUtils * )pFactoryList[i]( PROCESS_UTILS_INTERFACE_VERSION, NULL ); + } + } +} + +void DisconnectTier1Libraries() +{ + if ( !s_bConnected ) + return; + + g_pCVar = cvar = 0; + g_pProcessUtils = NULL; + s_bConnected = false; +} diff --git a/tier1/tier1.vpc b/tier1/tier1.vpc new file mode 100644 index 0000000..f4016ee --- /dev/null +++ b/tier1/tier1.vpc @@ -0,0 +1,166 @@ +//----------------------------------------------------------------------------- +// TIER1.VPC +// +// Project Script +//----------------------------------------------------------------------------- + +$macro SRCDIR ".." + +$include "$SRCDIR\vpc_scripts\source_lib_base.vpc" + +$Configuration +{ + $Compiler + { + $PreprocessorDefinitions "$BASE;TIER1_STATIC_LIB" + } + + $Librarian [$WINDOWS] + { + $AdditionalDependencies "$BASE Rpcrt4.lib" + } +} + +$Project "tier1" +{ + $Folder "Source Files" + { + $File "bitbuf.cpp" + $File "newbitbuf.cpp" + $File "byteswap.cpp" + $File "characterset.cpp" + $File "checksum_crc.cpp" + $File "checksum_md5.cpp" + $File "checksum_sha1.cpp" + $File "commandbuffer.cpp" + $File "convar.cpp" + $File "datamanager.cpp" + $File "diff.cpp" + $File "generichash.cpp" + $File "ilocalize.cpp" + $File "interface.cpp" + $File "KeyValues.cpp" + $File "keyvaluesjson.cpp" + $File "kvpacker.cpp" + $File "lzmaDecoder.cpp" + $File "lzss.cpp" [!$SOURCESDK] + $File "mempool.cpp" + $File "memstack.cpp" + $File "NetAdr.cpp" + $File "splitstring.cpp" + $File "processor_detect.cpp" [$WINDOWS||$X360] + { + $Configuration + { + $Compiler + { + $EnableC++Exceptions "Yes (/EHsc)" [!$X360] + } + } + } + + $File "processor_detect_linux.cpp" [$POSIX] + $File "qsort_s.cpp" [$LINUXALL||$PS3] + $File "rangecheckedvar.cpp" + $File "reliabletimer.cpp" + $File "stringpool.cpp" + $File "strtools.cpp" + $File "strtools_unicode.cpp" + $File "tier1.cpp" + $File "tokenreader.cpp" + $File "sparsematrix.cpp" + $File "uniqueid.cpp" + $File "utlbuffer.cpp" + $File "utlbufferutil.cpp" + $File "utlstring.cpp" + $File "utlsymbol.cpp" + $File "utlbinaryblock.cpp" + $File "pathmatch.cpp" [$LINUXALL] + $File "snappy.cpp" + $File "snappy-sinksource.cpp" + $File "snappy-stubs-internal.cpp" + } + + // Select bits from the LZMA SDK to support lzmaDecoder.h + // Encoding support requires the full lzma project + $Folder "LZMA Decompression Support" + { + $File "$SRCDIR\utils\lzma\C\LzmaDec.h" + $File "$SRCDIR\utils\lzma\C\LzmaDec.c" + $File "$SRCDIR\utils\lzma\C\7zTypes.h" + } + + $Folder "Header Files" + { + $Folder "Internal Header Files" + { + $File "snappy-internal.h" + $File "snappy-stubs-internal.h" + } + $File "$SRCDIR\public\tier1\bitbuf.h" + $File "$SRCDIR\public\tier1\byteswap.h" + $File "$SRCDIR\public\tier1\callqueue.h" + $File "$SRCDIR\public\tier1\characterset.h" + $File "$SRCDIR\public\tier1\checksum_crc.h" + $File "$SRCDIR\public\tier1\checksum_md5.h" + $File "$SRCDIR\public\tier1\checksum_sha1.h" + $File "$SRCDIR\public\tier1\CommandBuffer.h" + $File "$SRCDIR\public\tier1\convar.h" + $File "$SRCDIR\public\tier1\datamanager.h" + $File "$SRCDIR\public\datamap.h" + $File "$SRCDIR\public\tier1\delegates.h" + $File "$SRCDIR\public\tier1\diff.h" + $File "$SRCDIR\public\tier1\fmtstr.h" + $File "$SRCDIR\public\tier1\functors.h" + $File "$SRCDIR\public\tier1\generichash.h" + $File "$SRCDIR\public\tier1\iconvar.h" + $File "$SRCDIR\public\tier1\ilocalize.h" + $File "$SRCDIR\public\tier1\interface.h" + $File "$SRCDIR\public\tier1\KeyValues.h" + $File "$SRCDIR\public\tier1\keyvaluesjson.h" + $File "$SRCDIR\public\tier1\kvpacker.h" + $File "$SRCDIR\public\tier1\lzmaDecoder.h" + $File "$SRCDIR\public\tier1\lzss.h" + $File "$SRCDIR\public\tier1\mempool.h" + $File "$SRCDIR\public\tier1\memstack.h" + $File "$SRCDIR\public\tier1\netadr.h" + $File "$SRCDIR\public\tier1\processor_detect.h" + $File "$SRCDIR\public\tier1\rangecheckedvar.h" + $File "$SRCDIR\public\tier1\refcount.h" + $File "$SRCDIR\public\tier1\smartptr.h" + $File "$SRCDIR\public\tier1\snappy.h" + $File "$SRCDIR\public\tier1\snappy-sinksource.h" + $File "$SRCDIR\public\tier1\stringpool.h" + $File "$SRCDIR\public\tier1\strtools.h" + $File "$SRCDIR\public\tier1\tier1.h" + $File "$SRCDIR\public\tier1\tokenreader.h" + $File "$SRCDIR\public\tier1\uniqueid.h" [$WINDOWS] + $File "$SRCDIR\public\tier1\utlbidirectionalset.h" + $File "$SRCDIR\public\tier1\utlblockmemory.h" + $File "$SRCDIR\public\tier1\utlbuffer.h" + $File "$SRCDIR\public\tier1\utlbufferutil.h" + $File "$SRCDIR\public\tier1\utlcommon.h" + $File "$SRCDIR\public\tier1\utldict.h" + $File "$SRCDIR\public\tier1\utlenvelope.h" + $File "$SRCDIR\public\tier1\utlfixedmemory.h" + $File "$SRCDIR\public\tier1\utlhandletable.h" + $File "$SRCDIR\public\tier1\utlhash.h" + $File "$SRCDIR\public\tier1\utlhashtable.h" + $File "$SRCDIR\public\tier1\utllinkedlist.h" + $File "$SRCDIR\public\tier1\utlmap.h" + $File "$SRCDIR\public\tier1\utlmemory.h" + $File "$SRCDIR\public\tier1\utlmultilist.h" + $File "$SRCDIR\public\tier1\utlpriorityqueue.h" + $File "$SRCDIR\public\tier1\utlqueue.h" + $File "$SRCDIR\public\tier1\utlrbtree.h" + $File "$SRCDIR\public\tier1\UtlSortVector.h" + $File "$SRCDIR\public\tier1\utlstack.h" + $File "$SRCDIR\public\tier1\utlstring.h" + $File "$SRCDIR\public\tier1\UtlStringMap.h" + $File "$SRCDIR\public\tier1\utlsymbol.h" + $File "$SRCDIR\public\tier1\utlsymbollarge.h" + $File "$SRCDIR\public\tier1\utlvector.h" + $File "$SRCDIR\public\tier1\utlbinaryblock.h" + $File "$SRCDIR\common\xbox\xboxstubs.h" [$WINDOWS] + } +} diff --git a/tier1/tier1_exclude.vpc b/tier1/tier1_exclude.vpc new file mode 100644 index 0000000..5991f75 --- /dev/null +++ b/tier1/tier1_exclude.vpc @@ -0,0 +1,20 @@ +//----------------------------------------------------------------------------- +// tier1_exclude.vpc +// +// Project Script +//----------------------------------------------------------------------------- + +$Project +{ + $Folder "Link Libraries" + { + // Should match the sites that include this + -$Lib "$LIBPUBLIC\tier1" [$POSIX && !$IS_LIB_PROJECT] + } + + $Folder "Link Libraries" + { + -$File "$SRCDIR\lib\$PLATFORM\tier1$_STATICLIB_EXT" [!$WINDOWS] + -$File "$SRCDIR\lib\public\tier1$_STATICLIB_EXT" [$WINDOWS] + } +} diff --git a/tier1/tokenreader.cpp b/tier1/tokenreader.cpp new file mode 100644 index 0000000..39d0108 --- /dev/null +++ b/tier1/tokenreader.cpp @@ -0,0 +1,480 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//===========================================================================// + +#include <ctype.h> +#include <stdio.h> +#include <string.h> +#include "tokenreader.h" +#include "tier0/platform.h" +#include "tier1/strtools.h" +#include "tier0/dbg.h" + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +TokenReader::TokenReader(void) +{ + m_szFilename[0] = '\0'; + m_nLine = 1; + m_nErrorCount = 0; + m_bStuffed = false; +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *pszFilename - +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool TokenReader::Open(const char *pszFilename) +{ + open(pszFilename, std::ios::in | std::ios::binary ); + Q_strncpy(m_szFilename, pszFilename, sizeof( m_szFilename ) ); + m_nLine = 1; + m_nErrorCount = 0; + m_bStuffed = false; + return(is_open() != 0); +} + + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void TokenReader::Close() +{ + close(); +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *error - +// Output : const char +//----------------------------------------------------------------------------- +const char *TokenReader::Error(char *error, ...) +{ + static char szErrorBuf[256]; + Q_snprintf(szErrorBuf, sizeof( szErrorBuf ), "File %s, line %d: ", m_szFilename, m_nLine); + Q_strncat(szErrorBuf, error, sizeof( szErrorBuf ), COPY_ALL_CHARACTERS ); + m_nErrorCount++; + return(szErrorBuf); +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : pszStore - +// nSize - +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +trtoken_t TokenReader::GetString(char *pszStore, int nSize) +{ + if (nSize <= 0) + { + return TOKENERROR; + } + + char szBuf[1024]; + + // + // Until we reach the end of this string or run out of room in + // the destination buffer... + // + while (true) + { + // + // Fetch the next batch of text from the file. + // + get(szBuf, sizeof(szBuf), '\"'); + if (eof()) + { + return TOKENEOF; + } + + if (fail()) + { + // Just means nothing was read (empty string probably "") + clear(); + } + + // + // Transfer the text to the destination buffer. + // + char *pszSrc = szBuf; + while ((*pszSrc != '\0') && (nSize > 1)) + { + if (*pszSrc == 0x0d) + { + // + // Newline encountered before closing quote -- unterminated string. + // + *pszStore = '\0'; + return TOKENSTRINGTOOLONG; + } + else if (*pszSrc != '\\') + { + *pszStore = *pszSrc; + pszSrc++; + } + else + { + // + // Backslash sequence - replace with the appropriate character. + // + pszSrc++; + + if (*pszSrc == 'n') + { + *pszStore = '\n'; + } + + pszSrc++; + } + + pszStore++; + nSize--; + } + + if (*pszSrc != '\0') + { + // + // Ran out of room in the destination buffer. Skip to the close-quote, + // terminate the string, and exit. + // + ignore(1024, '\"'); + *pszStore = '\0'; + return TOKENSTRINGTOOLONG; + } + + // + // Check for closing quote. + // + if (peek() == '\"') + { + // + // Eat the close quote and any whitespace. + // + get(); + + bool bCombineStrings = SkipWhiteSpace(); + + // + // Combine consecutive quoted strings if the combine strings character was + // encountered between the two strings. + // + if (bCombineStrings && (peek() == '\"')) + { + // + // Eat the open quote and keep parsing this string. + // + get(); + } + else + { + // + // Done with this string, terminate the string and exit. + // + *pszStore = '\0'; + return STRING; + } + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns the next token, allocating enough memory to store the token +// plus a terminating NULL. +// Input : pszStore - Pointer to a string that will be allocated. +// Output : Returns the type of token that was read, or TOKENERROR. +//----------------------------------------------------------------------------- +trtoken_t TokenReader::NextTokenDynamic(char **ppszStore) +{ + char szTempBuffer[8192]; + trtoken_t eType = NextToken(szTempBuffer, sizeof(szTempBuffer)); + + int len = Q_strlen(szTempBuffer) + 1; + *ppszStore = new char [len]; + Assert( *ppszStore ); + Q_strncpy(*ppszStore, szTempBuffer, len ); + + return(eType); +} + + +//----------------------------------------------------------------------------- +// Purpose: Returns the next token. +// Input : pszStore - Pointer to a string that will receive the token. +// Output : Returns the type of token that was read, or TOKENERROR. +//----------------------------------------------------------------------------- +trtoken_t TokenReader::NextToken(char *pszStore, int nSize) +{ + char *pStart = pszStore; + + if (!is_open()) + { + return TOKENEOF; + } + + // + // If they stuffed a token, return that token. + // + if (m_bStuffed) + { + m_bStuffed = false; + Q_strncpy( pszStore, m_szStuffed, nSize ); + return m_eStuffed; + } + + SkipWhiteSpace(); + + if (eof()) + { + return TOKENEOF; + } + + if (fail()) + { + return TOKENEOF; + } + + char ch = get(); + + // + // Look for all the valid operators. + // + switch (ch) + { + case '@': + case ',': + case '!': + case '+': + case '&': + case '*': + case '$': + case '.': + case '=': + case ':': + case '[': + case ']': + case '(': + case ')': + case '{': + case '}': + case '\\': + { + pszStore[0] = ch; + pszStore[1] = 0; + return OPERATOR; + } + } + + // + // Look for the start of a quoted string. + // + if (ch == '\"') + { + return GetString(pszStore, nSize); + } + + // + // Integers consist of numbers with an optional leading minus sign. + // + if (isdigit(ch) || (ch == '-')) + { + do + { + if ( (pszStore - pStart + 1) < nSize ) + { + *pszStore = ch; + pszStore++; + } + + ch = get(); + if (ch == '-') + { + return TOKENERROR; + } + } while (isdigit(ch)); + + // + // No identifier characters are allowed contiguous with numbers. + // + if (isalpha(ch) || (ch == '_')) + { + return TOKENERROR; + } + + // + // Put back the non-numeric character for the next call. + // + putback(ch); + *pszStore = '\0'; + return INTEGER; + } + + // + // Identifiers consist of a consecutive string of alphanumeric + // characters and underscores. + // + while ( isalpha(ch) || isdigit(ch) || (ch == '_') ) + { + if ( (pszStore - pStart + 1) < nSize ) + { + *pszStore = ch; + pszStore++; + } + + ch = get(); + } + + // + // Put back the non-identifier character for the next call. + // + putback(ch); + *pszStore = '\0'; + return IDENT; +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : ttype - +// *pszToken - +//----------------------------------------------------------------------------- +void TokenReader::IgnoreTill(trtoken_t ttype, const char *pszToken) +{ + trtoken_t _ttype; + char szBuf[1024]; + + while(1) + { + _ttype = NextToken(szBuf, sizeof(szBuf)); + if(_ttype == TOKENEOF) + return; + if(_ttype == ttype) + { + if(IsToken(pszToken, szBuf)) + { + Stuff(ttype, pszToken); + return; + } + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : ttype - +// pszToken - +//----------------------------------------------------------------------------- +void TokenReader::Stuff(trtoken_t eType, const char *pszToken) +{ + m_eStuffed = eType; + Q_strncpy(m_szStuffed, pszToken, sizeof( m_szStuffed ) ); + m_bStuffed = true; +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : ttype - +// pszToken - +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool TokenReader::Expecting(trtoken_t ttype, const char *pszToken) +{ + char szBuf[1024]; + if (NextToken(szBuf, sizeof(szBuf)) != ttype || !IsToken(pszToken, szBuf)) + { + return false; + } + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : pszStore - +// Output : +//----------------------------------------------------------------------------- +trtoken_t TokenReader::PeekTokenType(char *pszStore, int maxlen ) +{ + if (!m_bStuffed) + { + m_eStuffed = NextToken(m_szStuffed, sizeof(m_szStuffed)); + m_bStuffed = true; + } + + if (pszStore) + { + Q_strncpy(pszStore, m_szStuffed, maxlen ); + } + + return(m_eStuffed); +} + + +//----------------------------------------------------------------------------- +// Purpose: Gets the next non-whitespace character from the file. +// Input : ch - Receives the character. +// Output : Returns true if the whitespace contained the combine strings +// character '\', which is used to merge consecutive quoted strings. +//----------------------------------------------------------------------------- +bool TokenReader::SkipWhiteSpace(void) +{ + bool bCombineStrings = false; + + while (true) + { + char ch = get(); + + if ((ch == ' ') || (ch == '\t') || (ch == '\r') || (ch == 0)) + { + continue; + } + + if (ch == '+') + { + bCombineStrings = true; + continue; + } + + if (ch == '\n') + { + m_nLine++; + continue; + } + + if (eof()) + { + return(bCombineStrings); + } + + // + // Check for the start of a comment. + // + if (ch == '/') + { + if (peek() == '/') + { + ignore(1024, '\n'); + m_nLine++; + } + } + else + { + // + // It is a worthy character. Put it back. + // + putback(ch); + return(bCombineStrings); + } + } +} + diff --git a/tier1/undiff.cpp b/tier1/undiff.cpp new file mode 100644 index 0000000..c39d8d4 --- /dev/null +++ b/tier1/undiff.cpp @@ -0,0 +1,94 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// UnDiff - Apply difference block +// +//=============================================================================// + +#include "tier0/platform.h" +#include "tier0/dbg.h" +#include "tier1/diff.h" +#include "mathlib/mathlib.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +void ApplyDiffs(uint8 const *OldBlock, uint8 const *DiffList, + int OldSize, int DiffListSize, int &ResultListSize,uint8 *Output,uint32 OutSize) +{ + uint8 const *copy_src=OldBlock; + uint8 const *end_of_diff_list=DiffList+DiffListSize; + uint8 const *obuf=Output; + while(DiffList<end_of_diff_list) + { + // printf("dptr=%x ",DiffList-d); + uint8 op=*(DiffList++); + if (op==0) + { + uint16 copy_sz=DiffList[0]+256*DiffList[1]; + int copy_ofs=DiffList[2]+DiffList[3]*256; + if (copy_ofs>32767) + copy_ofs|=0xffff0000; + // printf("long cp from %x to %x len=%d\n", copy_src+copy_ofs-OldBlock,Output-obuf,copy_sz); + + memcpy(Output,copy_src+copy_ofs,copy_sz); + Output+=copy_sz; + copy_src=copy_src+copy_ofs+copy_sz; + DiffList+=4; + } + else + { + if (op & 0x80) + { + int copy_sz=op & 0x7f; + int copy_ofs; + if (copy_sz==0) + { + copy_sz=DiffList[0]; + if (copy_sz==0) + { + // big raw copy + copy_sz=DiffList[1]+256*DiffList[2]+65536*DiffList[3]; + memcpy(Output,DiffList+4,copy_sz); + // printf("big rawcopy to %x len=%d\n", Output-obuf,copy_sz); + + DiffList+=copy_sz+4; + Output+=copy_sz; + } + else + { + copy_ofs=DiffList[1]+(DiffList[2]*256); + if (copy_ofs>32767) + copy_ofs|=0xffff0000; + // printf("long ofs cp from %x to %x len=%d\n", copy_src+copy_ofs-OldBlock,Output-obuf,copy_sz); + + memcpy(Output,copy_src+copy_ofs,copy_sz); + Output+=copy_sz; + copy_src=copy_src+copy_ofs+copy_sz; + DiffList+=3; + } + } + else + { + copy_ofs=DiffList[0]; + if (copy_ofs>127) + copy_ofs|=0xffffff80; + // printf("cp from %x to %x len=%d\n", copy_src+copy_ofs-OldBlock,Output-obuf,copy_sz); + + memcpy(Output,copy_src+copy_ofs,copy_sz); + Output+=copy_sz; + copy_src=copy_src+copy_ofs+copy_sz; + DiffList++; + } + } + else + { + // printf("raw copy %d to %x\n",op & 127,Output-obuf); + memcpy(Output,DiffList,op & 127); + Output+=op & 127; + DiffList+=(op & 127); + } + } + } + ResultListSize=Output-obuf; + +} diff --git a/tier1/uniqueid.cpp b/tier1/uniqueid.cpp new file mode 100644 index 0000000..d75e4fd --- /dev/null +++ b/tier1/uniqueid.cpp @@ -0,0 +1,177 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +// +// Unique ID generation +//=============================================================================// + +#include "tier0/platform.h" + +#ifdef IS_WINDOWS_PC +#include <windows.h> // UUIDCreate +#else +#include "checksum_crc.h" +#endif +#include "tier1/uniqueid.h" +#include "tier1/utlbuffer.h" + +//----------------------------------------------------------------------------- +// Creates a new unique id +//----------------------------------------------------------------------------- +void CreateUniqueId( UniqueId_t *pDest ) +{ +#ifdef IS_WINDOWS_PC + Assert( sizeof( UUID ) == sizeof( *pDest ) ); + UuidCreate( (UUID *)pDest ); +#else + // X360/linux TBD: Need a real UUID Implementation + Q_memset( pDest, 0, sizeof( UniqueId_t ) ); +#endif +} + + +//----------------------------------------------------------------------------- +// Creates a new unique id from a string representation of one +//----------------------------------------------------------------------------- +bool UniqueIdFromString( UniqueId_t *pDest, const char *pBuf, int nMaxLen ) +{ + if ( nMaxLen == 0 ) + { + nMaxLen = Q_strlen( pBuf ); + } + + char *pTemp = (char*)stackalloc( nMaxLen + 1 ); + V_strncpy( pTemp, pBuf, nMaxLen + 1 ); + --nMaxLen; + while( (nMaxLen >= 0) && isspace( pTemp[nMaxLen] ) ) + { + --nMaxLen; + } + pTemp[ nMaxLen + 1 ] = 0; + + while( *pTemp && isspace( *pTemp ) ) + { + ++pTemp; + } + +#ifdef IS_WINDOWS_PC + Assert( sizeof( UUID ) == sizeof( *pDest ) ); + + if ( RPC_S_OK != UuidFromString( (unsigned char *)pTemp, (UUID *)pDest ) ) + { + InvalidateUniqueId( pDest ); + return false; + } +#else + // X360TBD: Need a real UUID Implementation + // For now, use crc to generate a unique ID from the UUID string. + Q_memset( pDest, 0, sizeof( UniqueId_t ) ); + if ( nMaxLen > 0 ) + { + CRC32_t crc; + CRC32_Init( &crc ); + CRC32_ProcessBuffer( &crc, pBuf, nMaxLen ); + CRC32_Final( &crc ); + Q_memcpy( pDest, &crc, sizeof( CRC32_t ) ); + } +#endif + + return true; +} + +//----------------------------------------------------------------------------- +// Sets an object ID to be an invalid state +//----------------------------------------------------------------------------- +void InvalidateUniqueId( UniqueId_t *pDest ) +{ + Assert( pDest ); + memset( pDest, 0, sizeof( UniqueId_t ) ); +} + +bool IsUniqueIdValid( const UniqueId_t &id ) +{ + UniqueId_t invalidId; + memset( &invalidId, 0, sizeof( UniqueId_t ) ); + return !IsUniqueIdEqual( invalidId, id ); +} + +bool IsUniqueIdEqual( const UniqueId_t &id1, const UniqueId_t &id2 ) +{ + return memcmp( &id1, &id2, sizeof( UniqueId_t ) ) == 0; +} + +void UniqueIdToString( const UniqueId_t &id, char *pBuf, int nMaxLen ) +{ + pBuf[ 0 ] = 0; + +// X360TBD: Need a real UUID Implementation +#ifdef IS_WINDOWS_PC + UUID *self = ( UUID * )&id; + + unsigned char *outstring = NULL; + + UuidToString( self, &outstring ); + if ( outstring && *outstring ) + { + Q_strncpy( pBuf, (const char *)outstring, nMaxLen ); + RpcStringFree( &outstring ); + } +#endif +} + +void CopyUniqueId( const UniqueId_t &src, UniqueId_t *pDest ) +{ + memcpy( pDest, &src, sizeof( UniqueId_t ) ); +} + +bool Serialize( CUtlBuffer &buf, const UniqueId_t &src ) +{ +// X360TBD: Need a real UUID Implementation +#ifdef IS_WINDOWS_PC + if ( buf.IsText() ) + { + UUID *pId = ( UUID * )&src; + + unsigned char *outstring = NULL; + + UuidToString( pId, &outstring ); + if ( outstring && *outstring ) + { + buf.PutString( (const char *)outstring ); + RpcStringFree( &outstring ); + } + else + { + buf.PutChar( '\0' ); + } + } + else + { + buf.Put( &src, sizeof(UniqueId_t) ); + } + return buf.IsValid(); +#else + return false; +#endif +} + +bool Unserialize( CUtlBuffer &buf, UniqueId_t &dest ) +{ + if ( buf.IsText() ) + { + int nTextLen = buf.PeekStringLength(); + char *pBuf = (char*)stackalloc( nTextLen ); + buf.GetStringManualCharCount( pBuf, nTextLen ); + UniqueIdFromString( &dest, pBuf, nTextLen ); + } + else + { + buf.Get( &dest, sizeof(UniqueId_t) ); + } + return buf.IsValid(); +} + + + diff --git a/tier1/utlbinaryblock.cpp b/tier1/utlbinaryblock.cpp new file mode 100644 index 0000000..7f64c4e --- /dev/null +++ b/tier1/utlbinaryblock.cpp @@ -0,0 +1,116 @@ +//====== Copyright 1996-2004, Valve Corporation, All rights reserved. ======= +// +// Purpose: +// +//============================================================================= + +#include "tier1/utlbinaryblock.h" + +//----------------------------------------------------------------------------- +// Base class, containing simple memory management +//----------------------------------------------------------------------------- +CUtlBinaryBlock::CUtlBinaryBlock( int growSize, int initSize ) : m_Memory( growSize, initSize ) +{ + m_nActualLength = 0; +} + +CUtlBinaryBlock::CUtlBinaryBlock( void* pMemory, int nSizeInBytes, int nInitialLength ) : m_Memory( (unsigned char*)pMemory, nSizeInBytes ) +{ + m_nActualLength = nInitialLength; +} + +CUtlBinaryBlock::CUtlBinaryBlock( const void* pMemory, int nSizeInBytes ) : m_Memory( (const unsigned char*)pMemory, nSizeInBytes ) +{ + m_nActualLength = nSizeInBytes; +} + +CUtlBinaryBlock::CUtlBinaryBlock( const CUtlBinaryBlock& src ) +{ + Set( src.Get(), src.Length() ); +} + +void CUtlBinaryBlock::Get( void *pValue, int nLen ) const +{ + Assert( nLen > 0 ); + if ( m_nActualLength < nLen ) + { + nLen = m_nActualLength; + } + + if ( nLen > 0 ) + { + memcpy( pValue, m_Memory.Base(), nLen ); + } +} + +void CUtlBinaryBlock::SetLength( int nLength ) +{ + Assert( !m_Memory.IsReadOnly() ); + + m_nActualLength = nLength; + if ( nLength > m_Memory.NumAllocated() ) + { + int nOverFlow = nLength - m_Memory.NumAllocated(); + m_Memory.Grow( nOverFlow ); + + // If the reallocation failed, clamp length + if ( nLength > m_Memory.NumAllocated() ) + { + m_nActualLength = m_Memory.NumAllocated(); + } + } + +#ifdef _DEBUG + if ( m_Memory.NumAllocated() > m_nActualLength ) + { + memset( ( ( char * )m_Memory.Base() ) + m_nActualLength, 0xEB, m_Memory.NumAllocated() - m_nActualLength ); + } +#endif +} + + +void CUtlBinaryBlock::Set( const void *pValue, int nLen ) +{ + Assert( !m_Memory.IsReadOnly() ); + + if ( !pValue ) + { + nLen = 0; + } + + SetLength( nLen ); + + if ( m_nActualLength ) + { + if ( ( ( const char * )m_Memory.Base() ) >= ( ( const char * )pValue ) + nLen || + ( ( const char * )m_Memory.Base() ) + m_nActualLength <= ( ( const char * )pValue ) ) + { + memcpy( m_Memory.Base(), pValue, m_nActualLength ); + } + else + { + memmove( m_Memory.Base(), pValue, m_nActualLength ); + } + } +} + + +CUtlBinaryBlock &CUtlBinaryBlock::operator=( const CUtlBinaryBlock &src ) +{ + Assert( !m_Memory.IsReadOnly() ); + Set( src.Get(), src.Length() ); + return *this; +} + + +bool CUtlBinaryBlock::operator==( const CUtlBinaryBlock &src ) const +{ + if ( src.Length() != Length() ) + return false; + + return !memcmp( src.Get(), Get(), Length() ); +} + + + + diff --git a/tier1/utlbuffer.cpp b/tier1/utlbuffer.cpp new file mode 100644 index 0000000..ff03da0 --- /dev/null +++ b/tier1/utlbuffer.cpp @@ -0,0 +1,1796 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// $Header: $ +// $NoKeywords: $ +// +// Serialization buffer +//===========================================================================// + +#pragma warning (disable : 4514) + +#include "utlbuffer.h" +#include <stdio.h> +#include <stdarg.h> +#include <ctype.h> +#include <stdlib.h> +#include <limits.h> +#include "tier1/strtools.h" +#include "tier1/characterset.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + + +//----------------------------------------------------------------------------- +// Character conversions for C strings +//----------------------------------------------------------------------------- +class CUtlCStringConversion : public CUtlCharConversion +{ +public: + CUtlCStringConversion( char nEscapeChar, const char *pDelimiter, int nCount, ConversionArray_t *pArray ); + + // Finds a conversion for the passed-in string, returns length + virtual char FindConversion( const char *pString, int *pLength ); + +private: + char m_pConversion[256]; +}; + + +//----------------------------------------------------------------------------- +// Character conversions for no-escape sequence strings +//----------------------------------------------------------------------------- +class CUtlNoEscConversion : public CUtlCharConversion +{ +public: + CUtlNoEscConversion( char nEscapeChar, const char *pDelimiter, int nCount, ConversionArray_t *pArray ) : + CUtlCharConversion( nEscapeChar, pDelimiter, nCount, pArray ) {} + + // Finds a conversion for the passed-in string, returns length + virtual char FindConversion( const char *pString, int *pLength ) { *pLength = 0; return 0; } +}; + + +//----------------------------------------------------------------------------- +// List of character conversions +//----------------------------------------------------------------------------- +BEGIN_CUSTOM_CHAR_CONVERSION( CUtlCStringConversion, s_StringCharConversion, "\"", '\\' ) + { '\n', "n" }, + { '\t', "t" }, + { '\v', "v" }, + { '\b', "b" }, + { '\r', "r" }, + { '\f', "f" }, + { '\a', "a" }, + { '\\', "\\" }, + { '\?', "\?" }, + { '\'', "\'" }, + { '\"', "\"" }, +END_CUSTOM_CHAR_CONVERSION( CUtlCStringConversion, s_StringCharConversion, "\"", '\\' ) + +CUtlCharConversion *GetCStringCharConversion() +{ + return &s_StringCharConversion; +} + +BEGIN_CUSTOM_CHAR_CONVERSION( CUtlNoEscConversion, s_NoEscConversion, "\"", 0x7F ) + { 0x7F, "" }, +END_CUSTOM_CHAR_CONVERSION( CUtlNoEscConversion, s_NoEscConversion, "\"", 0x7F ) + +CUtlCharConversion *GetNoEscCharConversion() +{ + return &s_NoEscConversion; +} + + +//----------------------------------------------------------------------------- +// Constructor +//----------------------------------------------------------------------------- +CUtlCStringConversion::CUtlCStringConversion( char nEscapeChar, const char *pDelimiter, int nCount, ConversionArray_t *pArray ) : + CUtlCharConversion( nEscapeChar, pDelimiter, nCount, pArray ) +{ + memset( m_pConversion, 0x0, sizeof(m_pConversion) ); + for ( int i = 0; i < nCount; ++i ) + { + m_pConversion[ (unsigned char) pArray[i].m_pReplacementString[0] ] = pArray[i].m_nActualChar; + } +} + +// Finds a conversion for the passed-in string, returns length +char CUtlCStringConversion::FindConversion( const char *pString, int *pLength ) +{ + char c = m_pConversion[ (unsigned char) pString[0] ]; + *pLength = (c != '\0') ? 1 : 0; + return c; +} + + + +//----------------------------------------------------------------------------- +// Constructor +//----------------------------------------------------------------------------- +CUtlCharConversion::CUtlCharConversion( char nEscapeChar, const char *pDelimiter, int nCount, ConversionArray_t *pArray ) +{ + m_nEscapeChar = nEscapeChar; + m_pDelimiter = pDelimiter; + m_nCount = nCount; + m_nDelimiterLength = Q_strlen( pDelimiter ); + m_nMaxConversionLength = 0; + + memset( m_pReplacements, 0, sizeof(m_pReplacements) ); + + for ( int i = 0; i < nCount; ++i ) + { + m_pList[i] = pArray[i].m_nActualChar; + ConversionInfo_t &info = m_pReplacements[ (unsigned char) m_pList[i] ]; + Assert( info.m_pReplacementString == 0 ); + info.m_pReplacementString = pArray[i].m_pReplacementString; + info.m_nLength = Q_strlen( info.m_pReplacementString ); + if ( info.m_nLength > m_nMaxConversionLength ) + { + m_nMaxConversionLength = info.m_nLength; + } + } +} + + +//----------------------------------------------------------------------------- +// Escape character + delimiter +//----------------------------------------------------------------------------- +char CUtlCharConversion::GetEscapeChar() const +{ + return m_nEscapeChar; +} + +const char *CUtlCharConversion::GetDelimiter() const +{ + return m_pDelimiter; +} + +int CUtlCharConversion::GetDelimiterLength() const +{ + return m_nDelimiterLength; +} + + +//----------------------------------------------------------------------------- +// Constructor +//----------------------------------------------------------------------------- +const char *CUtlCharConversion::GetConversionString( char c ) const +{ + return m_pReplacements[ (unsigned char) c ].m_pReplacementString; +} + +int CUtlCharConversion::GetConversionLength( char c ) const +{ + return m_pReplacements[ (unsigned char) c ].m_nLength; +} + +int CUtlCharConversion::MaxConversionLength() const +{ + return m_nMaxConversionLength; +} + + +//----------------------------------------------------------------------------- +// Finds a conversion for the passed-in string, returns length +//----------------------------------------------------------------------------- +char CUtlCharConversion::FindConversion( const char *pString, int *pLength ) +{ + for ( int i = 0; i < m_nCount; ++i ) + { + if ( !Q_strcmp( pString, m_pReplacements[ (unsigned char) m_pList[i] ].m_pReplacementString ) ) + { + *pLength = m_pReplacements[ (unsigned char) m_pList[i] ].m_nLength; + return m_pList[i]; + } + } + + *pLength = 0; + return '\0'; +} + + +//----------------------------------------------------------------------------- +// constructors +//----------------------------------------------------------------------------- +CUtlBuffer::CUtlBuffer( int growSize, int initSize, int nFlags ) : + m_Error(0) +{ + MEM_ALLOC_CREDIT(); + m_Memory.Init( growSize, initSize ); + m_Get = 0; + m_Put = 0; + m_nTab = 0; + m_nOffset = 0; + m_Flags = nFlags; + if ( (initSize != 0) && !IsReadOnly() ) + { + m_nMaxPut = -1; + AddNullTermination(); + } + else + { + m_nMaxPut = 0; + } + SetOverflowFuncs( &CUtlBuffer::GetOverflow, &CUtlBuffer::PutOverflow ); +} + +CUtlBuffer::CUtlBuffer( const void *pBuffer, int nSize, int nFlags ) : + m_Memory( (unsigned char*)pBuffer, nSize ), m_Error(0) +{ + Assert( nSize != 0 ); + + m_Get = 0; + m_Put = 0; + m_nTab = 0; + m_nOffset = 0; + m_Flags = nFlags; + if ( IsReadOnly() ) + { + m_nMaxPut = nSize; + } + else + { + m_nMaxPut = -1; + AddNullTermination(); + } + SetOverflowFuncs( &CUtlBuffer::GetOverflow, &CUtlBuffer::PutOverflow ); +} + + +//----------------------------------------------------------------------------- +// Modifies the buffer to be binary or text; Blows away the buffer and the CONTAINS_CRLF value. +//----------------------------------------------------------------------------- +void CUtlBuffer::SetBufferType( bool bIsText, bool bContainsCRLF ) +{ +#ifdef _DEBUG + // If the buffer is empty, there is no opportunity for this stuff to fail + if ( TellMaxPut() != 0 ) + { + if ( IsText() ) + { + if ( bIsText ) + { + Assert( ContainsCRLF() == bContainsCRLF ); + } + else + { + Assert( ContainsCRLF() ); + } + } + else + { + if ( bIsText ) + { + Assert( bContainsCRLF ); + } + } + } +#endif + + if ( bIsText ) + { + m_Flags |= TEXT_BUFFER; + } + else + { + m_Flags &= ~TEXT_BUFFER; + } + if ( bContainsCRLF ) + { + m_Flags |= CONTAINS_CRLF; + } + else + { + m_Flags &= ~CONTAINS_CRLF; + } +} + + +//----------------------------------------------------------------------------- +// Attaches the buffer to external memory.... +//----------------------------------------------------------------------------- +void CUtlBuffer::SetExternalBuffer( void* pMemory, int nSize, int nInitialPut, int nFlags ) +{ + m_Memory.SetExternalBuffer( (unsigned char*)pMemory, nSize ); + + // Reset all indices; we just changed memory + m_Get = 0; + m_Put = nInitialPut; + m_nTab = 0; + m_Error = 0; + m_nOffset = 0; + m_Flags = nFlags; + m_nMaxPut = -1; + AddNullTermination(); +} + +//----------------------------------------------------------------------------- +// Assumes an external buffer but manages its deletion +//----------------------------------------------------------------------------- +void CUtlBuffer::AssumeMemory( void *pMemory, int nSize, int nInitialPut, int nFlags ) +{ + m_Memory.AssumeMemory( (unsigned char*) pMemory, nSize ); + + // Reset all indices; we just changed memory + m_Get = 0; + m_Put = nInitialPut; + m_nTab = 0; + m_Error = 0; + m_nOffset = 0; + m_Flags = nFlags; + m_nMaxPut = -1; + AddNullTermination(); +} + +//----------------------------------------------------------------------------- +// Makes sure we've got at least this much memory +//----------------------------------------------------------------------------- +void CUtlBuffer::EnsureCapacity( int num ) +{ + MEM_ALLOC_CREDIT(); + // Add one extra for the null termination + num += 1; + if ( m_Memory.IsExternallyAllocated() ) + { + if ( IsGrowable() && ( m_Memory.NumAllocated() < num ) ) + { + m_Memory.ConvertToGrowableMemory( 0 ); + } + else + { + num -= 1; + } + } + + m_Memory.EnsureCapacity( num ); +} + + +//----------------------------------------------------------------------------- +// Base get method from which all others derive +//----------------------------------------------------------------------------- +void CUtlBuffer::Get( void* pMem, int size ) +{ + if ( size > 0 && CheckGet( size ) ) + { + int Index = m_Get - m_nOffset; + Assert( m_Memory.IsIdxValid( Index ) && m_Memory.IsIdxValid( Index + size - 1 ) ); + + memcpy( pMem, &m_Memory[ Index ], size ); + m_Get += size; + } +} + + +//----------------------------------------------------------------------------- +// This will get at least 1 byte and up to nSize bytes. +// It will return the number of bytes actually read. +//----------------------------------------------------------------------------- +int CUtlBuffer::GetUpTo( void *pMem, int nSize ) +{ + if ( CheckArbitraryPeekGet( 0, nSize ) ) + { + int Index = m_Get - m_nOffset; + Assert( m_Memory.IsIdxValid( Index ) && m_Memory.IsIdxValid( Index + nSize - 1 ) ); + + memcpy( pMem, &m_Memory[ Index ], nSize ); + m_Get += nSize; + return nSize; + } + return 0; +} + + +//----------------------------------------------------------------------------- +// Eats whitespace +//----------------------------------------------------------------------------- +void CUtlBuffer::EatWhiteSpace() +{ + if ( IsText() && IsValid() ) + { + while ( CheckGet( sizeof(char) ) ) + { + if ( !isspace( *(const unsigned char*)PeekGet() ) ) + break; + m_Get += sizeof(char); + } + } +} + + +//----------------------------------------------------------------------------- +// Eats C++ style comments +//----------------------------------------------------------------------------- +bool CUtlBuffer::EatCPPComment() +{ + if ( IsText() && IsValid() ) + { + // If we don't have a a c++ style comment next, we're done + const char *pPeek = (const char *)PeekGet( 2 * sizeof(char), 0 ); + if ( !pPeek || ( pPeek[0] != '/' ) || ( pPeek[1] != '/' ) ) + return false; + + // Deal with c++ style comments + m_Get += 2; + + // read complete line + for ( char c = GetChar(); IsValid(); c = GetChar() ) + { + if ( c == '\n' ) + break; + } + return true; + } + return false; +} + + +//----------------------------------------------------------------------------- +// Peeks how much whitespace to eat +//----------------------------------------------------------------------------- +int CUtlBuffer::PeekWhiteSpace( int nOffset ) +{ + if ( !IsText() || !IsValid() ) + return 0; + + while ( CheckPeekGet( nOffset, sizeof(char) ) ) + { + if ( !isspace( *(unsigned char*)PeekGet( nOffset ) ) ) + break; + nOffset += sizeof(char); + } + + return nOffset; +} + + +//----------------------------------------------------------------------------- +// Peek size of sting to come, check memory bound +//----------------------------------------------------------------------------- +int CUtlBuffer::PeekStringLength() +{ + if ( !IsValid() ) + return 0; + + // Eat preceeding whitespace + int nOffset = 0; + if ( IsText() ) + { + nOffset = PeekWhiteSpace( nOffset ); + } + + int nStartingOffset = nOffset; + + do + { + int nPeekAmount = 128; + + // NOTE: Add 1 for the terminating zero! + if ( !CheckArbitraryPeekGet( nOffset, nPeekAmount ) ) + { + if ( nOffset == nStartingOffset ) + return 0; + return nOffset - nStartingOffset + 1; + } + + const char *pTest = (const char *)PeekGet( nOffset ); + + if ( !IsText() ) + { + for ( int i = 0; i < nPeekAmount; ++i ) + { + // The +1 here is so we eat the terminating 0 + if ( pTest[i] == 0 ) + return (i + nOffset - nStartingOffset + 1); + } + } + else + { + for ( int i = 0; i < nPeekAmount; ++i ) + { + // The +1 here is so we eat the terminating 0 + if ( isspace((unsigned char)pTest[i]) || (pTest[i] == 0) ) + return (i + nOffset - nStartingOffset + 1); + } + } + + nOffset += nPeekAmount; + + } while ( true ); +} + + +//----------------------------------------------------------------------------- +// Peek size of line to come, check memory bound +//----------------------------------------------------------------------------- +int CUtlBuffer::PeekLineLength() +{ + if ( !IsValid() ) + return 0; + + int nOffset = 0; + int nStartingOffset = nOffset; + + do + { + int nPeekAmount = 128; + + // NOTE: Add 1 for the terminating zero! + if ( !CheckArbitraryPeekGet( nOffset, nPeekAmount ) ) + { + if ( nOffset == nStartingOffset ) + return 0; + return nOffset - nStartingOffset + 1; + } + + const char *pTest = (const char *)PeekGet( nOffset ); + + for ( int i = 0; i < nPeekAmount; ++i ) + { + // The +2 here is so we eat the terminating '\n' and 0 + if ( pTest[i] == '\n' || pTest[i] == '\r' ) + return (i + nOffset - nStartingOffset + 2); + // The +1 here is so we eat the terminating 0 + if ( pTest[i] == 0 ) + return (i + nOffset - nStartingOffset + 1); + } + + nOffset += nPeekAmount; + + } while ( true ); +} + + +//----------------------------------------------------------------------------- +// Does the next bytes of the buffer match a pattern? +//----------------------------------------------------------------------------- +bool CUtlBuffer::PeekStringMatch( int nOffset, const char *pString, int nLen ) +{ + if ( !CheckPeekGet( nOffset, nLen ) ) + return false; + return !Q_strncmp( (const char*)PeekGet(nOffset), pString, nLen ); +} + + +//----------------------------------------------------------------------------- +// This version of PeekStringLength converts \" to \\ and " to \, etc. +// It also reads a " at the beginning and end of the string +//----------------------------------------------------------------------------- +int CUtlBuffer::PeekDelimitedStringLength( CUtlCharConversion *pConv, bool bActualSize ) +{ + if ( !IsText() || !pConv ) + return PeekStringLength(); + + // Eat preceeding whitespace + int nOffset = 0; + if ( IsText() ) + { + nOffset = PeekWhiteSpace( nOffset ); + } + + if ( !PeekStringMatch( nOffset, pConv->GetDelimiter(), pConv->GetDelimiterLength() ) ) + return 0; + + // Try to read ending ", but don't accept \" + int nActualStart = nOffset; + nOffset += pConv->GetDelimiterLength(); + int nLen = 1; // Starts at 1 for the '\0' termination + + do + { + if ( PeekStringMatch( nOffset, pConv->GetDelimiter(), pConv->GetDelimiterLength() ) ) + break; + + if ( !CheckPeekGet( nOffset, 1 ) ) + break; + + char c = *(const char*)PeekGet( nOffset ); + ++nLen; + ++nOffset; + if ( c == pConv->GetEscapeChar() ) + { + int nLength = pConv->MaxConversionLength(); + if ( !CheckArbitraryPeekGet( nOffset, nLength ) ) + break; + + pConv->FindConversion( (const char*)PeekGet(nOffset), &nLength ); + nOffset += nLength; + } + } while (true); + + return bActualSize ? nLen : nOffset - nActualStart + pConv->GetDelimiterLength() + 1; +} + + +//----------------------------------------------------------------------------- +// Reads a null-terminated string +//----------------------------------------------------------------------------- +void CUtlBuffer::GetStringInternal( char *pString, size_t maxLenInChars ) +{ + if ( !IsValid() ) + { + *pString = 0; + return; + } + + // This can legitimately be zero if we were told that the buffer is zero length, and + // we're asking to duplicate the buffer, so let that pass, too. + Assert( maxLenInChars != 0 || PeekStringLength() == 0 ); + + if ( maxLenInChars == 0 ) + { + return; + } + + // Remember, this *includes* the null character + // It will be 0, however, if the buffer is empty. + int nLen = PeekStringLength(); + + if ( IsText() ) + { + EatWhiteSpace(); + } + + if ( nLen <= 0 ) + { + *pString = 0; + m_Error |= GET_OVERFLOW; + return; + } + + const size_t nCharsToRead = min( (size_t)nLen, maxLenInChars ) - 1; + + Get( pString, nCharsToRead ); + pString[nCharsToRead] = 0; + + if ( (size_t)nLen > (nCharsToRead + 1) ) + { + SeekGet( SEEK_CURRENT, nLen - (nCharsToRead + 1) ); + } + + // Read the terminating NULL in binary formats + if ( !IsText() ) + { + VerifyEquals( GetChar(), 0 ); + } +} + + +//----------------------------------------------------------------------------- +// Reads up to and including the first \n +//----------------------------------------------------------------------------- +void CUtlBuffer::GetLine( char* pLine, int nMaxChars ) +{ + Assert( IsText() && !ContainsCRLF() ); + + if ( !IsValid() ) + { + *pLine = 0; + return; + } + + if ( nMaxChars == 0 ) + { + nMaxChars = INT_MAX; + } + + // Remember, this *includes* the null character + // It will be 0, however, if the buffer is empty. + int nLen = PeekLineLength(); + if ( nLen == 0 ) + { + *pLine = 0; + m_Error |= GET_OVERFLOW; + return; + } + + // Strip off the terminating NULL + if ( nLen <= nMaxChars ) + { + Get( pLine, nLen - 1 ); + pLine[ nLen - 1 ] = 0; + } + else + { + Get( pLine, nMaxChars - 1 ); + pLine[ nMaxChars - 1 ] = 0; + SeekGet( SEEK_CURRENT, nLen - 1 - nMaxChars ); + } +} + + +//----------------------------------------------------------------------------- +// This version of GetString converts \ to \\ and " to \", etc. +// It also places " at the beginning and end of the string +//----------------------------------------------------------------------------- +char CUtlBuffer::GetDelimitedCharInternal( CUtlCharConversion *pConv ) +{ + char c = GetChar(); + if ( c == pConv->GetEscapeChar() ) + { + int nLength = pConv->MaxConversionLength(); + if ( !CheckArbitraryPeekGet( 0, nLength ) ) + return '\0'; + + c = pConv->FindConversion( (const char *)PeekGet(), &nLength ); + SeekGet( SEEK_CURRENT, nLength ); + } + + return c; +} + +char CUtlBuffer::GetDelimitedChar( CUtlCharConversion *pConv ) +{ + if ( !IsText() || !pConv ) + return GetChar( ); + return GetDelimitedCharInternal( pConv ); +} + +void CUtlBuffer::GetDelimitedString( CUtlCharConversion *pConv, char *pString, int nMaxChars ) +{ + if ( !IsText() || !pConv ) + { + GetStringInternal( pString, nMaxChars ); + return; + } + + if (!IsValid()) + { + *pString = 0; + return; + } + + if ( nMaxChars == 0 ) + { + nMaxChars = INT_MAX; + } + + EatWhiteSpace(); + if ( !PeekStringMatch( 0, pConv->GetDelimiter(), pConv->GetDelimiterLength() ) ) + return; + + // Pull off the starting delimiter + SeekGet( SEEK_CURRENT, pConv->GetDelimiterLength() ); + + int nRead = 0; + while ( IsValid() ) + { + if ( PeekStringMatch( 0, pConv->GetDelimiter(), pConv->GetDelimiterLength() ) ) + { + SeekGet( SEEK_CURRENT, pConv->GetDelimiterLength() ); + break; + } + + char c = GetDelimitedCharInternal( pConv ); + + if ( nRead < nMaxChars ) + { + pString[nRead] = c; + ++nRead; + } + } + + if ( nRead >= nMaxChars ) + { + nRead = nMaxChars - 1; + } + pString[nRead] = '\0'; +} + + +//----------------------------------------------------------------------------- +// Checks if a get is ok +//----------------------------------------------------------------------------- +bool CUtlBuffer::CheckGet( int nSize ) +{ + if ( m_Error & GET_OVERFLOW ) + return false; + + if ( TellMaxPut() < m_Get + nSize ) + { + m_Error |= GET_OVERFLOW; + return false; + } + + if ( ( m_Get < m_nOffset ) || ( m_Memory.NumAllocated() < m_Get - m_nOffset + nSize ) ) + { + if ( !OnGetOverflow( nSize ) ) + { + m_Error |= GET_OVERFLOW; + return false; + } + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Checks if a peek get is ok +//----------------------------------------------------------------------------- +bool CUtlBuffer::CheckPeekGet( int nOffset, int nSize ) +{ + if ( m_Error & GET_OVERFLOW ) + return false; + + // Checking for peek can't set the overflow flag + bool bOk = CheckGet( nOffset + nSize ); + m_Error &= ~GET_OVERFLOW; + return bOk; +} + + +//----------------------------------------------------------------------------- +// Call this to peek arbitrarily long into memory. It doesn't fail unless +// it can't read *anything* new +//----------------------------------------------------------------------------- +bool CUtlBuffer::CheckArbitraryPeekGet( int nOffset, int &nIncrement ) +{ + if ( TellGet() + nOffset >= TellMaxPut() ) + { + nIncrement = 0; + return false; + } + + if ( TellGet() + nOffset + nIncrement > TellMaxPut() ) + { + nIncrement = TellMaxPut() - TellGet() - nOffset; + } + + // NOTE: CheckPeekGet could modify TellMaxPut for streaming files + // We have to call TellMaxPut again here + CheckPeekGet( nOffset, nIncrement ); + int nMaxGet = TellMaxPut() - TellGet(); + if ( nMaxGet < nIncrement ) + { + nIncrement = nMaxGet; + } + return (nIncrement != 0); +} + + +//----------------------------------------------------------------------------- +// Peek part of the butt +//----------------------------------------------------------------------------- +const void* CUtlBuffer::PeekGet( int nMaxSize, int nOffset ) +{ + if ( !CheckPeekGet( nOffset, nMaxSize ) ) + return NULL; + + int Index = m_Get + nOffset - m_nOffset; + Assert( m_Memory.IsIdxValid( Index ) && m_Memory.IsIdxValid( Index + nMaxSize - 1 ) ); + + return &m_Memory[ Index ]; +} + + +//----------------------------------------------------------------------------- +// Change where I'm reading +//----------------------------------------------------------------------------- +void CUtlBuffer::SeekGet( SeekType_t type, int offset ) +{ + switch( type ) + { + case SEEK_HEAD: + m_Get = offset; + break; + + case SEEK_CURRENT: + m_Get += offset; + break; + + case SEEK_TAIL: + m_Get = m_nMaxPut - offset; + break; + } + + if ( m_Get > m_nMaxPut ) + { + m_Error |= GET_OVERFLOW; + } + else + { + m_Error &= ~GET_OVERFLOW; + if ( m_Get < m_nOffset || m_Get >= m_nOffset + Size() ) + { + OnGetOverflow( -1 ); + } + } +} + + +//----------------------------------------------------------------------------- +// Parse... +//----------------------------------------------------------------------------- + +#pragma warning ( disable : 4706 ) + +int CUtlBuffer::VaScanf( const char* pFmt, va_list list ) +{ + Assert( pFmt ); + if ( m_Error || !IsText() ) + return 0; + + int numScanned = 0; + int nLength; + char c; + char* pEnd; + while ( (c = *pFmt++) ) + { + // Stop if we hit the end of the buffer + if ( m_Get >= TellMaxPut() ) + { + m_Error |= GET_OVERFLOW; + break; + } + + switch (c) + { + case ' ': + // eat all whitespace + EatWhiteSpace(); + break; + + case '%': + { + // Conversion character... try to convert baby! + char type = *pFmt++; + if (type == 0) + return numScanned; + + switch(type) + { + case 'c': + { + char* ch = va_arg( list, char * ); + if ( CheckPeekGet( 0, sizeof(char) ) ) + { + *ch = *(const char*)PeekGet(); + ++m_Get; + } + else + { + *ch = 0; + return numScanned; + } + } + break; + + case 'i': + case 'd': + { + int* i = va_arg( list, int * ); + + // NOTE: This is not bullet-proof; it assumes numbers are < 128 characters + nLength = 128; + if ( !CheckArbitraryPeekGet( 0, nLength ) ) + { + *i = 0; + return numScanned; + } + + *i = strtol( (char*)PeekGet(), &pEnd, 10 ); + int nBytesRead = (int)( pEnd - (char*)PeekGet() ); + if ( nBytesRead == 0 ) + return numScanned; + m_Get += nBytesRead; + } + break; + + case 'x': + { + int* i = va_arg( list, int * ); + + // NOTE: This is not bullet-proof; it assumes numbers are < 128 characters + nLength = 128; + if ( !CheckArbitraryPeekGet( 0, nLength ) ) + { + *i = 0; + return numScanned; + } + + *i = strtol( (char*)PeekGet(), &pEnd, 16 ); + int nBytesRead = (int)( pEnd - (char*)PeekGet() ); + if ( nBytesRead == 0 ) + return numScanned; + m_Get += nBytesRead; + } + break; + + case 'u': + { + unsigned int* u = va_arg( list, unsigned int *); + + // NOTE: This is not bullet-proof; it assumes numbers are < 128 characters + nLength = 128; + if ( !CheckArbitraryPeekGet( 0, nLength ) ) + { + *u = 0; + return numScanned; + } + + *u = strtoul( (char*)PeekGet(), &pEnd, 10 ); + int nBytesRead = (int)( pEnd - (char*)PeekGet() ); + if ( nBytesRead == 0 ) + return numScanned; + m_Get += nBytesRead; + } + break; + + case 'f': + { + float* f = va_arg( list, float *); + + // NOTE: This is not bullet-proof; it assumes numbers are < 128 characters + nLength = 128; + if ( !CheckArbitraryPeekGet( 0, nLength ) ) + { + *f = 0.0f; + return numScanned; + } + + *f = (float)strtod( (char*)PeekGet(), &pEnd ); + int nBytesRead = (int)( pEnd - (char*)PeekGet() ); + if ( nBytesRead == 0 ) + return numScanned; + m_Get += nBytesRead; + } + break; + + case 's': + { + char* s = va_arg( list, char * ); + GetStringInternal( s, 256 ); + } + break; + + default: + { + // unimplemented scanf type + Assert(0); + return numScanned; + } + break; + } + + ++numScanned; + } + break; + + default: + { + // Here we have to match the format string character + // against what's in the buffer or we're done. + if ( !CheckPeekGet( 0, sizeof(char) ) ) + return numScanned; + + if ( c != *(const char*)PeekGet() ) + return numScanned; + + ++m_Get; + } + } + } + return numScanned; +} + +#pragma warning ( default : 4706 ) + +int CUtlBuffer::Scanf( const char* pFmt, ... ) +{ + va_list args; + + va_start( args, pFmt ); + int count = VaScanf( pFmt, args ); + va_end( args ); + + return count; +} + + +//----------------------------------------------------------------------------- +// Advance the get index until after the particular string is found +// Do not eat whitespace before starting. Return false if it failed +//----------------------------------------------------------------------------- +bool CUtlBuffer::GetToken( const char *pToken ) +{ + Assert( pToken ); + + // Look for the token + int nLen = Q_strlen( pToken ); + + int nSizeToCheck = Size() - TellGet() - m_nOffset; + + int nGet = TellGet(); + do + { + int nMaxSize = TellMaxPut() - TellGet(); + if ( nMaxSize < nSizeToCheck ) + { + nSizeToCheck = nMaxSize; + } + if ( nLen > nSizeToCheck ) + break; + + if ( !CheckPeekGet( 0, nSizeToCheck ) ) + break; + + const char *pBufStart = (const char*)PeekGet(); + const char *pFoundEnd = Q_strnistr( pBufStart, pToken, nSizeToCheck ); + if ( pFoundEnd ) + { + size_t nOffset = (size_t)pFoundEnd - (size_t)pBufStart; + SeekGet( CUtlBuffer::SEEK_CURRENT, nOffset + nLen ); + return true; + } + + SeekGet( CUtlBuffer::SEEK_CURRENT, nSizeToCheck - nLen - 1 ); + nSizeToCheck = Size() - (nLen-1); + + } while ( true ); + + SeekGet( CUtlBuffer::SEEK_HEAD, nGet ); + return false; +} + + +//----------------------------------------------------------------------------- +// (For text buffers only) +// Parse a token from the buffer: +// Grab all text that lies between a starting delimiter + ending delimiter +// (skipping whitespace that leads + trails both delimiters). +// Note the delimiter checks are case-insensitive. +// If successful, the get index is advanced and the function returns true, +// otherwise the index is not advanced and the function returns false. +//----------------------------------------------------------------------------- +bool CUtlBuffer::ParseToken( const char *pStartingDelim, const char *pEndingDelim, char* pString, int nMaxLen ) +{ + int nCharsToCopy = 0; + int nCurrentGet = 0; + + size_t nEndingDelimLen; + + // Starting delimiter is optional + char emptyBuf = '\0'; + if ( !pStartingDelim ) + { + pStartingDelim = &emptyBuf; + } + + // Ending delimiter is not + Assert( pEndingDelim && pEndingDelim[0] ); + nEndingDelimLen = Q_strlen( pEndingDelim ); + + int nStartGet = TellGet(); + char nCurrChar; + int nTokenStart = -1; + EatWhiteSpace( ); + while ( *pStartingDelim ) + { + nCurrChar = *pStartingDelim++; + if ( !isspace((unsigned char)nCurrChar) ) + { + if ( tolower( GetChar() ) != tolower( nCurrChar ) ) + goto parseFailed; + } + else + { + EatWhiteSpace(); + } + } + + EatWhiteSpace(); + nTokenStart = TellGet(); + if ( !GetToken( pEndingDelim ) ) + goto parseFailed; + + nCurrentGet = TellGet(); + nCharsToCopy = (nCurrentGet - nEndingDelimLen) - nTokenStart; + if ( nCharsToCopy >= nMaxLen ) + { + nCharsToCopy = nMaxLen - 1; + } + + if ( nCharsToCopy > 0 ) + { + SeekGet( CUtlBuffer::SEEK_HEAD, nTokenStart ); + Get( pString, nCharsToCopy ); + if ( !IsValid() ) + goto parseFailed; + + // Eat trailing whitespace + for ( ; nCharsToCopy > 0; --nCharsToCopy ) + { + if ( !isspace( (unsigned char)pString[ nCharsToCopy-1 ] ) ) + break; + } + } + pString[ nCharsToCopy ] = '\0'; + + // Advance the Get index + SeekGet( CUtlBuffer::SEEK_HEAD, nCurrentGet ); + return true; + +parseFailed: + // Revert the get index + SeekGet( SEEK_HEAD, nStartGet ); + pString[0] = '\0'; + return false; +} + + +//----------------------------------------------------------------------------- +// Parses the next token, given a set of character breaks to stop at +//----------------------------------------------------------------------------- +int CUtlBuffer::ParseToken( characterset_t *pBreaks, char *pTokenBuf, int nMaxLen, bool bParseComments ) +{ + Assert( nMaxLen > 0 ); + pTokenBuf[0] = 0; + + // skip whitespace + comments + while ( true ) + { + if ( !IsValid() ) + return -1; + EatWhiteSpace(); + if ( bParseComments ) + { + if ( !EatCPPComment() ) + break; + } + else + { + break; + } + } + + char c = GetChar(); + + // End of buffer + if ( c == 0 ) + return -1; + + // handle quoted strings specially + if ( c == '\"' ) + { + int nLen = 0; + while( IsValid() ) + { + c = GetChar(); + if ( c == '\"' || !c ) + { + pTokenBuf[nLen] = 0; + return nLen; + } + pTokenBuf[nLen] = c; + if ( ++nLen == nMaxLen ) + { + pTokenBuf[nLen-1] = 0; + return nMaxLen; + } + } + + // In this case, we hit the end of the buffer before hitting the end qoute + pTokenBuf[nLen] = 0; + return nLen; + } + + // parse single characters + if ( IN_CHARACTERSET( *pBreaks, c ) ) + { + pTokenBuf[0] = c; + pTokenBuf[1] = 0; + return 1; + } + + // parse a regular word + int nLen = 0; + while ( true ) + { + pTokenBuf[nLen] = c; + if ( ++nLen == nMaxLen ) + { + pTokenBuf[nLen-1] = 0; + return nMaxLen; + } + c = GetChar(); + if ( !IsValid() ) + break; + + if ( IN_CHARACTERSET( *pBreaks, c ) || c == '\"' || c <= ' ' ) + { + SeekGet( SEEK_CURRENT, -1 ); + break; + } + } + + pTokenBuf[nLen] = 0; + return nLen; +} + + + +//----------------------------------------------------------------------------- +// Serialization +//----------------------------------------------------------------------------- +void CUtlBuffer::Put( const void *pMem, int size ) +{ + if ( size && CheckPut( size ) ) + { + int Index = m_Put - m_nOffset; + Assert( m_Memory.IsIdxValid( Index ) && m_Memory.IsIdxValid( Index + size - 1 ) ); + if( Index >= 0 ) + { + memcpy( &m_Memory[ Index ], pMem, size ); + m_Put += size; + + AddNullTermination(); + } + } +} + + +//----------------------------------------------------------------------------- +// Writes a null-terminated string +//----------------------------------------------------------------------------- +void CUtlBuffer::PutString( const char* pString ) +{ + if (!IsText()) + { + if ( pString ) + { + // Not text? append a null at the end. + size_t nLen = Q_strlen( pString ) + 1; + Put( pString, nLen * sizeof(char) ); + return; + } + else + { + PutTypeBin<char>( 0 ); + } + } + else if (pString) + { + int nTabCount = ( m_Flags & AUTO_TABS_DISABLED ) ? 0 : m_nTab; + if ( nTabCount > 0 ) + { + if ( WasLastCharacterCR() ) + { + PutTabs(); + } + + const char* pEndl = strchr( pString, '\n' ); + while ( pEndl ) + { + size_t nSize = (size_t)pEndl - (size_t)pString + sizeof(char); + Put( pString, nSize ); + pString = pEndl + 1; + if ( *pString ) + { + PutTabs(); + pEndl = strchr( pString, '\n' ); + } + else + { + pEndl = NULL; + } + } + } + size_t nLen = Q_strlen( pString ); + if ( nLen ) + { + Put( pString, nLen * sizeof(char) ); + } + } +} + + +//----------------------------------------------------------------------------- +// This version of PutString converts \ to \\ and " to \", etc. +// It also places " at the beginning and end of the string +//----------------------------------------------------------------------------- +inline void CUtlBuffer::PutDelimitedCharInternal( CUtlCharConversion *pConv, char c ) +{ + int l = pConv->GetConversionLength( c ); + if ( l == 0 ) + { + PutChar( c ); + } + else + { + PutChar( pConv->GetEscapeChar() ); + Put( pConv->GetConversionString( c ), l ); + } +} + +void CUtlBuffer::PutDelimitedChar( CUtlCharConversion *pConv, char c ) +{ + if ( !IsText() || !pConv ) + { + PutChar( c ); + return; + } + + PutDelimitedCharInternal( pConv, c ); +} + +void CUtlBuffer::PutDelimitedString( CUtlCharConversion *pConv, const char *pString ) +{ + if ( !IsText() || !pConv ) + { + PutString( pString ); + return; + } + + if ( WasLastCharacterCR() ) + { + PutTabs(); + } + Put( pConv->GetDelimiter(), pConv->GetDelimiterLength() ); + + int nLen = pString ? Q_strlen( pString ) : 0; + for ( int i = 0; i < nLen; ++i ) + { + PutDelimitedCharInternal( pConv, pString[i] ); + } + + if ( WasLastCharacterCR() ) + { + PutTabs(); + } + Put( pConv->GetDelimiter(), pConv->GetDelimiterLength() ); +} + + +void CUtlBuffer::VaPrintf( const char* pFmt, va_list list ) +{ + char temp[2048]; +#ifdef DBGFLAG_ASSERT + int nLen = +#endif + Q_vsnprintf( temp, sizeof( temp ), pFmt, list ); + Assert( nLen < 2048 ); + PutString( temp ); +} + +void CUtlBuffer::Printf( const char* pFmt, ... ) +{ + va_list args; + + va_start( args, pFmt ); + VaPrintf( pFmt, args ); + va_end( args ); +} + + +//----------------------------------------------------------------------------- +// Calls the overflow functions +//----------------------------------------------------------------------------- +void CUtlBuffer::SetOverflowFuncs( UtlBufferOverflowFunc_t getFunc, UtlBufferOverflowFunc_t putFunc ) +{ + m_GetOverflowFunc = getFunc; + m_PutOverflowFunc = putFunc; +} + + +//----------------------------------------------------------------------------- +// Calls the overflow functions +//----------------------------------------------------------------------------- +bool CUtlBuffer::OnPutOverflow( int nSize ) +{ + return (this->*m_PutOverflowFunc)( nSize ); +} + +bool CUtlBuffer::OnGetOverflow( int nSize ) +{ + return (this->*m_GetOverflowFunc)( nSize ); +} + + +//----------------------------------------------------------------------------- +// Checks if a put is ok +//----------------------------------------------------------------------------- +bool CUtlBuffer::PutOverflow( int nSize ) +{ + MEM_ALLOC_CREDIT(); + + if ( m_Memory.IsExternallyAllocated() ) + { + if ( !IsGrowable() ) + return false; + + m_Memory.ConvertToGrowableMemory( 0 ); + } + + while( Size() < m_Put - m_nOffset + nSize ) + { + m_Memory.Grow(); + } + + return true; +} + +bool CUtlBuffer::GetOverflow( int nSize ) +{ + return false; +} + + +//----------------------------------------------------------------------------- +// Checks if a put is ok +//----------------------------------------------------------------------------- +bool CUtlBuffer::CheckPut( int nSize ) +{ + if ( ( m_Error & PUT_OVERFLOW ) || IsReadOnly() ) + return false; + + if ( ( m_Put < m_nOffset ) || ( m_Memory.NumAllocated() < m_Put - m_nOffset + nSize ) ) + { + if ( !OnPutOverflow( nSize ) ) + { + m_Error |= PUT_OVERFLOW; + return false; + } + } + return true; +} + +void CUtlBuffer::SeekPut( SeekType_t type, int offset ) +{ + int nNextPut = m_Put; + switch( type ) + { + case SEEK_HEAD: + nNextPut = offset; + break; + + case SEEK_CURRENT: + nNextPut += offset; + break; + + case SEEK_TAIL: + nNextPut = m_nMaxPut - offset; + break; + } + + // Force a write of the data + // FIXME: We could make this more optimal potentially by writing out + // the entire buffer if you seek outside the current range + + // NOTE: This call will write and will also seek the file to nNextPut. + OnPutOverflow( -nNextPut-1 ); + m_Put = nNextPut; + + AddNullTermination(); +} + + +void CUtlBuffer::ActivateByteSwapping( bool bActivate ) +{ + m_Byteswap.ActivateByteSwapping( bActivate ); +} + +void CUtlBuffer::SetBigEndian( bool bigEndian ) +{ + m_Byteswap.SetTargetBigEndian( bigEndian ); +} + +bool CUtlBuffer::IsBigEndian( void ) +{ + return m_Byteswap.IsTargetBigEndian(); +} + + +//----------------------------------------------------------------------------- +// null terminate the buffer +//----------------------------------------------------------------------------- +void CUtlBuffer::AddNullTermination( void ) +{ + if ( m_Put > m_nMaxPut ) + { + if ( !IsReadOnly() && ((m_Error & PUT_OVERFLOW) == 0) ) + { + // Add null termination value + if ( CheckPut( 1 ) ) + { + int Index = m_Put - m_nOffset; + Assert( m_Memory.IsIdxValid( Index ) ); + if( Index >= 0 ) + { + m_Memory[ Index ] = 0; + } + } + else + { + // Restore the overflow state, it was valid before... + m_Error &= ~PUT_OVERFLOW; + } + } + m_nMaxPut = m_Put; + } +} + + +//----------------------------------------------------------------------------- +// Converts a buffer from a CRLF buffer to a CR buffer (and back) +// Returns false if no conversion was necessary (and outBuf is left untouched) +// If the conversion occurs, outBuf will be cleared. +//----------------------------------------------------------------------------- +bool CUtlBuffer::ConvertCRLF( CUtlBuffer &outBuf ) +{ + if ( !IsText() || !outBuf.IsText() ) + return false; + + if ( ContainsCRLF() == outBuf.ContainsCRLF() ) + return false; + + int nInCount = TellMaxPut(); + + outBuf.Purge(); + outBuf.EnsureCapacity( nInCount ); + + bool bFromCRLF = ContainsCRLF(); + + // Start reading from the beginning + int nGet = TellGet(); + int nPut = TellPut(); + int nGetDelta = 0; + int nPutDelta = 0; + + const char *pBase = (const char*)Base(); + int nCurrGet = 0; + while ( nCurrGet < nInCount ) + { + const char *pCurr = &pBase[nCurrGet]; + if ( bFromCRLF ) + { + const char *pNext = Q_strnistr( pCurr, "\r\n", nInCount - nCurrGet ); + if ( !pNext ) + { + outBuf.Put( pCurr, nInCount - nCurrGet ); + break; + } + + int nBytes = (size_t)pNext - (size_t)pCurr; + outBuf.Put( pCurr, nBytes ); + outBuf.PutChar( '\n' ); + nCurrGet += nBytes + 2; + if ( nGet >= nCurrGet - 1 ) + { + --nGetDelta; + } + if ( nPut >= nCurrGet - 1 ) + { + --nPutDelta; + } + } + else + { + const char *pNext = Q_strnchr( pCurr, '\n', nInCount - nCurrGet ); + if ( !pNext ) + { + outBuf.Put( pCurr, nInCount - nCurrGet ); + break; + } + + int nBytes = (size_t)pNext - (size_t)pCurr; + outBuf.Put( pCurr, nBytes ); + outBuf.PutChar( '\r' ); + outBuf.PutChar( '\n' ); + nCurrGet += nBytes + 1; + if ( nGet >= nCurrGet ) + { + ++nGetDelta; + } + if ( nPut >= nCurrGet ) + { + ++nPutDelta; + } + } + } + + Assert( nPut + nPutDelta <= outBuf.TellMaxPut() ); + + outBuf.SeekGet( SEEK_HEAD, nGet + nGetDelta ); + outBuf.SeekPut( SEEK_HEAD, nPut + nPutDelta ); + + return true; +} + +//----------------------------------------------------------------------------- +// Fast swap +//----------------------------------------------------------------------------- +void CUtlBuffer::Swap( CUtlBuffer &buf ) +{ + V_swap( m_Get, buf.m_Get ); + V_swap( m_Put, buf.m_Put ); + V_swap( m_nMaxPut, buf.m_nMaxPut ); + V_swap( m_Error, buf.m_Error ); + m_Memory.Swap( buf.m_Memory ); +} + + +//----------------------------------------------------------------------------- +// Fast swap w/ a CUtlMemory. +//----------------------------------------------------------------------------- +void CUtlBuffer::Swap( CUtlMemory<uint8> &mem ) +{ + m_Get = 0; + m_Put = mem.Count(); + m_nMaxPut = mem.Count(); + m_Error = 0; + m_Memory.Swap( mem ); +} + +//--------------------------------------------------------------------------- +// Implementation of CUtlInplaceBuffer +//--------------------------------------------------------------------------- + +CUtlInplaceBuffer::CUtlInplaceBuffer( int growSize /* = 0 */, int initSize /* = 0 */, int nFlags /* = 0 */ ) : + CUtlBuffer( growSize, initSize, nFlags ) +{ +} + +bool CUtlInplaceBuffer::InplaceGetLinePtr( char **ppszInBufferPtr, int *pnLineLength ) +{ + Assert( IsText() && !ContainsCRLF() ); + + int nLineLen = PeekLineLength(); + if ( nLineLen <= 1 ) + { + SeekGet( SEEK_TAIL, 0 ); + return false; + } + + -- nLineLen; // because it accounts for putting a terminating null-character + + char *pszLine = ( char * ) const_cast< void * >( PeekGet() ); + SeekGet( SEEK_CURRENT, nLineLen ); + + // Set the out args + if ( ppszInBufferPtr ) + *ppszInBufferPtr = pszLine; + + if ( pnLineLength ) + *pnLineLength = nLineLen; + + return true; +} + +char * CUtlInplaceBuffer::InplaceGetLinePtr( void ) +{ + char *pszLine = NULL; + int nLineLen = 0; + + if ( InplaceGetLinePtr( &pszLine, &nLineLen ) ) + { + Assert( nLineLen >= 1 ); + + switch ( pszLine[ nLineLen - 1 ] ) + { + case '\n': + case '\r': + pszLine[ nLineLen - 1 ] = 0; + if ( -- nLineLen ) + { + switch ( pszLine[ nLineLen - 1 ] ) + { + case '\n': + case '\r': + pszLine[ nLineLen - 1 ] = 0; + break; + } + } + break; + + default: + Assert( pszLine[ nLineLen ] == 0 ); + break; + } + } + + return pszLine; +} + diff --git a/tier1/utlbufferutil.cpp b/tier1/utlbufferutil.cpp new file mode 100644 index 0000000..79949bf --- /dev/null +++ b/tier1/utlbufferutil.cpp @@ -0,0 +1,560 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// $Header: $ +// $NoKeywords: $ +// +// Serialization buffer +//===========================================================================// + +#pragma warning (disable : 4514) + +#include "tier1/utlbufferutil.h" +#include "tier1/utlbuffer.h" +#include "mathlib/vector.h" +#include "mathlib/vector2d.h" +#include "mathlib/vector4d.h" +#include "mathlib/vmatrix.h" +#include "Color.h" +#include <stdio.h> +#include <stdarg.h> +#include <ctype.h> +#include <stdlib.h> +#include <limits.h> +#include "tier1/utlbinaryblock.h" +#include "tier1/utlstring.h" +#include "tier1/strtools.h" +#include "tier1/characterset.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + + +//----------------------------------------------------------------------------- +// For serialization, set the delimiter rules +//----------------------------------------------------------------------------- +CUtlCharConversion *s_pConv = NULL; +const char *s_pUtlBufferUtilArrayDelim = NULL; +void SetSerializationDelimiter( CUtlCharConversion *pConv ) +{ + s_pConv = pConv; +} + +void SetSerializationArrayDelimiter( const char *pDelimiter ) +{ + s_pUtlBufferUtilArrayDelim = pDelimiter; +} + + +//----------------------------------------------------------------------------- +// Serialize a floating point number in text mode in a readably friendly fashion +//----------------------------------------------------------------------------- +static void SerializeFloat( CUtlBuffer &buf, float f ) +{ + Assert( buf.IsText() ); + + // FIXME: Print this in a way that we never lose precision + char pTemp[256]; + int nLen = Q_snprintf( pTemp, sizeof(pTemp), "%.10f", f ); + while ( nLen > 0 && pTemp[nLen-1] == '0' ) + { + --nLen; + pTemp[nLen] = 0; + } + if ( nLen > 0 && pTemp[nLen-1] == '.' ) + { + --nLen; + pTemp[nLen] = 0; + } + buf.PutString( pTemp ); +} + +static void SerializeFloats( CUtlBuffer &buf, int nCount, const float *pFloats ) +{ + for ( int i = 0; i < nCount; ++i ) + { + SerializeFloat( buf, pFloats[i] ); + if ( i != nCount-1 ) + { + buf.PutChar( ' ' ); + } + } +} + + +//----------------------------------------------------------------------------- +// Serialization methods for basic types +//----------------------------------------------------------------------------- +bool Serialize( CUtlBuffer &buf, const bool &src ) +{ + if ( buf.IsText() ) + { + buf.Printf( "%d", src ); + } + else + { + buf.PutChar( src ); + } + return buf.IsValid(); +} + +bool Unserialize( CUtlBuffer &buf, bool &dest ) +{ + if ( buf.IsText() ) + { + int nValue = 0; + int nRetVal = buf.Scanf( "%d", &nValue ); + dest = ( nValue != 0 ); + return (nRetVal == 1) && buf.IsValid(); + } + + dest = ( buf.GetChar( ) != 0 ); + return buf.IsValid(); +} + + +bool Serialize( CUtlBuffer &buf, const int &src ) +{ + if ( buf.IsText() ) + { + buf.Printf( "%d", src ); + } + else + { + buf.PutInt( src ); + } + return buf.IsValid(); +} + +bool Unserialize( CUtlBuffer &buf, int &dest ) +{ + if ( buf.IsText() ) + { + int nRetVal = buf.Scanf( "%d", &dest ); + return (nRetVal == 1) && buf.IsValid(); + } + + dest = buf.GetInt( ); + return buf.IsValid(); +} + +bool Serialize( CUtlBuffer &buf, const float &src ) +{ + if ( buf.IsText() ) + { + SerializeFloat( buf, src ); + } + else + { + buf.PutFloat( src ); + } + return buf.IsValid(); +} + +bool Unserialize( CUtlBuffer &buf, float &dest ) +{ + if ( buf.IsText() ) + { + // FIXME: Print this in a way that we never lose precision + int nRetVal = buf.Scanf( "%f", &dest ); + return (nRetVal == 1) && buf.IsValid(); + } + + dest = buf.GetFloat( ); + return buf.IsValid(); +} + + +//----------------------------------------------------------------------------- +// Attribute types related to vector math +//----------------------------------------------------------------------------- +bool Serialize( CUtlBuffer &buf, const Vector2D &src ) +{ + if ( buf.IsText() ) + { + SerializeFloats( buf, 2, src.Base() ); + } + else + { + buf.PutFloat( src.x ); + buf.PutFloat( src.y ); + } + return buf.IsValid(); +} + +bool Unserialize( CUtlBuffer &buf, Vector2D &dest ) +{ + if ( buf.IsText() ) + { + // FIXME: Print this in a way that we never lose precision + int nRetVal = buf.Scanf( "%f %f", &dest.x, &dest.y ); + return (nRetVal == 2) && buf.IsValid(); + } + + dest.x = buf.GetFloat( ); + dest.y = buf.GetFloat( ); + return buf.IsValid(); +} + +bool Serialize( CUtlBuffer &buf, const Vector &src ) +{ + if ( buf.IsText() ) + { + SerializeFloats( buf, 3, src.Base() ); + } + else + { + buf.PutFloat( src.x ); + buf.PutFloat( src.y ); + buf.PutFloat( src.z ); + } + return buf.IsValid(); +} + +bool Unserialize( CUtlBuffer &buf, Vector &dest ) +{ + if ( buf.IsText() ) + { + // FIXME: Print this in a way that we never lose precision + int nRetVal = buf.Scanf( "%f %f %f", &dest.x, &dest.y, &dest.z ); + return (nRetVal == 3) && buf.IsValid(); + } + + dest.x = buf.GetFloat( ); + dest.y = buf.GetFloat( ); + dest.z = buf.GetFloat( ); + return buf.IsValid(); +} + +bool Serialize( CUtlBuffer &buf, const Vector4D &src ) +{ + if ( buf.IsText() ) + { + SerializeFloats( buf, 4, src.Base() ); + } + else + { + buf.PutFloat( src.x ); + buf.PutFloat( src.y ); + buf.PutFloat( src.z ); + buf.PutFloat( src.w ); + } + return buf.IsValid(); +} + +bool Unserialize( CUtlBuffer &buf, Vector4D &dest ) +{ + if ( buf.IsText() ) + { + // FIXME: Print this in a way that we never lose precision + int nRetVal = buf.Scanf( "%f %f %f %f", &dest.x, &dest.y, &dest.z, &dest.w ); + return (nRetVal == 4) && buf.IsValid(); + } + + dest.x = buf.GetFloat( ); + dest.y = buf.GetFloat( ); + dest.z = buf.GetFloat( ); + dest.w = buf.GetFloat( ); + return buf.IsValid(); +} + +bool Serialize( CUtlBuffer &buf, const QAngle &src ) +{ + if ( buf.IsText() ) + { + SerializeFloats( buf, 3, src.Base() ); + } + else + { + buf.PutFloat( src.x ); + buf.PutFloat( src.y ); + buf.PutFloat( src.z ); + } + return buf.IsValid(); +} + +bool Unserialize( CUtlBuffer &buf, QAngle &dest ) +{ + if ( buf.IsText() ) + { + // FIXME: Print this in a way that we never lose precision + int nRetVal = buf.Scanf( "%f %f %f", &dest.x, &dest.y, &dest.z ); + return (nRetVal == 3) && buf.IsValid(); + } + + dest.x = buf.GetFloat( ); + dest.y = buf.GetFloat( ); + dest.z = buf.GetFloat( ); + return buf.IsValid(); +} + +bool Serialize( CUtlBuffer &buf, const Quaternion &src ) +{ + if ( buf.IsText() ) + { + SerializeFloats( buf, 4, &src.x ); + } + else + { + buf.PutFloat( src.x ); + buf.PutFloat( src.y ); + buf.PutFloat( src.z ); + buf.PutFloat( src.w ); + } + return buf.IsValid(); +} + +bool Unserialize( CUtlBuffer &buf, Quaternion &dest ) +{ + if ( buf.IsText() ) + { + // FIXME: Print this in a way that we never lose precision + int nRetVal = buf.Scanf( "%f %f %f %f", &dest.x, &dest.y, &dest.z, &dest.w ); + return (nRetVal == 4) && buf.IsValid(); + } + + dest.x = buf.GetFloat( ); + dest.y = buf.GetFloat( ); + dest.z = buf.GetFloat( ); + dest.w = buf.GetFloat( ); + return buf.IsValid(); +} + +bool Serialize( CUtlBuffer &buf, const VMatrix &src ) +{ + if ( buf.IsText() ) + { + buf.Printf( "\n" ); + SerializeFloats( buf, 4, src[0] ); + buf.Printf( "\n" ); + SerializeFloats( buf, 4, src[1] ); + buf.Printf( "\n" ); + SerializeFloats( buf, 4, src[2] ); + buf.Printf( "\n" ); + SerializeFloats( buf, 4, src[3] ); + buf.Printf( "\n" ); + } + else + { + buf.Put( &src, sizeof(VMatrix) ); + } + return buf.IsValid(); +} + +bool Unserialize( CUtlBuffer &buf, VMatrix &dest ) +{ + if ( !buf.IsValid() ) + return false; + + if ( buf.IsText() ) + { + int nRetVal = buf.Scanf( "%f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f", + &dest[ 0 ][ 0 ], &dest[ 0 ][ 1 ], &dest[ 0 ][ 2 ], &dest[ 0 ][ 3 ], + &dest[ 1 ][ 0 ], &dest[ 1 ][ 1 ], &dest[ 1 ][ 2 ], &dest[ 1 ][ 3 ], + &dest[ 2 ][ 0 ], &dest[ 2 ][ 1 ], &dest[ 2 ][ 2 ], &dest[ 2 ][ 3 ], + &dest[ 3 ][ 0 ], &dest[ 3 ][ 1 ], &dest[ 3 ][ 2 ], &dest[ 3 ][ 3 ] ); + return (nRetVal == 16); + } + + buf.Get( &dest, sizeof(VMatrix) ); + return true; +} + + +//----------------------------------------------------------------------------- +// Color attribute +//----------------------------------------------------------------------------- +bool Serialize( CUtlBuffer &buf, const Color &src ) +{ + if ( buf.IsText() ) + { + buf.Printf( "%d %d %d %d", src[0], src[1], src[2], src[3] ); + } + else + { + buf.PutUnsignedChar( src[0] ); + buf.PutUnsignedChar( src[1] ); + buf.PutUnsignedChar( src[2] ); + buf.PutUnsignedChar( src[3] ); + } + return buf.IsValid(); +} + +bool Unserialize( CUtlBuffer &buf, Color &dest ) +{ + if ( buf.IsText() ) + { + int r = 0, g = 0, b = 0, a = 255; + int nRetVal = buf.Scanf( "%d %d %d %d", &r, &g, &b, &a ); + dest.SetColor( r, g, b, a ); + return (nRetVal == 4) && buf.IsValid(); + } + + dest[0] = buf.GetUnsignedChar( ); + dest[1] = buf.GetUnsignedChar( ); + dest[2] = buf.GetUnsignedChar( ); + dest[3] = buf.GetUnsignedChar( ); + return buf.IsValid(); +} + +/* +//----------------------------------------------------------------------------- +// Object ID attribute +//----------------------------------------------------------------------------- +bool Serialize( CUtlBuffer &buf, const DmObjectId_t &src ) +{ + return g_pDataModel->Serialize( buf, src ); +} + +bool Unserialize( CUtlBuffer &buf, DmObjectId_t &dest ) +{ + return g_pDataModel->Unserialize( buf, &dest ); +} +*/ + +//----------------------------------------------------------------------------- +// Binary buffer attribute +//----------------------------------------------------------------------------- +bool Serialize( CUtlBuffer &buf, const CUtlBinaryBlock &src ) +{ + int nLength = src.Length(); + if ( !buf.IsText() ) + { + buf.PutInt( nLength ); + if ( nLength != 0 ) + { + buf.Put( src.Get(), nLength ); + } + return buf.IsValid(); + } + + // Writes out uuencoded binaries + for ( int i = 0; i < nLength; ++i ) + { + if ( (i % 40) == 0 ) + { + buf.PutChar( '\n' ); + } + + char b1 = src[i] & 0xF; + char b2 = src[i] >> 4; + + char c1 = ( b1 <= 9 ) ? b1 + '0' : b1 - 10 + 'A'; + char c2 = ( b2 <= 9 ) ? b2 + '0' : b2 - 10 + 'A'; + + buf.PutChar( c2 ); + buf.PutChar( c1 ); + } + + buf.PutChar( '\n' ); + return buf.IsValid(); +} + +static int CountBinaryBytes( CUtlBuffer &buf, int *pEndGet ) +{ + // This counts the number of bytes in the uuencoded text + int nStartGet = buf.TellGet(); + buf.EatWhiteSpace(); + *pEndGet = buf.TellGet(); + int nByteCount = 0; + while ( buf.IsValid() ) + { + char c1 = buf.GetChar(); + char c2 = buf.GetChar(); + + bool bIsNum1 = ( c1 >= '0' ) && ( c1 <= '9' ); + bool bIsNum2 = ( c2 >= '0' ) && ( c2 <= '9' ); + + bool bIsAlpha1 = (( c1 >= 'A' ) && ( c1 <= 'F' )) || (( c1 >= 'a' ) && ( c1 <= 'f' )); + bool bIsAlpha2 = (( c2 >= 'A' ) && ( c2 <= 'F' )) || (( c2 >= 'a' ) && ( c2 <= 'f' )); + + if ( !(bIsNum1 || bIsAlpha1) || !(bIsNum2 || bIsAlpha2) ) + break; + + buf.EatWhiteSpace(); + *pEndGet = buf.TellGet(); + ++nByteCount; + } + buf.SeekGet( CUtlBuffer::SEEK_HEAD, nStartGet ); + return nByteCount; +} + +inline static unsigned char HexCharToInt( int c1 ) +{ + if (( c1 >= '0' ) && ( c1 <= '9' )) + return c1 - '0'; + + if (( c1 >= 'A' ) && ( c1 <= 'F' )) + return 10 + c1 - 'A'; + + if (( c1 >= 'a' ) && ( c1 <= 'f' )) + return 10 + c1 - 'a'; + + return 0xFF; +} + +bool Unserialize( CUtlBuffer &buf, CUtlBinaryBlock &dest ) +{ + if ( !buf.IsText() ) + { + int nLen = buf.GetInt( ); + dest.SetLength( nLen ); + if ( dest.Length() != 0 ) + { + buf.Get( dest.Get(), dest.Length() ); + } + + if ( nLen != dest.Length() ) + { + buf.SeekGet( CUtlBuffer::SEEK_CURRENT, nLen - dest.Length() ); + return false; + } + + return buf.IsValid(); + } + + int nEndGet; + int nByteCount = CountBinaryBytes( buf, &nEndGet ); + if ( nByteCount < 0 ) + return false; + + buf.EatWhiteSpace(); + int nDest = 0; + dest.SetLength( nByteCount ); + while( buf.TellGet() < nEndGet ) + { + char c1 = buf.GetChar(); + char c2 = buf.GetChar(); + + unsigned char b1 = HexCharToInt( c1 ); + unsigned char b2 = HexCharToInt( c2 ); + if ( b1 == 0xFF || b2 == 0xFF ) + return false; + + dest[ nDest++ ] = b2 | ( b1 << 4 ); + buf.EatWhiteSpace(); + } + + return true; +} + + +//----------------------------------------------------------------------------- +// String attribute +//----------------------------------------------------------------------------- +bool Serialize( CUtlBuffer &buf, const CUtlString &src ) +{ + buf.PutDelimitedString( s_pConv, src.Get() ); + return buf.IsValid(); +} + +bool Unserialize( CUtlBuffer &buf, CUtlString &dest ) +{ + int nLen = buf.PeekDelimitedStringLength( s_pConv ); + dest.SetLength( nLen - 1 ); // -1 because the length returned includes space for \0 + buf.GetDelimitedString( s_pConv, dest.GetForModify(), nLen ); + return buf.IsValid(); +} + + + + diff --git a/tier1/utlstring.cpp b/tier1/utlstring.cpp new file mode 100644 index 0000000..570aab8 --- /dev/null +++ b/tier1/utlstring.cpp @@ -0,0 +1,767 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +//============================================================================= + +#define __STDC_LIMIT_MACROS +#include <stdint.h> + +#include "tier1/utlstring.h" +#include "tier1/strtools.h" +#include <ctype.h> + +// NOTE: This has to be the last file included! +#include "tier0/memdbgon.h" + +//----------------------------------------------------------------------------- +// Simple string class. +//----------------------------------------------------------------------------- + +//----------------------------------------------------------------------------- +// Either allocates or reallocates memory to the length +// +// Allocated space for length characters. It automatically adds space for the +// nul and the cached length at the start of the memory block. Will adjust +// m_pString and explicitly set the nul at the end before returning. +void *CUtlString::AllocMemory( uint32 length ) +{ + void *pMemoryBlock; + if ( m_pString ) + { + pMemoryBlock = realloc( m_pString, length + 1 ); + } + else + { + pMemoryBlock = malloc( length + 1 ); + } + m_pString = (char*)pMemoryBlock; + m_pString[ length ] = 0; + + return pMemoryBlock; +} + +//----------------------------------------------------------------------------- +void CUtlString::SetDirect( const char *pValue, int nChars ) +{ + if ( pValue && nChars > 0 ) + { + if ( pValue == m_pString ) + { + AssertMsg( nChars == Q_strlen(m_pString), "CUtlString::SetDirect does not support resizing strings in place." ); + return; // Do nothing. Realloc in AllocMemory might move pValue's location resulting in a bad memcpy. + } + + Assert( nChars <= Min<int>( strnlen(pValue, nChars) + 1, nChars ) ); + AllocMemory( nChars ); + Q_memcpy( m_pString, pValue, nChars ); + } + else + { + Purge(); + } + +} + + +void CUtlString::Set( const char *pValue ) +{ + int length = pValue ? V_strlen( pValue ) : 0; + SetDirect( pValue, length ); +} + +// Sets the length (used to serialize into the buffer ) +void CUtlString::SetLength( int nLen ) +{ + if ( nLen > 0 ) + { +#ifdef _DEBUG + int prevLen = m_pString ? Length() : 0; +#endif + AllocMemory( nLen ); +#ifdef _DEBUG + if ( nLen > prevLen ) + { + V_memset( m_pString + prevLen, 0xEB, nLen - prevLen ); + } +#endif + } + else + { + Purge(); + } +} + +const char *CUtlString::Get( ) const +{ + if (!m_pString) + { + return ""; + } + return m_pString; +} + +char *CUtlString::GetForModify() +{ + if ( !m_pString ) + { + // In general, we optimise away small mallocs for empty strings + // but if you ask for the non-const bytes, they must be writable + // so we can't return "" here, like we do for the const version - jd + void *pMemoryBlock = malloc( 1 ); + m_pString = (char *)pMemoryBlock; + *m_pString = 0; + } + + return m_pString; +} + +char CUtlString::operator[]( int i ) const +{ + if ( !m_pString ) + return '\0'; + + if ( i >= Length() ) + { + return '\0'; + } + + return m_pString[i]; +} + +void CUtlString::Clear() +{ + Purge(); +} + +void CUtlString::Purge() +{ + free( m_pString ); + m_pString = NULL; +} + +bool CUtlString::IsEqual_CaseSensitive( const char *src ) const +{ + if ( !src ) + { + return (Length() == 0); + } + return ( V_strcmp( Get(), src ) == 0 ); +} + +bool CUtlString::IsEqual_CaseInsensitive( const char *src ) const +{ + if ( !src ) + { + return (Length() == 0); + } + return ( V_stricmp( Get(), src ) == 0 ); +} + + +void CUtlString::ToLower() +{ + if ( !m_pString ) + { + return; + } + + V_strlower( m_pString ); +} + +void CUtlString::ToUpper() +{ + if ( !m_pString ) + { + return; + } + + V_strupr( m_pString ); +} + +CUtlString &CUtlString::operator=( const CUtlString &src ) +{ + SetDirect( src.Get(), src.Length() ); + return *this; +} + +CUtlString &CUtlString::operator=( const char *src ) +{ + Set( src ); + return *this; +} + +bool CUtlString::operator==( const CUtlString &src ) const +{ + if ( IsEmpty() ) + { + if ( src.IsEmpty() ) + { + return true; + } + + return false; + } + else + { + if ( src.IsEmpty() ) + { + return false; + } + + return Q_strcmp( m_pString, src.m_pString ) == 0; + } +} + +CUtlString &CUtlString::operator+=( const CUtlString &rhs ) +{ + const int lhsLength( Length() ); + const int rhsLength( rhs.Length() ); + + if (!rhsLength) + { + return *this; + } + + const int requestedLength( lhsLength + rhsLength ); + + AllocMemory( requestedLength ); + Q_memcpy( m_pString + lhsLength, rhs.m_pString, rhsLength ); + + return *this; +} + +CUtlString &CUtlString::operator+=( const char *rhs ) +{ + const int lhsLength( Length() ); + const int rhsLength( V_strlen( rhs ) ); + const int requestedLength( lhsLength + rhsLength ); + + if (!requestedLength) + { + return *this; + } + + AllocMemory( requestedLength ); + Q_memcpy( m_pString + lhsLength, rhs, rhsLength ); + + return *this; +} + +CUtlString &CUtlString::operator+=( char c ) +{ + const int lhsLength( Length() ); + + AllocMemory( lhsLength + 1 ); + m_pString[ lhsLength ] = c; + + return *this; +} + +CUtlString &CUtlString::operator+=( int rhs ) +{ + Assert( sizeof( rhs ) == 4 ); + + char tmpBuf[ 12 ]; // Sufficient for a signed 32 bit integer [ -2147483648 to +2147483647 ] + V_snprintf( tmpBuf, sizeof( tmpBuf ), "%d", rhs ); + tmpBuf[ sizeof( tmpBuf ) - 1 ] = '\0'; + + return operator+=( tmpBuf ); +} + +CUtlString &CUtlString::operator+=( double rhs ) +{ + char tmpBuf[ 256 ]; // How big can doubles be??? Dunno. + V_snprintf( tmpBuf, sizeof( tmpBuf ), "%lg", rhs ); + tmpBuf[ sizeof( tmpBuf ) - 1 ] = '\0'; + + return operator+=( tmpBuf ); +} + +bool CUtlString::MatchesPattern( const CUtlString &Pattern, int nFlags ) const +{ + const char *pszSource = String(); + const char *pszPattern = Pattern.String(); + bool bExact = true; + + while( 1 ) + { + if ( ( *pszPattern ) == 0 ) + { + return ( (*pszSource ) == 0 ); + } + + if ( ( *pszPattern ) == '*' ) + { + pszPattern++; + + if ( ( *pszPattern ) == 0 ) + { + return true; + } + + bExact = false; + continue; + } + + int nLength = 0; + + while( ( *pszPattern ) != '*' && ( *pszPattern ) != 0 ) + { + nLength++; + pszPattern++; + } + + while( 1 ) + { + const char *pszStartPattern = pszPattern - nLength; + const char *pszSearch = pszSource; + + for( int i = 0; i < nLength; i++, pszSearch++, pszStartPattern++ ) + { + if ( ( *pszSearch ) == 0 ) + { + return false; + } + + if ( ( *pszSearch ) != ( *pszStartPattern ) ) + { + break; + } + } + + if ( pszSearch - pszSource == nLength ) + { + break; + } + + if ( bExact == true ) + { + return false; + } + + if ( ( nFlags & PATTERN_DIRECTORY ) != 0 ) + { + if ( ( *pszPattern ) != '/' && ( *pszSource ) == '/' ) + { + return false; + } + } + + pszSource++; + } + + pszSource += nLength; + } +} + + +int CUtlString::Format( const char *pFormat, ... ) +{ + va_list marker; + + va_start( marker, pFormat ); + int len = FormatV( pFormat, marker ); + va_end( marker ); + + return len; +} + +//-------------------------------------------------------------------------------------------------- +// This can be called from functions that take varargs. +//-------------------------------------------------------------------------------------------------- + +int CUtlString::FormatV( const char *pFormat, va_list marker ) +{ + char tmpBuf[ 4096 ]; //< Nice big 4k buffer, as much memory as my first computer had, a Radio Shack Color Computer + + //va_start( marker, pFormat ); + int len = V_vsprintf_safe( tmpBuf, pFormat, marker ); + //va_end( marker ); + Set( tmpBuf ); + return len; +} + +//----------------------------------------------------------------------------- +// Strips the trailing slash +//----------------------------------------------------------------------------- +void CUtlString::StripTrailingSlash() +{ + if ( IsEmpty() ) + return; + + int nLastChar = Length() - 1; + char c = m_pString[ nLastChar ]; + if ( c == '\\' || c == '/' ) + { + SetLength( nLastChar ); + } +} + +void CUtlString::FixSlashes( char cSeparator/*=CORRECT_PATH_SEPARATOR*/ ) +{ + if ( m_pString ) + { + V_FixSlashes( m_pString, cSeparator ); + } +} + +//----------------------------------------------------------------------------- +// Trim functions +//----------------------------------------------------------------------------- +void CUtlString::TrimLeft( char cTarget ) +{ + int nIndex = 0; + + if ( IsEmpty() ) + { + return; + } + + while( m_pString[nIndex] == cTarget ) + { + ++nIndex; + } + + // We have some whitespace to remove + if ( nIndex > 0 ) + { + memcpy( m_pString, &m_pString[nIndex], Length() - nIndex ); + SetLength( Length() - nIndex ); + } +} + + +void CUtlString::TrimLeft( const char *szTargets ) +{ + int i; + + if ( IsEmpty() ) + { + return; + } + + for( i = 0; m_pString[i] != 0; i++ ) + { + bool bWhitespace = false; + + for( int j = 0; szTargets[j] != 0; j++ ) + { + if ( m_pString[i] == szTargets[j] ) + { + bWhitespace = true; + break; + } + } + + if ( !bWhitespace ) + { + break; + } + } + + // We have some whitespace to remove + if ( i > 0 ) + { + memcpy( m_pString, &m_pString[i], Length() - i ); + SetLength( Length() - i ); + } +} + + +void CUtlString::TrimRight( char cTarget ) +{ + const int nLastCharIndex = Length() - 1; + int nIndex = nLastCharIndex; + + while ( nIndex >= 0 && m_pString[nIndex] == cTarget ) + { + --nIndex; + } + + // We have some whitespace to remove + if ( nIndex < nLastCharIndex ) + { + m_pString[nIndex + 1] = 0; + SetLength( nIndex + 2 ); + } +} + + +void CUtlString::TrimRight( const char *szTargets ) +{ + const int nLastCharIndex = Length() - 1; + int i; + + for( i = nLastCharIndex; i > 0; i-- ) + { + bool bWhitespace = false; + + for( int j = 0; szTargets[j] != 0; j++ ) + { + if ( m_pString[i] == szTargets[j] ) + { + bWhitespace = true; + break; + } + } + + if ( !bWhitespace ) + { + break; + } + } + + // We have some whitespace to remove + if ( i < nLastCharIndex ) + { + m_pString[i + 1] = 0; + SetLength( i + 2 ); + } +} + + +void CUtlString::Trim( char cTarget ) +{ + TrimLeft( cTarget ); + TrimRight( cTarget ); +} + + +void CUtlString::Trim( const char *szTargets ) +{ + TrimLeft( szTargets ); + TrimRight( szTargets ); +} + + +CUtlString CUtlString::Slice( int32 nStart, int32 nEnd ) const +{ + int length = Length(); + if ( length == 0 ) + { + return CUtlString(); + } + + if ( nStart < 0 ) + nStart = length - (-nStart % length); + else if ( nStart >= length ) + nStart = length; + + if ( nEnd == INT32_MAX ) + nEnd = length; + else if ( nEnd < 0 ) + nEnd = length - (-nEnd % length); + else if ( nEnd >= length ) + nEnd = length; + + if ( nStart >= nEnd ) + return CUtlString(); + + const char *pIn = String(); + + CUtlString ret; + ret.SetDirect( pIn + nStart, nEnd - nStart ); + return ret; +} + +// Grab a substring starting from the left or the right side. +CUtlString CUtlString::Left( int32 nChars ) const +{ + return Slice( 0, nChars ); +} + +CUtlString CUtlString::Right( int32 nChars ) const +{ + return Slice( -nChars ); +} + +CUtlString CUtlString::Replace( char cFrom, char cTo ) const +{ + if (!m_pString) + { + return CUtlString(); + } + + CUtlString ret = *this; + int len = ret.Length(); + for ( int i=0; i < len; i++ ) + { + if ( ret.m_pString[i] == cFrom ) + ret.m_pString[i] = cTo; + } + + return ret; +} + +CUtlString CUtlString::Replace( const char *pszFrom, const char *pszTo ) const +{ + Assert( pszTo ); // Can be 0 length, but not null + Assert( pszFrom && *pszFrom ); // Must be valid and have one character. + + + const char *pos = V_strstr( String(), pszFrom ); + if ( !pos ) + { + return *this; + } + + const char *pFirstFound = pos; + + // count number of search string + int nSearchCount = 0; + int nSearchLength = V_strlen( pszFrom ); + while ( pos ) + { + nSearchCount++; + int nSrcOffset = ( pos - String() ) + nSearchLength; + pos = V_strstr( String() + nSrcOffset, pszFrom ); + } + + // allocate the new string + int nReplaceLength = V_strlen( pszTo ); + int nAllocOffset = nSearchCount * ( nReplaceLength - nSearchLength ); + size_t srcLength = Length(); + CUtlString strDest; + size_t destLength = srcLength + nAllocOffset; + strDest.SetLength( destLength ); + + // find and replace the search string + pos = pFirstFound; + int nDestOffset = 0; + int nSrcOffset = 0; + while ( pos ) + { + // Found an instance + int nCurrentSearchOffset = pos - String(); + int nCopyLength = nCurrentSearchOffset - nSrcOffset; + V_strncpy( strDest.GetForModify() + nDestOffset, String() + nSrcOffset, nCopyLength + 1 ); + nDestOffset += nCopyLength; + V_strncpy( strDest.GetForModify() + nDestOffset, pszTo, nReplaceLength + 1 ); + nDestOffset += nReplaceLength; + + nSrcOffset = nCurrentSearchOffset + nSearchLength; + pos = V_strstr( String() + nSrcOffset, pszFrom ); + } + + // making sure that the left over string from the source is the same size as the left over dest buffer + Assert( destLength - nDestOffset == srcLength - nSrcOffset ); + if ( destLength - nDestOffset > 0 ) + { + V_strncpy( strDest.GetForModify() + nDestOffset, String() + nSrcOffset, destLength - nDestOffset + 1 ); + } + + return strDest; +} + +CUtlString CUtlString::AbsPath( const char *pStartingDir ) const +{ + char szNew[MAX_PATH]; + V_MakeAbsolutePath( szNew, sizeof( szNew ), this->String(), pStartingDir ); + return CUtlString( szNew ); +} + +CUtlString CUtlString::UnqualifiedFilename() const +{ + const char *pFilename = V_UnqualifiedFileName( this->String() ); + return CUtlString( pFilename ); +} + +CUtlString CUtlString::DirName() const +{ + CUtlString ret( this->String() ); + V_StripLastDir( (char*)ret.Get(), ret.Length() + 1 ); + V_StripTrailingSlash( (char*)ret.Get() ); + return ret; +} + +CUtlString CUtlString::StripExtension() const +{ + char szTemp[MAX_PATH]; + V_StripExtension( String(), szTemp, sizeof( szTemp ) ); + return CUtlString( szTemp ); +} + +CUtlString CUtlString::StripFilename() const +{ + const char *pFilename = V_UnqualifiedFileName( Get() ); // NOTE: returns 'Get()' on failure, never NULL + int nCharsToCopy = pFilename - Get(); + CUtlString result; + result.SetDirect( Get(), nCharsToCopy ); + result.StripTrailingSlash(); + return result; +} + +CUtlString CUtlString::GetBaseFilename() const +{ + char szTemp[MAX_PATH]; + V_FileBase( String(), szTemp, sizeof( szTemp ) ); + return CUtlString( szTemp ); +} + +CUtlString CUtlString::GetExtension() const +{ + char szTemp[MAX_PATH]; + V_ExtractFileExtension( String(), szTemp, sizeof( szTemp ) ); + return CUtlString( szTemp ); +} + + +CUtlString CUtlString::PathJoin( const char *pStr1, const char *pStr2 ) +{ + char szPath[MAX_PATH]; + V_ComposeFileName( pStr1, pStr2, szPath, sizeof( szPath ) ); + return CUtlString( szPath ); +} + +CUtlString CUtlString::operator+( const char *pOther ) const +{ + CUtlString s = *this; + s += pOther; + return s; +} + +CUtlString CUtlString::operator+( const CUtlString &other ) const +{ + CUtlString s = *this; + s += other; + return s; +} + +CUtlString CUtlString::operator+( int rhs ) const +{ + CUtlString ret = *this; + ret += rhs; + return ret; +} + +//----------------------------------------------------------------------------- +// Purpose: concatenate the provided string to our current content +//----------------------------------------------------------------------------- +void CUtlString::Append( const char *pchAddition ) +{ + (*this) += pchAddition; +} + +void CUtlString::Append( const char *pchAddition, int nChars ) +{ + nChars = Min<int>( nChars, V_strlen( pchAddition ) ); + + const int lhsLength( Length() ); + const int rhsLength( nChars ); + const int requestedLength( lhsLength + rhsLength ); + + AllocMemory( requestedLength ); + const int allocatedLength( requestedLength ); + const int copyLength( allocatedLength - lhsLength < rhsLength ? allocatedLength - lhsLength : rhsLength ); + memcpy( GetForModify() + lhsLength, pchAddition, copyLength ); + m_pString[ allocatedLength ] = '\0'; +} + +// Shared static empty string. +const CUtlString &CUtlString::GetEmptyString() +{ + static const CUtlString s_emptyString; + + return s_emptyString; +} diff --git a/tier1/utlsymbol.cpp b/tier1/utlsymbol.cpp new file mode 100644 index 0000000..d75eaa5 --- /dev/null +++ b/tier1/utlsymbol.cpp @@ -0,0 +1,436 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: Defines a symbol table +// +// $Header: $ +// $NoKeywords: $ +//=============================================================================// + +#pragma warning (disable:4514) + +#include "utlsymbol.h" +#include "KeyValues.h" +#include "tier0/threadtools.h" +#include "tier0/memdbgon.h" +#include "stringpool.h" +#include "utlhashtable.h" +#include "utlstring.h" + +// Ensure that everybody has the right compiler version installed. The version +// number can be obtained by looking at the compiler output when you type 'cl' +// and removing the last two digits and the periods: 16.00.40219.01 becomes 160040219 +#ifdef _MSC_FULL_VER + #if _MSC_FULL_VER > 160000000 + // VS 2010 + #if _MSC_FULL_VER < 160040219 + #error You must install VS 2010 SP1 + #endif + #else + // VS 2005 + #if _MSC_FULL_VER < 140050727 + #error You must install VS 2005 SP1 + #endif + #endif +#endif + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +#define INVALID_STRING_INDEX CStringPoolIndex( 0xFFFF, 0xFFFF ) + +#define MIN_STRING_POOL_SIZE 2048 + +//----------------------------------------------------------------------------- +// globals +//----------------------------------------------------------------------------- + +CUtlSymbolTableMT* CUtlSymbol::s_pSymbolTable = 0; +bool CUtlSymbol::s_bAllowStaticSymbolTable = true; + + +//----------------------------------------------------------------------------- +// symbol methods +//----------------------------------------------------------------------------- + +void CUtlSymbol::Initialize() +{ + // If this assert fails, then the module that this call is in has chosen to disallow + // use of the static symbol table. Usually, it's to prevent confusion because it's easy + // to accidentally use the global symbol table when you really want to use a specific one. + Assert( s_bAllowStaticSymbolTable ); + + // necessary to allow us to create global symbols + static bool symbolsInitialized = false; + if (!symbolsInitialized) + { + s_pSymbolTable = new CUtlSymbolTableMT; + symbolsInitialized = true; + } +} + +//----------------------------------------------------------------------------- +// Purpose: Singleton to delete table on exit from module +//----------------------------------------------------------------------------- +class CCleanupUtlSymbolTable +{ +public: + ~CCleanupUtlSymbolTable() + { + delete CUtlSymbol::s_pSymbolTable; + CUtlSymbol::s_pSymbolTable = NULL; + } +}; + +static CCleanupUtlSymbolTable g_CleanupSymbolTable; + +CUtlSymbolTableMT* CUtlSymbol::CurrTable() +{ + Initialize(); + return s_pSymbolTable; +} + + +//----------------------------------------------------------------------------- +// string->symbol->string +//----------------------------------------------------------------------------- + +CUtlSymbol::CUtlSymbol( const char* pStr ) +{ + m_Id = CurrTable()->AddString( pStr ); +} + +const char* CUtlSymbol::String( ) const +{ + return CurrTable()->String(m_Id); +} + +void CUtlSymbol::DisableStaticSymbolTable() +{ + s_bAllowStaticSymbolTable = false; +} + +//----------------------------------------------------------------------------- +// checks if the symbol matches a string +//----------------------------------------------------------------------------- + +bool CUtlSymbol::operator==( const char* pStr ) const +{ + if (m_Id == UTL_INVAL_SYMBOL) + return false; + return strcmp( String(), pStr ) == 0; +} + + + +//----------------------------------------------------------------------------- +// symbol table stuff +//----------------------------------------------------------------------------- + +inline const char* CUtlSymbolTable::StringFromIndex( const CStringPoolIndex &index ) const +{ + Assert( index.m_iPool < m_StringPools.Count() ); + Assert( index.m_iOffset < m_StringPools[index.m_iPool]->m_TotalLen ); + + return &m_StringPools[index.m_iPool]->m_Data[index.m_iOffset]; +} + + +bool CUtlSymbolTable::CLess::operator()( const CStringPoolIndex &i1, const CStringPoolIndex &i2 ) const +{ + // Need to do pointer math because CUtlSymbolTable is used in CUtlVectors, and hence + // can be arbitrarily moved in memory on a realloc. Yes, this is portable. In reality, + // right now at least, because m_LessFunc is the first member of CUtlRBTree, and m_Lookup + // is the first member of CUtlSymbolTabke, this == pTable + CUtlSymbolTable *pTable = (CUtlSymbolTable *)( (byte *)this - offsetof(CUtlSymbolTable::CTree, m_LessFunc) ) - offsetof(CUtlSymbolTable, m_Lookup ); + const char* str1 = (i1 == INVALID_STRING_INDEX) ? pTable->m_pUserSearchString : + pTable->StringFromIndex( i1 ); + const char* str2 = (i2 == INVALID_STRING_INDEX) ? pTable->m_pUserSearchString : + pTable->StringFromIndex( i2 ); + + if ( !str1 && str2 ) + return false; + if ( !str2 && str1 ) + return true; + if ( !str1 && !str2 ) + return false; + if ( !pTable->m_bInsensitive ) + return V_strcmp( str1, str2 ) < 0; + else + return V_stricmp( str1, str2 ) < 0; +} + + +//----------------------------------------------------------------------------- +// constructor, destructor +//----------------------------------------------------------------------------- +CUtlSymbolTable::CUtlSymbolTable( int growSize, int initSize, bool caseInsensitive ) : + m_Lookup( growSize, initSize ), m_bInsensitive( caseInsensitive ), m_StringPools( 8 ) +{ +} + +CUtlSymbolTable::~CUtlSymbolTable() +{ + // Release the stringpool string data + RemoveAll(); +} + + +CUtlSymbol CUtlSymbolTable::Find( const char* pString ) const +{ + if (!pString) + return CUtlSymbol(); + + // Store a special context used to help with insertion + m_pUserSearchString = pString; + + // Passing this special invalid symbol makes the comparison function + // use the string passed in the context + UtlSymId_t idx = m_Lookup.Find( INVALID_STRING_INDEX ); + +#ifdef _DEBUG + m_pUserSearchString = NULL; +#endif + + return CUtlSymbol( idx ); +} + + +int CUtlSymbolTable::FindPoolWithSpace( int len ) const +{ + for ( int i=0; i < m_StringPools.Count(); i++ ) + { + StringPool_t *pPool = m_StringPools[i]; + + if ( (pPool->m_TotalLen - pPool->m_SpaceUsed) >= len ) + { + return i; + } + } + + return -1; +} + + +//----------------------------------------------------------------------------- +// Finds and/or creates a symbol based on the string +//----------------------------------------------------------------------------- + +CUtlSymbol CUtlSymbolTable::AddString( const char* pString ) +{ + if (!pString) + return CUtlSymbol( UTL_INVAL_SYMBOL ); + + CUtlSymbol id = Find( pString ); + + if (id.IsValid()) + return id; + + int len = V_strlen(pString) + 1; + + // Find a pool with space for this string, or allocate a new one. + int iPool = FindPoolWithSpace( len ); + if ( iPool == -1 ) + { + // Add a new pool. + int newPoolSize = max( len, MIN_STRING_POOL_SIZE ); + StringPool_t *pPool = (StringPool_t*)malloc( sizeof( StringPool_t ) + newPoolSize - 1 ); + pPool->m_TotalLen = newPoolSize; + pPool->m_SpaceUsed = 0; + iPool = m_StringPools.AddToTail( pPool ); + } + + // Copy the string in. + StringPool_t *pPool = m_StringPools[iPool]; + Assert( pPool->m_SpaceUsed < 0xFFFF ); // This should never happen, because if we had a string > 64k, it + // would have been given its entire own pool. + + unsigned short iStringOffset = pPool->m_SpaceUsed; + + memcpy( &pPool->m_Data[pPool->m_SpaceUsed], pString, len ); + pPool->m_SpaceUsed += len; + + // didn't find, insert the string into the vector. + CStringPoolIndex index; + index.m_iPool = iPool; + index.m_iOffset = iStringOffset; + + UtlSymId_t idx = m_Lookup.Insert( index ); + return CUtlSymbol( idx ); +} + + +//----------------------------------------------------------------------------- +// Look up the string associated with a particular symbol +//----------------------------------------------------------------------------- + +const char* CUtlSymbolTable::String( CUtlSymbol id ) const +{ + if (!id.IsValid()) + return ""; + + Assert( m_Lookup.IsValidIndex((UtlSymId_t)id) ); + return StringFromIndex( m_Lookup[id] ); +} + + +//----------------------------------------------------------------------------- +// Remove all symbols in the table. +//----------------------------------------------------------------------------- + +void CUtlSymbolTable::RemoveAll() +{ + m_Lookup.Purge(); + + for ( int i=0; i < m_StringPools.Count(); i++ ) + free( m_StringPools[i] ); + + m_StringPools.RemoveAll(); +} + + + +class CUtlFilenameSymbolTable::HashTable : public CUtlStableHashtable<CUtlConstString> +{ +}; + +CUtlFilenameSymbolTable::CUtlFilenameSymbolTable() +{ + m_Strings = new HashTable; +} + +CUtlFilenameSymbolTable::~CUtlFilenameSymbolTable() +{ + delete m_Strings; +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *pFileName - +// Output : FileNameHandle_t +//----------------------------------------------------------------------------- +FileNameHandle_t CUtlFilenameSymbolTable::FindOrAddFileName( const char *pFileName ) +{ + if ( !pFileName ) + { + return NULL; + } + + // find first + FileNameHandle_t hFileName = FindFileName( pFileName ); + if ( hFileName ) + { + return hFileName; + } + + // Fix slashes+dotslashes and make lower case first.. + char fn[ MAX_PATH ]; + Q_strncpy( fn, pFileName, sizeof( fn ) ); + Q_RemoveDotSlashes( fn ); +#ifdef _WIN32 + Q_strlower( fn ); +#endif + + // Split the filename into constituent parts + char basepath[ MAX_PATH ]; + Q_ExtractFilePath( fn, basepath, sizeof( basepath ) ); + char filename[ MAX_PATH ]; + Q_strncpy( filename, fn + Q_strlen( basepath ), sizeof( filename ) ); + + // not found, lock and look again + FileNameHandleInternal_t handle; + m_lock.LockForWrite(); + handle.path = m_Strings->Insert( basepath ) + 1; + handle.file = m_Strings->Insert( filename ) + 1; + //handle.path = m_StringPool.FindStringHandle( basepath ); + //handle.file = m_StringPool.FindStringHandle( filename ); + //if ( handle.path != m_Strings.InvalidHandle() && handle.file ) + //{ + // found + // m_lock.UnlockWrite(); + // return *( FileNameHandle_t * )( &handle ); + //} + + // safely add it + //handle.path = m_StringPool.ReferenceStringHandle( basepath ); + //handle.file = m_StringPool.ReferenceStringHandle( filename ); + m_lock.UnlockWrite(); + + return *( FileNameHandle_t * )( &handle ); +} + +FileNameHandle_t CUtlFilenameSymbolTable::FindFileName( const char *pFileName ) +{ + if ( !pFileName ) + { + return NULL; + } + + // Fix slashes+dotslashes and make lower case first.. + char fn[ MAX_PATH ]; + Q_strncpy( fn, pFileName, sizeof( fn ) ); + Q_RemoveDotSlashes( fn ); +#ifdef _WIN32 + Q_strlower( fn ); +#endif + + // Split the filename into constituent parts + char basepath[ MAX_PATH ]; + Q_ExtractFilePath( fn, basepath, sizeof( basepath ) ); + char filename[ MAX_PATH ]; + Q_strncpy( filename, fn + Q_strlen( basepath ), sizeof( filename ) ); + + FileNameHandleInternal_t handle; + + Assert( (uint16)(m_Strings->InvalidHandle() + 1) == 0 ); + + m_lock.LockForRead(); + handle.path = m_Strings->Find(basepath) + 1; + handle.file = m_Strings->Find(filename) + 1; + //handle.path = m_StringPool.FindStringHandle(basepath); + //handle.file = m_StringPool.FindStringHandle(filename); + m_lock.UnlockRead(); + + if ( handle.path == 0 || handle.file == 0 ) + return NULL; + + return *( FileNameHandle_t * )( &handle ); +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : handle - +// Output : const char +//----------------------------------------------------------------------------- +bool CUtlFilenameSymbolTable::String( const FileNameHandle_t& handle, char *buf, int buflen ) +{ + buf[ 0 ] = 0; + + FileNameHandleInternal_t *internal = ( FileNameHandleInternal_t * )&handle; + if ( !internal || !internal->file || !internal->path ) + { + return false; + } + + m_lock.LockForRead(); + //const char *path = m_StringPool.HandleToString(internal->path); + //const char *fn = m_StringPool.HandleToString(internal->file); + const char *path = (*m_Strings)[ internal->path - 1 ].Get(); + const char *fn = (*m_Strings)[ internal->file - 1].Get(); + m_lock.UnlockRead(); + + if ( !path || !fn ) + { + return false; + } + + Q_strncpy( buf, path, buflen ); + Q_strncat( buf, fn, buflen, COPY_ALL_CHARACTERS ); + + return true; +} + +void CUtlFilenameSymbolTable::RemoveAll() +{ + m_Strings->Purge(); +} |