diff options
Diffstat (limited to 'mp/src/tier1')
| -rw-r--r-- | mp/src/tier1/KeyValues.cpp | 331 | ||||
| -rw-r--r-- | mp/src/tier1/convar.cpp | 20 | ||||
| -rw-r--r-- | mp/src/tier1/ilocalize.cpp | 5 | ||||
| -rw-r--r-- | mp/src/tier1/interface.cpp | 2 | ||||
| -rw-r--r-- | mp/src/tier1/kvpacker.cpp | 29 | ||||
| -rw-r--r-- | mp/src/tier1/lzmaDecoder.cpp | 956 | ||||
| -rw-r--r-- | mp/src/tier1/strtools.cpp | 94 | ||||
| -rw-r--r-- | mp/src/tier1/strtools_unicode.cpp | 572 | ||||
| -rw-r--r-- | mp/src/tier1/tier1.vpc | 10 | ||||
| -rw-r--r-- | mp/src/tier1/uniqueid.cpp | 2 | ||||
| -rw-r--r-- | mp/src/tier1/utlbuffer.cpp | 35 |
11 files changed, 1157 insertions, 899 deletions
diff --git a/mp/src/tier1/KeyValues.cpp b/mp/src/tier1/KeyValues.cpp index c72172ca..ad4af096 100644 --- a/mp/src/tier1/KeyValues.cpp +++ b/mp/src/tier1/KeyValues.cpp @@ -18,14 +18,16 @@ #include <KeyValues.h> #include "filesystem.h" #include <vstdlib/IKeyValuesSystem.h> +#include "tier0/icommandline.h" #include <Color.h> #include <stdlib.h> #include "tier0/dbg.h" #include "tier0/mem.h" -#include "utlvector.h" #include "utlbuffer.h" #include "utlhash.h" +#include "utlvector.h" +#include "utlqueue.h" #include "UtlSortVector.h" #include "convar.h" @@ -84,16 +86,19 @@ public: { Assert( stackLevel >= 0 ); Assert( stackLevel < m_errorIndex ); - m_errorStack[stackLevel] = symName; + if ( stackLevel < MAX_ERROR_STACK ) + m_errorStack[stackLevel] = symName; } // Hit an error, report it and the parsing stack for context void ReportError( const char *pError ) { + bool bSpewCR = false; + Warning( "KeyValues Error: %s in file %s\n", pError, m_pFilename ); for ( int i = 0; i < m_maxErrorIndex; i++ ) { - if ( m_errorStack[i] != INVALID_KEY_SYMBOL ) + if ( i < MAX_ERROR_STACK && m_errorStack[i] != INVALID_KEY_SYMBOL ) { if ( i < m_errorIndex ) { @@ -103,9 +108,13 @@ public: { Warning( "(*%s*), ", KeyValues::CallGetStringForSymbol(m_errorStack[i]) ); } + + bSpewCR = true; } } - Warning( "\n" ); + + if ( bSpewCR ) + Warning( "\n" ); } private: @@ -137,6 +146,10 @@ public: { g_KeyValuesErrorStack.Reset( m_stackLevel, symName ); } + int GetStackLevel() const + { + return m_stackLevel; + } private: void Init( int symName ) { @@ -630,15 +643,62 @@ void KeyValues::UsesConditionals(bool state) //----------------------------------------------------------------------------- // Purpose: Load keyValues from disk //----------------------------------------------------------------------------- -bool KeyValues::LoadFromFile( IBaseFileSystem *filesystem, const char *resourceName, const char *pathID ) +bool KeyValues::LoadFromFile( IBaseFileSystem *filesystem, const char *resourceName, const char *pathID, bool refreshCache ) { Assert(filesystem); #ifdef WIN32 Assert( IsX360() || ( IsPC() && _heapchk() == _HEAPOK ) ); #endif + +#ifdef STAGING_ONLY + static bool s_bCacheEnabled = !!CommandLine()->FindParm( "-enable_keyvalues_cache" ); + const bool bUseCache = s_bCacheEnabled && ( s_pfGetSymbolForString == KeyValues::GetSymbolForStringClassic ); +#else + /* + People are cheating with the keyvalue cache enabled by doing the below, so disable it. + + For example if one is to allow a blue demoman texture on sv_pure they + change it to this, "$basetexture" "temp/demoman_blue". Remember to move the + demoman texture to the temp folder in the materials folder. It will likely + not be there so make a new folder for it. Once the directory in the + demoman_blue vmt is changed to the temp folder and the vtf texture is in + the temp folder itself you are finally done. + + I packed my mods into a vpk but I don't think it's required. Once in game + you must create a server via the create server button and select the map + that will load the custom texture before you join a valve server. I suggest + you only do this with player textures and such as they are always loaded. + After you load the map you join the valve server and the textures should + appear and work on valve servers. + + This can be done on any sv_pure 1 server but it depends on what is type of + files are allowed. All valve servers allow temp files so that is the + example I used here." + + So all vmt's files can bypass sv_pure 1. And I believe this mod is mostly + made of vmt files, so valve's sv_pure 1 bull is pretty redundant. + */ + const bool bUseCache = false; +#endif + + // If pathID is null, we cannot cache the result because that has a weird iterate-through-a-bunch-of-locations behavior. + const bool bUseCacheForRead = bUseCache && !refreshCache && pathID != NULL; + const bool bUseCacheForWrite = bUseCache && pathID != NULL; + + COM_TimestampedLog( "KeyValues::LoadFromFile(%s%s%s): Begin", pathID ? pathID : "", pathID && resourceName ? "/" : "", resourceName ? resourceName : "" ); + + // Keep a cache of keyvalues, try to load it here. + if ( bUseCacheForRead && KeyValuesSystem()->LoadFileKeyValuesFromCache( this, resourceName, pathID, filesystem ) ) { + COM_TimestampedLog( "KeyValues::LoadFromFile(%s%s%s): End / CacheHit", pathID ? pathID : "", pathID && resourceName ? "/" : "", resourceName ? resourceName : "" ); + return true; + } + FileHandle_t f = filesystem->Open(resourceName, "rb", pathID); if ( !f ) + { + COM_TimestampedLog("KeyValues::LoadFromFile(%s%s%s): End / FileNotFound", pathID ? pathID : "", pathID && resourceName ? "/" : "", resourceName ? resourceName : ""); return false; + } s_LastFileLoadingFrom = (char*)resourceName; @@ -660,28 +720,41 @@ bool KeyValues::LoadFromFile( IBaseFileSystem *filesystem, const char *resourceN buffer[fileSize+1] = 0; // double NULL terminating in case this is a unicode file bRetOK = LoadFromBuffer( resourceName, buffer, filesystem ); } + + // The cache relies on the KeyValuesSystem string table, which will only be valid if we're + // using classic mode. + if ( bUseCacheForWrite && bRetOK ) + { + KeyValuesSystem()->AddFileKeyValuesToCache( this, resourceName, pathID ); + } - ((IFileSystem *)filesystem)->FreeOptimalReadBuffer( buffer ); + ( (IFileSystem *)filesystem )->FreeOptimalReadBuffer( buffer ); + + COM_TimestampedLog("KeyValues::LoadFromFile(%s%s%s): End / Success", pathID ? pathID : "", pathID && resourceName ? "/" : "", resourceName ? resourceName : ""); return bRetOK; } //----------------------------------------------------------------------------- // Purpose: Save the keyvalues to disk -// Creates the path to the file if it doesn't exist +// Creates the path to the file if it doesn't exist //----------------------------------------------------------------------------- -bool KeyValues::SaveToFile( IBaseFileSystem *filesystem, const char *resourceName, const char *pathID, bool sortKeys /*= false*/, bool bAllowEmptyString /*= false*/ ) +bool KeyValues::SaveToFile( IBaseFileSystem *filesystem, const char *resourceName, const char *pathID, bool sortKeys /*= false*/, bool bAllowEmptyString /*= false*/, bool bCacheResult /*= false*/ ) { // create a write file FileHandle_t f = filesystem->Open(resourceName, "wb", pathID); if ( f == FILESYSTEM_INVALID_HANDLE ) { - DevMsg(1, "KeyValues::SaveToFile: couldn't open file \"%s\" in path \"%s\".\n", + DevMsg(1, "KeyValues::SaveToFile: couldn't open file \"%s\" in path \"%s\".\n", resourceName?resourceName:"NULL", pathID?pathID:"NULL" ); return false; } + KeyValuesSystem()->InvalidateCacheForFile( resourceName, pathID ); + if ( bCacheResult ) { + KeyValuesSystem()->AddFileKeyValuesToCache( this, resourceName, pathID ); + } RecursiveSaveToFile(filesystem, f, NULL, 0, sortKeys, bAllowEmptyString ); filesystem->Close(f); @@ -1671,115 +1744,138 @@ void KeyValues::SetPtr( const char *keyName, void *value ) } } -void KeyValues::RecursiveCopyKeyValues( KeyValues& src ) +//----------------------------------------------------------------------------- +// Purpose: Copies the tree from the other KeyValues into this one, recursively +// beginning with the root specified by rootSrc. +//----------------------------------------------------------------------------- +void KeyValues::CopyKeyValuesFromRecursive( const KeyValues& rootSrc ) { - // garymcthack - need to check this code for possible buffer overruns. - - m_iKeyName = src.GetNameSymbol(); + // This code used to be recursive, which was more elegant. Unfortunately, it also blew the stack for large + // KeyValues. So now we have the iterative version which is uglier but doesn't blow the stack. + // This uses breadth-first traversal. + + struct CopyStruct + { + KeyValues* dst; + const KeyValues* src; + }; + + char tmp[256]; + KeyValues* localDst = NULL; - if( !src.m_pSub ) + CUtlQueue<CopyStruct> nodeQ; + nodeQ.Insert({ this, &rootSrc }); + + while ( nodeQ.Count() > 0 ) { - m_iDataType = src.m_iDataType; - char buf[256]; - switch( src.m_iDataType ) + CopyStruct cs = nodeQ.RemoveAtHead(); + + // Process all the siblings of the current node. If anyone has a child, add it to the queue. + while (cs.src) { - case TYPE_NONE: - break; - case TYPE_STRING: - if( src.m_sValue ) - { - int len = Q_strlen(src.m_sValue) + 1; - m_sValue = new char[len]; - Q_strncpy( m_sValue, src.m_sValue, len ); - } - break; - case TYPE_INT: - { - m_iValue = src.m_iValue; - Q_snprintf( buf,sizeof(buf), "%d", m_iValue ); - int len = Q_strlen(buf) + 1; - m_sValue = new char[len]; - Q_strncpy( m_sValue, buf, len ); - } - break; - case TYPE_FLOAT: - { - m_flValue = src.m_flValue; - Q_snprintf( buf,sizeof(buf), "%f", m_flValue ); - int len = Q_strlen(buf) + 1; - m_sValue = new char[len]; - Q_strncpy( m_sValue, buf, len ); - } - break; - case TYPE_PTR: - { - m_pValue = src.m_pValue; - } - break; - case TYPE_UINT64: - { - m_sValue = new char[sizeof(uint64)]; - Q_memcpy( m_sValue, src.m_sValue, sizeof(uint64) ); + Assert( (cs.src != NULL) == (cs.dst != NULL) ); + + // Copy the node contents + cs.dst->CopyKeyValue( *cs.src, sizeof(tmp), tmp ); + + // Add children to the queue to process later. + if (cs.src->m_pSub) { + cs.dst->m_pSub = localDst = new KeyValues( NULL ); + nodeQ.Insert({ localDst, cs.src->m_pSub }); } - break; - case TYPE_COLOR: - { - m_Color[0] = src.m_Color[0]; - m_Color[1] = src.m_Color[1]; - m_Color[2] = src.m_Color[2]; - m_Color[3] = src.m_Color[3]; + + // Process siblings until we hit the end of the line. + if (cs.src->m_pPeer) { + cs.dst->m_pPeer = new KeyValues( NULL ); } - break; - - default: - { - // do nothing . .what the heck is this? - Assert( 0 ); + else { + cs.dst->m_pPeer = NULL; } - break; - } + // Advance to the next peer. + cs.src = cs.src->m_pPeer; + cs.dst = cs.dst->m_pPeer; + } } -#if 0 - KeyValues *pDst = this; - for ( KeyValues *pSrc = src.m_pSub; pSrc; pSrc = pSrc->m_pPeer ) +} + +//----------------------------------------------------------------------------- +// Purpose: Copies a single KeyValue from src to this, using the provided temporary +// buffer if the keytype requires it. Does NOT recurse. +//----------------------------------------------------------------------------- +void KeyValues::CopyKeyValue( const KeyValues& src, size_t tmpBufferSizeB, char* tmpBuffer ) +{ + m_iKeyName = src.GetNameSymbol(); + + if ( src.m_pSub ) + return; + + m_iDataType = src.m_iDataType; + + switch( src.m_iDataType ) { - if ( pSrc->m_pSub ) + case TYPE_NONE: + break; + case TYPE_STRING: + if( src.m_sValue ) { - pDst->m_pSub = new KeyValues( pSrc->m_pSub->getName() ); - pDst->m_pSub->RecursiveCopyKeyValues( *pSrc->m_pSub ); + int len = Q_strlen(src.m_sValue) + 1; + m_sValue = new char[len]; + Q_strncpy( m_sValue, src.m_sValue, len ); } - else + break; + case TYPE_INT: { - // copy non-empty keys - if ( pSrc->m_sValue && *(pSrc->m_sValue) ) - { - pDst->m_pPeer = new KeyValues( - } + m_iValue = src.m_iValue; + Q_snprintf( tmpBuffer, tmpBufferSizeB, "%d", m_iValue ); + int len = Q_strlen(tmpBuffer) + 1; + m_sValue = new char[len]; + Q_strncpy( m_sValue, tmpBuffer, len ); } - } -#endif - - // Handle the immediate child - if( src.m_pSub ) - { - m_pSub = new KeyValues( NULL ); - m_pSub->RecursiveCopyKeyValues( *src.m_pSub ); - } - - // Handle the immediate peer - if( src.m_pPeer ) - { - m_pPeer = new KeyValues( NULL ); - m_pPeer->RecursiveCopyKeyValues( *src.m_pPeer ); + break; + case TYPE_FLOAT: + { + m_flValue = src.m_flValue; + Q_snprintf( tmpBuffer, tmpBufferSizeB, "%f", m_flValue ); + int len = Q_strlen(tmpBuffer) + 1; + m_sValue = new char[len]; + Q_strncpy( m_sValue, tmpBuffer, len ); + } + break; + case TYPE_PTR: + { + m_pValue = src.m_pValue; + } + break; + case TYPE_UINT64: + { + m_sValue = new char[sizeof(uint64)]; + Q_memcpy( m_sValue, src.m_sValue, sizeof(uint64) ); + } + break; + case TYPE_COLOR: + { + m_Color[0] = src.m_Color[0]; + m_Color[1] = src.m_Color[1]; + m_Color[2] = src.m_Color[2]; + m_Color[3] = src.m_Color[3]; + } + break; + + default: + { + // do nothing . .what the heck is this? + Assert( 0 ); + } + break; } } -KeyValues& KeyValues::operator=( KeyValues& src ) +KeyValues& KeyValues::operator=( const KeyValues& src ) { RemoveEverything(); Init(); // reset all values - RecursiveCopyKeyValues( src ); + CopyKeyValuesFromRecursive( src ); return *this; } @@ -1878,6 +1974,25 @@ KeyValues *KeyValues::MakeCopy( void ) const return newKeyValue; } +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +KeyValues *KeyValues::MakeCopy( bool copySiblings ) const +{ + KeyValues* rootDest = MakeCopy(); + if ( !copySiblings ) + return rootDest; + + const KeyValues* curSrc = GetNextKey(); + KeyValues* curDest = rootDest; + while (curSrc) { + curDest->SetNextKey( curSrc->MakeCopy() ); + curDest = curDest->GetNextKey(); + curSrc = curSrc->GetNextKey(); + } + + return rootDest; +} //----------------------------------------------------------------------------- // Purpose: Check if a keyName has no value assigned to it. @@ -2243,6 +2358,8 @@ bool KeyValues::LoadFromBuffer( char const *resourceName, const char *pBuffer, I if ( !pBuffer ) return true; + COM_TimestampedLog("KeyValues::LoadFromBuffer(%s%s%s): Begin", pPathID ? pPathID : "", pPathID && resourceName ? "/" : "", resourceName ? resourceName : ""); + int nLen = Q_strlen( pBuffer ); CUtlBuffer buf( pBuffer, nLen, CUtlBuffer::READ_ONLY | CUtlBuffer::TEXT_BUFFER ); @@ -2255,7 +2372,11 @@ bool KeyValues::LoadFromBuffer( char const *resourceName, const char *pBuffer, I buf.AssumeMemory( pUTF8Buf, nUTF8Len, nUTF8Len, CUtlBuffer::READ_ONLY | CUtlBuffer::TEXT_BUFFER ); } - return LoadFromBuffer( resourceName, buf, pFileSystem, pPathID ); + bool retVal = LoadFromBuffer( resourceName, buf, pFileSystem, pPathID ); + + COM_TimestampedLog("KeyValues::LoadFromBuffer(%s%s%s): End", pPathID ? pPathID : "", pPathID && resourceName ? "/" : "", resourceName ? resourceName : ""); + + return retVal; } //----------------------------------------------------------------------------- @@ -2266,6 +2387,12 @@ void KeyValues::RecursiveLoadFromBuffer( char const *resourceName, CUtlBuffer &b CKeyErrorContext errorReport(this); bool wasQuoted; bool wasConditional; + if ( errorReport.GetStackLevel() > 100 ) + { + g_KeyValuesErrorStack.ReportError( "RecursiveLoadFromBuffer: recursion overflow" ); + return; + } + // keep this out of the stack until a key is parsed CKeyErrorContext errorKey( INVALID_KEY_SYMBOL ); @@ -2570,7 +2697,7 @@ bool KeyValues::ReadAsBinary( CUtlBuffer &buffer, int nStackDepth ) { char token[KEYVALUES_TOKEN_SIZE]; - buffer.GetString( token, KEYVALUES_TOKEN_SIZE-1 ); + buffer.GetString( token ); token[KEYVALUES_TOKEN_SIZE-1] = 0; dat->SetName( token ); } @@ -2586,7 +2713,7 @@ bool KeyValues::ReadAsBinary( CUtlBuffer &buffer, int nStackDepth ) case TYPE_STRING: { char token[KEYVALUES_TOKEN_SIZE]; - buffer.GetString( token, KEYVALUES_TOKEN_SIZE-1 ); + buffer.GetString( token ); token[KEYVALUES_TOKEN_SIZE-1] = 0; int len = Q_strlen( token ); @@ -3039,4 +3166,4 @@ bool CKeyValuesDumpContextAsDevMsg::KvWriteText( char const *szText ) Msg( "%s", szText ); } return true; -}
\ No newline at end of file +} diff --git a/mp/src/tier1/convar.cpp b/mp/src/tier1/convar.cpp index 956ad91d..772586f2 100644 --- a/mp/src/tier1/convar.cpp +++ b/mp/src/tier1/convar.cpp @@ -592,7 +592,7 @@ void ConCommand::Dispatch( const CCommand &command ) } // Command without callback!!! - AssertMsg( 0, ( "Encountered ConCommand '%s' without a callback!\n", GetName() ) ); + AssertMsg( 0, "Encountered ConCommand '%s' without a callback!\n", GetName() ); } @@ -777,10 +777,10 @@ void ConVar::InternalSetValue( const char *value ) Q_snprintf( tempVal,sizeof(tempVal), "%f", fNewValue ); val = tempVal; } - + // Redetermine value m_fValue = fNewValue; - m_nValue = ( int )( m_fValue ); + m_nValue = ( int )( fNewValue ); if ( !( m_nFlags & FCVAR_NEVER_AS_STRING ) ) { @@ -821,13 +821,17 @@ void ConVar::ChangeStringValue( const char *tempVal, float flOldValue ) *m_pszString = 0; } - // Invoke any necessary callback function - if ( m_fnChangeCallback ) + // If nothing has changed, don't do the callbacks. + if (V_strcmp(pszOldValue, m_pszString) != 0) { - m_fnChangeCallback( this, pszOldValue, flOldValue ); - } + // Invoke any necessary callback function + if ( m_fnChangeCallback ) + { + m_fnChangeCallback( this, pszOldValue, flOldValue ); + } - g_pCVar->CallGlobalChangeCallbacks( this, pszOldValue, flOldValue ); + g_pCVar->CallGlobalChangeCallbacks( this, pszOldValue, flOldValue ); + } stackfree( pszOldValue ); } diff --git a/mp/src/tier1/ilocalize.cpp b/mp/src/tier1/ilocalize.cpp index d95bbf29..66673e64 100644 --- a/mp/src/tier1/ilocalize.cpp +++ b/mp/src/tier1/ilocalize.cpp @@ -17,7 +17,8 @@ int ILocalize::ConvertANSIToUnicode(const char *ansi, wchar_t *unicode, int unicodeBufferSizeInBytes) { #ifdef POSIX - return Q_UTF8ToUnicode(ansi, unicode, unicodeBufferSizeInBytes); + // Q_UTF8ToUnicode returns the number of bytes. This function is expected to return the number of chars. + return Q_UTF8ToUnicode(ansi, unicode, unicodeBufferSizeInBytes) / sizeof( wchar_t ); #else int chars = MultiByteToWideChar(CP_UTF8, 0, ansi, -1, unicode, unicodeBufferSizeInBytes / sizeof(wchar_t)); unicode[(unicodeBufferSizeInBytes / sizeof(wchar_t)) - 1] = 0; @@ -256,4 +257,4 @@ void ILocalize::ConstructStringKeyValuesInternal(char *unicodeOutput, int unicod void ILocalize::ConstructStringKeyValuesInternal(wchar_t *unicodeOutput, int unicodeBufferSizeInBytes, const wchar_t *formatString, KeyValues *localizationVariables) { ConstructStringKeyValuesInternal_Impl<wchar_t>( unicodeOutput, unicodeBufferSizeInBytes, formatString, localizationVariables ); -}
\ No newline at end of file +} diff --git a/mp/src/tier1/interface.cpp b/mp/src/tier1/interface.cpp index c970e5c6..031a49a7 100644 --- a/mp/src/tier1/interface.cpp +++ b/mp/src/tier1/interface.cpp @@ -284,7 +284,7 @@ CSysModule *Sys_LoadModule( const char *pModuleName, Sys_Flags flags /* = SYS_NO int i = CommandLine()->FindParm( "-basedir" ); if ( i ) { - strcpy( szCwd, CommandLine()->GetParm( i+1 ) ); + V_strcpy_safe( szCwd, CommandLine()->GetParm( i + 1 ) ); } } if (szCwd[strlen(szCwd) - 1] == '/' || szCwd[strlen(szCwd) - 1] == '\\' ) diff --git a/mp/src/tier1/kvpacker.cpp b/mp/src/tier1/kvpacker.cpp index 3672a2d0..53f7672e 100644 --- a/mp/src/tier1/kvpacker.cpp +++ b/mp/src/tier1/kvpacker.cpp @@ -181,7 +181,7 @@ bool KVPacker::ReadAsBinary( KeyValues *pNode, CUtlBuffer &buffer ) if ( ePackType == PACKTYPE_NULLMARKER ) break; // no more peers - buffer.GetString( token, KEYVALUES_TOKEN_SIZE-1 ); + buffer.GetString( token ); token[KEYVALUES_TOKEN_SIZE-1] = 0; dat->SetName( token ); @@ -198,7 +198,7 @@ bool KVPacker::ReadAsBinary( KeyValues *pNode, CUtlBuffer &buffer ) } case PACKTYPE_STRING: { - buffer.GetString( token, KEYVALUES_TOKEN_SIZE-1 ); + buffer.GetString( token ); token[KEYVALUES_TOKEN_SIZE-1] = 0; dat->SetStringValue( token ); break; @@ -206,15 +206,26 @@ bool KVPacker::ReadAsBinary( KeyValues *pNode, CUtlBuffer &buffer ) case PACKTYPE_WSTRING: { int nLength = buffer.GetShort(); - wchar_t *pTemp = (wchar_t *)stackalloc( sizeof(wchar_t) * ( 1 + nLength ) ); - - for( int k = 0; k < nLength; ++ k ) + if ( nLength >= 0 && nLength*sizeof( uint16 ) <= (uint)buffer.GetBytesRemaining() ) { - pTemp[k] = buffer.GetShort(); - } - pTemp[ nLength ] = 0; + if ( nLength > 0 ) + { + wchar_t *pTemp = (wchar_t *)malloc( sizeof( wchar_t ) * (1 + nLength) ); + + for ( int k = 0; k < nLength; ++k ) + { + pTemp[k] = buffer.GetShort(); // ugly, but preserving existing behavior + } - dat->SetWString( NULL, pTemp ); + pTemp[nLength] = 0; + dat->SetWString( NULL, pTemp ); + + free( pTemp ); + } + else + dat->SetWString( NULL, L"" ); + + } break; } diff --git a/mp/src/tier1/lzmaDecoder.cpp b/mp/src/tier1/lzmaDecoder.cpp index d547c2bc..473f9c36 100644 --- a/mp/src/tier1/lzmaDecoder.cpp +++ b/mp/src/tier1/lzmaDecoder.cpp @@ -1,764 +1,348 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// // -// LZMA Codec. +// LZMA Codec interface for engine. // -// LZMA SDK 4.43 Copyright (c) 1999-2006 Igor Pavlov (2006-05-01) -// http://www.7-zip.org/ +// LZMA SDK 9.38 beta +// 2015-01-03 : Igor Pavlov : Public domain +// http://www.7-zip.org/ // -// Modified to use Source platform utilities and memory allocation overrides. -//=====================================================================================// +//========================================================================// + +#define _LZMADECODER_CPP #include "tier0/platform.h" +#include "tier0/basetypes.h" #include "tier0/dbg.h" + +#include "../utils/lzma/C/7zTypes.h" +#include "../utils/lzma/C/LzmaEnc.h" +#include "../utils/lzma/C/LzmaDec.h" + +// Ugly define to let us forward declare the anonymous-struct-typedef that is CLzmaDec in the header. +#define CLzmaDec_t CLzmaDec #include "tier1/lzmaDecoder.h" // memdbgon must be the last include file in a .cpp file!!! #include "tier0/memdbgon.h" -#ifndef _7ZIP_BYTE_DEFINED -#define _7ZIP_BYTE_DEFINED -typedef unsigned char Byte; -#endif - -#ifndef _7ZIP_UINT16_DEFINED -#define _7ZIP_UINT16_DEFINED -typedef unsigned short UInt16; -#endif - -#ifndef _7ZIP_UINT32_DEFINED -#define _7ZIP_UINT32_DEFINED -#ifdef _LZMA_UINT32_IS_ULONG -typedef unsigned long UInt32; -#else -typedef unsigned int UInt32; -#endif -#endif - -/* #define _LZMA_SYSTEM_SIZE_T */ -/* Use system's size_t. You can use it to enable 64-bit sizes supporting */ - -#ifndef _7ZIP_SIZET_DEFINED -#define _7ZIP_SIZET_DEFINED -#ifdef _LZMA_SYSTEM_SIZE_T -#include <stddef.h> -typedef size_t SizeT; -#else -typedef UInt32 SizeT; -#endif -#endif - -/* #define _LZMA_IN_CB */ -/* Use callback for input data */ - -/* #define _LZMA_OUT_READ */ -/* Use read function for output data */ - -#define _LZMA_PROB32 -/* It can increase speed on some 32-bit CPUs, -but memory usage will be doubled in that case */ - -/* #define _LZMA_LOC_OPT */ -/* Enable local speed optimizations inside code */ - -#ifdef _LZMA_PROB32 -#define CProb UInt32 -#else -#define CProb UInt16 -#endif - -#define LZMA_RESULT_OK 0 -#define LZMA_RESULT_DATA_ERROR 1 - -#ifdef _LZMA_IN_CB -typedef struct _ILzmaInCallback -{ - int (*Read)(void *object, const unsigned char **buffer, SizeT *bufferSize); -} ILzmaInCallback; -#endif - -#define LZMA_BASE_SIZE 1846 -#define LZMA_LIT_SIZE 768 +// Allocator to pass to LZMA functions +static void *SzAlloc(void *p, size_t size) { return malloc(size); } +static void SzFree(void *p, void *address) { free(address); } +static ISzAlloc g_Alloc = { SzAlloc, SzFree }; -#define LZMA_PROPERTIES_SIZE 5 - -typedef struct _CLzmaProperties +//----------------------------------------------------------------------------- +// Returns true if buffer is compressed. +//----------------------------------------------------------------------------- +/* static */ +bool CLZMA::IsCompressed( unsigned char *pInput ) { - int lc; - int lp; - int pb; -#ifdef _LZMA_OUT_READ - UInt32 DictionarySize; -#endif -}CLzmaProperties; - -int LzmaDecodeProperties(CLzmaProperties *propsRes, const unsigned char *propsData, int size); - -#define LzmaGetNumProbs(Properties) (LZMA_BASE_SIZE + (LZMA_LIT_SIZE << ((Properties)->lc + (Properties)->lp))) + lzma_header_t *pHeader = (lzma_header_t *)pInput; + if ( pHeader && pHeader->id == LZMA_ID ) + { + return true; + } -#define kLzmaNeedInitId (-2) + // unrecognized + return false; +} -typedef struct _CLzmaDecoderState +//----------------------------------------------------------------------------- +// Returns uncompressed size of compressed input buffer. Used for allocating output +// buffer for decompression. Returns 0 if input buffer is not compressed. +//----------------------------------------------------------------------------- +/* static */ +unsigned int CLZMA::GetActualSize( unsigned char *pInput ) { - CLzmaProperties Properties; - CProb *Probs; - -#ifdef _LZMA_IN_CB - const unsigned char *Buffer; - const unsigned char *BufferLim; -#endif - -#ifdef _LZMA_OUT_READ - unsigned char *Dictionary; - UInt32 Range; - UInt32 Code; - UInt32 DictionaryPos; - UInt32 GlobalPos; - UInt32 DistanceLimit; - UInt32 Reps[4]; - int State; - int RemainLen; - unsigned char TempDictionary[4]; -#endif -} CLzmaDecoderState; - -#ifdef _LZMA_OUT_READ -#define LzmaDecoderInit(vs) { (vs)->RemainLen = kLzmaNeedInitId; } -#endif - -int LzmaDecode(CLzmaDecoderState *vs, -#ifdef _LZMA_IN_CB - ILzmaInCallback *inCallback, -#else - const unsigned char *inStream, SizeT inSize, SizeT *inSizeProcessed, -#endif - unsigned char *outStream, SizeT outSize, SizeT *outSizeProcessed); - -#define kNumTopBits 24 -#define kTopValue ((UInt32)1 << kNumTopBits) - -#define kNumBitModelTotalBits 11 -#define kBitModelTotal (1 << kNumBitModelTotalBits) -#define kNumMoveBits 5 - -#define RC_READ_BYTE (*Buffer++) - -#define RC_INIT2 Code = 0; Range = 0xFFFFFFFF; \ -{ int i; for(i = 0; i < 5; i++) { RC_TEST; Code = (Code << 8) | RC_READ_BYTE; }} - -#ifdef _LZMA_IN_CB - -#define RC_TEST { if (Buffer == BufferLim) \ -{ SizeT size; int result = InCallback->Read(InCallback, &Buffer, &size); if (result != LZMA_RESULT_OK) return result; \ - BufferLim = Buffer + size; if (size == 0) return LZMA_RESULT_DATA_ERROR; }} - -#define RC_INIT Buffer = BufferLim = 0; RC_INIT2 - -#else - -#define RC_TEST { if (Buffer == BufferLim) return LZMA_RESULT_DATA_ERROR; } - -#define RC_INIT(buffer, bufferSize) Buffer = buffer; BufferLim = buffer + bufferSize; RC_INIT2 - -#endif - -#define RC_NORMALIZE if (Range < kTopValue) { RC_TEST; Range <<= 8; Code = (Code << 8) | RC_READ_BYTE; } - -#define IfBit0(p) RC_NORMALIZE; bound = (Range >> kNumBitModelTotalBits) * *(p); if (Code < bound) -#define UpdateBit0(p) Range = bound; *(p) += (kBitModelTotal - *(p)) >> kNumMoveBits; -#define UpdateBit1(p) Range -= bound; Code -= bound; *(p) -= (*(p)) >> kNumMoveBits; - -#define RC_GET_BIT2(p, mi, A0, A1) IfBit0(p) \ -{ UpdateBit0(p); mi <<= 1; A0; } else \ -{ UpdateBit1(p); mi = (mi + mi) + 1; A1; } - -#define RC_GET_BIT(p, mi) RC_GET_BIT2(p, mi, ; , ;) - -#define RangeDecoderBitTreeDecode(probs, numLevels, res) \ -{ int i = numLevels; res = 1; \ - do { CProb *p = probs + res; RC_GET_BIT(p, res) } while(--i != 0); \ - res -= (1 << numLevels); } - - -#define kNumPosBitsMax 4 -#define kNumPosStatesMax (1 << kNumPosBitsMax) - -#define kLenNumLowBits 3 -#define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumMidBits 3 -#define kLenNumMidSymbols (1 << kLenNumMidBits) -#define kLenNumHighBits 8 -#define kLenNumHighSymbols (1 << kLenNumHighBits) - -#define LenChoice 0 -#define LenChoice2 (LenChoice + 1) -#define LenLow (LenChoice2 + 1) -#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits)) -#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits)) -#define kNumLenProbs (LenHigh + kLenNumHighSymbols) + lzma_header_t *pHeader = (lzma_header_t *)pInput; + if ( pHeader && pHeader->id == LZMA_ID ) + { + return LittleLong( pHeader->actualSize ); + } + // unrecognized + return 0; +} -#define kNumStates 12 -#define kNumLitStates 7 +//----------------------------------------------------------------------------- +// Uncompress a buffer, Returns the uncompressed size. Caller must provide an +// adequate sized output buffer or memory corruption will occur. +//----------------------------------------------------------------------------- +/* static */ +unsigned int CLZMA::Uncompress( unsigned char *pInput, unsigned char *pOutput ) +{ + lzma_header_t *pHeader = (lzma_header_t *)pInput; + if ( pHeader->id != LZMA_ID ) + { + // not ours + return false; + } -#define kStartPosModelIndex 4 -#define kEndPosModelIndex 14 -#define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) + CLzmaDec state; -#define kNumPosSlotBits 6 -#define kNumLenToPosStates 4 + LzmaDec_Construct(&state); -#define kNumAlignBits 4 -#define kAlignTableSize (1 << kNumAlignBits) + if ( LzmaDec_Allocate(&state, pHeader->properties, LZMA_PROPS_SIZE, &g_Alloc) != SZ_OK ) + { + Assert( false ); + return 0; + } -#define kMatchMinLen 2 + // These are in/out variables + SizeT outProcessed = pHeader->actualSize; + SizeT inProcessed = pHeader->lzmaSize; + ELzmaStatus status; + SRes result = LzmaDecode( (Byte *)pOutput, &outProcessed, (Byte *)(pInput + sizeof( lzma_header_t ) ), + &inProcessed, (Byte *)pHeader->properties, LZMA_PROPS_SIZE, LZMA_FINISH_END, &status, &g_Alloc ); -#define IsMatch 0 -#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax)) -#define IsRepG0 (IsRep + kNumStates) -#define IsRepG1 (IsRepG0 + kNumStates) -#define IsRepG2 (IsRepG1 + kNumStates) -#define IsRep0Long (IsRepG2 + kNumStates) -#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax)) -#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) -#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex) -#define LenCoder (Align + kAlignTableSize) -#define RepLenCoder (LenCoder + kNumLenProbs) -#define Literal (RepLenCoder + kNumLenProbs) -#if Literal != LZMA_BASE_SIZE -StopCompilingDueBUG -#endif + LzmaDec_Free(&state, &g_Alloc); -int LzmaDecodeProperties(CLzmaProperties *propsRes, const unsigned char *propsData, int size) -{ - unsigned char prop0; - if (size < LZMA_PROPERTIES_SIZE) - return LZMA_RESULT_DATA_ERROR; - prop0 = propsData[0]; - if (prop0 >= (9 * 5 * 5)) - return LZMA_RESULT_DATA_ERROR; + if ( result != SZ_OK || pHeader->actualSize != outProcessed ) { - for (propsRes->pb = 0; prop0 >= (9 * 5); propsRes->pb++, prop0 -= (9 * 5)); - for (propsRes->lp = 0; prop0 >= 9; propsRes->lp++, prop0 -= 9); - propsRes->lc = prop0; - /* - unsigned char remainder = (unsigned char)(prop0 / 9); - propsRes->lc = prop0 % 9; - propsRes->pb = remainder / 5; - propsRes->lp = remainder % 5; - */ + Warning( "LZMA Decompression failed (%i)\n", result ); + return 0; } -#ifdef _LZMA_OUT_READ - { - int i; - propsRes->DictionarySize = 0; - for (i = 0; i < 4; i++) - propsRes->DictionarySize += (UInt32)(propsData[1 + i]) << (i * 8); - if (propsRes->DictionarySize == 0) - propsRes->DictionarySize = 1; - } -#endif - return LZMA_RESULT_OK; + return outProcessed; } -#define kLzmaStreamWasFinishedId (-1) +CLZMAStream::CLZMAStream() + : m_pDecoderState( NULL ), + m_nActualSize( 0 ), + m_nActualBytesRead ( 0 ), + m_nCompressedSize( 0 ), + m_nCompressedBytesRead ( 0 ), + m_bParsedHeader( false ), + m_bZIPStyleHeader( false ) +{} + +CLZMAStream::~CLZMAStream() +{ + FreeDecoderState(); +} -int LzmaDecode(CLzmaDecoderState *vs, -#ifdef _LZMA_IN_CB - ILzmaInCallback *InCallback, -#else - const unsigned char *inStream, SizeT inSize, SizeT *inSizeProcessed, -#endif - unsigned char *outStream, SizeT outSize, SizeT *outSizeProcessed) +void CLZMAStream::FreeDecoderState() { - CProb *p = vs->Probs; - SizeT nowPos = 0; - Byte previousByte = 0; - UInt32 posStateMask = (1 << (vs->Properties.pb)) - 1; - UInt32 literalPosMask = (1 << (vs->Properties.lp)) - 1; - int lc = vs->Properties.lc; - -#ifdef _LZMA_OUT_READ - - UInt32 Range = vs->Range; - UInt32 Code = vs->Code; -#ifdef _LZMA_IN_CB - const Byte *Buffer = vs->Buffer; - const Byte *BufferLim = vs->BufferLim; -#else - const Byte *Buffer = inStream; - const Byte *BufferLim = inStream + inSize; -#endif - int state = vs->State; - UInt32 rep0 = vs->Reps[0], rep1 = vs->Reps[1], rep2 = vs->Reps[2], rep3 = vs->Reps[3]; - int len = vs->RemainLen; - UInt32 globalPos = vs->GlobalPos; - UInt32 distanceLimit = vs->DistanceLimit; - - Byte *dictionary = vs->Dictionary; - UInt32 dictionarySize = vs->Properties.DictionarySize; - UInt32 dictionaryPos = vs->DictionaryPos; - - Byte tempDictionary[4]; - -#ifndef _LZMA_IN_CB - *inSizeProcessed = 0; -#endif - *outSizeProcessed = 0; - if (len == kLzmaStreamWasFinishedId) - return LZMA_RESULT_OK; - - if (dictionarySize == 0) + if ( m_pDecoderState ) { - dictionary = tempDictionary; - dictionarySize = 1; - tempDictionary[0] = vs->TempDictionary[0]; + LzmaDec_Free( m_pDecoderState, &g_Alloc ); + m_pDecoderState = NULL; } +} - if (len == kLzmaNeedInitId) - { - { - UInt32 numProbs = Literal + ((UInt32)LZMA_LIT_SIZE << (lc + vs->Properties.lp)); - UInt32 i; - for (i = 0; i < numProbs; i++) - p[i] = kBitModelTotal >> 1; - rep0 = rep1 = rep2 = rep3 = 1; - state = 0; - globalPos = 0; - distanceLimit = 0; - dictionaryPos = 0; - dictionary[dictionarySize - 1] = 0; -#ifdef _LZMA_IN_CB - RC_INIT; -#else - RC_INIT(inStream, inSize); -#endif - } - len = 0; - } - while(len != 0 && nowPos < outSize) +bool CLZMAStream::CreateDecoderState( const unsigned char *pProperties ) +{ + if ( m_pDecoderState ) { - UInt32 pos = dictionaryPos - rep0; - if (pos >= dictionarySize) - pos += dictionarySize; - outStream[nowPos++] = dictionary[dictionaryPos] = dictionary[pos]; - if (++dictionaryPos == dictionarySize) - dictionaryPos = 0; - len--; + Assert( !m_pDecoderState ); + FreeDecoderState(); } - if (dictionaryPos == 0) - previousByte = dictionary[dictionarySize - 1]; - else - previousByte = dictionary[dictionaryPos - 1]; - -#else /* if !_LZMA_OUT_READ */ - int state = 0; - UInt32 rep0 = 1, rep1 = 1, rep2 = 1, rep3 = 1; - int len = 0; - const Byte *Buffer; - const Byte *BufferLim; - UInt32 Range; - UInt32 Code; - -#ifndef _LZMA_IN_CB - *inSizeProcessed = 0; -#endif - *outSizeProcessed = 0; + m_pDecoderState = new CLzmaDec(); + LzmaDec_Construct( m_pDecoderState ); + if ( LzmaDec_Allocate( m_pDecoderState, pProperties, LZMA_PROPS_SIZE, &g_Alloc) != SZ_OK ) { - UInt32 i; - UInt32 numProbs = Literal + ((UInt32)LZMA_LIT_SIZE << (lc + vs->Properties.lp)); - for (i = 0; i < numProbs; i++) - p[i] = kBitModelTotal >> 1; + AssertMsg( false, "Failed to allocate lzma decoder state" ); + m_pDecoderState = NULL; + return false; } -#ifdef _LZMA_IN_CB - RC_INIT; -#else - RC_INIT(inStream, inSize); -#endif + LzmaDec_Init( m_pDecoderState ); -#endif /* _LZMA_OUT_READ */ + return true; +} - while(nowPos < outSize) +// Attempt to read up to nMaxInputBytes from the compressed stream, writing up to nMaxOutputBytes to pOutput. +// Returns false if read stops due to an error. +bool CLZMAStream::Read( unsigned char *pInput, unsigned int nMaxInputBytes, + unsigned char *pOutput, unsigned int nMaxOutputBytes, + /* out */ unsigned int &nCompressedBytesRead, + /* out */ unsigned int &nOutputBytesWritten ) +{ + nCompressedBytesRead = 0; + nOutputBytesWritten = 0; + bool bStartedWithHeader = m_bParsedHeader; + + // Check for initial chunk of data + if ( !m_bParsedHeader ) { - CProb *prob; - UInt32 bound; - int posState = (int)( - (nowPos -#ifdef _LZMA_OUT_READ - + globalPos -#endif - ) - & posStateMask); - - prob = p + IsMatch + (state << kNumPosBitsMax) + posState; - IfBit0(prob) + unsigned int nBytesConsumed = 0; + eHeaderParse parseResult = TryParseHeader( pInput, nMaxInputBytes, nBytesConsumed ); + + if ( parseResult == eHeaderParse_NeedMoreBytes ) { - int symbol = 1; - UpdateBit0(prob) - prob = p + Literal + (LZMA_LIT_SIZE * - ((( - (nowPos -#ifdef _LZMA_OUT_READ - + globalPos -#endif - ) - & literalPosMask) << lc) + (previousByte >> (8 - lc)))); - - if (state >= kNumLitStates) - { - int matchByte; -#ifdef _LZMA_OUT_READ - UInt32 pos = dictionaryPos - rep0; - if (pos >= dictionarySize) - pos += dictionarySize; - matchByte = dictionary[pos]; -#else - matchByte = outStream[nowPos - rep0]; -#endif - do - { - int bit; - CProb *probLit; - matchByte <<= 1; - bit = (matchByte & 0x100); - probLit = prob + 0x100 + bit + symbol; - RC_GET_BIT2(probLit, symbol, if (bit != 0) break, if (bit == 0) break) - } - while (symbol < 0x100); - } - while (symbol < 0x100) - { - CProb *probLit = prob + symbol; - RC_GET_BIT(probLit, symbol) - } - previousByte = (Byte)symbol; - - outStream[nowPos++] = previousByte; -#ifdef _LZMA_OUT_READ - if (distanceLimit < dictionarySize) - distanceLimit++; - - dictionary[dictionaryPos] = previousByte; - if (++dictionaryPos == dictionarySize) - dictionaryPos = 0; -#endif - if (state < 4) state = 0; - else if (state < 10) state -= 3; - else state -= 6; + // Not an error, just need more data to continue + return true; } -else -{ - UpdateBit1(prob); - prob = p + IsRep + state; - IfBit0(prob) - { - UpdateBit0(prob); - rep3 = rep2; - rep2 = rep1; - rep1 = rep0; - state = state < kNumLitStates ? 0 : 3; - prob = p + LenCoder; - } - else - { - UpdateBit1(prob); - prob = p + IsRepG0 + state; - IfBit0(prob) - { - UpdateBit0(prob); - prob = p + IsRep0Long + (state << kNumPosBitsMax) + posState; - IfBit0(prob) - { -#ifdef _LZMA_OUT_READ - UInt32 pos; -#endif - UpdateBit0(prob); - -#ifdef _LZMA_OUT_READ - if (distanceLimit == 0) -#else - if (nowPos == 0) -#endif - return LZMA_RESULT_DATA_ERROR; - - state = state < kNumLitStates ? 9 : 11; -#ifdef _LZMA_OUT_READ - pos = dictionaryPos - rep0; - if (pos >= dictionarySize) - pos += dictionarySize; - previousByte = dictionary[pos]; - dictionary[dictionaryPos] = previousByte; - if (++dictionaryPos == dictionarySize) - dictionaryPos = 0; -#else - previousByte = outStream[nowPos - rep0]; -#endif - outStream[nowPos++] = previousByte; -#ifdef _LZMA_OUT_READ - if (distanceLimit < dictionarySize) - distanceLimit++; -#endif - - continue; - } - else - { - UpdateBit1(prob); - } - } - else + else if ( parseResult != eHeaderParse_OK ) { - UInt32 distance; - UpdateBit1(prob); - prob = p + IsRepG1 + state; - IfBit0(prob) - { - UpdateBit0(prob); - distance = rep1; - } - else - { - UpdateBit1(prob); - prob = p + IsRepG2 + state; - IfBit0(prob) - { - UpdateBit0(prob); - distance = rep2; - } - else - { - UpdateBit1(prob); - distance = rep3; - rep3 = rep2; - } - rep2 = rep1; - } - rep1 = rep0; - rep0 = distance; + Assert( parseResult == eHeaderParse_Fail ); + // Invalid header + return false; } - state = state < kNumLitStates ? 8 : 11; - prob = p + RepLenCoder; - } - { - int numBits, offset; - CProb *probLen = prob + LenChoice; - IfBit0(probLen) - { - UpdateBit0(probLen); - probLen = prob + LenLow + (posState << kLenNumLowBits); - offset = 0; - numBits = kLenNumLowBits; - } - else + + // Header consumed, fall through to continue read after it + nCompressedBytesRead += nBytesConsumed; + pInput += nBytesConsumed; + nMaxInputBytes -= nBytesConsumed; + } + + // These are input ( available size ) *and* output ( size processed ) vars for lzma + SizeT expectedInputRemaining = m_nCompressedSize - Min( m_nCompressedBytesRead + nCompressedBytesRead, m_nCompressedSize ); + SizeT expectedOutputRemaining = m_nActualSize - m_nActualBytesRead; + SizeT inSize = Min( (SizeT)nMaxInputBytes, expectedInputRemaining ); + SizeT outSize = Min( (SizeT)nMaxOutputBytes, expectedOutputRemaining ); + ELzmaStatus status; + ELzmaFinishMode finishMode = LZMA_FINISH_ANY; + if ( inSize == expectedInputRemaining && outSize == expectedOutputRemaining ) + { + // Expect to finish decoding this call. + finishMode = LZMA_FINISH_END; + } + SRes result = LzmaDec_DecodeToBuf( m_pDecoderState, pOutput, &outSize, + pInput, &inSize, finishMode, &status ); + + // DevMsg("[%p] Running lzmaDecode:\n" + // " pInput: %p\n" + // " nMaxInputBytes: %i\n" + // " pOutput: %p\n" + // " nMaxOutputBytes: %u\n" + // " inSize: %u\n" + // " outSize: %u\n" + // " result: %u\n" + // " status: %i\n" + // " m_nActualSize: %u\n" + // " m_nActualBytesRead: %u\n", + // this, pInput, nMaxInputBytes, pOutput, nMaxOutputBytes, + // inSize, outSize, result, status, m_nActualSize, m_nActualBytesRead); + + if ( result != SZ_OK ) + { + if ( !bStartedWithHeader ) { - UpdateBit1(probLen); - probLen = prob + LenChoice2; - IfBit0(probLen) - { - UpdateBit0(probLen); - probLen = prob + LenMid + (posState << kLenNumMidBits); - offset = kLenNumLowSymbols; - numBits = kLenNumMidBits; - } - else - { - UpdateBit1(probLen); - probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; - numBits = kLenNumHighBits; - } + // If we're returning false, we need to pretend we didn't consume anything. + FreeDecoderState(); + m_bParsedHeader = false; } - RangeDecoderBitTreeDecode(probLen, numBits, len); - len += offset; - } - - if (state < 4) - { - int posSlot; - state += kNumLitStates; - prob = p + PosSlot + - ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << - kNumPosSlotBits); - RangeDecoderBitTreeDecode(prob, kNumPosSlotBits, posSlot); - if (posSlot >= kStartPosModelIndex) - { - int numDirectBits = ((posSlot >> 1) - 1); - rep0 = (2 | ((UInt32)posSlot & 1)); - if (posSlot < kEndPosModelIndex) - { - rep0 <<= numDirectBits; - prob = p + SpecPos + rep0 - posSlot - 1; - } - else - { - numDirectBits -= kNumAlignBits; - do - { - RC_NORMALIZE - Range >>= 1; - rep0 <<= 1; - if (Code >= Range) - { - Code -= Range; - rep0 |= 1; - } - } - while (--numDirectBits != 0); - prob = p + Align; - rep0 <<= kNumAlignBits; - numDirectBits = kNumAlignBits; - } - { - int i = 1; - int mi = 1; - do - { - CProb *prob3 = prob + mi; - RC_GET_BIT2(prob3, mi, ; , rep0 |= i); - i <<= 1; - } - while(--numDirectBits != 0); - } - } - else - rep0 = posSlot; - if (++rep0 == (UInt32)(0)) - { - /* it's for stream version */ - len = kLzmaStreamWasFinishedId; - break; - } - } - - len += kMatchMinLen; -#ifdef _LZMA_OUT_READ - if (rep0 > distanceLimit) -#else - if (rep0 > nowPos) -#endif - return LZMA_RESULT_DATA_ERROR; - -#ifdef _LZMA_OUT_READ - if (dictionarySize - distanceLimit > (UInt32)len) - distanceLimit += len; - else - distanceLimit = dictionarySize; -#endif - - do - { -#ifdef _LZMA_OUT_READ - UInt32 pos = dictionaryPos - rep0; - if (pos >= dictionarySize) - pos += dictionarySize; - previousByte = dictionary[pos]; - dictionary[dictionaryPos] = previousByte; - if (++dictionaryPos == dictionarySize) - dictionaryPos = 0; -#else - previousByte = outStream[nowPos - rep0]; -#endif - len--; - outStream[nowPos++] = previousByte; - } - while(len != 0 && nowPos < outSize); -} + return false; } - RC_NORMALIZE; - -#ifdef _LZMA_OUT_READ - vs->Range = Range; - vs->Code = Code; - vs->DictionaryPos = dictionaryPos; - vs->GlobalPos = globalPos + (UInt32)nowPos; - vs->DistanceLimit = distanceLimit; - vs->Reps[0] = rep0; - vs->Reps[1] = rep1; - vs->Reps[2] = rep2; - vs->Reps[3] = rep3; - vs->State = state; - vs->RemainLen = len; - vs->TempDictionary[0] = tempDictionary[0]; -#endif - -#ifdef _LZMA_IN_CB - vs->Buffer = Buffer; - vs->BufferLim = BufferLim; -#else - *inSizeProcessed = (SizeT)(Buffer - inStream); -#endif - *outSizeProcessed = nowPos; - return LZMA_RESULT_OK; + + nCompressedBytesRead += inSize; + nOutputBytesWritten += outSize; + + m_nCompressedBytesRead += nCompressedBytesRead; + m_nActualBytesRead += nOutputBytesWritten; + + Assert( m_nCompressedBytesRead <= m_nCompressedSize ); + return true; } -//----------------------------------------------------------------------------- -// Returns true if buffer is compressed. -//----------------------------------------------------------------------------- -bool CLZMA::IsCompressed( unsigned char *pInput ) +bool CLZMAStream::GetExpectedBytesRemaining( /* out */ unsigned int &nBytesRemaining ) { - lzma_header_t *pHeader = (lzma_header_t *)pInput; - if ( pHeader && pHeader->id == LZMA_ID ) - { - return true; + if ( !m_bParsedHeader && !m_bZIPStyleHeader ) { + return false; } - // unrecognized - return false; + nBytesRemaining = m_nActualSize - m_nActualBytesRead; + + return true; } -//----------------------------------------------------------------------------- -// Returns uncompressed size of compressed input buffer. Used for allocating output -// buffer for decompression. Returns 0 if input buffer is not compressed. -//----------------------------------------------------------------------------- -unsigned int CLZMA::GetActualSize( unsigned char *pInput ) +void CLZMAStream::InitZIPHeader( unsigned int nCompressedSize, unsigned int nOriginalSize ) { - lzma_header_t *pHeader = (lzma_header_t *)pInput; - if ( pHeader && pHeader->id == LZMA_ID ) + if ( m_bParsedHeader || m_bZIPStyleHeader ) { - return LittleLong( pHeader->actualSize ); + AssertMsg( !m_bParsedHeader && !m_bZIPStyleHeader, + "LZMA Stream: InitZIPHeader() called on stream past header" ); + return; } - // unrecognized - return 0; + m_nCompressedSize = nCompressedSize; + m_nActualSize = nOriginalSize; + // Signal to TryParseHeader to expect a zip-style header (which wont have the size values) + m_bZIPStyleHeader = true; } -//----------------------------------------------------------------------------- -// Uncompress a buffer, Returns the uncompressed size. Caller must provide an -// adequate sized output buffer or memory corruption will occur. -//----------------------------------------------------------------------------- -unsigned int CLZMA::Uncompress( unsigned char *pInput, unsigned char *pOutput ) +CLZMAStream::eHeaderParse CLZMAStream::TryParseHeader( unsigned char *pInput, unsigned int nBytesAvailable, /* out */ unsigned int &nBytesConsumed ) { - unsigned int actualSize = GetActualSize( pInput ); - if ( !actualSize ) + nBytesConsumed = 0; + + if ( m_bParsedHeader ) { - // unrecognized - return 0; + AssertMsg( !m_bParsedHeader, "CLZMAStream::ReadSourceHeader called on already initialized stream" ); + return eHeaderParse_Fail; } - CLzmaDecoderState state; - if ( LzmaDecodeProperties( &state.Properties, ((lzma_header_t *)pInput)->properties, LZMA_PROPERTIES_SIZE ) != LZMA_RESULT_OK ) + if ( m_bZIPStyleHeader ) { - Assert( 0 ); - } - state.Probs = (CProb *)malloc( LzmaGetNumProbs( &state.Properties ) * sizeof( CProb ) ); + // ZIP Spec, 5.8.8 + // LZMA Version Information 2 bytes + // LZMA Properties Size 2 bytes + // LZMA Properties Data variable, defined by "LZMA Properties Size" + + if ( nBytesAvailable < 4 ) + { + // No error, but need more input to continue + return eHeaderParse_NeedMoreBytes; + } - unsigned int lzmaSize = LittleLong( ((lzma_header_t *)pInput)->lzmaSize ); + // Should probably check this + // unsigned char nLZMAVer[2] = { pInput[0], pInput[1] }; - SizeT inProcessed; - SizeT outProcessed; - int result = LzmaDecode( &state, pInput + sizeof( lzma_header_t ), lzmaSize, &inProcessed, pOutput, actualSize, &outProcessed ); + uint16 nLZMAPropertiesSize = LittleWord( *(uint16 *)(pInput + 2) ); - free( state.Probs ); + nBytesConsumed += 4; - if ( result != LZMA_RESULT_OK || outProcessed != (SizeT)actualSize ) + if ( nLZMAPropertiesSize != LZMA_PROPS_SIZE ) + { + Warning( "LZMA stream: Unexpected LZMA properties size: %hu, expecting %u. Version mismatch?\n", + nLZMAPropertiesSize, LZMA_PROPS_SIZE ); + return eHeaderParse_Fail; + } + + if ( nBytesAvailable < static_cast<unsigned int>(nLZMAPropertiesSize) + 4 ) + { + return eHeaderParse_NeedMoreBytes; + } + + // Looks reasonable, try to parse + if ( !CreateDecoderState( (Byte *)pInput + 4 ) ) + { + AssertMsg( false, "Failed decoding Lzma properties" ); + return eHeaderParse_Fail; + } + + nBytesConsumed += nLZMAPropertiesSize; + } + else { - Assert( 0 ); - return 0; + // Else native source engine style header + if ( nBytesAvailable < sizeof( lzma_header_t ) ) + { + // need more input to continue + return eHeaderParse_NeedMoreBytes; + } + + m_nActualSize = CLZMA::GetActualSize( pInput ); + + if ( !m_nActualSize ) + { + // unrecognized + Warning( "Unrecognized LZMA data\n" ); + return eHeaderParse_Fail; + } + + if ( !CreateDecoderState( ((lzma_header_t *)pInput)->properties ) ) + { + AssertMsg( false, "Failed decoding Lzma properties" ); + return eHeaderParse_Fail; + } + + m_nCompressedSize = LittleLong( ((lzma_header_t *)pInput)->lzmaSize ) + sizeof( lzma_header_t ); + nBytesConsumed += sizeof( lzma_header_t ); } - return outProcessed; + m_bParsedHeader = true; + return eHeaderParse_OK; } - diff --git a/mp/src/tier1/strtools.cpp b/mp/src/tier1/strtools.cpp index ad301750..2c5f72fa 100644 --- a/mp/src/tier1/strtools.cpp +++ b/mp/src/tier1/strtools.cpp @@ -1251,7 +1251,7 @@ bool Q_RemoveAllEvilCharacters( char *pch ) int cch = Q_strlen( pch ); int cubDest = (cch + 1 ) * sizeof( wchar_t ); wchar_t *pwch = (wchar_t *)stackalloc( cubDest ); - int cwch = Q_UTF8ToUnicode( pch, pwch, cubDest ); + int cwch = Q_UTF8ToUnicode( pch, pwch, cubDest ) / sizeof( wchar_t ); bool bStrippedWhitespace = false; @@ -1289,8 +1289,13 @@ bool Q_RemoveAllEvilCharacters( char *pch ) //----------------------------------------------------------------------------- bool Q_StripPrecedingAndTrailingWhitespaceW( wchar_t *pwch ) { - // duplicate on stack int cch = Q_wcslen( pwch ); + + // Early out and don't convert if we don't have any chars or leading/trailing ws. + if ( ( cch < 1 ) || ( !iswspace( pwch[ 0 ] ) && !iswspace( pwch[ cch - 1 ] ) ) ) + return false; + + // duplicate on stack int cubDest = ( cch + 1 ) * sizeof( wchar_t ); wchar_t *pwchT = (wchar_t *)stackalloc( cubDest ); Q_wcsncpy( pwchT, pwch, cubDest ); @@ -1340,11 +1345,16 @@ bool Q_AggressiveStripPrecedingAndTrailingWhitespaceW( wchar_t *pwch ) //----------------------------------------------------------------------------- bool Q_StripPrecedingAndTrailingWhitespace( char *pch ) { - // convert to unicode int cch = Q_strlen( pch ); + + // Early out and don't convert if we don't have any chars or leading/trailing ws. + if ( ( cch < 1 ) || ( !isspace( (unsigned char)pch[ 0 ] ) && !isspace( (unsigned char)pch[ cch - 1 ] ) ) ) + return false; + + // convert to unicode int cubDest = (cch + 1 ) * sizeof( wchar_t ); wchar_t *pwch = (wchar_t *)stackalloc( cubDest ); - int cwch = Q_UTF8ToUnicode( pch, pwch, cubDest ); + int cwch = Q_UTF8ToUnicode( pch, pwch, cubDest ) / sizeof( wchar_t ); bool bStrippedWhitespace = false; pwch = StripWhitespaceWorker( cwch-1, pwch, &bStrippedWhitespace, false /* not aggressive */ ); @@ -1367,7 +1377,7 @@ bool Q_AggressiveStripPrecedingAndTrailingWhitespace( char *pch ) int cch = Q_strlen( pch ); int cubDest = (cch + 1 ) * sizeof( wchar_t ); wchar_t *pwch = (wchar_t *)stackalloc( cubDest ); - int cwch = Q_UTF8ToUnicode( pch, pwch, cubDest ); + int cwch = Q_UTF8ToUnicode( pch, pwch, cubDest ) / sizeof( wchar_t ); bool bStrippedWhitespace = false; pwch = StripWhitespaceWorker( cwch-1, pwch, &bStrippedWhitespace, true /* is aggressive */ ); @@ -1381,72 +1391,10 @@ bool Q_AggressiveStripPrecedingAndTrailingWhitespace( char *pch ) return bStrippedWhitespace; } - -//----------------------------------------------------------------------------- -// Purpose: Converts a UTF8 string into a unicode string -//----------------------------------------------------------------------------- -int V_UTF8ToUnicode( const char *pUTF8, wchar_t *pwchDest, int cubDestSizeInBytes ) -{ - // pwchDest can be null to allow for getting the length of the string - if ( cubDestSizeInBytes > 0 ) - { - AssertValidWritePtr(pwchDest); - pwchDest[0] = 0; - } - - if ( !pUTF8 ) - return 0; - - AssertValidStringPtr(pUTF8); - -#ifdef _WIN32 - int cchResult = MultiByteToWideChar( CP_UTF8, 0, pUTF8, -1, pwchDest, cubDestSizeInBytes / sizeof(wchar_t) ); -#elif POSIX - int cchResult = mbstowcs( pwchDest, pUTF8, cubDestSizeInBytes / sizeof(wchar_t) ) + 1; -#endif - - if ( cubDestSizeInBytes > 0 ) - { - pwchDest[(cubDestSizeInBytes / sizeof(wchar_t)) - 1] = 0; - } - - return cchResult; -} - -//----------------------------------------------------------------------------- -// Purpose: Converts a unicode string into a UTF8 (standard) string -//----------------------------------------------------------------------------- -int V_UnicodeToUTF8( const wchar_t *pUnicode, char *pUTF8, int cubDestSizeInBytes ) -{ - //AssertValidStringPtr(pUTF8, cubDestSizeInBytes); // no, we are sometimes pasing in NULL to fetch the length of the buffer needed. - AssertValidReadPtr(pUnicode); - - if ( cubDestSizeInBytes > 0 ) - { - pUTF8[0] = 0; - } - -#ifdef _WIN32 - int cchResult = WideCharToMultiByte( CP_UTF8, 0, pUnicode, -1, pUTF8, cubDestSizeInBytes, NULL, NULL ); -#elif POSIX - int cchResult = 0; - if ( pUnicode && pUTF8 ) - cchResult = wcstombs( pUTF8, pUnicode, cubDestSizeInBytes ) + 1; -#endif - - if ( cubDestSizeInBytes > 0 ) - { - pUTF8[cubDestSizeInBytes - 1] = 0; - } - - return cchResult; -} - - //----------------------------------------------------------------------------- // Purpose: Converts a ucs2 string to a unicode (wchar_t) one, no-op on win32 //----------------------------------------------------------------------------- -int V_UCS2ToUnicode( const ucs2 *pUCS2, wchar_t *pUnicode, int cubDestSizeInBytes ) +int _V_UCS2ToUnicode( const ucs2 *pUCS2, wchar_t *pUnicode, int cubDestSizeInBytes ) { Assert( cubDestSizeInBytes >= sizeof( *pUnicode ) ); AssertValidWritePtr(pUnicode); @@ -1455,7 +1403,7 @@ int V_UCS2ToUnicode( const ucs2 *pUCS2, wchar_t *pUnicode, int cubDestSizeInByte pUnicode[0] = 0; #ifdef _WIN32 int cchResult = V_wcslen( pUCS2 ); - Q_memcpy( pUnicode, pUCS2, cubDestSizeInBytes ); + V_memcpy( pUnicode, pUCS2, cubDestSizeInBytes ); #else iconv_t conv_t = iconv_open( "UCS-4LE", "UCS-2LE" ); int cchResult = -1; @@ -1486,7 +1434,7 @@ int V_UCS2ToUnicode( const ucs2 *pUCS2, wchar_t *pUnicode, int cubDestSizeInByte //----------------------------------------------------------------------------- // Purpose: Converts a wchar_t string into a UCS2 string -noop on windows //----------------------------------------------------------------------------- -int V_UnicodeToUCS2( const wchar_t *pUnicode, int cubSrcInBytes, char *pUCS2, int cubDestSizeInBytes ) +int _V_UnicodeToUCS2( const wchar_t *pUnicode, int cubSrcInBytes, char *pUCS2, int cubDestSizeInBytes ) { #ifdef _WIN32 // Figure out which buffer is smaller and convert from bytes to character @@ -1512,6 +1460,8 @@ int V_UnicodeToUCS2( const wchar_t *pUnicode, int cubSrcInBytes, char *pUCS2, in else cchResult = cubSrcInBytes / sizeof( wchar_t ); } +#else + #error Must be implemented for this platform #endif return cchResult; } @@ -1520,7 +1470,7 @@ int V_UnicodeToUCS2( const wchar_t *pUnicode, int cubSrcInBytes, char *pUCS2, in //----------------------------------------------------------------------------- // Purpose: Converts a ucs-2 (windows wchar_t) string into a UTF8 (standard) string //----------------------------------------------------------------------------- -int V_UCS2ToUTF8( const ucs2 *pUCS2, char *pUTF8, int cubDestSizeInBytes ) +int _V_UCS2ToUTF8( const ucs2 *pUCS2, char *pUTF8, int cubDestSizeInBytes ) { AssertValidStringPtr(pUTF8, cubDestSizeInBytes); AssertValidReadPtr(pUCS2); @@ -1574,7 +1524,7 @@ int V_UCS2ToUTF8( const ucs2 *pUCS2, char *pUTF8, int cubDestSizeInBytes ) //----------------------------------------------------------------------------- // Purpose: Converts a UTF8 to ucs-2 (windows wchar_t) //----------------------------------------------------------------------------- -int V_UTF8ToUCS2( const char *pUTF8, int cubSrcInBytes, ucs2 *pUCS2, int cubDestSizeInBytes ) +int _V_UTF8ToUCS2( const char *pUTF8, int cubSrcInBytes, ucs2 *pUCS2, int cubDestSizeInBytes ) { Assert( cubDestSizeInBytes >= sizeof(pUCS2[0]) ); AssertValidStringPtr(pUTF8, cubDestSizeInBytes); diff --git a/mp/src/tier1/strtools_unicode.cpp b/mp/src/tier1/strtools_unicode.cpp new file mode 100644 index 00000000..77685ec3 --- /dev/null +++ b/mp/src/tier1/strtools_unicode.cpp @@ -0,0 +1,572 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +// +//=============================================================================// + +#include <limits.h> +#include "tier0/dbg.h" +#include "tier1/strtools.h" + +// This code was copied from steam +#define DbgAssert Assert + +//----------------------------------------------------------------------------- +// Purpose: determine if a uchar32 represents a valid Unicode code point +//----------------------------------------------------------------------------- +bool Q_IsValidUChar32( uchar32 uVal ) +{ + // Values > 0x10FFFF are explicitly invalid; ditto for UTF-16 surrogate halves, + // values ending in FFFE or FFFF, or values in the 0x00FDD0-0x00FDEF reserved range + return ( uVal < 0x110000u ) && ( (uVal - 0x00D800u) > 0x7FFu ) && ( (uVal & 0xFFFFu) < 0xFFFEu ) && ( ( uVal - 0x00FDD0u ) > 0x1Fu ); +} + +//----------------------------------------------------------------------------- +// Purpose: return number of UTF-8 bytes required to encode a Unicode code point +//----------------------------------------------------------------------------- +int Q_UChar32ToUTF8Len( uchar32 uVal ) +{ + DbgAssert( Q_IsValidUChar32( uVal ) ); + if ( uVal <= 0x7F ) + return 1; + if ( uVal <= 0x7FF ) + return 2; + if ( uVal <= 0xFFFF ) + return 3; + return 4; +} + + +//----------------------------------------------------------------------------- +// Purpose: return number of UTF-16 elements required to encode a Unicode code point +//----------------------------------------------------------------------------- +int Q_UChar32ToUTF16Len( uchar32 uVal ) +{ + DbgAssert( Q_IsValidUChar32( uVal ) ); + if ( uVal <= 0xFFFF ) + return 1; + return 2; +} + + +//----------------------------------------------------------------------------- +// Purpose: encode Unicode code point as UTF-8, returns number of bytes written +//----------------------------------------------------------------------------- +int Q_UChar32ToUTF8( uchar32 uVal, char *pUTF8Out ) +{ + DbgAssert( Q_IsValidUChar32( uVal ) ); + if ( uVal <= 0x7F ) + { + pUTF8Out[0] = (unsigned char) uVal; + return 1; + } + if ( uVal <= 0x7FF ) + { + pUTF8Out[0] = (unsigned char)(uVal >> 6) | 0xC0; + pUTF8Out[1] = (unsigned char)(uVal & 0x3F) | 0x80; + return 2; + } + if ( uVal <= 0xFFFF ) + { + pUTF8Out[0] = (unsigned char)(uVal >> 12) | 0xE0; + pUTF8Out[1] = (unsigned char)((uVal >> 6) & 0x3F) | 0x80; + pUTF8Out[2] = (unsigned char)(uVal & 0x3F) | 0x80; + return 3; + } + pUTF8Out[0] = (unsigned char)((uVal >> 18) & 0x07) | 0xF0; + pUTF8Out[1] = (unsigned char)((uVal >> 12) & 0x3F) | 0x80; + pUTF8Out[2] = (unsigned char)((uVal >> 6) & 0x3F) | 0x80; + pUTF8Out[3] = (unsigned char)(uVal & 0x3F) | 0x80; + return 4; +} + +//----------------------------------------------------------------------------- +// Purpose: encode Unicode code point as UTF-16, returns number of elements written +//----------------------------------------------------------------------------- +int Q_UChar32ToUTF16( uchar32 uVal, uchar16 *pUTF16Out ) +{ + DbgAssert( Q_IsValidUChar32( uVal ) ); + if ( uVal <= 0xFFFF ) + { + pUTF16Out[0] = (uchar16) uVal; + return 1; + } + uVal -= 0x010000; + pUTF16Out[0] = (uchar16)(uVal >> 10) | 0xD800; + pUTF16Out[1] = (uchar16)(uVal & 0x3FF) | 0xDC00; + return 2; +} + + +// Decode one character from a UTF-8 encoded string. Treats 6-byte CESU-8 sequences +// as a single character, as if they were a correctly-encoded 4-byte UTF-8 sequence. +int Q_UTF8ToUChar32( const char *pUTF8_, uchar32 &uValueOut, bool &bErrorOut ) +{ + const uint8 *pUTF8 = (const uint8 *)pUTF8_; + + int nBytes = 1; + uint32 uValue = pUTF8[0]; + uint32 uMinValue = 0; + + // 0....... single byte + if ( uValue < 0x80 ) + goto decodeFinishedNoCheck; + + // Expecting at least a two-byte sequence with 0xC0 <= first <= 0xF7 (110...... and 11110...) + if ( (uValue - 0xC0u) > 0x37u || ( pUTF8[1] & 0xC0 ) != 0x80 ) + goto decodeError; + + uValue = (uValue << 6) - (0xC0 << 6) + pUTF8[1] - 0x80; + nBytes = 2; + uMinValue = 0x80; + + // 110..... two-byte lead byte + if ( !( uValue & (0x20 << 6) ) ) + goto decodeFinished; + + // Expecting at least a three-byte sequence + if ( ( pUTF8[2] & 0xC0 ) != 0x80 ) + goto decodeError; + + uValue = (uValue << 6) - (0x20 << 12) + pUTF8[2] - 0x80; + nBytes = 3; + uMinValue = 0x800; + + // 1110.... three-byte lead byte + if ( !( uValue & (0x10 << 12) ) ) + goto decodeFinishedMaybeCESU8; + + // Expecting a four-byte sequence, longest permissible in UTF-8 + if ( ( pUTF8[3] & 0xC0 ) != 0x80 ) + goto decodeError; + + uValue = (uValue << 6) - (0x10 << 18) + pUTF8[3] - 0x80; + nBytes = 4; + uMinValue = 0x10000; + + // 11110... four-byte lead byte. fall through to finished. + +decodeFinished: + if ( uValue >= uMinValue && Q_IsValidUChar32( uValue ) ) + { +decodeFinishedNoCheck: + uValueOut = uValue; + bErrorOut = false; + return nBytes; + } +decodeError: + uValueOut = '?'; + bErrorOut = true; + return nBytes; + +decodeFinishedMaybeCESU8: + // Do we have a full UTF-16 surrogate pair that's been UTF-8 encoded afterwards? + // That is, do we have 0xD800-0xDBFF followed by 0xDC00-0xDFFF? If so, decode it all. + if ( ( uValue - 0xD800u ) < 0x400u && pUTF8[3] == 0xED && (uint8)( pUTF8[4] - 0xB0 ) < 0x10 && ( pUTF8[5] & 0xC0 ) == 0x80 ) + { + uValue = 0x10000 + ( ( uValue - 0xD800u ) << 10 ) + ( (uint8)( pUTF8[4] - 0xB0 ) << 6 ) + pUTF8[5] - 0x80; + nBytes = 6; + uMinValue = 0x10000; + } + goto decodeFinished; +} + +// Decode one character from a UTF-16 encoded string. +int Q_UTF16ToUChar32( const uchar16 *pUTF16, uchar32 &uValueOut, bool &bErrorOut ) +{ + if ( Q_IsValidUChar32( pUTF16[0] ) ) + { + uValueOut = pUTF16[0]; + bErrorOut = false; + return 1; + } + else if ( (pUTF16[0] - 0xD800u) < 0x400u && (pUTF16[1] - 0xDC00u) < 0x400u ) + { + // Valid surrogate pair, but maybe not encoding a valid Unicode code point... + uchar32 uVal = 0x010000 + ((pUTF16[0] - 0xD800u) << 10) + (pUTF16[1] - 0xDC00); + if ( Q_IsValidUChar32( uVal ) ) + { + uValueOut = uVal; + bErrorOut = false; + return 2; + } + else + { + uValueOut = '?'; + bErrorOut = true; + return 2; + } + } + else + { + uValueOut = '?'; + bErrorOut = true; + return 1; + } +} + +namespace // internal use only +{ + // Identity transformations and validity tests for use with Q_UnicodeConvertT + int Q_UTF32ToUChar32( const uchar32 *pUTF32, uchar32 &uVal, bool &bErr ) + { + bErr = !Q_IsValidUChar32( *pUTF32 ); + uVal = bErr ? '?' : *pUTF32; + return 1; + } + + int Q_UChar32ToUTF32Len( uchar32 uVal ) + { + return 1; + } + + int Q_UChar32ToUTF32( uchar32 uVal, uchar32 *pUTF32 ) + { + *pUTF32 = uVal; + return 1; + } + + // A generic Unicode processing loop: decode one character from input to uchar32, handle errors, encode uchar32 to output + template < typename SrcType, typename DstType, bool bStopAtNull, int (&DecodeSrc)( const SrcType*, uchar32&, bool& ), int (&EncodeDstLen)( uchar32 ), int (&EncodeDst)( uchar32, DstType* ) > + int Q_UnicodeConvertT( const SrcType *pIn, int nInChars, DstType *pOut, int nOutBytes, EStringConvertErrorPolicy ePolicy ) + { + if ( !pIn ) + { + // For now, assert and return 0. Once these are cleaned out a bit + // we should remove this return and just leave in the assert... + AssertMsg( pIn, "We shouldn't be passing in NULL!" ); + return 0; + } + + int nOut = 0; + + if ( !pOut ) + { + while ( bStopAtNull ? ( *pIn ) : ( nInChars-- > 0 ) ) + { + uchar32 uVal; + // Initialize in order to avoid /analyze warnings. + bool bErr = false; + pIn += DecodeSrc( pIn, uVal, bErr ); + nOut += EncodeDstLen( uVal ); + if ( bErr ) + { +#ifdef _DEBUG + AssertMsg( !(ePolicy & _STRINGCONVERTFLAG_ASSERT), "invalid Unicode byte sequence" ); +#endif + if ( ePolicy & _STRINGCONVERTFLAG_SKIP ) + { + nOut -= EncodeDstLen( uVal ); + } + else if ( ePolicy & _STRINGCONVERTFLAG_FAIL ) + { + pOut[0] = 0; + return 0; + } + } + } + } + else + { + int nOutElems = nOutBytes / sizeof( DstType ); + if ( nOutElems <= 0 ) + return 0; + + int nMaxOut = nOutElems - 1; + while ( bStopAtNull ? ( *pIn ) : ( nInChars-- > 0 ) ) + { + uchar32 uVal; + // Initialize in order to avoid /analyze warnings. + bool bErr = false; + pIn += DecodeSrc( pIn, uVal, bErr ); + if ( nOut + EncodeDstLen( uVal ) > nMaxOut ) + break; + nOut += EncodeDst( uVal, pOut + nOut ); + if ( bErr ) + { +#ifdef _DEBUG + AssertMsg( !(ePolicy & _STRINGCONVERTFLAG_ASSERT), "invalid Unicode byte sequence" ); +#endif + if ( ePolicy & _STRINGCONVERTFLAG_SKIP ) + { + nOut -= EncodeDstLen( uVal ); + } + else if ( ePolicy & _STRINGCONVERTFLAG_FAIL ) + { + pOut[0] = 0; + return 0; + } + } + } + pOut[nOut] = 0; + } + + return (nOut + 1) * sizeof( DstType ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: Returns true if UTF-8 string contains invalid sequences. +//----------------------------------------------------------------------------- +bool Q_UnicodeValidate( const char *pUTF8 ) +{ + bool bError = false; + while ( *pUTF8 ) + { + uchar32 uVal; + // Our UTF-8 decoder silently fixes up 6-byte CESU-8 (improperly re-encoded UTF-16) sequences. + // However, these are technically not valid UTF-8. So if we eat 6 bytes at once, it's an error. + int nCharSize = Q_UTF8ToUChar32( pUTF8, uVal, bError ); + if ( bError || nCharSize == 6 ) + return false; + pUTF8 += nCharSize; + } + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Returns true if UTF-16 string contains invalid sequences. +//----------------------------------------------------------------------------- +bool Q_UnicodeValidate( const uchar16 *pUTF16 ) +{ + bool bError = false; + while ( *pUTF16 ) + { + uchar32 uVal; + pUTF16 += Q_UTF16ToUChar32( pUTF16, uVal, bError ); + if ( bError ) + return false; + } + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Returns true if UTF-32 string contains invalid sequences. +//----------------------------------------------------------------------------- +bool Q_UnicodeValidate( const uchar32 *pUTF32 ) +{ + while ( *pUTF32 ) + { + if ( !Q_IsValidUChar32( *pUTF32++ ) ) + return false; + ++pUTF32; + } + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Returns number of Unicode code points (aka glyphs / characters) encoded in the UTF-8 string +//----------------------------------------------------------------------------- +int Q_UnicodeLength( const char *pUTF8 ) +{ + int nChars = 0; + while ( *pUTF8 ) + { + bool bError; + uchar32 uVal; + pUTF8 += Q_UTF8ToUChar32( pUTF8, uVal, bError ); + ++nChars; + } + return nChars; +} + +//----------------------------------------------------------------------------- +// Purpose: Returns number of Unicode code points (aka glyphs / characters) encoded in the UTF-16 string +//----------------------------------------------------------------------------- +int Q_UnicodeLength( const uchar16 *pUTF16 ) +{ + int nChars = 0; + while ( *pUTF16 ) + { + bool bError; + uchar32 uVal; + pUTF16 += Q_UTF16ToUChar32( pUTF16, uVal, bError ); + ++nChars; + } + return nChars; +} + +//----------------------------------------------------------------------------- +// Purpose: Returns number of Unicode code points (aka glyphs / characters) encoded in the UTF-32 string +//----------------------------------------------------------------------------- +int Q_UnicodeLength( const uchar32 *pUTF32 ) +{ + int nChars = 0; + while ( *pUTF32++ ) + ++nChars; + return nChars; +} + +//----------------------------------------------------------------------------- +// Purpose: Advance a UTF-8 string pointer by a certain number of Unicode code points, stopping at end of string +//----------------------------------------------------------------------------- +char *Q_UnicodeAdvance( char *pUTF8, int nChars ) +{ + while ( nChars > 0 && *pUTF8 ) + { + uchar32 uVal; + bool bError; + pUTF8 += Q_UTF8ToUChar32( pUTF8, uVal, bError ); + --nChars; + } + return pUTF8; +} + +//----------------------------------------------------------------------------- +// Purpose: Advance a UTF-16 string pointer by a certain number of Unicode code points, stopping at end of string +//----------------------------------------------------------------------------- +uchar16 *Q_UnicodeAdvance( uchar16 *pUTF16, int nChars ) +{ + while ( nChars > 0 && *pUTF16 ) + { + uchar32 uVal; + bool bError; + pUTF16 += Q_UTF16ToUChar32( pUTF16, uVal, bError ); + --nChars; + } + return pUTF16; +} + +//----------------------------------------------------------------------------- +// Purpose: Advance a UTF-32 string pointer by a certain number of Unicode code points, stopping at end of string +//----------------------------------------------------------------------------- +uchar32 *Q_UnicodeAdvance( uchar32 *pUTF32, int nChars ) +{ + while ( nChars > 0 && *pUTF32 ) + { + ++pUTF32; + --nChars; + } + return pUTF32; +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF8ToUTF16( const char *pUTF8, uchar16 *pUTF16, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< char, uchar16, true, Q_UTF8ToUChar32, Q_UChar32ToUTF16Len, Q_UChar32ToUTF16 >( pUTF8, 0, pUTF16, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF8ToUTF32( const char *pUTF8, uchar32 *pUTF32, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< char, uchar32, true, Q_UTF8ToUChar32, Q_UChar32ToUTF32Len, Q_UChar32ToUTF32 >( pUTF8, 0, pUTF32, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF16ToUTF8( const uchar16 *pUTF16, char *pUTF8, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar16, char, true, Q_UTF16ToUChar32, Q_UChar32ToUTF8Len, Q_UChar32ToUTF8 >( pUTF16, 0, pUTF8, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF16ToUTF32( const uchar16 *pUTF16, uchar32 *pUTF32, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar16, uchar32, true, Q_UTF16ToUChar32, Q_UChar32ToUTF32Len, Q_UChar32ToUTF32 >( pUTF16, 0, pUTF32, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF32ToUTF8( const uchar32 *pUTF32, char *pUTF8, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar32, char, true, Q_UTF32ToUChar32, Q_UChar32ToUTF8Len, Q_UChar32ToUTF8 >( pUTF32, 0, pUTF8, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF32ToUTF16( const uchar32 *pUTF32, uchar16 *pUTF16, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar32, uchar16, true, Q_UTF32ToUChar32, Q_UChar32ToUTF16Len, Q_UChar32ToUTF16 >( pUTF32, 0, pUTF16, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF32ToUTF32( const uchar32 *pUTF32Source, uchar32 *pUTF32Dest, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar32, uchar32, true, Q_UTF32ToUChar32, Q_UChar32ToUTF32Len, Q_UChar32ToUTF32 >( pUTF32Source, 0, pUTF32Dest, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF8CharsToUTF16( const char *pUTF8, int nElements, uchar16 *pUTF16, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< char, uchar16, false, Q_UTF8ToUChar32, Q_UChar32ToUTF16Len, Q_UChar32ToUTF16 >( pUTF8, nElements, pUTF16, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF8CharsToUTF32( const char *pUTF8, int nElements, uchar32 *pUTF32, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< char, uchar32, false, Q_UTF8ToUChar32, Q_UChar32ToUTF32Len, Q_UChar32ToUTF32 >( pUTF8, nElements, pUTF32, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF16CharsToUTF8( const uchar16 *pUTF16, int nElements, char *pUTF8, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar16, char, false, Q_UTF16ToUChar32, Q_UChar32ToUTF8Len, Q_UChar32ToUTF8 >( pUTF16, nElements, pUTF8, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF16CharsToUTF32( const uchar16 *pUTF16, int nElements, uchar32 *pUTF32, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar16, uchar32, false, Q_UTF16ToUChar32, Q_UChar32ToUTF32Len, Q_UChar32ToUTF32 >( pUTF16, nElements, pUTF32, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF32CharsToUTF8( const uchar32 *pUTF32, int nElements, char *pUTF8, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar32, char, false, Q_UTF32ToUChar32, Q_UChar32ToUTF8Len, Q_UChar32ToUTF8 >( pUTF32, nElements, pUTF8, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL. +//----------------------------------------------------------------------------- +int Q_UTF32CharsToUTF16( const uchar32 *pUTF32, int nElements, uchar16 *pUTF16, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar32, uchar16, false, Q_UTF32ToUChar32, Q_UChar32ToUTF16Len, Q_UChar32ToUTF16 >( pUTF32, nElements, pUTF16, cubDestSizeInBytes, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Repair a UTF-8 string by removing or replacing invalid seqeuences. Returns non-zero on success. +//----------------------------------------------------------------------------- +int Q_UnicodeRepair( char *pUTF8, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< char, char, true, Q_UTF8ToUChar32, Q_UChar32ToUTF8Len, Q_UChar32ToUTF8 >( pUTF8, 0, pUTF8, INT_MAX, ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Repair a UTF-16 string by removing or replacing invalid seqeuences. Returns non-zero on success. +//----------------------------------------------------------------------------- +int Q_UnicodeRepair( uchar16 *pUTF16, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar16, uchar16, true, Q_UTF16ToUChar32, Q_UChar32ToUTF16Len, Q_UChar32ToUTF16 >( pUTF16, 0, pUTF16, INT_MAX/sizeof(uchar16), ePolicy ); +} + +//----------------------------------------------------------------------------- +// Purpose: Repair a UTF-32 string by removing or replacing invalid seqeuences. Returns non-zero on success. +//----------------------------------------------------------------------------- +int Q_UnicodeRepair( uchar32 *pUTF32, EStringConvertErrorPolicy ePolicy ) +{ + return Q_UnicodeConvertT< uchar32, uchar32, true, Q_UTF32ToUChar32, Q_UChar32ToUTF32Len, Q_UChar32ToUTF32 >( pUTF32, 0, pUTF32, INT_MAX/sizeof(uchar32), ePolicy ); +} + diff --git a/mp/src/tier1/tier1.vpc b/mp/src/tier1/tier1.vpc index bc3438a7..6f163a8b 100644 --- a/mp/src/tier1/tier1.vpc +++ b/mp/src/tier1/tier1.vpc @@ -64,6 +64,7 @@ $Project "tier1" $File "reliabletimer.cpp" $File "stringpool.cpp" $File "strtools.cpp" + $File "strtools_unicode.cpp" $File "tier1.cpp" $File "tokenreader.cpp" $File "sparsematrix.cpp" @@ -79,6 +80,15 @@ $Project "tier1" $File "snappy-stubs-internal.cpp" } + // Select bits from the LZMA SDK to support lzmaDecoder.h + // Encoding support requires the full lzma project + $Folder "LZMA Decompression Support" + { + $File "$SRCDIR\utils\lzma\C\LzmaDec.h" + $File "$SRCDIR\utils\lzma\C\LzmaDec.c" + $File "$SRCDIR\utils\lzma\C\7zTypes.h" + } + $Folder "Header Files" { $Folder "Internal Header Files" diff --git a/mp/src/tier1/uniqueid.cpp b/mp/src/tier1/uniqueid.cpp index 0d51c4cf..d75e4fd9 100644 --- a/mp/src/tier1/uniqueid.cpp +++ b/mp/src/tier1/uniqueid.cpp @@ -163,7 +163,7 @@ bool Unserialize( CUtlBuffer &buf, UniqueId_t &dest ) { int nTextLen = buf.PeekStringLength(); char *pBuf = (char*)stackalloc( nTextLen ); - buf.GetString( pBuf, nTextLen ); + buf.GetStringManualCharCount( pBuf, nTextLen ); UniqueIdFromString( &dest, pBuf, nTextLen ); } else diff --git a/mp/src/tier1/utlbuffer.cpp b/mp/src/tier1/utlbuffer.cpp index 55dd8f6b..fc89d773 100644 --- a/mp/src/tier1/utlbuffer.cpp +++ b/mp/src/tier1/utlbuffer.cpp @@ -607,17 +607,19 @@ int CUtlBuffer::PeekDelimitedStringLength( CUtlCharConversion *pConv, bool bActu //----------------------------------------------------------------------------- // Reads a null-terminated string //----------------------------------------------------------------------------- -void CUtlBuffer::GetString( char* pString, int nMaxChars ) +void CUtlBuffer::GetStringInternal( char *pString, size_t maxLenInChars ) { - if (!IsValid()) + if ( !IsValid() ) { *pString = 0; return; } - if ( nMaxChars == 0 ) + Assert( maxLenInChars != 0 ); + + if ( maxLenInChars == 0 ) { - nMaxChars = INT_MAX; + return; } // Remember, this *includes* the null character @@ -629,24 +631,21 @@ void CUtlBuffer::GetString( char* pString, int nMaxChars ) EatWhiteSpace(); } - if ( nLen == 0 ) + if ( nLen <= 0 ) { *pString = 0; m_Error |= GET_OVERFLOW; return; } - - // Strip off the terminating NULL - if ( nLen <= nMaxChars ) - { - Get( pString, nLen - 1 ); - pString[ nLen - 1 ] = 0; - } - else + + const size_t nCharsToRead = min( (size_t)nLen, maxLenInChars ) - 1; + + Get( pString, nCharsToRead ); + pString[nCharsToRead] = 0; + + if ( (size_t)nLen > (nCharsToRead + 1) ) { - Get( pString, nMaxChars - 1 ); - pString[ nMaxChars - 1 ] = 0; - SeekGet( SEEK_CURRENT, nLen - 1 - nMaxChars ); + SeekGet( SEEK_CURRENT, nLen - (nCharsToRead + 1) ); } // Read the terminating NULL in binary formats @@ -731,7 +730,7 @@ void CUtlBuffer::GetDelimitedString( CUtlCharConversion *pConv, char *pString, i { if ( !IsText() || !pConv ) { - GetString( pString, nMaxChars ); + GetStringInternal( pString, nMaxChars ); return; } @@ -1041,7 +1040,7 @@ int CUtlBuffer::VaScanf( const char* pFmt, va_list list ) case 's': { char* s = va_arg( list, char * ); - GetString( s ); + GetStringInternal( s, 256 ); } break; |