diff options
Diffstat (limited to 'tier1/lzmaDecoder.cpp')
| -rw-r--r-- | tier1/lzmaDecoder.cpp | 405 |
1 files changed, 405 insertions, 0 deletions
diff --git a/tier1/lzmaDecoder.cpp b/tier1/lzmaDecoder.cpp new file mode 100644 index 0000000..209c7fc --- /dev/null +++ b/tier1/lzmaDecoder.cpp @@ -0,0 +1,405 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// LZMA Codec interface for engine. +// +// LZMA SDK 9.38 beta +// 2015-01-03 : Igor Pavlov : Public domain +// http://www.7-zip.org/ +// +//========================================================================// + +#define _LZMADECODER_CPP + +#include "tier0/platform.h" +#include "tier0/basetypes.h" +#include "tier0/dbg.h" + +#include "../utils/lzma/C/7zTypes.h" +#include "../utils/lzma/C/LzmaEnc.h" +#include "../utils/lzma/C/LzmaDec.h" + +// Ugly define to let us forward declare the anonymous-struct-typedef that is CLzmaDec in the header. +#define CLzmaDec_t CLzmaDec +#include "tier1/lzmaDecoder.h" +#include "tier1/convar.h" + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +#ifdef OSX +// OS X is having fragmentation issues, and I suspect this 16meg buffer being recreated many times during load is +// hitting a bad case in the default allocator. So this is an experiment to see if it reduces crash rates there. +#define LZMA_DEFAULT_PERSISTENT_BUFFER "1" +#else +#define LZMA_DEFAULT_PERSISTENT_BUFFER "0" +#endif + +ConVar lzma_persistent_buffer( "lzma_persistent_buffer", LZMA_DEFAULT_PERSISTENT_BUFFER, FCVAR_NONE, + "If set, attempt to keep a persistent buffer for the LZMA decoder dictionary. " \ + "This avoids re-allocating a ~16-64meg buffer for each operation, " \ + "at the expensive of keeping extra memory around when it is not in-use." ); + +// Allocator to pass to LZMA functions +static void *g_pStaticLZMABuf = NULL; +static size_t g_unStaticLZMABufSize = 0; +static uint32 g_unStaticLZMABufRef = 0; +static void *SzAlloc(void *p, size_t size) { + // Don't touch static buffer on other threads. + if ( ThreadInMainThread() ) + { + // If nobody is using the persistent buffer and size is above a threshold, use it. + bool bPersistentBuf = (g_pStaticLZMABuf || lzma_persistent_buffer.GetBool()) && size >= (1024 * 1024 * 8) && g_unStaticLZMABufRef == 0; + if ( bPersistentBuf ) + { + if ( g_unStaticLZMABufSize < size ) + { + g_pStaticLZMABuf = g_pStaticLZMABuf ? realloc( g_pStaticLZMABuf, size ) : malloc( size ); + g_unStaticLZMABufSize = size; + } + g_unStaticLZMABufRef++; + return g_pStaticLZMABuf; + } + } + + // Not using the persistent buffer + return malloc(size); +} +static void SzFree(void *p, void *address) { + // Don't touch static buffer on other threads. + if ( ThreadInMainThread() ) + { + if ( address != NULL && g_unStaticLZMABufRef && address == g_pStaticLZMABuf ) + { + g_unStaticLZMABufRef--; + // If the convar was turned off, free the buffer + if ( g_pStaticLZMABuf && g_unStaticLZMABufRef == 0 && !lzma_persistent_buffer.GetBool() ) + { + free( g_pStaticLZMABuf ); + g_pStaticLZMABuf = NULL; + g_unStaticLZMABufSize = 0; + } + return; + } + } + + // Not the static buffer + free(address); +} +static ISzAlloc g_Alloc = { SzAlloc, SzFree }; + +//----------------------------------------------------------------------------- +// Returns true if buffer is compressed. +//----------------------------------------------------------------------------- +/* static */ +bool CLZMA::IsCompressed( unsigned char *pInput ) +{ + lzma_header_t *pHeader = (lzma_header_t *)pInput; + if ( pHeader && pHeader->id == LZMA_ID ) + { + return true; + } + + // unrecognized + return false; +} + +//----------------------------------------------------------------------------- +// Returns uncompressed size of compressed input buffer. Used for allocating output +// buffer for decompression. Returns 0 if input buffer is not compressed. +//----------------------------------------------------------------------------- +/* static */ +unsigned int CLZMA::GetActualSize( unsigned char *pInput ) +{ + lzma_header_t *pHeader = (lzma_header_t *)pInput; + if ( pHeader && pHeader->id == LZMA_ID ) + { + return LittleLong( pHeader->actualSize ); + } + + // unrecognized + return 0; +} + +//----------------------------------------------------------------------------- +// Uncompress a buffer, Returns the uncompressed size. Caller must provide an +// adequate sized output buffer or memory corruption will occur. +//----------------------------------------------------------------------------- +/* static */ +unsigned int CLZMA::Uncompress( unsigned char *pInput, unsigned char *pOutput ) +{ + lzma_header_t *pHeader = (lzma_header_t *)pInput; + if ( pHeader->id != LZMA_ID ) + { + // not ours + return false; + } + + CLzmaDec state; + + LzmaDec_Construct(&state); + + if ( LzmaDec_Allocate(&state, pHeader->properties, LZMA_PROPS_SIZE, &g_Alloc) != SZ_OK ) + { + Assert( false ); + return 0; + } + + // These are in/out variables + SizeT outProcessed = pHeader->actualSize; + SizeT inProcessed = pHeader->lzmaSize; + ELzmaStatus status; + SRes result = LzmaDecode( (Byte *)pOutput, &outProcessed, (Byte *)(pInput + sizeof( lzma_header_t ) ), + &inProcessed, (Byte *)pHeader->properties, LZMA_PROPS_SIZE, LZMA_FINISH_END, &status, &g_Alloc ); + + + LzmaDec_Free(&state, &g_Alloc); + + if ( result != SZ_OK || pHeader->actualSize != outProcessed ) + { + Warning( "LZMA Decompression failed (%i)\n", result ); + return 0; + } + + return outProcessed; +} + +CLZMAStream::CLZMAStream() + : m_pDecoderState( NULL ), + m_nActualSize( 0 ), + m_nActualBytesRead ( 0 ), + m_nCompressedSize( 0 ), + m_nCompressedBytesRead ( 0 ), + m_bParsedHeader( false ), + m_bZIPStyleHeader( false ) +{} + +CLZMAStream::~CLZMAStream() +{ + FreeDecoderState(); +} + +void CLZMAStream::FreeDecoderState() +{ + if ( m_pDecoderState ) + { + LzmaDec_Free( m_pDecoderState, &g_Alloc ); + delete m_pDecoderState; + m_pDecoderState = NULL; + } +} + +bool CLZMAStream::CreateDecoderState( const unsigned char *pProperties ) +{ + CLzmaDec *pDecoderState = new CLzmaDec(); + + LzmaDec_Construct( pDecoderState ); + if ( LzmaDec_Allocate( pDecoderState, pProperties, LZMA_PROPS_SIZE, &g_Alloc) != SZ_OK ) + { + AssertMsg( false, "Failed to allocate lzma decoder state" ); + delete pDecoderState; + return false; + } + + LzmaDec_Init( pDecoderState ); + + // Replace current state + Assert( !m_pDecoderState ); + FreeDecoderState(); + + m_pDecoderState = pDecoderState; + return true; +} + +// Attempt to read up to nMaxInputBytes from the compressed stream, writing up to nMaxOutputBytes to pOutput. +// Returns false if read stops due to an error. +bool CLZMAStream::Read( unsigned char *pInput, unsigned int nMaxInputBytes, + unsigned char *pOutput, unsigned int nMaxOutputBytes, + /* out */ unsigned int &nCompressedBytesRead, + /* out */ unsigned int &nOutputBytesWritten ) +{ + nCompressedBytesRead = 0; + nOutputBytesWritten = 0; + bool bStartedWithHeader = m_bParsedHeader; + + // Check for initial chunk of data + if ( !m_bParsedHeader ) + { + unsigned int nBytesConsumed = 0; + eHeaderParse parseResult = TryParseHeader( pInput, nMaxInputBytes, nBytesConsumed ); + + if ( parseResult == eHeaderParse_NeedMoreBytes ) + { + // Not an error, just need more data to continue + return true; + } + else if ( parseResult != eHeaderParse_OK ) + { + Assert( parseResult == eHeaderParse_Fail ); + // Invalid header + return false; + } + + // Header consumed, fall through to continue read after it + nCompressedBytesRead += nBytesConsumed; + pInput += nBytesConsumed; + nMaxInputBytes -= nBytesConsumed; + } + + // These are input ( available size ) *and* output ( size processed ) vars for lzma + SizeT expectedInputRemaining = m_nCompressedSize - Min( m_nCompressedBytesRead + nCompressedBytesRead, m_nCompressedSize ); + SizeT expectedOutputRemaining = m_nActualSize - m_nActualBytesRead; + SizeT inSize = Min( (SizeT)nMaxInputBytes, expectedInputRemaining ); + SizeT outSize = Min( (SizeT)nMaxOutputBytes, expectedOutputRemaining ); + ELzmaStatus status; + ELzmaFinishMode finishMode = LZMA_FINISH_ANY; + if ( inSize == expectedInputRemaining && outSize == expectedOutputRemaining ) + { + // Expect to finish decoding this call. + finishMode = LZMA_FINISH_END; + } + SRes result = LzmaDec_DecodeToBuf( m_pDecoderState, pOutput, &outSize, + pInput, &inSize, finishMode, &status ); + + // DevMsg("[%p] Running lzmaDecode:\n" + // " pInput: %p\n" + // " nMaxInputBytes: %i\n" + // " pOutput: %p\n" + // " nMaxOutputBytes: %u\n" + // " inSize: %u\n" + // " outSize: %u\n" + // " result: %u\n" + // " status: %i\n" + // " m_nActualSize: %u\n" + // " m_nActualBytesRead: %u\n", + // this, pInput, nMaxInputBytes, pOutput, nMaxOutputBytes, + // inSize, outSize, result, status, m_nActualSize, m_nActualBytesRead); + + if ( result != SZ_OK ) + { + if ( !bStartedWithHeader ) + { + // If we're returning false, we need to pretend we didn't consume anything. + FreeDecoderState(); + m_bParsedHeader = false; + } + return false; + } + + nCompressedBytesRead += inSize; + nOutputBytesWritten += outSize; + + m_nCompressedBytesRead += nCompressedBytesRead; + m_nActualBytesRead += nOutputBytesWritten; + + Assert( m_nCompressedBytesRead <= m_nCompressedSize ); + return true; +} + +bool CLZMAStream::GetExpectedBytesRemaining( /* out */ unsigned int &nBytesRemaining ) +{ + if ( !m_bParsedHeader && !m_bZIPStyleHeader ) { + return false; + } + + nBytesRemaining = m_nActualSize - m_nActualBytesRead; + + return true; +} + +void CLZMAStream::InitZIPHeader( unsigned int nCompressedSize, unsigned int nOriginalSize ) +{ + if ( m_bParsedHeader || m_bZIPStyleHeader ) + { + AssertMsg( !m_bParsedHeader && !m_bZIPStyleHeader, + "LZMA Stream: InitZIPHeader() called on stream past header" ); + return; + } + + m_nCompressedSize = nCompressedSize; + m_nActualSize = nOriginalSize; + // Signal to TryParseHeader to expect a zip-style header (which wont have the size values) + m_bZIPStyleHeader = true; +} + +CLZMAStream::eHeaderParse CLZMAStream::TryParseHeader( unsigned char *pInput, unsigned int nBytesAvailable, /* out */ unsigned int &nBytesConsumed ) +{ + nBytesConsumed = 0; + + if ( m_bParsedHeader ) + { + AssertMsg( !m_bParsedHeader, "CLZMAStream::ReadSourceHeader called on already initialized stream" ); + return eHeaderParse_Fail; + } + + if ( m_bZIPStyleHeader ) + { + // ZIP Spec, 5.8.8 + // LZMA Version Information 2 bytes + // LZMA Properties Size 2 bytes + // LZMA Properties Data variable, defined by "LZMA Properties Size" + + if ( nBytesAvailable < 4 ) + { + // No error, but need more input to continue + return eHeaderParse_NeedMoreBytes; + } + + // Should probably check this + // unsigned char nLZMAVer[2] = { pInput[0], pInput[1] }; + + uint16 nLZMAPropertiesSize = LittleWord( *(uint16 *)(pInput + 2) ); + + nBytesConsumed += 4; + + if ( nLZMAPropertiesSize != LZMA_PROPS_SIZE ) + { + Warning( "LZMA stream: Unexpected LZMA properties size: %hu, expecting %u. Version mismatch?\n", + nLZMAPropertiesSize, LZMA_PROPS_SIZE ); + return eHeaderParse_Fail; + } + + if ( nBytesAvailable < static_cast<unsigned int>(nLZMAPropertiesSize) + 4 ) + { + return eHeaderParse_NeedMoreBytes; + } + + // Looks reasonable, try to parse + if ( !CreateDecoderState( (Byte *)pInput + 4 ) ) + { + AssertMsg( false, "Failed decoding Lzma properties" ); + return eHeaderParse_Fail; + } + + nBytesConsumed += nLZMAPropertiesSize; + } + else + { + // Else native source engine style header + if ( nBytesAvailable < sizeof( lzma_header_t ) ) + { + // need more input to continue + return eHeaderParse_NeedMoreBytes; + } + + m_nActualSize = CLZMA::GetActualSize( pInput ); + + if ( !m_nActualSize ) + { + // unrecognized + Warning( "Unrecognized LZMA data\n" ); + return eHeaderParse_Fail; + } + + if ( !CreateDecoderState( ((lzma_header_t *)pInput)->properties ) ) + { + AssertMsg( false, "Failed decoding Lzma properties" ); + return eHeaderParse_Fail; + } + + m_nCompressedSize = LittleLong( ((lzma_header_t *)pInput)->lzmaSize ) + sizeof( lzma_header_t ); + nBytesConsumed += sizeof( lzma_header_t ); + } + + m_bParsedHeader = true; + return eHeaderParse_OK; +} |