diff options
| author | FluorescentCIAAfricanAmerican <[email protected]> | 2020-04-22 12:56:21 -0400 |
|---|---|---|
| committer | FluorescentCIAAfricanAmerican <[email protected]> | 2020-04-22 12:56:21 -0400 |
| commit | 3bf9df6b2785fa6d951086978a3e66f49427166a (patch) | |
| tree | 2c0f1f0c63c4832882bc93814ebd2c2b1c6224e5 /datamodel/dmserializerbinary.cpp | |
| download | archived-source-engine-2018-hl2-src-3bf9df6b2785fa6d951086978a3e66f49427166a.tar.xz archived-source-engine-2018-hl2-src-3bf9df6b2785fa6d951086978a3e66f49427166a.zip | |
Diffstat (limited to 'datamodel/dmserializerbinary.cpp')
| -rw-r--r-- | datamodel/dmserializerbinary.cpp | 587 |
1 files changed, 587 insertions, 0 deletions
diff --git a/datamodel/dmserializerbinary.cpp b/datamodel/dmserializerbinary.cpp new file mode 100644 index 0000000..f036317 --- /dev/null +++ b/datamodel/dmserializerbinary.cpp @@ -0,0 +1,587 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +//============================================================================= + +#include "dmserializerbinary.h" +#include "datamodel/idatamodel.h" +#include "datamodel.h" +#include "datamodel/dmelement.h" +#include "datamodel/dmattributevar.h" +#include "dmattributeinternal.h" +#include "dmelementdictionary.h" +#include "tier1/utlbuffer.h" +#include "DmElementFramework.h" + + +//----------------------------------------------------------------------------- +// Forward declarations +//----------------------------------------------------------------------------- +class CUtlBuffer; +class CBaseSceneObject; + + +//----------------------------------------------------------------------------- +// special element indices +//----------------------------------------------------------------------------- +enum +{ + ELEMENT_INDEX_NULL = -1, + ELEMENT_INDEX_EXTERNAL = -2, +}; + + +//----------------------------------------------------------------------------- +// Serialization class for Binary output +//----------------------------------------------------------------------------- +class CDmSerializerBinary : public IDmSerializer +{ +public: + CDmSerializerBinary() {} + // Inherited from IDMSerializer + virtual const char *GetName() const { return "binary"; } + virtual const char *GetDescription() const { return "Binary"; } + virtual bool StoresVersionInFile() const { return true; } + virtual bool IsBinaryFormat() const { return true; } + virtual int GetCurrentVersion() const { return 2; } + virtual bool Serialize( CUtlBuffer &buf, CDmElement *pRoot ); + virtual bool Unserialize( CUtlBuffer &buf, const char *pEncodingName, int nEncodingVersion, + const char *pSourceFormatName, int nFormatVersion, + DmFileId_t fileid, DmConflictResolution_t idConflictResolution, CDmElement **ppRoot ); + +private: + // Methods related to serialization + void SerializeElementIndex( CUtlBuffer& buf, CDmElementSerializationDictionary& list, DmElementHandle_t hElement, DmFileId_t fileid ); + void SerializeElementAttribute( CUtlBuffer& buf, CDmElementSerializationDictionary& list, CDmAttribute *pAttribute ); + void SerializeElementArrayAttribute( CUtlBuffer& buf, CDmElementSerializationDictionary& list, CDmAttribute *pAttribute ); + bool SerializeAttributes( CUtlBuffer& buf, CDmElementSerializationDictionary& list, unsigned short *symbolToIndexMap, CDmElement *pElement ); + bool SaveElementDict( CUtlBuffer& buf, unsigned short *symbolToIndexMap, CDmElement *pElement ); + bool SaveElement( CUtlBuffer& buf, CDmElementSerializationDictionary& dict, unsigned short *symbolToIndexMap, CDmElement *pElement); + + // Methods related to unserialization + DmElementHandle_t UnserializeElementIndex( CUtlBuffer &buf, CUtlVector<CDmElement*> &elementList ); + void UnserializeElementAttribute( CUtlBuffer &buf, CDmAttribute *pAttribute, CUtlVector<CDmElement*> &elementList ); + void UnserializeElementArrayAttribute( CUtlBuffer &buf, CDmAttribute *pAttribute, CUtlVector<CDmElement*> &elementList ); + bool UnserializeAttributes( CUtlBuffer &buf, CDmElement *pElement, CUtlVector<CDmElement*> &elementList, UtlSymId_t *symbolTable ); + bool UnserializeElements( CUtlBuffer &buf, DmFileId_t fileid, DmConflictResolution_t idConflictResolution, CDmElement **ppRoot, UtlSymId_t *symbolTable ); +}; + + +//----------------------------------------------------------------------------- +// Singleton instance +//----------------------------------------------------------------------------- +static CDmSerializerBinary s_DMSerializerBinary; + +void InstallBinarySerializer( IDataModel *pFactory ) +{ + pFactory->AddSerializer( &s_DMSerializerBinary ); +} + + +//----------------------------------------------------------------------------- +// Write out the index of the element to avoid looks at read time +//----------------------------------------------------------------------------- +void CDmSerializerBinary::SerializeElementIndex( CUtlBuffer& buf, CDmElementSerializationDictionary& list, DmElementHandle_t hElement, DmFileId_t fileid ) +{ + if ( hElement == DMELEMENT_HANDLE_INVALID ) + { + buf.PutInt( ELEMENT_INDEX_NULL ); // invalid handle + } + else + { + CDmElement *pElement = g_pDataModel->GetElement( hElement ); + if ( pElement ) + { + if ( pElement->GetFileId() == fileid ) + { + buf.PutInt( list.Find( pElement ) ); + } + else + { + buf.PutInt( ELEMENT_INDEX_EXTERNAL ); + char idstr[ 40 ]; + UniqueIdToString( pElement->GetId(), idstr, sizeof( idstr ) ); + buf.PutString( idstr ); + } + } + else + { + DmObjectId_t *pId = NULL; + DmElementReference_t *pRef = g_pDataModelImp->FindElementReference( hElement, &pId ); + if ( pRef && pId ) + { + buf.PutInt( ELEMENT_INDEX_EXTERNAL ); + char idstr[ 40 ]; + UniqueIdToString( *pId, idstr, sizeof( idstr ) ); + buf.PutString( idstr ); + } + else + { + buf.PutInt( ELEMENT_INDEX_NULL ); // invalid handle + } + } + } +} + + +//----------------------------------------------------------------------------- +// Writes out element attributes +//----------------------------------------------------------------------------- +void CDmSerializerBinary::SerializeElementAttribute( CUtlBuffer& buf, CDmElementSerializationDictionary& list, CDmAttribute *pAttribute ) +{ + SerializeElementIndex( buf, list, pAttribute->GetValue< DmElementHandle_t >(), pAttribute->GetOwner()->GetFileId() ); +} + + +//----------------------------------------------------------------------------- +// Writes out element array attributes +//----------------------------------------------------------------------------- +void CDmSerializerBinary::SerializeElementArrayAttribute( CUtlBuffer& buf, CDmElementSerializationDictionary& list, CDmAttribute *pAttribute ) +{ + DmFileId_t fileid = pAttribute->GetOwner()->GetFileId(); + CDmrElementArray<> vec( pAttribute ); + + int nCount = vec.Count(); + buf.PutInt( nCount ); + for ( int i = 0; i < nCount; ++i ) + { + SerializeElementIndex( buf, list, vec.GetHandle(i), fileid ); + } +} + + +//----------------------------------------------------------------------------- +// Writes out all attributes +//----------------------------------------------------------------------------- +bool CDmSerializerBinary::SerializeAttributes( CUtlBuffer& buf, CDmElementSerializationDictionary& list, unsigned short *symbolToIndexMap, CDmElement *pElement ) +{ + // Collect the attributes to be written + CDmAttribute **ppAttributes = ( CDmAttribute** )_alloca( pElement->AttributeCount() * sizeof( CDmAttribute* ) ); + int nAttributes = 0; + for ( CDmAttribute *pAttribute = pElement->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() ) + { + if ( pAttribute->IsFlagSet( FATTRIB_DONTSAVE | FATTRIB_STANDARD ) ) + continue; + + ppAttributes[ nAttributes++ ] = pAttribute; + } + + // Now write them all out in reverse order, since FirstAttribute is actually the *last* attribute for perf reasons + buf.PutInt( nAttributes ); + for ( int i = nAttributes - 1; i >= 0; --i ) + { + CDmAttribute *pAttribute = ppAttributes[ i ]; + Assert( pAttribute ); + + buf.PutShort( symbolToIndexMap[ pAttribute->GetNameSymbol() ] ); + buf.PutChar( pAttribute->GetType() ); + switch( pAttribute->GetType() ) + { + default: + pAttribute->Serialize( buf ); + break; + + case AT_ELEMENT: + SerializeElementAttribute( buf, list, pAttribute ); + break; + + case AT_ELEMENT_ARRAY: + SerializeElementArrayAttribute( buf, list, pAttribute ); + break; + } + } + + return buf.IsValid(); +} + + +bool CDmSerializerBinary::SaveElement( CUtlBuffer& buf, CDmElementSerializationDictionary& list, unsigned short *symbolToIndexMap, CDmElement *pElement ) +{ + SerializeAttributes( buf, list, symbolToIndexMap, pElement ); + return buf.IsValid(); +} + +bool CDmSerializerBinary::SaveElementDict( CUtlBuffer& buf, unsigned short *symbolToIndexMap, CDmElement *pElement ) +{ + buf.PutShort( symbolToIndexMap[ pElement->GetType() ] ); + buf.PutString( pElement->GetName() ); + buf.Put( &pElement->GetId(), sizeof(DmObjectId_t) ); + return buf.IsValid(); +} + +void MarkSymbol( UtlSymId_t *indexToSymbolMap, unsigned short *symbolToIndexMap, unsigned short &nSymbols, UtlSymId_t sym ) +{ + if ( symbolToIndexMap[ sym ] != 0xffff ) + return; + + symbolToIndexMap[ sym ] = nSymbols; + indexToSymbolMap[ nSymbols ] = sym; + nSymbols++; +} + +void MarkSymbols( UtlSymId_t *indexToSymbolMap, unsigned short *symbolToIndexMap, unsigned short &nSymbols, CDmElement *pElement ) +{ + MarkSymbol( indexToSymbolMap, symbolToIndexMap, nSymbols, pElement->GetType() ); + for ( CDmAttribute *pAttr = pElement->FirstAttribute(); pAttr; pAttr = pAttr->NextAttribute() ) + { + MarkSymbol( indexToSymbolMap, symbolToIndexMap, nSymbols, pAttr->GetNameSymbol() ); + } +} + +bool CDmSerializerBinary::Serialize( CUtlBuffer &outBuf, CDmElement *pRoot ) +{ + // Save elements, attribute links + CDmElementSerializationDictionary dict; + dict.BuildElementList( pRoot, true ); + + // TODO - consider allowing dmxconvert to skip the collection step, since the only datamodel symbols will be the ones from the file + + unsigned short nTotalSymbols = g_pDataModelImp->GetSymbolCount(); + UtlSymId_t *indexToSymbolMap = ( UtlSymId_t * )stackalloc( nTotalSymbols * sizeof( UtlSymId_t ) ); + unsigned short *symbolToIndexMap = ( unsigned short* )stackalloc( nTotalSymbols * sizeof( unsigned short ) ); + V_memset( indexToSymbolMap, 0xff, nTotalSymbols * sizeof( UtlSymId_t ) ); + V_memset( symbolToIndexMap, 0xff, nTotalSymbols * sizeof( unsigned short ) ); + + // collect list of attribute names and element types into string table + unsigned short nUsedSymbols = 0; + DmElementDictHandle_t i; + for ( i = dict.FirstRootElement(); i != ELEMENT_DICT_HANDLE_INVALID; i = dict.NextRootElement(i) ) + { + MarkSymbols( indexToSymbolMap, symbolToIndexMap, nUsedSymbols, dict.GetRootElement( i ) ); + } + Assert( nUsedSymbols <= nTotalSymbols ); + + // write out the symbol table for this file (may be significantly smaller than datamodel's full symbol table) + outBuf.PutShort( nUsedSymbols ); + for ( int si = 0; si < nUsedSymbols; ++si ) + { + UtlSymId_t sym = indexToSymbolMap[ si ]; + const char *pStr = g_pDataModel->GetString( sym ); + outBuf.PutString( pStr ); + } + + // First write out the dictionary of all elements (to avoid later stitching up in unserialize) + outBuf.PutInt( dict.RootElementCount() ); + for ( i = dict.FirstRootElement(); i != ELEMENT_DICT_HANDLE_INVALID; i = dict.NextRootElement(i) ) + { + SaveElementDict( outBuf, symbolToIndexMap, dict.GetRootElement( i ) ); + } + + // Now write out the attributes of each of those elements + for ( i = dict.FirstRootElement(); i != ELEMENT_DICT_HANDLE_INVALID; i = dict.NextRootElement(i) ) + { + SaveElement( outBuf, dict, symbolToIndexMap, dict.GetRootElement( i ) ); + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Reads an element index and converts it to a handle (local or external) +//----------------------------------------------------------------------------- +DmElementHandle_t CDmSerializerBinary::UnserializeElementIndex( CUtlBuffer &buf, CUtlVector<CDmElement*> &elementList ) +{ + int nElementIndex = buf.GetInt(); + Assert( nElementIndex < elementList.Count() ); + if ( nElementIndex == ELEMENT_INDEX_EXTERNAL ) + { + char idstr[ 40 ]; + buf.GetString( idstr ); + DmObjectId_t id; + UniqueIdFromString( &id, idstr, sizeof( idstr ) ); + return g_pDataModelImp->FindOrCreateElementHandle( id ); + } + + Assert( nElementIndex >= 0 || nElementIndex == ELEMENT_INDEX_NULL ); + if ( nElementIndex < 0 || !elementList[ nElementIndex ] ) + return DMELEMENT_HANDLE_INVALID; + + return elementList[ nElementIndex ]->GetHandle(); +} + + +//----------------------------------------------------------------------------- +// Reads an element attribute +//----------------------------------------------------------------------------- +void CDmSerializerBinary::UnserializeElementAttribute( CUtlBuffer &buf, CDmAttribute *pAttribute, CUtlVector<CDmElement*> &elementList ) +{ + DmElementHandle_t hElement = UnserializeElementIndex( buf, elementList ); + if ( !pAttribute ) + return; + + pAttribute->SetValue( hElement ); +} + + +//----------------------------------------------------------------------------- +// Reads an element array attribute +//----------------------------------------------------------------------------- +void CDmSerializerBinary::UnserializeElementArrayAttribute( CUtlBuffer &buf, CDmAttribute *pAttribute, CUtlVector<CDmElement*> &elementList ) +{ + int nElementCount = buf.GetInt(); + + if ( !pAttribute || pAttribute->GetType() != AT_ELEMENT_ARRAY ) + { + // Parse past the data + for ( int i = 0; i < nElementCount; ++i ) + { + UnserializeElementIndex( buf, elementList ); + } + return; + } + + CDmrElementArray<> array( pAttribute ); + array.RemoveAll(); + array.EnsureCapacity( nElementCount ); + for ( int i = 0; i < nElementCount; ++i ) + { + DmElementHandle_t hElement = UnserializeElementIndex( buf, elementList ); + array.AddToTail( hElement ); + } +} + + +//----------------------------------------------------------------------------- +// Reads a single element +//----------------------------------------------------------------------------- +bool CDmSerializerBinary::UnserializeAttributes( CUtlBuffer &buf, CDmElement *pElement, CUtlVector<CDmElement*> &elementList, UtlSymId_t *symbolTable ) +{ + char nameBuf[ 1024 ]; + + int nAttributeCount = buf.GetInt(); + for ( int i = 0; i < nAttributeCount; ++i ) + { + const char *pName = NULL; + if ( symbolTable ) + { + unsigned short nName = buf.GetShort(); + pName = g_pDataModel->GetString( symbolTable[ nName ] ); + } + else + { + buf.GetString( nameBuf ); + pName = nameBuf; + } + DmAttributeType_t nAttributeType = (DmAttributeType_t)buf.GetChar(); + + Assert( pName != NULL && pName[ 0 ] != '\0' ); + Assert( nAttributeType != AT_UNKNOWN ); + + CDmAttribute *pAttribute = pElement ? pElement->AddAttribute( pName, nAttributeType ) : NULL; + if ( pElement && !pAttribute ) + { + Warning("Dm: Attempted to read an attribute (\"%s\") of an inappropriate type!\n", pName ); + return false; + } + + switch( nAttributeType ) + { + default: + if ( !pAttribute ) + { + SkipUnserialize( buf, nAttributeType ); + } + else + { + pAttribute->Unserialize( buf ); + } + break; + + case AT_ELEMENT: + UnserializeElementAttribute( buf, pAttribute, elementList ); + break; + + case AT_ELEMENT_ARRAY: + UnserializeElementArrayAttribute( buf, pAttribute, elementList ); + break; + } + } + + return buf.IsValid(); +} + +struct DmIdPair_t +{ + DmObjectId_t m_oldId; + DmObjectId_t m_newId; + DmIdPair_t &operator=( const DmIdPair_t &that ) + { + CopyUniqueId( that.m_oldId, &m_oldId ); + CopyUniqueId( that.m_newId, &m_newId ); + return *this; + } + static unsigned int HashKey( const DmIdPair_t& that ) + { + return *( unsigned int* )&that.m_oldId.m_Value; + } + static bool Compare( const DmIdPair_t& a, const DmIdPair_t& b ) + { + return IsUniqueIdEqual( a.m_oldId, b.m_oldId ); + } +}; + +DmElementHandle_t CreateElementWithFallback( const char *pType, const char *pName, DmFileId_t fileid, const DmObjectId_t &id ) +{ + DmElementHandle_t hElement = g_pDataModel->CreateElement( pType, pName, fileid, &id ); + if ( hElement == DMELEMENT_HANDLE_INVALID ) + { + Warning("Binary: Element uses unknown element type %s\n", pType ); + hElement = g_pDataModel->CreateElement( "DmElement", pName, fileid, &id ); + Assert( hElement != DMELEMENT_HANDLE_INVALID ); + } + return hElement; +} + +//----------------------------------------------------------------------------- +// Main entry point for the unserialization +//----------------------------------------------------------------------------- +bool CDmSerializerBinary::Unserialize( CUtlBuffer &buf, const char *pEncodingName, int nEncodingVersion, + const char *pSourceFormatName, int nSourceFormatVersion, + DmFileId_t fileid, DmConflictResolution_t idConflictResolution, CDmElement **ppRoot ) +{ + Assert( !V_stricmp( pEncodingName, GetName() ) ); + if ( V_stricmp( pEncodingName, GetName() ) != 0 ) + return false; + + Assert( nEncodingVersion >= 0 && nEncodingVersion <= 2 ); + if ( nEncodingVersion < 0 || nEncodingVersion > 2 ) + return false; + + bool bReadSymbolTable = nEncodingVersion >= 2; + + // Read string table + unsigned short nStrings = 0; + UtlSymId_t *symbolTable = NULL; + if ( bReadSymbolTable ) + { + char stringBuf[ 256 ]; + + nStrings = buf.GetShort(); + symbolTable = ( UtlSymId_t* )stackalloc( nStrings * sizeof( UtlSymId_t ) ); + for ( int i = 0; i < nStrings; ++i ) + { + buf.GetString( stringBuf ); + symbolTable[ i ] = g_pDataModel->GetSymbol( stringBuf ); + } + } + + bool bSuccess = UnserializeElements( buf, fileid, idConflictResolution, ppRoot, symbolTable ); + if ( !bSuccess ) + return false; + + return g_pDataModel->UpdateUnserializedElements( pSourceFormatName, nSourceFormatVersion, fileid, idConflictResolution, ppRoot ); +} + +bool CDmSerializerBinary::UnserializeElements( CUtlBuffer &buf, DmFileId_t fileid, DmConflictResolution_t idConflictResolution, CDmElement **ppRoot, UtlSymId_t *symbolTable ) +{ + *ppRoot = NULL; + + // Read in the element count. + int nElementCount = buf.GetInt(); + if ( !nElementCount ) + return true; + + int nMaxIdConflicts = min( nElementCount, g_pDataModel->GetAllocatedElementCount() ); + int nExpectedIdCopyConflicts = ( idConflictResolution == CR_FORCE_COPY || idConflictResolution == CR_COPY_NEW ) ? nMaxIdConflicts : 0; + int nBuckets = min( 0x10000, max( 16, nExpectedIdCopyConflicts / 16 ) ); // CUtlHash can only address up to 65k buckets + CUtlHash< DmIdPair_t > idmap( nBuckets, 0, 0, DmIdPair_t::Compare, DmIdPair_t::HashKey ); + + // Read + create all elements + CUtlVector<CDmElement*> elementList( 0, nElementCount ); + for ( int i = 0; i < nElementCount; ++i ) + { + char pName[2048]; + DmObjectId_t id; + + char typeBuf[ 256 ]; + const char *pType = NULL; + if ( symbolTable ) + { + unsigned short nType = buf.GetShort(); + pType = g_pDataModel->GetString( symbolTable[ nType ] ); + } + else + { + buf.GetString( typeBuf ); + pType = typeBuf; + } + + buf.GetString( pName ); + buf.Get( &id, sizeof(DmObjectId_t) ); + + if ( idConflictResolution == CR_FORCE_COPY ) + { + DmIdPair_t idpair; + CopyUniqueId( id, &idpair.m_oldId ); + CreateUniqueId( &idpair.m_newId ); + idmap.Insert( idpair ); + + CopyUniqueId( idpair.m_newId, &id ); + } + + DmElementHandle_t hElement = DMELEMENT_HANDLE_INVALID; + DmElementHandle_t hExistingElement = g_pDataModel->FindElement( id ); + if ( hExistingElement != DMELEMENT_HANDLE_INVALID ) + { + // id is already in use - need to resolve conflict + + if ( idConflictResolution == CR_DELETE_NEW ) + { + elementList.AddToTail( g_pDataModel->GetElement( hExistingElement ) ); + continue; // just don't create this element + } + else if ( idConflictResolution == CR_DELETE_OLD ) + { + g_pDataModelImp->DeleteElement( hExistingElement, HR_NEVER ); // keep the handle around until CreateElementWithFallback + hElement = CreateElementWithFallback( pType, pName, fileid, id ); + Assert( hElement == hExistingElement ); + } + else if ( idConflictResolution == CR_COPY_NEW ) + { + DmIdPair_t idpair; + CopyUniqueId( id, &idpair.m_oldId ); + CreateUniqueId( &idpair.m_newId ); + idmap.Insert( idpair ); + + hElement = CreateElementWithFallback( pType, pName, fileid, idpair.m_newId ); + } + else + Assert( 0 ); + } + + // if not found, then create it + if ( hElement == DMELEMENT_HANDLE_INVALID ) + { + hElement = CreateElementWithFallback( pType, pName, fileid, id ); + } + + CDmElement *pElement = g_pDataModel->GetElement( hElement ); + CDmeElementAccessor::MarkBeingUnserialized( pElement, true ); + elementList.AddToTail( pElement ); + } + + // The root is the 0th element + *ppRoot = elementList[ 0 ]; + + // Now read all attributes + for ( int i = 0; i < nElementCount; ++i ) + { + CDmElement *pInternal = elementList[ i ]; + UnserializeAttributes( buf, pInternal->GetFileId() == fileid ? pInternal : NULL, elementList, symbolTable ); + } + + for ( int i = 0; i < nElementCount; ++i ) + { + CDmElement *pElement = elementList[ i ]; + if ( pElement->GetFileId() == fileid ) + { + // mark all unserialized elements as done unserializing, and call Resolve() + CDmeElementAccessor::MarkBeingUnserialized( pElement, false ); + } + } + + g_pDmElementFrameworkImp->RemoveCleanElementsFromDirtyList( ); + return buf.IsValid(); +} |