diff options
Diffstat (limited to 'filesystem')
| -rw-r--r-- | filesystem/QueuedLoader.cpp | 1978 | ||||
| -rw-r--r-- | filesystem/basefilesystem.cpp | 5807 | ||||
| -rw-r--r-- | filesystem/basefilesystem.h | 1001 | ||||
| -rw-r--r-- | filesystem/filesystem_async.cpp | 1538 | ||||
| -rw-r--r-- | filesystem/filesystem_stdio.cpp | 1606 | ||||
| -rw-r--r-- | filesystem/filesystem_stdio.vpc | 107 | ||||
| -rw-r--r-- | filesystem/filesystem_stdio/ThreadSafeRefCountedObject.h | 85 | ||||
| -rw-r--r-- | filesystem/filesystem_steam.cpp | 1536 | ||||
| -rw-r--r-- | filesystem/filesystem_steam.vpc | 80 | ||||
| -rw-r--r-- | filesystem/filetracker.cpp | 596 | ||||
| -rw-r--r-- | filesystem/filetracker.h | 232 | ||||
| -rw-r--r-- | filesystem/linux_support.cpp | 266 | ||||
| -rw-r--r-- | filesystem/linux_support.h | 58 | ||||
| -rw-r--r-- | filesystem/packfile.cpp | 1127 | ||||
| -rw-r--r-- | filesystem/packfile.h | 277 | ||||
| -rw-r--r-- | filesystem/threadsaferefcountedobject.h | 85 | ||||
| -rw-r--r-- | filesystem/xbox/xbox.def | 3 |
17 files changed, 16382 insertions, 0 deletions
diff --git a/filesystem/QueuedLoader.cpp b/filesystem/QueuedLoader.cpp new file mode 100644 index 0000000..15ab20f --- /dev/null +++ b/filesystem/QueuedLoader.cpp @@ -0,0 +1,1978 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Queued Loading of map resources. !!!!Specifically!!! designed for the map loading process. +// +// Not designed for application startup or gameplay time. Layered on top of async i/o. +// Queued loading is allowed during the map load process until full connection only, +// but can complete the remaining low priority jobs during the game render. +// The normal loading path can run in units of seconds if it does not have to do I/O, +// which is why this system runs first and gets all the data in memory unhindered +// by dependency blocking. The I/O delivery process achieves its speed by having all the I/O +// requests at once, performing the I/O, and handing the actual consumption +// of the I/O buffer to another available core/thread (via job pool) for computation work. +// The I/O (should be all unbuffered) is then only throttled by physical transfer rates. +// +// The Load process is broken into three phases. The first phase build up I/O requests. +// The second phase fulfills only the high priority I/O requests. This gets the critical +// data in memory, that has to be there for the normal load path to query, or the renderer +// to run (i.e. models and shaders). The third phase is the normal load process. +// The low priority jobs run concurrently with the normal load process. Low priority jobs +// are those that have been specially built such that the game or loading can operate unblocked +// without the actual data (i.e. d3d texture bits). +// +// Phase 1: The reslist is parsed into seperate lists based on handled extensions. Each list +// call its own loader which in turn generates its own dictionaries and I/O requests through +// "AddJob". A single reslist entry could cause a laoder to request multiple jobs. ( i.e. models ) +// A loader marks its jobs as high or low priority. +// Phase 2: The I/O requests are sorted (which achieves seek offset order) and +// async i/o commences. Phase 2 does not end until all the high priority jobs +// are complete. This ensures critical data is resident. +// Phase 3: The !!!NORMAL!!! loading path can commence. The legacy loading path then +// is not expected to do I/O (it can, but that's a hole in the reslist), as all of the data +// that it queries, should be resident. +// +// Late added jobs are non-optimal (should have been in reslist), warned, but handled. +// +//===========================================================================// + +#include "basefilesystem.h" + +#include "tier0/vprof.h" +#include "tier0/tslist.h" +#include "tier1/utlbuffer.h" +#include "tier1/convar.h" +#include "tier1/KeyValues.h" +#include "tier1/utllinkedlist.h" +#include "tier1/utlstring.h" +#include "tier1/UtlSortVector.h" +#include "tier1/utldict.h" +#include "basefilesystem.h" +#include "tier0/icommandline.h" +#include "vstdlib/jobthread.h" +#include "filesystem/IQueuedLoader.h" +#include "tier2/tier2.h" +#include "characterset.h" +#if !defined( _X360 ) +#include "xbox/xboxstubs.h" +#endif + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + + + + +#define PRIORITY_HIGH 1 +#define PRIORITY_NORMAL 0 +#define PRIORITY_LOW -1 + +// main thread has reason to block and wait for thread pool to finish jobs +#define MAIN_THREAD_YIELD_TIME 20 + +// discrete stages in the preload process to tick the progress bar +#define PROGRESS_START 0.10f +#define PROGRESS_GOTRESLIST 0.12f +#define PROGRESS_PARSEDRESLIST 0.15f +#define PROGRESS_CREATEDRESOURCES 0.20f +#define PROGRESS_PREPURGE 0.22f +#define PROGRESS_IO 0.25f // up to 1.0 + +struct FileJob_t +{ + FileJob_t() + { + Q_memset( this, 0, sizeof( FileJob_t ) ); + } + + FileNameHandle_t m_hFilename; + QueuedLoaderCallback_t m_pCallback; + FSAsyncControl_t m_hAsyncControl; + void *m_pContext; + void *m_pContext2; + void *m_pTargetData; + int m_nBytesToRead; + unsigned int m_nStartOffset; + LoaderPriority_t m_Priority; + + unsigned int m_SubmitTime; + unsigned int m_FinishTime; + int m_SubmitTag; + int m_nActualBytesRead; + LoaderError_t m_LoaderError; + unsigned int m_ThreadId; + + unsigned int m_bFinished : 1; + unsigned int m_bFreeTargetAfterIO : 1; + unsigned int m_bFileExists : 1; + unsigned int m_bClaimed : 1; +}; + +// dummy stubbed progress interface +class CDummyProgress : public ILoaderProgress +{ + void BeginProgress() {} + void UpdateProgress( float progress ) {} + void EndProgress() {} +}; +static CDummyProgress s_DummyProgress; + +class CQueuedLoader : public CTier2AppSystem< IQueuedLoader > +{ + typedef CTier2AppSystem< IQueuedLoader > BaseClass; + +public: + CQueuedLoader(); + virtual ~CQueuedLoader(); + + // Inherited from IAppSystem + virtual InitReturnVal_t Init(); + virtual void Shutdown(); + + // IQueuedLoader + virtual void InstallLoader( ResourcePreload_t type, IResourcePreload *pLoader ); + virtual void InstallProgress( ILoaderProgress *pProgress ); + // Set bOptimizeReload if you want appropriate data (such as static prop lighting) + // to persist - rather than being purged and reloaded - when going from map A to map A. + virtual bool BeginMapLoading( const char *pMapName, bool bLoadForHDR, bool bOptimizeMapReload ); + virtual void EndMapLoading( bool bAbort ); + virtual bool AddJob( const LoaderJob_t *pLoaderJob ); + virtual void AddMapResource( const char *pFilename ); + virtual void DynamicLoadMapResource( const char *pFilename, DynamicResourceCallback_t pCallback, void *pContext, void *pContext2 ); + virtual void QueueDynamicLoadFunctor( CFunctor* pFunctor ); + virtual bool CompleteDynamicLoad(); + virtual void QueueCleanupDynamicLoadFunctor( CFunctor* pFunctor ); + virtual bool CleanupDynamicLoad(); + virtual bool ClaimAnonymousJob( const char *pFilename, QueuedLoaderCallback_t pCallback, void *pContext, void *pContext2 ); + virtual bool ClaimAnonymousJob( const char *pFilename, void **pData, int *pDataSize, LoaderError_t *pError ); + virtual bool IsMapLoading() const; + virtual bool IsSameMapLoading() const; + virtual bool IsFinished() const; + virtual bool IsBatching() const; + virtual bool IsDynamic() const; + virtual int GetSpewDetail() const; + + char *GetFilename( const FileNameHandle_t hFilename, char *pBuff, int nBuffSize ); + FileNameHandle_t FindFilename( const char *pFilename ); + void SpewInfo(); + + // submit any queued jobs to the async loader, called by main or async thread to get more work + void SubmitPendingJobs(); + + void PurgeAll(); + + +private: + + class CFileJobsLessFunc + { + public: + int GetLayoutOrderForFilename( const char *pFilename ); + bool Less( FileJob_t* const &pFileJobLHS, FileJob_t* const &pFileJobRHS, void *pCtx ); + }; + + class CResourceNameLessFunc + { + public: + bool Less( const FileNameHandle_t &hFilenameLHS, const FileNameHandle_t &hFilenameRHS, void *pCtx ); + }; + typedef CUtlSortVector< FileNameHandle_t, CResourceNameLessFunc > ResourceList_t; + + static void BuildResources( IResourcePreload *pLoader, ResourceList_t *pList, float *pBuildTime ); + static void BuildMaterialResources( IResourcePreload *pLoader, ResourceList_t *pList, float *pBuildTime ); + + void PurgeQueue(); + void CleanQueue(); + void SubmitBatchedJobs(); + void SubmitBatchedJobsAndWait(); + void ParseResourceList( CUtlBuffer &resourceList ); + void GetJobRequests(); + void PurgeUnreferencedResources(); + void AddResourceToTable( const char *pFilename ); + + bool m_bStarted; + bool m_bActive; + bool m_bBatching; + bool m_bDynamic; + bool m_bCanBatch; + bool m_bLoadForHDR; + bool m_bDoProgress; + bool m_bSameMap; + int m_nSubmitCount; + unsigned int m_StartTime; + unsigned int m_EndTime; + char m_szMapNameToCompareSame[MAX_PATH]; + + DynamicResourceCallback_t m_pfnDynamicCallback; + CUtlString m_DynamicFileName; + void* m_pDynamicContext; + void* m_pDynamicContext2; + CThreadFastMutex m_FunctorQueueMutex; + CUtlVector< CFunctor* > m_FunctorQueue; + CUtlVector< CFunctor* > m_CleanupFunctorQueue; + + CUtlFilenameSymbolTable m_Filenames; + CTSList< FileJob_t* > m_PendingJobs; + CTSList< FileJob_t* > m_BatchedJobs; + CUtlLinkedList< FileJob_t* > m_SubmittedJobs; + CUtlDict< FileJob_t*, int > m_AnonymousJobs; + CUtlSymbolTable m_AdditionalResources; + + CUtlSortVector< FileNameHandle_t, CResourceNameLessFunc > m_ResourceNames[RESOURCEPRELOAD_COUNT]; + IResourcePreload *m_pLoaders[RESOURCEPRELOAD_COUNT]; + float m_LoaderTimes[RESOURCEPRELOAD_COUNT]; + ILoaderProgress *m_pProgress; + CThreadFastMutex m_Mutex; +}; +static CQueuedLoader g_QueuedLoader; +EXPOSE_SINGLE_INTERFACE_GLOBALVAR( CQueuedLoader, IQueuedLoader, QUEUEDLOADER_INTERFACE_VERSION, g_QueuedLoader ); + + +class CResourcePreloadAnonymous : public IResourcePreload +{ + virtual bool CreateResource( const char *pName ) + { + // create an anonymous job to get the data in memory, claimed during load, or auto-freed + LoaderJob_t loaderJob; + loaderJob.m_pFilename = pName; + loaderJob.m_pPathID = "GAME"; + loaderJob.m_Priority = LOADERPRIORITY_DURINGPRELOAD; + g_QueuedLoader.AddJob( &loaderJob ); + return true; + } + + virtual void PurgeUnreferencedResources() {} + virtual void OnEndMapLoading( bool bAbort ) {} + virtual void PurgeAll() {} +}; +static CResourcePreloadAnonymous s_ResourcePreloadAnonymous; + +const char *g_ResourceLoaderNames[RESOURCEPRELOAD_COUNT] = +{ + "???", // RESOURCEPRELOAD_UNKNOWN + "Sounds", // RESOURCEPRELOAD_SOUND + "Materials", // RESOURCEPRELOAD_MATERIAL + "Models", // RESOURCEPRELOAD_MODEL + "Cubemaps", // RESOURCEPRELOAD_CUBEMAP + "PropLighting", // RESOURCEPRELOAD_STATICPROPLIGHTING + "Anonymous", // RESOURCEPRELOAD_ANONYMOUS +}; + +static CInterlockedInt g_nActiveJobs; +static CInterlockedInt g_nQueuedJobs; +static CInterlockedInt g_nHighPriorityJobs; // tracks jobs that must finish during preload +static CInterlockedInt g_nJobsToFinishBeforePlay; // tracks jobs that must finish before gameplay +static CInterlockedInt g_nIOMemory; // tracks I/O data from async delivery until consumed +static CInterlockedInt g_nAnonymousIOMemory; // tracks anonymous I/O data from async delivery until consumed +static CInterlockedInt g_SuspendIO; // used to throttle the I/O +static int g_nIOMemoryPeak; +static int g_nAnonymousIOMemoryPeak; +static int g_nHighIOSuspensionMark; +static int g_nLowIOSuspensionMark; + +ConVar loader_spew_info( "loader_spew_info", "0", 0, "0:Off, 1:Timing, 2:Completions, 3:Late Completions, 4:Purges, -1:All " ); + +// Kyle says: this is here only to change the DLL size to force clients to update! This should be removed +// by whoever sees this comment after we've shipped a DLL using it! +ConVar loader_sped_info_ex( "loader_spew_info_ex", "0", 0, "(internal)" ); + +CON_COMMAND( loader_dump_table, "" ) +{ + g_QueuedLoader.SpewInfo(); +} + +//----------------------------------------------------------------------------- +// Constructor +//----------------------------------------------------------------------------- +CQueuedLoader::CQueuedLoader() : BaseClass( false ) +{ + m_bStarted = false; + m_bActive = false; + m_bBatching = false; + m_bDynamic = false; + m_bCanBatch = false; + m_bLoadForHDR = false; + m_bDoProgress = false; + m_bSameMap = false; + + m_nSubmitCount = 0; + + m_pfnDynamicCallback = NULL; + m_pDynamicContext = NULL; + m_pDynamicContext2 = NULL; + + m_szMapNameToCompareSame[0] = '\0'; + + m_pProgress = &s_DummyProgress; + V_memset( m_pLoaders, 0, sizeof( m_pLoaders ) ); + + // set resource dictionaries sort context + for ( int i = 0; i < RESOURCEPRELOAD_COUNT; i++ ) + { + m_ResourceNames[i].SetLessContext( (void *)i ); + } + + InstallLoader( RESOURCEPRELOAD_ANONYMOUS, &s_ResourcePreloadAnonymous ); +} + +//----------------------------------------------------------------------------- +// Destructor +//----------------------------------------------------------------------------- +CQueuedLoader::~CQueuedLoader() +{ +} + +//----------------------------------------------------------------------------- +// Computation job to build out objects +//----------------------------------------------------------------------------- +void CQueuedLoader::BuildResources( IResourcePreload *pLoader, ResourceList_t *pList, float *pBuildTime ) +{ + float t0 = Plat_FloatTime(); + + if ( pLoader ) + { + pList->RedoSort(); + + for ( int i = 0; i < pList->Count(); i++ ) + { + char szFilename[MAX_PATH]; + g_QueuedLoader.GetFilename( pList->Element( i ), szFilename, sizeof( szFilename ) ); + if ( szFilename[0] ) + { + if ( !pLoader->CreateResource( szFilename ) ) + { + Warning( "QueuedLoader: Failed to create resource %s\n", szFilename ); + } + } + } + } + + // finished with list + pList->Purge(); + + *pBuildTime = Plat_FloatTime() - t0; +} + +//----------------------------------------------------------------------------- +// Computation job to build out material objects +//----------------------------------------------------------------------------- +void CQueuedLoader::BuildMaterialResources( IResourcePreload *pLoader, ResourceList_t *pList, float *pBuildTime ) +{ + float t0 = Plat_FloatTime(); + + char szLastFilename[MAX_PATH]; + szLastFilename[0] = '\0'; + + // ensure cubemaps are first + pList->RedoSort(); + + // run a clean operation to cull the non-patched env_cubemap materials, which are not built directly + for ( int i = 0; i < pList->Count(); i++ ) + { + char szFilename[MAX_PATH]; + char *pFilename = g_QueuedLoader.GetFilename( pList->Element( i ), szFilename, sizeof( szFilename ) ); + if ( !V_stristr( pFilename, "maps\\" ) ) + { + // list is sorted, first non-cubemap marks end of relevant list + break; + } + + // skip past maps/mapname/ + pFilename += 5; + pFilename = strchr( pFilename, '\\' ) + 1; + // back up until end of material name is found, need to strip off _%d_%d_%d.vmt + char *pEndFilename = V_stristr( pFilename, ".vmt" ); + if ( !pEndFilename ) + { + pEndFilename = pFilename + strlen( pFilename ); + } + int numUnderscores = 3; + while ( pEndFilename != pFilename && numUnderscores > 0 ) + { + pEndFilename--; + if ( pEndFilename[0] == '_' ) + { + numUnderscores--; + } + } + if ( numUnderscores == 0 ) + { + *pEndFilename = '\0'; + if ( !V_strcmp( szLastFilename, pFilename ) ) + { + // same cubemap material base already processed, skip it + continue; + } + V_strncpy( szLastFilename, pFilename, sizeof( szLastFilename ) ); + + strcat( pFilename, ".vmt" ); + FileNameHandle_t hFilename = g_QueuedLoader.FindFilename( pFilename ); + if ( hFilename ) + { + pList->Remove( hFilename ); + } + } + } + + // process clean list + BuildResources( pLoader, pList, pBuildTime ); + + *pBuildTime = Plat_FloatTime() - t0; +} + +//----------------------------------------------------------------------------- +// Called by multiple worker threads. Throttle the I/O to ensure too many +// buffers don't flood the work queue. Anonymous I/O is allowed to grow unbounded. +//----------------------------------------------------------------------------- +void AdjustAsyncIOSpeed() +{ + if ( g_QueuedLoader.IsDynamic() == true ) + { + return; + } + + // throttle back the I/O to keep the pending buffers from exhausting memory + if ( g_SuspendIO == 0 ) + { + if ( g_nIOMemory >= g_nHighIOSuspensionMark && g_nActiveJobs != 0 ) + { + // protect against another worker thread + if ( g_SuspendIO.AssignIf( 0, 1 ) ) + { + if ( g_QueuedLoader.GetSpewDetail() ) + { + Msg( "QueuedLoader: Suspending I/O at %.2f MB\n", (float)g_nIOMemory / ( 1024.0f * 1024.0f ) ); + } + g_pFullFileSystem->AsyncSuspend(); + } + } + } + else if ( g_SuspendIO == 1 ) + { + if ( g_nIOMemory <= g_nLowIOSuspensionMark ) + { + // protect against another worker thread + if ( g_SuspendIO.AssignIf( 1, 0 ) ) + { + if ( g_QueuedLoader.GetSpewDetail() ) + { + Msg( "QueuedLoader: Resuming I/O at %.2f MB\n", (float)g_nIOMemory / ( 1024.0f * 1024.0f ) ); + } + g_pFullFileSystem->AsyncResume(); + } + } + } +} + +//----------------------------------------------------------------------------- +// Computation job to do work after IO, runs callback +//----------------------------------------------------------------------------- +void IOComputationJob( FileJob_t *pFileJob, void *pData, int nSize, LoaderError_t loaderError ) +{ + int spewDetail = g_QueuedLoader.GetSpewDetail(); + if ( spewDetail & ( LOADER_DETAIL_COMPLETIONS|LOADER_DETAIL_LATECOMPLETIONS ) ) + { + const char *pLateString = ""; + if ( !g_QueuedLoader.IsMapLoading() ) + { + // completed outside of load process + pLateString = "(Late) "; + } + + if ( ( spewDetail & LOADER_DETAIL_COMPLETIONS ) || ( ( spewDetail & LOADER_DETAIL_LATECOMPLETIONS ) && pLateString[0] ) ) + { + char szFilename[MAX_PATH]; + g_QueuedLoader.GetFilename( pFileJob->m_hFilename, szFilename, sizeof( szFilename ) ); + Msg( "QueuedLoader: Computation:%8.8x, Size:%7d %s%s\n", ThreadGetCurrentId(), nSize, pLateString, szFilename ); + } + } + + if ( loaderError != LOADERERROR_NONE && pFileJob->m_bFileExists ) + { + char szFilename[MAX_PATH]; + g_QueuedLoader.GetFilename( pFileJob->m_hFilename, szFilename, sizeof( szFilename ) ); + Warning( "QueuedLoader:: I/O Error on %s\n", szFilename ); + } + + pFileJob->m_nActualBytesRead = nSize; + pFileJob->m_LoaderError = loaderError; + + if ( !pFileJob->m_pCallback ) + { + // absent callback means resource loader want this system to delay buffer until ready for it + if ( !pFileJob->m_pTargetData ) + { + // track it for later, unclaimed buffers will get freed + pFileJob->m_pTargetData = pData; + } + } + else + { + // regardless of error, call job callback so caller can do cleanup of their context + pFileJob->m_pCallback( pFileJob->m_pContext, pFileJob->m_pContext2, pData, nSize, loaderError ); + if ( pFileJob->m_bFreeTargetAfterIO && pData ) + { + // free our data only + g_pFullFileSystem->FreeOptimalReadBuffer( pData ); + } + + // memory has been consumed + g_nIOMemory -= nSize; + } + + // mark as completed + pFileJob->m_bFinished = true; + pFileJob->m_FinishTime = Plat_MSTime(); + pFileJob->m_ThreadId = ThreadGetCurrentId(); + + if ( pFileJob->m_Priority == LOADERPRIORITY_DURINGPRELOAD ) + { + g_nHighPriorityJobs--; + } + else if ( pFileJob->m_Priority == LOADERPRIORITY_BEFOREPLAY ) + { + g_nJobsToFinishBeforePlay--; + } + + g_nQueuedJobs--; + + if ( g_nQueuedJobs == 0 && ( spewDetail & LOADER_DETAIL_TIMING ) ) + { + Msg( "QueuedLoader: Finished I/O of all queued jobs!\n" ); + } + + AdjustAsyncIOSpeed(); +} + +//----------------------------------------------------------------------------- +// Computation job to do work after anonymous job was asynchronously claimed, runs callback. +//----------------------------------------------------------------------------- +void FinishAnonymousJob( FileJob_t *pFileJob, QueuedLoaderCallback_t pCallback, void *pContext, void *pContext2 ) +{ + // regardless of error, call job callback so caller can do cleanup of their context + pCallback( pContext, pContext2, pFileJob->m_pTargetData, pFileJob->m_nActualBytesRead, pFileJob->m_LoaderError ); + if ( pFileJob->m_bFreeTargetAfterIO && pFileJob->m_pTargetData ) + { + // free our data only + g_pFullFileSystem->FreeOptimalReadBuffer( pFileJob->m_pTargetData ); + pFileJob->m_pTargetData = NULL; + } + + pFileJob->m_bClaimed = true; + + // memory has been consumed + g_nAnonymousIOMemory -= pFileJob->m_nActualBytesRead; +} + +//----------------------------------------------------------------------------- +// Callback from I/O job thread. Purposely lightweight as possible to keep i/o from stalling. +//----------------------------------------------------------------------------- +void IOAsyncCallback( const FileAsyncRequest_t &asyncRequest, int numReadBytes, FSAsyncStatus_t asyncStatus ) +{ + FileJob_t *pFileJob = (FileJob_t *)asyncRequest.pContext; + + // interpret the async error + LoaderError_t loaderError; + switch ( asyncStatus ) + { + case FSASYNC_OK: + loaderError = LOADERERROR_NONE; + break; + case FSASYNC_ERR_FILEOPEN: + loaderError = LOADERERROR_FILEOPEN; + break; + default: + loaderError = LOADERERROR_READING; + } + + // track how much i/o data is in flight, consumption will decrement + if ( !pFileJob->m_pCallback ) + { + // anonymous io memory is tracked seperatley + g_nAnonymousIOMemory += numReadBytes; + if ( g_nAnonymousIOMemory > g_nAnonymousIOMemoryPeak ) + { + g_nAnonymousIOMemoryPeak = g_nAnonymousIOMemory; + } + } + else + { + g_nIOMemory += numReadBytes; + if ( g_nIOMemory > g_nIOMemoryPeak ) + { + g_nIOMemoryPeak = g_nIOMemory; + } + } + + // have data or error, do callback as a computation job + if ( !g_QueuedLoader.IsDynamic() ) + { + g_pThreadPool->QueueCall( IOComputationJob, pFileJob, asyncRequest.pData, numReadBytes, loaderError )->Release(); + } + else + { + g_QueuedLoader.QueueDynamicLoadFunctor( CreateFunctor( IOComputationJob, pFileJob, asyncRequest.pData, numReadBytes, loaderError ) ); + } + + // don't let the i/o starve, possibly get some more work from the pending queue + g_QueuedLoader.SubmitPendingJobs(); + + // possibly goes to zero atomically, AFTER submission + // prevents contention between main thread + --g_nActiveJobs; +} + +//----------------------------------------------------------------------------- +// Public method to filename dictionary +//----------------------------------------------------------------------------- +char *CQueuedLoader::GetFilename( const FileNameHandle_t hFilename, char *pBuff, int nBuffSize ) +{ + m_Filenames.String( hFilename, pBuff, nBuffSize ); + return pBuff; +} + +//----------------------------------------------------------------------------- +// Public method to filename dictionary +//----------------------------------------------------------------------------- +FileNameHandle_t CQueuedLoader::FindFilename( const char *pFilename ) +{ + return m_Filenames.FindFileName( pFilename ); +} + +//----------------------------------------------------------------------------- +// Sort function for resource names. +//----------------------------------------------------------------------------- +bool CQueuedLoader::CResourceNameLessFunc::Less( const FileNameHandle_t &hFilenameLHS, const FileNameHandle_t &hFilenameRHS, void *pCtx ) +{ + switch ( (int)pCtx ) + { + case RESOURCEPRELOAD_MATERIAL: + { + // Cubemap materials are expected to be at top of list + char szNameLHS[MAX_PATH]; + char szNameRHS[MAX_PATH]; + + const char *pNameLHS = g_QueuedLoader.GetFilename( hFilenameLHS, szNameLHS, sizeof( szNameLHS ) ); + const char *pNameRHS = g_QueuedLoader.GetFilename( hFilenameRHS, szNameRHS, sizeof( szNameRHS ) ); + + bool bIsCubemapLHS = V_stristr( pNameLHS, "maps\\" ) != NULL; + bool bIsCubemapRHS = V_stristr( pNameRHS, "maps\\" ) != NULL; + if ( bIsCubemapLHS != bIsCubemapRHS ) + { + return ( bIsCubemapLHS == true && bIsCubemapRHS == false ); + } + return ( V_stricmp( pNameLHS, pNameRHS ) < 0 ); + } + break; + + default: + // sort not really needed, just use numeric handles + return ( hFilenameLHS < hFilenameRHS ); + } +} + +//----------------------------------------------------------------------------- +// Resolve filenames to expected disc layout order as... +// bsp, graphs, platform, hl2, episodic, ep2, tf, portal, non-zip +// see XGD layout. +//----------------------------------------------------------------------------- +int CQueuedLoader::CFileJobsLessFunc::GetLayoutOrderForFilename( const char *pFilename ) +{ + bool bIsLocalizedZip = false; + if ( XBX_IsLocalized() ) + { + if ( V_stristr( pFilename, "\\zip" ) && V_stristr( pFilename, XBX_GetLanguageString() ) ) + { + bIsLocalizedZip = true; + } + } + + int order; + if ( V_stristr( pFilename, "\\maps\\" ) ) + { + // bsp's and graphs on the opposite layer, these must be topmost + // the queued loader is expecting to do these first, all at once + // this allows for a single layer switch + if ( V_stristr( pFilename, "\\graphs\\" ) ) + { + order = 1; + } + else + { + order = 0; + } + } + else if ( V_stristr( pFilename, "\\platform\\zip" ) ) + { + order = 2; + } + else if ( V_stristr( pFilename, "\\hl2\\zip" ) ) + { + order = 3; + } + else if ( V_stristr( pFilename, "\\episodic\\zip" ) ) + { + order = 4; + } + else if ( V_stristr( pFilename, "\\ep2\\zip" ) ) + { + order = 5; + } + else if ( V_stristr( pFilename, "\\tf\\zip" ) ) + { + order = 6; + } + else if ( V_stristr( pFilename, "\\portal\\zip" ) ) + { + order = 7; + } + else + { + // other + order = 8; + } + + // localized zips have same relative sort order, but after all other zips + return bIsLocalizedZip ? 10*order : order; +} + +//----------------------------------------------------------------------------- +// Sort function, high priority jobs sort first, then offset, then zip +//----------------------------------------------------------------------------- +bool CQueuedLoader::CFileJobsLessFunc::Less( FileJob_t* const &pFileJobLHS, FileJob_t* const &pFileJobRHS, void *pCtx ) +{ + if ( pFileJobLHS->m_Priority != pFileJobRHS->m_Priority ) + { + // higher priorities sort to top + return ( pFileJobLHS->m_Priority > pFileJobRHS->m_Priority ); + } + + if ( pFileJobLHS->m_hFilename == pFileJobRHS->m_hFilename ) + { + // same file (zip), sort by offset + return pFileJobLHS->m_nStartOffset < pFileJobRHS->m_nStartOffset; + } + + char szFilenameLHS[MAX_PATH]; + char szFilenameRHS[MAX_PATH]; + g_QueuedLoader.GetFilename( pFileJobLHS->m_hFilename, szFilenameLHS, sizeof( szFilenameLHS ) ); + g_QueuedLoader.GetFilename( pFileJobRHS->m_hFilename, szFilenameRHS, sizeof( szFilenameRHS ) ); + + // resolve filename to match disk layout of zips + int layoutLHS = GetLayoutOrderForFilename( szFilenameLHS ); + int layoutRHS = GetLayoutOrderForFilename( szFilenameRHS ); + if ( layoutLHS != layoutRHS ) + { + return layoutLHS < layoutRHS; + } + + return CaselessStringLessThan( szFilenameLHS, szFilenameRHS ); +} + +//----------------------------------------------------------------------------- +// Dump the queue contents to the file system. +//----------------------------------------------------------------------------- +void CQueuedLoader::SubmitPendingJobs() +{ + // prevents contention between I/O and main thread attempting to submit + if ( ThreadInMainThread() && g_nActiveJobs != 0 && m_bDynamic == false ) + { + // main thread can only kick start work if the I/O is idle + // once the I/O is kicked off, the I/O thread is responsible for continual draining + return; + } + else if ( !ThreadInMainThread() && g_nActiveJobs != 1 && m_bDynamic == false ) + { + // I/O thread requests more work, but will only fall through and get some when it expects to go idle + // I/O thread still has jobs and doesn't need any more yet + return; + } + + CTSList<FileJob_t *>::Node_t *pNode = m_PendingJobs.Detach(); + if ( !pNode ) + { + return; + } + + // used by spew to indicate submission blocks + m_nSubmitCount++; + + // sort entries + CUtlSortVector< FileJob_t*, CFileJobsLessFunc > sortedFiles( 0, 128 ); + while ( pNode ) + { + FileJob_t *pFileJob = pNode->elem; + + sortedFiles.InsertNoSort( pFileJob ); + + CTSList<FileJob_t *>::Node_t *pNext = (CTSList<FileJob_t *>::Node_t*)pNode->Next; + delete pNode; + pNode = pNext; + } + sortedFiles.RedoSort(); + + FileAsyncRequest_t asyncRequest; + asyncRequest.pfnCallback = IOAsyncCallback; + + char szFilename[MAX_PATH]; + for ( int i = 0; i<sortedFiles.Count(); i++ ) + { + FileJob_t *pFileJob = sortedFiles[i]; + + pFileJob->m_SubmitTag = m_nSubmitCount; + pFileJob->m_SubmitTime = Plat_MSTime(); + + m_SubmittedJobs.AddToTail( pFileJob ); + + // build an async request + if ( pFileJob->m_Priority == LOADERPRIORITY_DURINGPRELOAD ) + { + // must finish during preload + asyncRequest.priority = PRIORITY_HIGH; + g_nHighPriorityJobs++; + } + else if ( pFileJob->m_Priority == LOADERPRIORITY_BEFOREPLAY ) + { + // must finish before gameplay + asyncRequest.priority = PRIORITY_NORMAL; + g_nJobsToFinishBeforePlay++; + } + else + { + // can finish during gameplay, normal priority + asyncRequest.priority = PRIORITY_NORMAL; + } + + // async will allocate unless caller provided a target + // loader always takes ownership of buffer + asyncRequest.pData = pFileJob->m_pTargetData; + asyncRequest.flags = pFileJob->m_pTargetData ? 0 : FSASYNC_FLAGS_ALLOCNOFREE; + asyncRequest.nOffset = pFileJob->m_nStartOffset; + asyncRequest.nBytes = pFileJob->m_nBytesToRead; + asyncRequest.pszFilename = GetFilename( pFileJob->m_hFilename, szFilename, sizeof( szFilename ) ); + asyncRequest.pContext = (void *)pFileJob; + + if ( pFileJob->m_bFileExists ) + { + // start the valid async request + g_nActiveJobs++; + g_pFullFileSystem->AsyncRead( asyncRequest, &pFileJob->m_hAsyncControl ); + } + else + { + // prevent dragging the i/o system down for known failures + // still need to do callback so subsystems can do the right thing based on file absence + if ( IsDynamic() ) + QueueDynamicLoadFunctor( CreateFunctor( IOComputationJob, pFileJob, pFileJob->m_pTargetData, 0, LOADERERROR_FILEOPEN ) ); + else + g_pThreadPool->QueueCall( IOComputationJob, pFileJob, pFileJob->m_pTargetData, 0, LOADERERROR_FILEOPEN )->Release(); + } + } +} + +//----------------------------------------------------------------------------- +// Add to queue +//----------------------------------------------------------------------------- +bool CQueuedLoader::AddJob( const LoaderJob_t *pLoaderJob ) +{ + if ( !m_bActive ) + { + return false; + } + + Assert( pLoaderJob && pLoaderJob->m_pFilename ); + + if ( m_bCanBatch && !m_bBatching ) + { + // should have been part of pre-load batch + DevWarning( "QueuedLoader: Late Queued Job: %s\n", pLoaderJob->m_pFilename ); + } + + // anonymous jobs lack callbacks and are heavily restricted to ensure their stability + // the caller is expected to claim these before load ends (which auto-purges them) + if ( !pLoaderJob->m_pCallback && pLoaderJob->m_Priority == LOADERPRIORITY_ANYTIME ) + { + Assert( 0 ); + DevWarning( "QueuedLoader: Ignoring Anonymous Job: %s\n", pLoaderJob->m_pFilename ); + return false; + } + + MEM_ALLOC_CREDIT(); + + // all bsp based files get forced to a higher priority in order to achieve a clustered sort + // the bsp files are not going to be anywhere near the zips, thus we don't want head thrashing + bool bFileIsFromBSP; + bool bExists = false; + + char *pFullPath; + char szFullPath[MAX_PATH]; + if ( V_IsAbsolutePath( pLoaderJob->m_pFilename ) ) + { + // an absolute path is trusted, take as is + pFullPath = (char *)pLoaderJob->m_pFilename; + bFileIsFromBSP = V_stristr( pFullPath, ".bsp" ) != NULL; + bExists = true; + } + else + { + // must resolve now, all submitted paths must be absolute for proper sort which achieves seek linearization + // a resolved absolute file ensures its existence + PathTypeFilter_t pathFilter = FILTER_NONE; + if ( IsX360() && ( g_pFullFileSystem->GetDVDMode() == DVDMODE_STRICT ) ) + { + if ( V_stristr( pLoaderJob->m_pFilename, ".bsp" ) || V_stristr( pLoaderJob->m_pFilename, ".ain" ) ) + { + // only the bsp/ain are allowed to be external + pathFilter = FILTER_CULLPACK; + } + else + { + // all files are expected to be in zip + pathFilter = FILTER_CULLNONPACK; + } + } + + PathTypeQuery_t pathType; + g_pFullFileSystem->RelativePathToFullPath( pLoaderJob->m_pFilename, pLoaderJob->m_pPathID, szFullPath, sizeof( szFullPath ), pathFilter, &pathType ); + bExists = V_IsAbsolutePath( szFullPath ); + pFullPath = szFullPath; + bFileIsFromBSP = ( (pathType & PATH_IS_MAPPACKFILE) != 0 ); + } + + // create a file job + FileJob_t *pFileJob = new FileJob_t; + + pFileJob->m_hFilename = m_Filenames.FindOrAddFileName( pFullPath ); + pFileJob->m_bFileExists = bExists; + pFileJob->m_pCallback = pLoaderJob->m_pCallback; + pFileJob->m_pContext = pLoaderJob->m_pContext; + pFileJob->m_pContext2 = pLoaderJob->m_pContext2; + pFileJob->m_pTargetData = pLoaderJob->m_pTargetData; + pFileJob->m_nBytesToRead = pLoaderJob->m_nBytesToRead; + pFileJob->m_nStartOffset = pLoaderJob->m_nStartOffset; + pFileJob->m_Priority = bFileIsFromBSP ? LOADERPRIORITY_DURINGPRELOAD : pLoaderJob->m_Priority; + + if ( pLoaderJob->m_pTargetData ) + { + // never free caller's buffer, if they provide, they have to free it + pFileJob->m_bFreeTargetAfterIO = false; + } + else + { + // caller can take over ownership, otherwise it gets freed after I/O + pFileJob->m_bFreeTargetAfterIO = ( pLoaderJob->m_bPersistTargetData == false ); + } + + if ( !pLoaderJob->m_pCallback ) + { + // track anonymous jobs + AUTO_LOCK( m_Mutex ); + char szFixedName[MAX_PATH]; + V_strncpy( szFixedName, pLoaderJob->m_pFilename, sizeof( szFixedName ) ); + V_FixSlashes( szFixedName ); + m_AnonymousJobs.Insert( szFixedName, pFileJob ); + } + + g_nQueuedJobs++; + + if ( m_bBatching ) + { + m_BatchedJobs.PushItem( pFileJob ); + } + else + { + m_PendingJobs.PushItem( pFileJob ); + SubmitPendingJobs(); + } + + return true; +} + +//----------------------------------------------------------------------------- +// Allows an external system to append to a map's reslist. The next map load +// will append these specified files. Unhandled resources will just get +// quietly discarded. An external system could use this to patch a hole +// or prevent a purge. +//----------------------------------------------------------------------------- +void CQueuedLoader::AddMapResource( const char *pFilename ) +{ + if ( !pFilename || !pFilename[0] ) + { + // pointless + return; + } + + // normalize the provided name as a filename + char szFilename[MAX_PATH]; + V_strncpy( szFilename, pFilename, sizeof( szFilename ) ); + V_FixSlashes( szFilename ); + V_strlower( szFilename ); + + if ( m_AdditionalResources.Find( szFilename ) != UTL_INVAL_SYMBOL ) + { + // already added + return; + } + + m_AdditionalResources.AddString( szFilename ); +} + +//----------------------------------------------------------------------------- +// Asynchronous claim for an anonymous job. +// This allows loaders with deep dependencies to get their data in flight, and then claim it +// when the they are in a state to consume it. +//----------------------------------------------------------------------------- +bool CQueuedLoader::ClaimAnonymousJob( const char *pFilename, QueuedLoaderCallback_t pCallback, void *pContext, void *pContext2 ) +{ + Assert( ThreadInMainThread() ); + Assert( pFilename && pCallback && !m_bBatching ); + + char szFixedName[MAX_PATH]; + V_strncpy( szFixedName, pFilename, sizeof( szFixedName ) ); + V_FixSlashes( szFixedName ); + pFilename = szFixedName; + + int iIndex = m_AnonymousJobs.Find( pFilename ); + if ( iIndex == m_AnonymousJobs.InvalidIndex() ) + { + // unknown + DevWarning( "QueuedLoader: Anonymous Job '%s' not found\n", pFilename ); + return false; + } + + // caller is claiming + FileJob_t *pFileJob = m_AnonymousJobs[iIndex]; + if ( !pFileJob->m_bFinished ) + { + // unfinished shouldn't happen and caller can't have it + // anonymous jobs and their claims are very restrictive in such a way to provide stability + // this dead job will get auto-cleaned at end of map loading + Assert( 0 ); + return false; + } + + m_AnonymousJobs.RemoveAt( iIndex ); + g_pThreadPool->QueueCall( FinishAnonymousJob, pFileJob, pCallback, pContext, pContext2 )->Release(); + + return true; +} + +//----------------------------------------------------------------------------- +// Synchronous claim for an anonymous job. This allows loaders +// with deep dependencies to get their data in flight, and then claim it +// when the they are in a state to consume it. +//----------------------------------------------------------------------------- +bool CQueuedLoader::ClaimAnonymousJob( const char *pFilename, void **pData, int *pDataSize, LoaderError_t *pError ) +{ + Assert( ThreadInMainThread() ); + Assert( pFilename && !m_bBatching ); + + char szFixedName[MAX_PATH]; + V_strncpy( szFixedName, pFilename, sizeof( szFixedName ) ); + V_FixSlashes( szFixedName ); + pFilename = szFixedName; + + int iIndex = m_AnonymousJobs.Find( pFilename ); + if ( iIndex == m_AnonymousJobs.InvalidIndex() ) + { + // unknown + DevWarning( "QueuedLoader: Anonymous Job '%s' not found\n", pFilename ); + return false; + } + + // caller is claiming + FileJob_t *pFileJob = m_AnonymousJobs[iIndex]; + if ( !pFileJob->m_bFinished ) + { + // unfinished shouldn't happen and caller can't have it + // anonymous jobs and their claims are very restrictive in such a way to provide stability + // this dead job will get auto-cleaned at end of map loading + Assert( 0 ); + return false; + } + + pFileJob->m_bClaimed = true; + + m_AnonymousJobs.RemoveAt( iIndex ); + + *pData = pFileJob->m_pTargetData; + *pDataSize = pFileJob->m_LoaderError == LOADERERROR_NONE ? pFileJob->m_nActualBytesRead : 0; + if ( pError ) + { + *pError = pFileJob->m_LoaderError; + } + + // caller owns the data, regardless of how the job was setup + pFileJob->m_pTargetData = NULL; + + // memory has been consumed + g_nAnonymousIOMemory -= pFileJob->m_nActualBytesRead; + + return true; +} + +//----------------------------------------------------------------------------- +// End of batching. Moves jobs into pending queue and submits but does not wait +//----------------------------------------------------------------------------- +void CQueuedLoader::SubmitBatchedJobs() +{ + // end of batching + m_bBatching = false; + + CTSList<FileJob_t *>::Node_t *pNode = m_BatchedJobs.Detach(); + if ( !pNode ) + { + return; + } + + // must wait for any initial i/o to finish + // i/o thread must stop in order to submit all the batched jobs atomically + // and get an accurate accounting of high priority jobs + while ( g_nActiveJobs != 0 ) + { + g_pThreadPool->Yield( MAIN_THREAD_YIELD_TIME ); + } + + // dump batched jobs to pending jobs + while ( pNode ) + { + FileJob_t *pFileJob = pNode->elem; + + m_PendingJobs.PushItem( pFileJob ); + + CTSList<FileJob_t *>::Node_t *pNext = (CTSList<FileJob_t *>::Node_t*)pNode->Next; + delete pNode; + pNode = pNext; + } + + SubmitPendingJobs(); + + if ( GetSpewDetail() ) + { + Msg( "QueuedLoader: High Priority Jobs: %d\n", (int)g_nHighPriorityJobs ); + } +} + +//----------------------------------------------------------------------------- +// End of batching. High priority jobs are guaranteed completed before function returns. +//----------------------------------------------------------------------------- +void CQueuedLoader::SubmitBatchedJobsAndWait() +{ + SubmitBatchedJobs(); + + // finish only the high priority jobs + // high priority jobs are expected to be complete at the conclusion of batching + int total = g_nHighPriorityJobs; + while ( g_nHighPriorityJobs != 0 ) + { + float t = (float)( total - g_nHighPriorityJobs ) / (float)total; + m_pProgress->UpdateProgress( PROGRESS_IO + t * ( 1.0f - PROGRESS_IO ) ); + + // yield some time + g_pThreadPool->Yield( MAIN_THREAD_YIELD_TIME ); + } +} + +//----------------------------------------------------------------------------- +// Clean queue of stale entries. Active entries are skipped. +//----------------------------------------------------------------------------- +void CQueuedLoader::CleanQueue() +{ + for ( int i = 0; i<RESOURCEPRELOAD_COUNT; i++ ) + { + m_ResourceNames[i].Purge(); + } + + m_BatchedJobs.Purge(); + + int iIndex = m_SubmittedJobs.Head(); + while ( iIndex != m_SubmittedJobs.InvalidIndex() ) + { + int iNext = m_SubmittedJobs.Next( iIndex ); + + FileJob_t *pFileJob = m_SubmittedJobs[iIndex]; + if ( pFileJob->m_bFinished ) + { + // job is complete, safe to free + m_SubmittedJobs.Free( iIndex ); + g_pFullFileSystem->AsyncRelease( pFileJob->m_hAsyncControl ); + delete pFileJob; + } + iIndex = iNext; + } + + m_Filenames.RemoveAll(); +} + +//----------------------------------------------------------------------------- +// Abandon queue +//----------------------------------------------------------------------------- +void CQueuedLoader::PurgeQueue() +{ +} + +//----------------------------------------------------------------------------- +// Spew info abut queued load +//----------------------------------------------------------------------------- +void CQueuedLoader::SpewInfo() +{ + Msg( "Queued Loader:\n\n" ); + + int totalClaimed = 0; + int totalUnclaimed = 0; + + if ( IsFinished() || m_bDynamic == true ) + { + // can only access submitted jobs safely when io thread complete + int lastPriority = -1; + int iIndex = m_SubmittedJobs.Head(); + while ( iIndex != m_SubmittedJobs.InvalidIndex() ) + { + FileJob_t *pFileJob = m_SubmittedJobs[iIndex]; + + int asyncDuration = -1; + if ( pFileJob->m_FinishTime ) + { + asyncDuration = pFileJob->m_FinishTime - pFileJob->m_SubmitTime; + } + + if ( pFileJob->m_Priority != lastPriority ) + { + switch ( pFileJob->m_Priority ) + { + case LOADERPRIORITY_DURINGPRELOAD: + Msg( "---- FINISH DURING PRELOAD ( HIGH PRIORITY )----\n" ); + break; + case LOADERPRIORITY_BEFOREPLAY: + Msg( "---- FINISH BEFORE GAMEPLAY ( NORMAL PRIORITY )----\n" ); + break; + case LOADERPRIORITY_ANYTIME: + Msg( "---- FINISH ANYTIME ( NORMAL PRIORITY )----\n" ); + break; + } + lastPriority = pFileJob->m_Priority; + } + + char szAnonymousString[MAX_PATH]; + const char *pAnonymousStatus = ""; + if ( !pFileJob->m_pCallback ) + { + V_snprintf( szAnonymousString, sizeof( szAnonymousString ), "(%s) ", pFileJob->m_bClaimed ? "Claimed" : "Unclaimed" ); + pAnonymousStatus = szAnonymousString; + + if ( pFileJob->m_bClaimed ) + { + totalClaimed += pFileJob->m_nActualBytesRead; + } + else + { + totalUnclaimed += pFileJob->m_nActualBytesRead; + } + } + + char szFilename[MAX_PATH]; + Msg( "Submit:%5dms AsyncDuration:%5dms Tag:%d Thread:%8.8x Size:%7d %s%s\n", + pFileJob->m_SubmitTime - m_StartTime, + asyncDuration, + pFileJob->m_SubmitTag, + pFileJob->m_ThreadId, + pFileJob->m_nActualBytesRead, + pAnonymousStatus, + GetFilename( pFileJob->m_hFilename, szFilename, sizeof( szFilename ) ) ); + + iIndex = m_SubmittedJobs.Next( iIndex ); + } + + Msg( "%d Total Jobs\n", m_SubmittedJobs.Count() ); + } + + Msg( "%d Queued Jobs\n", (int)g_nQueuedJobs ); + Msg( "%d Active Jobs\n", (int)g_nActiveJobs ); + Msg( "Peak IO Memory: %.2f MB\n", (float)g_nIOMemoryPeak / ( 1024.0f * 1024.0f ) ); + Msg( "Peak Anonymous IO Memory: %.2f MB\n", (float)g_nAnonymousIOMemoryPeak / ( 1024.0f * 1024.0f ) ); + Msg( " Total Anonymous Claimed: %d\n", totalClaimed ); + Msg( " Total Anonymous Unclaimed: %d\n", totalUnclaimed ); + if ( m_EndTime ) + { + Msg( "Queuing Duration: %dms\n", m_EndTime - m_StartTime ); + } +} + +//----------------------------------------------------------------------------- +// Initialization +//----------------------------------------------------------------------------- +InitReturnVal_t CQueuedLoader::Init() +{ + InitReturnVal_t nRetVal = BaseClass::Init(); + if ( nRetVal != INIT_OK ) + { + return nRetVal; + } + + return INIT_OK; +} + +//----------------------------------------------------------------------------- +// Shutdown +//----------------------------------------------------------------------------- +void CQueuedLoader::Shutdown() +{ + BaseClass::Shutdown(); +} + +//----------------------------------------------------------------------------- +// Install a type specific interface from managing system. +//----------------------------------------------------------------------------- +void CQueuedLoader::InstallLoader( ResourcePreload_t type, IResourcePreload *pLoader ) +{ + m_pLoaders[type] = pLoader; +} + +void CQueuedLoader::InstallProgress( ILoaderProgress *pProgress ) +{ + m_pProgress = pProgress; +} + +//----------------------------------------------------------------------------- +// Invoke the loader systems to purge dead resources +//----------------------------------------------------------------------------- +void CQueuedLoader::PurgeUnreferencedResources() +{ + ResourcePreload_t purgeOrder[RESOURCEPRELOAD_COUNT]; + + // the purge operations require a specific order (models and cubemaps before materials) + int numPurges = 0; + purgeOrder[numPurges++] = RESOURCEPRELOAD_SOUND; + purgeOrder[numPurges++] = RESOURCEPRELOAD_STATICPROPLIGHTING; + purgeOrder[numPurges++] = RESOURCEPRELOAD_MODEL; + purgeOrder[numPurges++] = RESOURCEPRELOAD_CUBEMAP; + purgeOrder[numPurges++] = RESOURCEPRELOAD_MATERIAL; + + // iterate according to order + for ( int i = 0; i < numPurges; i++ ) + { + ResourcePreload_t loader = purgeOrder[i]; + if ( m_pLoaders[loader] ) + { + m_pLoaders[loader]->PurgeUnreferencedResources(); + } + } + + m_pProgress->UpdateProgress( PROGRESS_PREPURGE ); +} + + +//----------------------------------------------------------------------------- +// Invoke the loader systems to purge all resources, if possible +//----------------------------------------------------------------------------- +void CQueuedLoader::PurgeAll() +{ + ResourcePreload_t purgeOrder[RESOURCEPRELOAD_COUNT]; + + // the purge operations require a specific order (models and cubemaps before materials) + int numPurges = 0; + purgeOrder[numPurges++] = RESOURCEPRELOAD_SOUND; + purgeOrder[numPurges++] = RESOURCEPRELOAD_STATICPROPLIGHTING; + purgeOrder[numPurges++] = RESOURCEPRELOAD_MODEL; + purgeOrder[numPurges++] = RESOURCEPRELOAD_CUBEMAP; + purgeOrder[numPurges++] = RESOURCEPRELOAD_MATERIAL; + + // iterate according to order + for ( int i = 0; i < numPurges; i++ ) + { + ResourcePreload_t loader = purgeOrder[i]; + if ( m_pLoaders[loader] ) + { + m_pLoaders[loader]->PurgeAll(); + } + } + *m_szMapNameToCompareSame = 0; +} + + +//----------------------------------------------------------------------------- +// Invoke the loader systems to request i/o jobs, which are batched. +//----------------------------------------------------------------------------- +void CQueuedLoader::GetJobRequests() +{ + COM_TimestampedLog( "CQueuedLoader::GetJobRequests - Start" ); + + // causes the batch queue to fill with i/o requests + m_bCanBatch = true; + m_bBatching = true; + + float t0 = Plat_FloatTime(); + + if ( !IsPC() && !m_bDynamic ) + { + // cubemap textures must be first to install correctly before their cubemap materials are built (and precache the cubmeap textures) + // cannot be overlapped, must run serially + BuildResources( m_pLoaders[RESOURCEPRELOAD_CUBEMAP], &m_ResourceNames[RESOURCEPRELOAD_CUBEMAP], &m_LoaderTimes[RESOURCEPRELOAD_CUBEMAP] ); + + // Overlapping these is not critical in any way, total time is currently < 2 seconds. + // These operations flood calls (AddJob) back into the queued loader (which has to mutex its lists), + // so in fact it's slightly slower to queue these at this stage. As these routines age they may become more heavyweight. + CJob *jobs[5]; + jobs[0] = g_pThreadPool->QueueCall( BuildResources, m_pLoaders[RESOURCEPRELOAD_SOUND], &m_ResourceNames[RESOURCEPRELOAD_SOUND], &m_LoaderTimes[RESOURCEPRELOAD_SOUND] ); + jobs[1] = g_pThreadPool->QueueCall( BuildMaterialResources, m_pLoaders[RESOURCEPRELOAD_MATERIAL], &m_ResourceNames[RESOURCEPRELOAD_MATERIAL], &m_LoaderTimes[RESOURCEPRELOAD_MATERIAL] ); + jobs[2] = g_pThreadPool->QueueCall( BuildResources, m_pLoaders[RESOURCEPRELOAD_STATICPROPLIGHTING], &m_ResourceNames[RESOURCEPRELOAD_STATICPROPLIGHTING], &m_LoaderTimes[RESOURCEPRELOAD_STATICPROPLIGHTING] ); + jobs[3] = g_pThreadPool->QueueCall( BuildResources, m_pLoaders[RESOURCEPRELOAD_MODEL], &m_ResourceNames[RESOURCEPRELOAD_MODEL], &m_LoaderTimes[RESOURCEPRELOAD_MODEL] ); + jobs[4] = g_pThreadPool->QueueCall( BuildResources, m_pLoaders[RESOURCEPRELOAD_ANONYMOUS], &m_ResourceNames[RESOURCEPRELOAD_ANONYMOUS], &m_LoaderTimes[RESOURCEPRELOAD_ANONYMOUS] ); + + // all jobs must finish + float flLastUpdateT = -1000.0f; + // Update as if this takes 2 seconds + float flDelta = ( PROGRESS_CREATEDRESOURCES - PROGRESS_PARSEDRESLIST ) * 0.03 / 2.0f; + float flProgress = PROGRESS_PARSEDRESLIST; + while( true ) + { + bool bIsDone = true; + for ( int i=0; i<ARRAYSIZE( jobs ); i++ ) + { + if ( !jobs[i]->IsFinished() ) + { + bIsDone = false; + break; + } + } + if ( bIsDone ) + break; + + // Can't sleep; that will allow this thread to be used by the thread pool + float newt = Plat_FloatTime(); + if ( newt - flLastUpdateT > .03 ) + { + m_pProgress->UpdateProgress( flProgress ); + flProgress = clamp( flProgress + flDelta, PROGRESS_PARSEDRESLIST, PROGRESS_CREATEDRESOURCES ); + + // Necessary to take into account any waits for vsync + flLastUpdateT = Plat_FloatTime(); + } + } + + for ( int i=0; i<ARRAYSIZE( jobs ); i++ ) + { + jobs[i]->Release(); + } + } + else + { + BuildResources( m_pLoaders[RESOURCEPRELOAD_CUBEMAP], &m_ResourceNames[RESOURCEPRELOAD_CUBEMAP], &m_LoaderTimes[RESOURCEPRELOAD_CUBEMAP] ); + BuildResources( m_pLoaders[RESOURCEPRELOAD_SOUND], &m_ResourceNames[RESOURCEPRELOAD_SOUND], &m_LoaderTimes[RESOURCEPRELOAD_SOUND] ); + BuildMaterialResources( m_pLoaders[RESOURCEPRELOAD_MATERIAL], &m_ResourceNames[RESOURCEPRELOAD_MATERIAL], &m_LoaderTimes[RESOURCEPRELOAD_MATERIAL] ); + BuildResources( m_pLoaders[RESOURCEPRELOAD_STATICPROPLIGHTING], &m_ResourceNames[RESOURCEPRELOAD_STATICPROPLIGHTING], &m_LoaderTimes[RESOURCEPRELOAD_STATICPROPLIGHTING] ); + BuildResources( m_pLoaders[RESOURCEPRELOAD_MODEL], &m_ResourceNames[RESOURCEPRELOAD_MODEL], &m_LoaderTimes[RESOURCEPRELOAD_MODEL] ); + BuildResources( m_pLoaders[RESOURCEPRELOAD_ANONYMOUS], &m_ResourceNames[RESOURCEPRELOAD_ANONYMOUS], &m_LoaderTimes[RESOURCEPRELOAD_ANONYMOUS] ); + } + + if ( g_QueuedLoader.GetSpewDetail() & LOADER_DETAIL_TIMING ) + { + for ( int i = RESOURCEPRELOAD_UNKNOWN+1; i<RESOURCEPRELOAD_COUNT; i++ ) + { + Msg( "QueuedLoader: %s Creating: %.2f seconds\n", g_ResourceLoaderNames[i], m_LoaderTimes[i] ); + } + Msg( "QueuedLoader: Total Creating: %.2f seconds\n", Plat_FloatTime() - t0 ); + } + + m_pProgress->UpdateProgress( PROGRESS_CREATEDRESOURCES ); + + COM_TimestampedLog( "CQueuedLoader::GetJobRequests - End" ); +} + +void CQueuedLoader::AddResourceToTable( const char *pFilename ) +{ + const char *pExt = V_GetFileExtension( pFilename ); + if ( !pExt ) + { + // unknown + // all resources are identified by their extension + return; + } + + const char *pTypeDir = NULL; + const char *pName = pFilename; + ResourcePreload_t type = RESOURCEPRELOAD_UNKNOWN; + + if ( !V_stricmp( pExt, "wav" ) ) + { + type = RESOURCEPRELOAD_SOUND; + pTypeDir = "sound\\"; + } + else if ( !V_stricmp( pExt, "vmt" ) ) + { + type = RESOURCEPRELOAD_MATERIAL; + pTypeDir = "materials\\"; + } + else if ( !V_stricmp( pExt, "vtf" ) ) + { + if ( V_stristr( pFilename, "maps\\" ) ) + { + // only want cubemap textures + if ( !m_bLoadForHDR && V_stristr( pFilename, ".hdr." ) ) + { + return; + } + else if ( m_bLoadForHDR && !V_stristr( pFilename, ".hdr." ) ) + { + return; + } + type = RESOURCEPRELOAD_CUBEMAP; + pTypeDir = "materials\\"; + } + else + { + return; + } + } + else if ( !V_stricmp( pExt, "mdl" ) ) + { + type = RESOURCEPRELOAD_MODEL; + pTypeDir = "models\\"; + } + else if ( !V_stricmp( pExt, "vhv" ) ) + { + // want static props only + pName = V_stristr( pFilename, "sp_" ); + if ( !pName ) + { + return; + } + + if ( !m_bLoadForHDR && V_stristr( pFilename, "_hdr_" ) ) + { + return; + } + else if ( m_bLoadForHDR && !V_stristr( pFilename, "_hdr_" ) ) + { + return; + } + type = RESOURCEPRELOAD_STATICPROPLIGHTING; + } + else + { + // unknown, ignored + return; + } + + if ( pTypeDir ) + { + // want object name only + // skip past game/type directory prefixing + const char *pDir = V_stristr( pName, pTypeDir ); + if ( pDir ) + { + pName = pDir + strlen( pTypeDir ); + } + } + + FileNameHandle_t hFilename = m_Filenames.FindOrAddFileName( pName ); + m_ResourceNames[type].InsertNoSort( hFilename ); +} + + +//----------------------------------------------------------------------------- +// Parse the raw resource list into resource dictionaries +//----------------------------------------------------------------------------- +void CQueuedLoader::ParseResourceList( CUtlBuffer &resourceList ) +{ + // parse resource list into known types + characterset_t breakSet; + CharacterSetBuild( &breakSet, "" ); + char szToken[MAX_PATH]; + for ( ;; ) + { + int nTokenSize = resourceList.ParseToken( &breakSet, szToken, sizeof( szToken ) ); + if ( nTokenSize <= 0 ) + { + break; + } + + AddResourceToTable( szToken ); + } + + // add any additional resources + // duplicates don't need to be culled, loaders are supposed to handle resources that already exist + for ( int i = 0; i < m_AdditionalResources.GetNumStrings(); i++ ) + { + if ( g_QueuedLoader.GetSpewDetail() ) + { + Msg( "QueuedLoader: Appending: %s\n", m_AdditionalResources.String( i ) ); + } + AddResourceToTable( m_AdditionalResources.String( i ) ); + } + + if ( g_QueuedLoader.GetSpewDetail() ) + { + for ( int i = RESOURCEPRELOAD_UNKNOWN+1; i < RESOURCEPRELOAD_COUNT; i++ ) + { + Msg( "QueuedLoader: %s: %d Entries\n", g_ResourceLoaderNames[i], m_ResourceNames[i].Count() ); + } + } + + m_pProgress->UpdateProgress( PROGRESS_PARSEDRESLIST ); +} + +//----------------------------------------------------------------------------- +// Mark the start of the queued loading process. +//----------------------------------------------------------------------------- +bool CQueuedLoader::BeginMapLoading( const char *pMapName, bool bLoadForHDR, bool bOptimizeMapReload ) +{ + if ( IsPC() ) + { + return false; + } + + if ( CommandLine()->FindParm( "-noqueuedload" ) || ( g_pFullFileSystem->GetDVDMode() != DVDMODE_STRICT ) ) + { + return false; + } + + if ( m_bStarted ) + { + // already started, shouldn't be started more than once + Assert( 0 ); + return true; + } + + COM_TimestampedLog( "CQueuedLoader::BeginMapLoading" ); + + // set the IO throttle markers based on available memory + // these safety watermarks throttle the i/o from flooding memory, when the cores cannot keep up + // the delta must be larger than any single operation, otherwise deadlock + // markers that are too close will cause excessive suspension + size_t usedMemory, freeMemory; + MemAlloc_GlobalMemoryStatus( &usedMemory, &freeMemory ); + if ( freeMemory >= 64*1024*1024 ) + { + // lots of available memory, can afford to have let the i/o get ahead + g_nHighIOSuspensionMark = 10*1024*1024; + g_nLowIOSuspensionMark = 2*1024*1024; + } + else + { + // low memory, suspend the i/o more frequently + g_nHighIOSuspensionMark = 5*1024*1024; + g_nLowIOSuspensionMark = 1*1024*1024; + } + + if ( GetSpewDetail() ) + { + Msg( "QueuedLoader: Suspend I/O at [%.2f,%.2f] MB\n", (float)g_nLowIOSuspensionMark/(1024.0f*1024.0f), (float)g_nHighIOSuspensionMark/(1024.0f*1024.0f) ); + } + + m_bStarted = true; + m_bDynamic = false; + m_bLoadForHDR = bLoadForHDR; + + // map pak will be accessed asynchronously throughout loading and into game frame + g_pFullFileSystem->BeginMapAccess(); + + // remove any prior stale entries + CleanQueue(); + Assert( m_SubmittedJobs.Count() == 0 && g_nActiveJobs == 0 && g_nQueuedJobs == 0 ); + + m_bActive = true; + m_nSubmitCount = 0; + m_StartTime = Plat_MSTime(); + m_EndTime = 0; + m_bCanBatch = false; + m_bBatching = false; + m_bDoProgress = false; + + g_nIOMemory = 0; + g_nAnonymousIOMemory = 0; + g_nIOMemoryPeak = 0; + g_nAnonymousIOMemoryPeak = 0; + + m_bSameMap = bOptimizeMapReload && ( V_stricmp( pMapName, m_szMapNameToCompareSame ) == 0 ); + if ( m_bSameMap ) + { + // Data will persist (so reloading a map is v. fast) + } + else + { + // Full load of the new map's data + V_strncpy( m_szMapNameToCompareSame, pMapName, sizeof( m_szMapNameToCompareSame ) ); + } + + m_pProgress->BeginProgress(); + m_pProgress->UpdateProgress( PROGRESS_START ); + + // load this map's resource list before any other i/o + char szBaseName[MAX_PATH]; + char szFilename[MAX_PATH]; + V_FileBase( pMapName, szBaseName, sizeof( szBaseName ) ); + V_snprintf( szFilename, sizeof( szFilename ), "reslists_xbox/%s%s.lst", szBaseName, GetPlatformExt() ); + + MEM_ALLOC_CREDIT(); + + CUtlBuffer resListBuffer( 0, 0, CUtlBuffer::TEXT_BUFFER ); + if ( !g_pFullFileSystem->ReadFile( szFilename, "GAME", resListBuffer, 0, 0 ) ) + { + // very bad, a valid reslist is critical + DevWarning( "QueuedLoader: Failed to get reslist '%s', Non-Optimal Loading.\n", szFilename ); + m_bActive = false; + return false; + } + + if ( XBX_IsLocalized() ) + { + // find optional localized reslist fixup + V_snprintf( szFilename, sizeof( szFilename ), "reslists_xbox/%s%s.lst", XBX_GetLanguageString(), GetPlatformExt() ); + CUtlBuffer localizedBuffer( 0, 0, CUtlBuffer::TEXT_BUFFER ); + if ( g_pFullFileSystem->ReadFile( szFilename, "GAME", localizedBuffer, 0, 0 ) ) + { + // append it + resListBuffer.EnsureCapacity( resListBuffer.TellPut() + localizedBuffer.TellPut() ); + resListBuffer.Put( localizedBuffer.PeekGet(), localizedBuffer.TellPut() ); + } + } + + m_pProgress->UpdateProgress( PROGRESS_GOTRESLIST ); + + // due to its size, the bsp load is a lengthy i/o operation + // this causes a non-batched async i/o operation to commence immediately + if ( !m_pLoaders[RESOURCEPRELOAD_MODEL]->CreateResource( pMapName ) ) + { + // very bad, a valid bsp is critical + DevWarning( "QueuedLoader: Failed to mount BSP '%s', Non-Optimal Loading.\n", pMapName ); + m_bActive = false; + return false; + } + + // parse the raw resource list into loader specific dictionaries + ParseResourceList( resListBuffer ); + + // run the distributed precache loaders, generating a batch of i/o requests + GetJobRequests(); + + // event each loader to discard dead resources + PurgeUnreferencedResources(); + + // sort and start async fulfilling the i/o requests + // waits for all "must complete" jobs to finish + SubmitBatchedJobsAndWait(); + + // progress is only relevant during preload + // normal load process takes over any progress bar + // disable progress tracking to prevent any late queued operation from updating + m_pProgress->EndProgress(); + + return m_bActive; +} + +//----------------------------------------------------------------------------- +// Signal the end of the queued loading process, i/o will still be in progress. +//----------------------------------------------------------------------------- +void CQueuedLoader::EndMapLoading( bool bAbort ) +{ + if ( !m_bStarted ) + { + // already stopped or never started + return; + } + + ///////////////////////////////////////////////////// + // TBD: Cannot abort!!!! feature has not been done // + ///////////////////////////////////////////////////// + bAbort = false; + + if ( m_bActive ) + { + if ( bAbort ) + { + PurgeQueue(); + } + else + { + // finish all outstanding priority jobs + SubmitPendingJobs(); + while ( g_nHighPriorityJobs != 0 || g_nJobsToFinishBeforePlay != 0 ) + { + // yield some time + g_pThreadPool->Yield( MAIN_THREAD_YIELD_TIME ); + } + } + + m_EndTime = Plat_MSTime(); + m_bActive = false; + + // transmit the end map event + for ( int i = RESOURCEPRELOAD_UNKNOWN+1; i < RESOURCEPRELOAD_COUNT; i++ ) + { + if ( m_pLoaders[i] ) + { + m_pLoaders[i]->OnEndMapLoading( bAbort ); + } + } + + // free any unclaimed anonymous buffers + int iIndex = m_AnonymousJobs.First(); + while ( iIndex != m_AnonymousJobs.InvalidIndex() ) + { + FileJob_t *pFileJob = m_AnonymousJobs[iIndex]; + if ( pFileJob->m_bFreeTargetAfterIO && pFileJob->m_pTargetData ) + { + g_pFullFileSystem->FreeOptimalReadBuffer( pFileJob->m_pTargetData ); + pFileJob->m_pTargetData = NULL; + } + g_nAnonymousIOMemory -= pFileJob->m_nActualBytesRead; + iIndex = m_AnonymousJobs.Next( iIndex ); + } + m_AnonymousJobs.Purge(); + + if ( g_nIOMemory || g_nAnonymousIOMemory ) + { + // expected to be zero, otherwise logic flaw + DevWarning( "CQueuedLoader: Unclaimed I/O memory: total:%d anonymous:%d\n", (int)g_nIOMemory, (int)g_nAnonymousIOMemory ); + g_nIOMemory = 0; + g_nAnonymousIOMemory = 0; + } + + // no longer needed + m_AdditionalResources.RemoveAll(); + } + + g_pFullFileSystem->EndMapAccess(); + m_bStarted = false; +} + +//----------------------------------------------------------------------------- +// Returns true if loader is accepting queue requests. +//----------------------------------------------------------------------------- +bool CQueuedLoader::IsMapLoading() const +{ + return m_bActive; +} + +//----------------------------------------------------------------------------- +// Returns true if loader is working on same map as last load +//----------------------------------------------------------------------------- +bool CQueuedLoader::IsSameMapLoading() const +{ + return m_bActive && m_bSameMap; +} + +//----------------------------------------------------------------------------- +// Returns true if the loader is idle, indicates all i/o and work has completed. +//----------------------------------------------------------------------------- +bool CQueuedLoader::IsFinished() const +{ + return ( m_bActive == false && g_nActiveJobs == 0 && g_nQueuedJobs == 0 ); +} + +//----------------------------------------------------------------------------- +// Returns true if loader is batching +//----------------------------------------------------------------------------- +bool CQueuedLoader::IsBatching() const +{ + return m_bBatching; +} + + +//----------------------------------------------------------------------------- +// Returns true if loader is batching +//----------------------------------------------------------------------------- +bool CQueuedLoader::IsDynamic() const +{ + return m_bDynamic; +} + + +int CQueuedLoader::GetSpewDetail() const +{ + int spewDetail = loader_spew_info.GetInt(); + if ( spewDetail <= 0 ) + { + return spewDetail; + } + + return 1 << ( spewDetail - 1 ); +} + + +void CQueuedLoader::DynamicLoadMapResource( const char *pFilename, DynamicResourceCallback_t pCallback, void *pContext, void *pContext2 ) +{ + Assert( m_bActive == false ); + + m_bActive = true; + m_bDynamic = true; + m_DynamicFileName = pFilename; + m_pfnDynamicCallback = pCallback; + m_pDynamicContext = pContext; + m_pDynamicContext2 = pContext2; + + CleanQueue(); + AddResourceToTable( m_DynamicFileName ); + + // run the distributed precache loaders, generating a batch of i/o requests + GetJobRequests(); + + // sort and start async fulfilling the i/o requests + Assert( m_bBatching && g_nActiveJobs == 0 ); + SubmitBatchedJobs(); + Assert( !m_bBatching ); +} + +void CQueuedLoader::QueueDynamicLoadFunctor( CFunctor* pFunctor ) +{ + AUTO_LOCK( m_FunctorQueueMutex ); + m_FunctorQueue.AddToTail( pFunctor ); +} + +bool CQueuedLoader::CompleteDynamicLoad() +{ + Assert( m_bActive && m_bDynamic && !m_bBatching ); + bool bDone = true; + if ( m_bDynamic ) + { + CUtlVector< CFunctor* > functors; + { + AUTO_LOCK( m_FunctorQueueMutex ); + functors.Swap( m_FunctorQueue ); + } + FOR_EACH_VEC( functors, i ) + { + ( *functors[i] )(); + functors[i]->Release(); + } + + { + AUTO_LOCK( m_FunctorQueueMutex ); + bDone = m_FunctorQueue.Count() == 0 && g_nQueuedJobs == 0 && g_nActiveJobs == 0; + } + + if ( bDone ) + { + if ( m_pfnDynamicCallback ) + { + ( *m_pfnDynamicCallback )( m_DynamicFileName, m_pDynamicContext, m_pDynamicContext2 ); + } + m_DynamicFileName.Clear(); + m_bActive = false; + m_bDynamic = false; + } + } + return bDone; +} + +void CQueuedLoader::QueueCleanupDynamicLoadFunctor( CFunctor* pFunctor ) +{ + Assert( ThreadInMainThread() ); + + m_CleanupFunctorQueue.AddToTail( pFunctor ); +} + +bool CQueuedLoader::CleanupDynamicLoad() +{ + Assert( ThreadInMainThread() ); + + FOR_EACH_VEC( m_CleanupFunctorQueue, i ) + { + ( *m_CleanupFunctorQueue[i] )(); + m_CleanupFunctorQueue[i]->Release(); + } + m_CleanupFunctorQueue.Purge(); + + return true; +} + diff --git a/filesystem/basefilesystem.cpp b/filesystem/basefilesystem.cpp new file mode 100644 index 0000000..b699836 --- /dev/null +++ b/filesystem/basefilesystem.cpp @@ -0,0 +1,5807 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//===========================================================================// + +#ifdef POSIX +#define _snwprintf swprintf +#endif + +#include "basefilesystem.h" +#include "tier0/vprof.h" +#include "tier1/characterset.h" +#include "tier1/utlbuffer.h" +#include "tier1/convar.h" +#include "tier1/KeyValues.h" +#include "tier0/icommandline.h" +#include "generichash.h" +#include "tier1/utllinkedlist.h" +#include "filesystem/IQueuedLoader.h" +#include "tier2/tier2.h" +#include "zip_utils.h" +#include "packfile.h" +#ifdef _X360 +#include "xbox/xbox_launch.h" +#endif + +#ifndef DEDICATED +#include "keyvaluescompiler.h" +#endif +#include "ifilelist.h" + +#ifdef IS_WINDOWS_PC +// Needed for getting file type string +#define WIN32_LEAN_AND_MEAN +#include <shellapi.h> +#endif + +#if defined( _X360 ) +#include "xbox\xbox_win32stubs.h" +#undef GetCurrentDirectory +#endif + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + + +#pragma warning( disable : 4355 ) // warning C4355: 'this' : used in base member initializer list + + +ConVar fs_report_sync_opens( "fs_report_sync_opens", "0", 0, "0:Off, 1:Blocking only, 2:All" ); +ConVar fs_warning_mode( "fs_warning_mode", "0", 0, "0:Off, 1:Warn main thread, 2:Warn other threads" ); + +#define BSPOUTPUT 0 // bsp output flag -- determines type of fs_log output to generate + +static void AddSeperatorAndFixPath( char *str ); + +// Case-insensitive symbol table for path IDs. +CUtlSymbolTableMT g_PathIDTable( 0, 32, true ); + +int g_iNextSearchPathID = 1; + +// This can be used to easily fix a filename on the stack. +#ifdef DBGFLAG_ASSERT +// Look for cases like materials\\blah.vmt. +static bool V_CheckDoubleSlashes( const char *pStr ) +{ + int len = V_strlen( pStr ); + + for ( int i=1; i < len-1; i++ ) + { + if ( (pStr[i] == '/' || pStr[i] == '\\') && (pStr[i+1] == '/' || pStr[i+1] == '\\') ) + { + Msg( "Double slashes found in path '%s'\n", pStr ); + return true; + } + } + return false; +} + +#define CHECK_DOUBLE_SLASHES( x ) V_CheckDoubleSlashes(x) +#else +#define CHECK_DOUBLE_SLASHES( x ) +#endif + +static void LogFileOpen( const char *vpk, const char *pFilename, const char *pAbsPath ) +{ + static const char *mode = NULL; + + // Figure out if we should be doing this at all, the first time we are acalled + if ( mode == NULL ) + { + if ( CommandLine()->FindParm( "-log_opened_files" ) ) + mode = "wt"; + else + mode = ""; + } + if ( *mode == '\0' ) + return; + + // Open file for write or append + FILE *f = fopen( "opened_files.txt", mode ); + Assert( f ); + if ( f ) + { + fprintf( f, "%s, %s, %s\n", vpk, pFilename, pAbsPath ); + //fprintf( f, "%s\n", pFilename ); + fclose(f); + + // If this was the first time, switch from write to append for further writes + mode = "at"; + } +} + + +static CBaseFileSystem *g_pBaseFileSystem; +CBaseFileSystem *BaseFileSystem() +{ + return g_pBaseFileSystem; +} + +ConVar filesystem_buffer_size( "filesystem_buffer_size", "0", 0, "Size of per file buffers. 0 for none" ); + +#if defined( TRACK_BLOCKING_IO ) + +// If we hit more than 100 items in a frame, we're probably doing a level load... +#define MAX_ITEMS 100 + +class CBlockingFileItemList : public IBlockingFileItemList +{ +public: + CBlockingFileItemList( CBaseFileSystem *fs ) + : + m_pFS( fs ), + m_bLocked( false ) + { + } + + // You can't call any of the below calls without calling these methods!!!! + virtual void LockMutex() + { + Assert( !m_bLocked ); + if ( m_bLocked ) + return; + m_bLocked = true; + m_pFS->BlockingFileAccess_EnterCriticalSection(); + } + + virtual void UnlockMutex() + { + Assert( m_bLocked ); + if ( !m_bLocked ) + return; + + m_pFS->BlockingFileAccess_LeaveCriticalSection(); + m_bLocked = false; + } + + virtual int First() const + { + if ( !m_bLocked ) + { + Error( "CBlockingFileItemList::First() w/o calling EnterCriticalSectionFirst!" ); + } + return m_Items.Head(); + } + + virtual int Next( int i ) const + { + if ( !m_bLocked ) + { + Error( "CBlockingFileItemList::Next() w/o calling EnterCriticalSectionFirst!" ); + } + return m_Items.Next( i ); + } + + virtual int InvalidIndex() const + { + return m_Items.InvalidIndex(); + } + + virtual const FileBlockingItem& Get( int index ) const + { + if ( !m_bLocked ) + { + Error( "CBlockingFileItemList::Get( %d ) w/o calling EnterCriticalSectionFirst!", index ); + } + return m_Items[ index ]; + } + + virtual void Reset() + { + if ( !m_bLocked ) + { + Error( "CBlockingFileItemList::Reset() w/o calling EnterCriticalSectionFirst!" ); + } + m_Items.RemoveAll(); + } + + void Add( const FileBlockingItem& item ) + { + // Ack, should use a linked list probably... + while ( m_Items.Count() > MAX_ITEMS ) + { + m_Items.Remove( m_Items.Head() ); + } + m_Items.AddToTail( item ); + } + + +private: + CUtlLinkedList< FileBlockingItem, unsigned short > m_Items; + CBaseFileSystem *m_pFS; + bool m_bLocked; +}; +#endif + +CUtlSymbol CBaseFileSystem::m_GamePathID; +CUtlSymbol CBaseFileSystem::m_BSPPathID; +DVDMode_t CBaseFileSystem::m_DVDMode; +CUtlVector< FileNameHandle_t > CBaseFileSystem::m_ExcludePaths; + + +class CStoreIDEntry +{ +public: + CStoreIDEntry() {} + CStoreIDEntry( const char *pPathIDStr, int storeID ) + { + m_PathIDString = pPathIDStr; + m_StoreID = storeID; + } + +public: + CUtlSymbol m_PathIDString; + int m_StoreID; +}; + +#if 0 +static CStoreIDEntry* FindPrevFileByStoreID( CUtlDict< CUtlVector<CStoreIDEntry>* ,int> &filesByStoreID, const char *pFilename, const char *pPathIDStr, int foundStoreID ) +{ + int iEntry = filesByStoreID.Find( pFilename ); + if ( iEntry == filesByStoreID.InvalidIndex() ) + { + CUtlVector<CStoreIDEntry> *pList = new CUtlVector<CStoreIDEntry>; + pList->AddToTail( CStoreIDEntry(pPathIDStr, foundStoreID) ); + filesByStoreID.Insert( pFilename, pList ); + return NULL; + } + else + { + // Now is there a previous entry with a different path ID string and the same store ID? + CUtlVector<CStoreIDEntry> *pList = filesByStoreID[iEntry]; + for ( int i=0; i < pList->Count(); i++ ) + { + CStoreIDEntry &entry = pList->Element( i ); + if ( entry.m_StoreID == foundStoreID && V_stricmp( entry.m_PathIDString.String(), pPathIDStr ) != 0 ) + return &entry; + } + return NULL; + } +} +#endif + +//----------------------------------------------------------------------------- + +class CBaseFileSystem::CFileCacheObject +{ +public: + CFileCacheObject( CBaseFileSystem* pFS ); + ~CFileCacheObject(); + void AddFiles( const char **ppFileNames, int nFileNames ); + bool IsReady() const { return m_nPending == 0; } + + static void IOCallback( const FileAsyncRequest_t &request, int nBytesRead, FSAsyncStatus_t err ); + + struct Info_t + { + const char* pFileName; + FSAsyncControl_t hIOAsync; + CMemoryFileBacking* pBacking; + CFileCacheObject* pOwner; + }; + + CBaseFileSystem* m_pFS; + CInterlockedInt m_nPending; + CThreadFastMutex m_InfosMutex; + CUtlVector< Info_t* > m_Infos; + +private: + void ProcessNewEntries( int start ); + + CFileCacheObject( const CFileCacheObject& ); // not implemented + CFileCacheObject & operator=(const CFileCacheObject& ); // not implemented +}; + +//----------------------------------------------------------------------------- +// constructor +//----------------------------------------------------------------------------- + +CBaseFileSystem::CBaseFileSystem() + : m_FileTracker2( this ) +{ + g_pBaseFileSystem = this; + g_pFullFileSystem = this; + + m_WhitelistFileTrackingEnabled = -1; + + // If this changes then FileNameHandleInternal_t/FileNameHandle_t needs to be fixed!!! + Assert( sizeof( CUtlSymbol ) == sizeof( short ) ); + + // Clear out statistics + memset( &m_Stats, 0, sizeof(m_Stats) ); + + m_fwLevel = FILESYSTEM_WARNING_REPORTUNCLOSED; + m_pfnWarning = NULL; + m_pLogFile = NULL; + m_bOutputDebugString = false; + m_WhitelistSpewFlags = 0; + m_DirtyDiskReportFunc = NULL; + m_pPureServerWhitelist = NULL; + + m_pThreadPool = NULL; +#if defined( TRACK_BLOCKING_IO ) + m_pBlockingItems = new CBlockingFileItemList( this ); + m_bBlockingFileAccessReportingEnabled = false; + m_bAllowSynchronousLogging = true; +#endif + + m_iMapLoad = 0; + + Q_memset( m_PreloadData, 0, sizeof( m_PreloadData ) ); + + // allows very specifc constrained behavior + m_DVDMode = DVDMODE_OFF; + if ( IsX360() ) + { + if ( CommandLine()->FindParm( "-dvd" ) ) + { + m_DVDMode = DVDMODE_STRICT; + } + else if ( CommandLine()->FindParm( "-dvddev" ) ) + { + m_DVDMode = DVDMODE_DEV; + } + } +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +CBaseFileSystem::~CBaseFileSystem() +{ + m_PathIDInfos.PurgeAndDeleteElements(); +#if defined( TRACK_BLOCKING_IO ) + delete m_pBlockingItems; +#endif + + // Free the whitelist. + RegisterFileWhitelist( NULL, NULL ); +} + + +//----------------------------------------------------------------------------- +// Methods of IAppSystem +//----------------------------------------------------------------------------- +void *CBaseFileSystem::QueryInterface( const char *pInterfaceName ) +{ + // We also implement the IMatSystemSurface interface + if (!Q_strncmp( pInterfaceName, BASEFILESYSTEM_INTERFACE_VERSION, Q_strlen(BASEFILESYSTEM_INTERFACE_VERSION) + 1)) + return (IBaseFileSystem*)this; + + return NULL; +} + +InitReturnVal_t CBaseFileSystem::Init() +{ + InitReturnVal_t nRetVal = BaseClass::Init(); + if ( nRetVal != INIT_OK ) + return nRetVal; + + // This is a special tag to allow iterating just the BSP file, it doesn't show up in the list per se, but gets converted to "GAME" in the filter function + m_BSPPathID = g_PathIDTable.AddString( "BSP" ); + m_GamePathID = g_PathIDTable.AddString( "GAME" ); + + if ( getenv( "fs_debug" ) ) + { + m_bOutputDebugString = true; + } + + const char *logFileName = CommandLine()->ParmValue( "-fs_log" ); + if( logFileName ) + { + m_pLogFile = fopen( logFileName, "w" ); // STEAM OK + if ( !m_pLogFile ) + return INIT_FAILED; + fprintf( m_pLogFile, "@echo off\n" ); + fprintf( m_pLogFile, "setlocal\n" ); + const char *fs_target = CommandLine()->ParmValue( "-fs_target" ); + if( fs_target ) + { + fprintf( m_pLogFile, "set fs_target=\"%s\"\n", fs_target ); + } + fprintf( m_pLogFile, "if \"%%fs_target%%\" == \"\" goto error\n" ); + fprintf( m_pLogFile, "@echo on\n" ); + } + + InitAsync(); + + if ( IsX360() && m_DVDMode == DVDMODE_DEV ) + { + // exclude paths are valid ony in dvddev mode + char szExcludeFile[MAX_PATH]; + const char *pRemotePath = CommandLine()->ParmValue( "-remote" ); + const char *pBasePath = CommandLine()->ParmValue( "-basedir" ); + if ( pRemotePath && pBasePath ) + { + // the optional exclude path file only exists at the remote path + V_ComposeFileName( pRemotePath, "xbox_exclude_paths.txt", szExcludeFile, sizeof( szExcludeFile ) ); + + // populate the exclusion list + CUtlBuffer buf( 0, 0, CUtlBuffer::TEXT_BUFFER ); + if ( ReadFile( szExcludeFile, NULL, buf, 0, 0 ) ) + { + characterset_t breakSet; + CharacterSetBuild( &breakSet, "" ); + char szPath[MAX_PATH]; + char szToken[MAX_PATH]; + for ( ;; ) + { + int nTokenSize = buf.ParseToken( &breakSet, szToken, sizeof( szToken ) ); + if ( nTokenSize <= 0 ) + { + break; + } + + char *pToken = szToken; + if ( pToken[0] == '\\' ) + { + // skip past possible initial seperator + pToken++; + } + + V_ComposeFileName( pBasePath, pToken, szPath, sizeof( szPath ) ); + V_AppendSlash( szPath, sizeof( szPath ) ); + + FileNameHandle_t hFileName = FindOrAddFileName( szPath ); + if ( m_ExcludePaths.Find( hFileName ) == -1 ) + { + m_ExcludePaths.AddToTail( hFileName ); + } + } + } + } + } + + return INIT_OK; +} + +void CBaseFileSystem::Shutdown() +{ + ShutdownAsync(); + m_FileTracker2.ShutdownAsync(); + +#ifndef _X360 + if( m_pLogFile ) + { + if( CommandLine()->FindParm( "-fs_logbins" ) >= 0 ) + { + char cwd[MAX_FILEPATH]; + getcwd( cwd, MAX_FILEPATH-1 ); + fprintf( m_pLogFile, "set binsrc=\"%s\"\n", cwd ); + fprintf( m_pLogFile, "mkdir \"%%fs_target%%\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\hl2.exe\" \"%%fs_target%%\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\hl2.dat\" \"%%fs_target%%\"\n" ); + fprintf( m_pLogFile, "mkdir \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\bin\\*.asi\" \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\bin\\materialsystem.dll\" \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\bin\\shaderapidx9.dll\" \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\bin\\filesystem_stdio.dll\" \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\bin\\soundemittersystem.dll\" \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\bin\\stdshader*.dll\" \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\bin\\shader_nv*.dll\" \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\bin\\launcher.dll\" \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\bin\\engine.dll\" \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\bin\\mss32.dll\" \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\bin\\tier0.dll\" \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\bin\\vgui2.dll\" \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\bin\\vguimatsurface.dll\" \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\bin\\voice_miles.dll\" \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\bin\\vphysics.dll\" \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\bin\\vstdlib.dll\" \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\bin\\studiorender.dll\" \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\bin\\vaudio_miles.dll\" \"%%fs_target%%\\bin\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\hl2\\resource\\*.ttf\" \"%%fs_target%%\\hl2\\resource\"\n" ); + fprintf( m_pLogFile, "copy \"%%binsrc%%\\hl2\\bin\\gameui.dll\" \"%%fs_target%%\\hl2\\bin\"\n" ); + } + fprintf( m_pLogFile, "goto done\n" ); + fprintf( m_pLogFile, ":error\n" ); + fprintf( m_pLogFile, "echo !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\"\n" ); + fprintf( m_pLogFile, "echo ERROR: must set fs_target=targetpath (ie. \"set fs_target=u:\\destdir\")!\n" ); + fprintf( m_pLogFile, "echo !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\"\n" ); + fprintf( m_pLogFile, ":done\n" ); + fclose( m_pLogFile ); // STEAM OK + } +#endif + + UnloadCompiledKeyValues(); + + RemoveAllSearchPaths(); + Trace_DumpUnclosedFiles(); + BaseClass::Shutdown(); +} + + +//----------------------------------------------------------------------------- +// Computes a full write path +//----------------------------------------------------------------------------- +inline void CBaseFileSystem::ComputeFullWritePath( char* pDest, int maxlen, const char *pRelativePath, const char *pWritePathID ) +{ + Q_strncpy( pDest, GetWritePath( pRelativePath, pWritePathID ), maxlen ); + Q_strncat( pDest, pRelativePath, maxlen, COPY_ALL_CHARACTERS ); + Q_FixSlashes( pDest ); +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : src1 - +// src2 - +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CBaseFileSystem::OpenedFileLessFunc( COpenedFile const& src1, COpenedFile const& src2 ) +{ + return src1.m_pFile < src2.m_pFile; +} + + +void CBaseFileSystem::InstallDirtyDiskReportFunc( FSDirtyDiskReportFunc_t func ) +{ + m_DirtyDiskReportFunc = func; +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *fullpath - +//----------------------------------------------------------------------------- +void CBaseFileSystem::LogAccessToFile( char const *accesstype, char const *fullpath, char const *options ) +{ + LOCAL_THREAD_LOCK(); + + if ( m_fwLevel >= FILESYSTEM_WARNING_REPORTALLACCESSES && !V_stristr( fullpath, "console.log" ) ) + { + Warning( FILESYSTEM_WARNING_REPORTALLACCESSES, "---FS%s: %s %s (%.3f)\n", ThreadInMainThread() ? "" : "[a]", accesstype, fullpath, Plat_FloatTime() ); + } + + int c = m_LogFuncs.Count(); + if ( !c ) + return; + + for ( int i = 0; i < c; ++i ) + { + ( m_LogFuncs[ i ] )( fullpath, options ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *filename - +// *options - +// Output : FILE +//----------------------------------------------------------------------------- +FILE *CBaseFileSystem::Trace_FOpen( const char *filenameT, const char *options, unsigned flags, int64 *size ) +{ + AUTOBLOCKREPORTER_FN( Trace_FOpen, this, true, filenameT, FILESYSTEM_BLOCKING_SYNCHRONOUS, FileBlockingItem::FB_ACCESS_OPEN ); + + char filename[MAX_PATH]; + + FixUpPath ( filenameT, filename, sizeof( filename ) ); + + FILE *fp = FS_fopen( filename, options, flags, size ); + + if ( fp ) + { + if ( options[0] == 'r' ) + { + FS_setbufsize(fp, filesystem_buffer_size.GetInt() ); + } + else + { + FS_setbufsize(fp, 32*1024 ); + } + + AUTO_LOCK( m_OpenedFilesMutex ); + COpenedFile file; + + file.SetName( filename ); + file.m_pFile = fp; + + m_OpenedFiles.AddToTail( file ); + + LogAccessToFile( "open", filename, options ); + } + + return fp; +} + +void CBaseFileSystem::GetFileNameForHandle( FileHandle_t handle, char *buf, size_t buflen ) +{ + V_strncpy( buf, "Unknown", buflen ); + /* + CFileHandle *fh = ( CFileHandle *)handle; + if ( !fh ) + { + buf[ 0 ] = 0; + return; + } + +#if !defined( _RETAIL ) + // Pack file filehandles store the underlying name for convenience + if ( fh->IsPack() ) + { + Q_strncpy( buf, fh->Name(), buflen ); + return; + } +#endif + + AUTO_LOCK( m_OpenedFilesMutex ); + + COpenedFile file; + file.m_pFile = fh->GetFileHandle(); + + int result = m_OpenedFiles.Find( file ); + if ( result != -1 ) + { + COpenedFile found = m_OpenedFiles[ result ]; + Q_strncpy( buf, found.GetName(), buflen ); + } + else + { + buf[ 0 ] = 0; + } + */ +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *fp - +//----------------------------------------------------------------------------- +void CBaseFileSystem::Trace_FClose( FILE *fp ) +{ + if ( fp ) + { + m_OpenedFilesMutex.Lock(); + + COpenedFile file; + file.m_pFile = fp; + + int result = m_OpenedFiles.Find( file ); + if ( result != -1 /*m_OpenedFiles.InvalidIdx()*/ ) + { + COpenedFile found = m_OpenedFiles[ result ]; + if ( m_fwLevel >= FILESYSTEM_WARNING_REPORTALLACCESSES && !V_stristr( found.GetName(), "console.log" ) ) + { + Warning( FILESYSTEM_WARNING_REPORTALLACCESSES, "---FS%s: close %s %p %i (%.3f)\n", ThreadInMainThread() ? "" : "[a]", found.GetName(), fp, m_OpenedFiles.Count(), Plat_FloatTime() ); + } + m_OpenedFiles.Remove( result ); + } + else + { + Assert( 0 ); + + if ( m_fwLevel >= FILESYSTEM_WARNING_REPORTALLACCESSES ) + { + Warning( FILESYSTEM_WARNING_REPORTALLACCESSES, "Tried to close unknown file pointer %p\n", fp ); + } + } + + m_OpenedFilesMutex.Unlock(); + + FS_fclose( fp ); + } +} + + +void CBaseFileSystem::Trace_FRead( int size, FILE* fp ) +{ + if ( !fp || m_fwLevel < FILESYSTEM_WARNING_REPORTALLACCESSES_READ ) + return; + + AUTO_LOCK( m_OpenedFilesMutex ); + + COpenedFile file; + file.m_pFile = fp; + + int result = m_OpenedFiles.Find( file ); + + if( result != -1 ) + { + COpenedFile found = m_OpenedFiles[ result ]; + + Warning( FILESYSTEM_WARNING_REPORTALLACCESSES_READ, "---FS%s: read %s %i %p (%.3f)\n", ThreadInMainThread() ? "" : "[a]", found.GetName(), size, fp, Plat_FloatTime() ); + } + else + { + Warning( FILESYSTEM_WARNING_REPORTALLACCESSES_READ, "Tried to read %i bytes from unknown file pointer %p\n", size, fp ); + + } +} + +void CBaseFileSystem::Trace_FWrite( int size, FILE* fp ) +{ + if ( !fp || m_fwLevel < FILESYSTEM_WARNING_REPORTALLACCESSES_READWRITE ) + return; + + COpenedFile file; + file.m_pFile = fp; + + AUTO_LOCK( m_OpenedFilesMutex ); + + int result = m_OpenedFiles.Find( file ); + + if( result != -1 ) + { + COpenedFile found = m_OpenedFiles[ result ]; + + Warning( FILESYSTEM_WARNING_REPORTALLACCESSES_READWRITE, "---FS%s: write %s %i %p\n", ThreadInMainThread() ? "" : "[a]", found.GetName(), size, fp ); + } + else + { + Warning( FILESYSTEM_WARNING_REPORTALLACCESSES_READWRITE, "Tried to write %i bytes from unknown file pointer %p\n", size, fp ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void CBaseFileSystem::Trace_DumpUnclosedFiles( void ) +{ + /* + AUTO_LOCK( m_OpenedFilesMutex ); + for ( int i = 0 ; i < m_OpenedFiles.Count(); i++ ) + { + COpenedFile *found = &m_OpenedFiles[ i ]; + + if ( m_fwLevel >= FILESYSTEM_WARNING_REPORTUNCLOSED ) + { + // This warning can legitimately trigger because the log file is, by design, not + // closed. It is not closed at shutdown in order to avoid race conditions. It is + // not closed at each call to log information because the performance costs, + // especially if Microsoft Security Essentials is running, are extreme. + // In addition, when this warning triggers it can, due to the state of the process, + // actually cause a crash. + Warning( FILESYSTEM_WARNING_REPORTUNCLOSED, "File %s was never closed\n", found->GetName() ); + } + } + */ +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void CBaseFileSystem::PrintOpenedFiles( void ) +{ + FileWarningLevel_t saveLevel = m_fwLevel; + m_fwLevel = FILESYSTEM_WARNING_REPORTUNCLOSED; + Trace_DumpUnclosedFiles(); + m_fwLevel = saveLevel; +} + +#if defined( SUPPORT_PACKED_STORE ) + +CPackedStoreRefCount::CPackedStoreRefCount( char const *pFileBasename, char *pszFName, IBaseFileSystem *pFS ) +: CPackedStore( pFileBasename, pszFName, pFS, false ) +{ + // If the VPK is signed, check the signature + // + // !FIXME! This code is simple a linchpin that a hacker could detour to bypass sv_pure + #ifdef VPK_ENABLE_SIGNING + CPackedStore::ESignatureCheckResult eSigResult = CPackedStore::CheckSignature( 0, NULL ); + m_bSignatureValid = ( eSigResult == CPackedStore::eSignatureCheckResult_ValidSignature ); + #else + m_bSignatureValid = false; + #endif +} + +#endif + +void CBaseFileSystem::AddVPKFile( char const *pPath, const char *pPathID, SearchPathAdd_t addType ) +{ +#if defined( SUPPORT_PACKED_STORE ) + char nameBuf[MAX_PATH]; + + Q_MakeAbsolutePath( nameBuf, sizeof( nameBuf ), pPath ); + Q_FixSlashes( nameBuf ); + + CUtlSymbol pathIDSym = g_PathIDTable.AddString( pPathID ); + + // See if we already have this vpk file as a search path + CPackedStoreRefCount *pVPK = NULL; + for ( int i = 0; i < m_SearchPaths.Count(); i++ ) + { + CPackedStoreRefCount *p = m_SearchPaths[i].GetPackedStore(); + if ( p && V_stricmp( p->FullPathName(), nameBuf ) == 0 ) + { + // Already present + if ( m_SearchPaths[i].GetPath() == pathIDSym ) + return; + + // Present, but for a different path + pVPK = p; + } + } + + // Create a new VPK if we didn't don't already have it opened + if ( pVPK == NULL ) + { + char pszFName[MAX_PATH]; + pVPK = new CPackedStoreRefCount( nameBuf, pszFName, this ); + if ( pVPK->IsEmpty() ) + { + delete pVPK; + return; + } + pVPK->RegisterFileTracker( (IThreadedFileMD5Processor *)&m_FileTracker2 ); + + pVPK->m_PackFileID = m_FileTracker2.NotePackFileOpened( pVPK->FullPathName(), pPathID, 0 ); + } + else + { + pVPK->AddRef(); + } + + // Crete a search path for this + CSearchPath *sp = &m_SearchPaths[ ( addType == PATH_ADD_TO_TAIL ) ? m_SearchPaths.AddToTail() : m_SearchPaths.AddToHead() ]; + sp->SetPackedStore( pVPK ); + sp->m_storeId = g_iNextSearchPathID++; + sp->SetPath( pathIDSym ); + sp->m_pPathIDInfo = FindOrAddPathIDInfo( g_PathIDTable.AddString( pPathID ), -1 ); + + // Check if we're trusted or not + SetSearchPathIsTrustedSource( sp ); +#endif // SUPPORT_PACKED_STORE +} + + +bool CBaseFileSystem::RemoveVPKFile( const char *pPath, const char *pPathID ) +{ +#if defined( SUPPORT_PACKED_STORE ) + char nameBuf[MAX_PATH]; + + Q_MakeAbsolutePath( nameBuf, sizeof( nameBuf ), pPath ); + Q_FixSlashes( nameBuf ); + + CUtlSymbol pathIDSym = g_PathIDTable.AddString( pPathID ); + + // See if we already have this vpk file as a search path + for ( int i = 0; i < m_SearchPaths.Count(); i++ ) + { + CPackedStoreRefCount *p = m_SearchPaths[i].GetPackedStore(); + if ( p && V_stricmp( p->FullPathName(), nameBuf ) == 0 ) + { + // remove if we find one + if ( m_SearchPaths[i].GetPath() == pathIDSym ) + { + m_SearchPaths.Remove( i ); + return true; + } + } + } +#endif // SUPPORT_PACKED_STORE + + return false; +} + + +//----------------------------------------------------------------------------- +// Purpose: Adds the specified pack file to the list +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CBaseFileSystem::AddPackFile( const char *pFileName, const char *pathID ) +{ + CHECK_DOUBLE_SLASHES( pFileName ); + + AsyncFinishAll(); + return AddPackFileFromPath( "", pFileName, true, pathID ); +} + +//----------------------------------------------------------------------------- +// Purpose: Adds a pack file from the specified path +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CBaseFileSystem::AddPackFileFromPath( const char *pPath, const char *pakfile, bool bCheckForAppendedPack, const char *pathID ) +{ + char fullpath[ MAX_PATH ]; + _snprintf( fullpath, sizeof(fullpath), "%s%s", pPath, pakfile ); + Q_FixSlashes( fullpath ); + + struct _stat buf; + if ( FS_stat( fullpath, &buf ) == -1 ) + return false; + + CPackFile *pf = new CZipPackFile( this ); + pf->m_hPackFileHandleFS = Trace_FOpen( fullpath, "rb", 0, NULL ); + if ( !pf->m_hPackFileHandleFS ) + { + delete pf; + return false; + } + + // NOTE: Opening .bsp fiels inside of VPK files is not supported + + // Get the length of the pack file: + FS_fseek( ( FILE * )pf->m_hPackFileHandleFS, 0, FILESYSTEM_SEEK_TAIL ); + int64 len = FS_ftell( ( FILE * )pf->m_hPackFileHandleFS ); + FS_fseek( ( FILE * )pf->m_hPackFileHandleFS, 0, FILESYSTEM_SEEK_HEAD ); + + if ( !pf->Prepare( len ) ) + { + // Failed for some reason, ignore it + Trace_FClose( pf->m_hPackFileHandleFS ); + pf->m_hPackFileHandleFS = NULL; + delete pf; + + return false; + } + + // Add this pack file to the search path: + CSearchPath *sp = &m_SearchPaths[ m_SearchPaths.AddToTail() ]; + pf->SetPath( sp->GetPath() ); + pf->m_lPackFileTime = GetFileTime( pakfile ); + + sp->SetPath( pPath ); + sp->m_pPathIDInfo->SetPathID( pathID ); + sp->SetPackFile( pf ); + + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Search pPath for pak?.pak files and add to search path if found +// Input : *pPath - +//----------------------------------------------------------------------------- +#if !defined( _X360 ) +#define PACK_NAME_FORMAT "zip%i.zip" +#else +#define PACK_NAME_FORMAT "zip%i.360.zip" +#define PACK_LOCALIZED_NAME_FORMAT "zip%i_%s.360.zip" +#endif + +void CBaseFileSystem::AddPackFiles( const char *pPath, const CUtlSymbol &pathID, SearchPathAdd_t addType ) +{ + Assert( ThreadInMainThread() ); + DISK_INTENSIVE(); + + CUtlVector< CUtlString > pakNames; + CUtlVector< int64 > pakSizes; + + // determine pak files, [zip0..zipN] + for ( int i = 0; ; i++ ) + { + char pakfile[MAX_PATH]; + char fullpath[MAX_PATH]; + V_snprintf( pakfile, sizeof( pakfile ), PACK_NAME_FORMAT, i ); + V_ComposeFileName( pPath, pakfile, fullpath, sizeof( fullpath ) ); + + struct _stat buf; + if ( FS_stat( fullpath, &buf ) == -1 ) + break; + + MEM_ALLOC_CREDIT(); + + pakNames.AddToTail( pakfile ); + pakSizes.AddToTail( (int64)((unsigned int)buf.st_size) ); + } + +#if defined( _X360 ) + // localized zips are added last to ensure they appear first in the search path construction + // localized zips can only appear in the game or mod directories + bool bUseEnglishAudio = XboxLaunch()->GetForceEnglish(); + + if ( XBX_IsLocalized() && ( bUseEnglishAudio == false ) && + ( V_stricmp( g_PathIDTable.String( pathID ), "game" ) == 0 || V_stricmp( g_PathIDTable.String( pathID ), "mod" ) == 0 ) ) + { + // determine localized pak files, [zip0_language..zipN_language] + for ( int i = 0; ; i++ ) + { + char pakfile[MAX_PATH]; + char fullpath[MAX_PATH]; + V_snprintf( pakfile, sizeof( pakfile ), PACK_LOCALIZED_NAME_FORMAT, i, XBX_GetLanguageString() ); + V_ComposeFileName( pPath, pakfile, fullpath, sizeof( fullpath ) ); + + struct _stat buf; + if ( FS_stat( fullpath, &buf ) == -1 ) + break; + + pakNames.AddToTail( pakfile ); + pakSizes.AddToTail( (int64)((unsigned int)buf.st_size) ); + } + } +#endif + + // Add any zip files in the format zip1.zip ... zip0.zip + // Add them backwards so zip(N) is higher priority than zip(N-1), etc. + int pakcount = pakSizes.Count(); + int nCount = 0; + for ( int i = pakcount-1; i >= 0; i-- ) + { + char fullpath[MAX_PATH]; + V_ComposeFileName( pPath, pakNames[i].Get(), fullpath, sizeof( fullpath ) ); + + int nIndex; + if ( addType == PATH_ADD_TO_TAIL ) + { + nIndex = m_SearchPaths.AddToTail(); + } + else + { + nIndex = m_SearchPaths.InsertBefore( nCount ); + ++nCount; + } + + CSearchPath *sp = &m_SearchPaths[ nIndex ]; + + sp->m_pPathIDInfo = FindOrAddPathIDInfo( pathID, -1 ); + sp->m_storeId = g_iNextSearchPathID++; + sp->SetPath( g_PathIDTable.AddString( pPath ) ); + + CPackFile *pf = NULL; + for ( int iPackFile = 0; iPackFile < m_ZipFiles.Count(); iPackFile++ ) + { + if ( !Q_stricmp( m_ZipFiles[iPackFile]->m_ZipName.Get(), fullpath ) ) + { + pf = m_ZipFiles[iPackFile]; + sp->SetPackFile( pf ); + pf->AddRef(); + } + } + + if ( !pf ) + { + MEM_ALLOC_CREDIT(); + + pf = new CZipPackFile( this ); + + pf->SetPath( sp->GetPath() ); + pf->m_ZipName = fullpath; + + m_ZipFiles.AddToTail( pf ); + sp->SetPackFile( pf ); + pf->m_lPackFileTime = GetFileTime( fullpath ); + + pf->m_hPackFileHandleFS = Trace_FOpen( fullpath, "rb", 0, NULL ); + + if ( pf->m_hPackFileHandleFS ) + { + FS_setbufsize( pf->m_hPackFileHandleFS, 32*1024 ); // 32k buffer. + + if ( pf->Prepare( pakSizes[i] ) ) + { + FS_setbufsize( pf->m_hPackFileHandleFS, filesystem_buffer_size.GetInt() ); + } + else + { + // Failed for some reason, ignore it + if ( pf->m_hPackFileHandleFS ) + { + Trace_FClose( pf->m_hPackFileHandleFS ); + pf->m_hPackFileHandleFS = NULL; + } + m_SearchPaths.Remove( nIndex ); + } + } + else + { + // !NOTE! Pack files inside of VPK not supported + //pf->m_hPackFileHandleVPK = FindFileInVPKs( fullpath ); + //if ( !pf->m_hPackFileHandleVPK || !pf->Prepare( pakSizes[i] ) ) + { + m_SearchPaths.Remove( nIndex ); + } + } + } + } +} + +//----------------------------------------------------------------------------- +// Purpose: Wipe all map (.bsp) pak file search paths +//----------------------------------------------------------------------------- +void CBaseFileSystem::RemoveAllMapSearchPaths( void ) +{ + AsyncFinishAll(); + + int c = m_SearchPaths.Count(); + for ( int i = c - 1; i >= 0; i-- ) + { + if ( !( m_SearchPaths[i].GetPackFile() && m_SearchPaths[i].GetPackFile()->m_bIsMapPath ) ) + { + continue; + } + + m_SearchPaths.Remove( i ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void CBaseFileSystem::AddMapPackFile( const char *pPath, const char *pPathID, SearchPathAdd_t addType ) +{ + char tempPathID[MAX_PATH]; + ParsePathID( pPath, pPathID, tempPathID ); + + // Security nightmares already, should not let things explicitly loading from e.g. "MOD" get surprise untrusted + // files unless you really really know what you're doing. + AssertMsg( V_strcasecmp( pPathID, "GAME" ) == 0, + "Mounting map files anywhere besides GAME is asking for pain" ); + + char newPath[ MAX_FILEPATH ]; + // +2 for '\0' and potential slash added at end. + V_strncpy( newPath, pPath, sizeof( newPath ) ); +#ifdef _WIN32 // don't do this on linux! + V_strlower( newPath ); +#endif + V_FixSlashes( newPath ); + + // Open the .bsp and find the map lump + char fullpath[ MAX_FILEPATH ]; + if ( V_IsAbsolutePath( newPath ) ) // If it's an absolute path, just use that. + { + V_strncpy( fullpath, newPath, sizeof(fullpath) ); + } + else + { + if ( !GetLocalPath( newPath, fullpath, sizeof(fullpath) ) ) + { + // Couldn't find that .bsp file!!! + return; + } + } + + int c = m_SearchPaths.Count(); + for ( int i = c - 1; i >= 0; i-- ) + { + if ( !( m_SearchPaths[i].GetPackFile() && m_SearchPaths[i].GetPackFile()->m_bIsMapPath ) ) + continue; + + if ( V_stricmp( m_SearchPaths[i].GetPackFile()->m_ZipName.Get(), fullpath ) == 0 ) + { + // Already set as map path + return; + } + } + + RemoveAllMapSearchPaths(); + + CUtlSymbol pathSymbol = g_PathIDTable.AddString( newPath ); + + int iStoreId = CRC32_ProcessSingleBuffer( fullpath, V_strlen(fullpath) ) | 0x80000000; // set store ID based on .bsp filename, so if we remount the same map file, it will have the same storeid + + // Look through already-open packfiles in case we intentionally + // preserved this ZIP across a map reload via refcount holding + FOR_EACH_VEC( m_ZipFiles, i ) + { + CPackFile* pf = m_ZipFiles[i]; + if ( pf && pf->m_bIsMapPath && pf->GetPath() == pathSymbol && V_stricmp( pf->m_ZipName.Get(), fullpath ) == 0 ) + { + CSearchPath *sp = &m_SearchPaths[ ( addType == PATH_ADD_TO_TAIL ) ? m_SearchPaths.AddToTail() : m_SearchPaths.AddToHead() ]; + pf->AddRef(); + sp->SetPackFile( pf ); + sp->m_storeId = iStoreId; + sp->SetPath( pathSymbol ); + sp->m_pPathIDInfo = FindOrAddPathIDInfo( g_PathIDTable.AddString( pPathID ), -1 ); + if ( IsX360() && !V_strnicmp( newPath, "net:", 4 ) ) + { + sp->m_bIsRemotePath = true; + } + SetSearchPathIsTrustedSource( sp ); + return; + } + } + + { + FILE *fp = Trace_FOpen( fullpath, "rb", 0, NULL ); + if ( !fp ) + { + // Couldn't open it + Warning( FILESYSTEM_WARNING, "Couldn't open .bsp %s for embedded pack file check\n", fullpath ); + return; + } + + // Get the .bsp file header + dheader_t header; + memset( &header, 0, sizeof(dheader_t) ); + m_Stats.nBytesRead += FS_fread( &header, sizeof( header ), fp ); + m_Stats.nReads++; + + if ( header.ident != IDBSPHEADER || header.version < MINBSPVERSION || header.version > BSPVERSION ) + { + Trace_FClose( fp ); + return; + } + + // Find the LUMP_PAKFILE offset + lump_t *packfile = &header.lumps[ LUMP_PAKFILE ]; + if ( packfile->filelen <= sizeof( lump_t ) ) + { + // It's empty or only contains a file header ( so there are no entries ), so don't add to search paths + Trace_FClose( fp ); + return; + } + + // Seek to correct position + FS_fseek( fp, packfile->fileofs, FILESYSTEM_SEEK_HEAD ); + + CPackFile *pf = new CZipPackFile( this ); + + pf->m_bIsMapPath = true; + pf->m_hPackFileHandleFS = fp; + + MEM_ALLOC_CREDIT(); + pf->m_ZipName = fullpath; + + if ( pf->Prepare( packfile->filelen, packfile->fileofs ) ) + { + int nIndex; + if ( addType == PATH_ADD_TO_TAIL ) + { + nIndex = m_SearchPaths.AddToTail(); + } + else + { + nIndex = m_SearchPaths.AddToHead(); + } + + CSearchPath *sp = &m_SearchPaths[ nIndex ]; + + sp->SetPackFile( pf ); + sp->m_storeId = iStoreId; + sp->SetPath( pathSymbol ); + sp->m_pPathIDInfo = FindOrAddPathIDInfo( g_PathIDTable.AddString( pPathID ), -1 ); + + if ( IsX360() && !V_strnicmp( newPath, "net:", 4 ) ) + { + sp->m_bIsRemotePath = true; + } + + pf->SetPath( pathSymbol ); + pf->m_lPackFileTime = GetFileTime( newPath ); + + Trace_FClose( pf->m_hPackFileHandleFS ); + pf->m_hPackFileHandleFS = NULL; + + m_ZipFiles.AddToTail( pf ); + + SetSearchPathIsTrustedSource( sp ); + } + else + { + delete pf; + } + } +} + + +void CBaseFileSystem::BeginMapAccess() +{ + if ( m_iMapLoad++ == 0 ) + { + int c = m_SearchPaths.Count(); + for( int i = 0; i < c; i++ ) + { + CSearchPath *pSearchPath = &m_SearchPaths[i]; + CPackFile *pPackFile = pSearchPath->GetPackFile(); + + if ( pPackFile && pPackFile->m_bIsMapPath ) + { + pPackFile->AddRef(); + pPackFile->m_mutex.Lock(); + +#if defined( SUPPORT_PACKED_STORE ) + if ( pPackFile->m_nOpenFiles == 0 && pPackFile->m_hPackFileHandleFS == NULL && !pPackFile->m_hPackFileHandleVPK ) +#else + if ( pPackFile->m_nOpenFiles == 0 && pPackFile->m_hPackFileHandleFS == NULL ) +#endif + { + // Try opening the file as a regular file + pPackFile->m_hPackFileHandleFS = Trace_FOpen( pPackFile->m_ZipName, "rb", 0, NULL ); + +// !NOTE! Pack files inside of VPK not supported +//#if defined( SUPPORT_PACKED_STORE ) +// if ( !pPackFile->m_hPackFileHandleFS ) +// { +// pPackFile->m_hPackFileHandleVPK = FindFileInVPKs( pPackFile->m_ZipName ); +// } +//#endif + } + pPackFile->m_nOpenFiles++; + pPackFile->m_mutex.Unlock(); + } + } + } +} + + +void CBaseFileSystem::EndMapAccess() +{ + if ( m_iMapLoad-- == 1 ) + { + int c = m_SearchPaths.Count(); + for( int i = 0; i < c; i++ ) + { + CSearchPath *pSearchPath = &m_SearchPaths[i]; + CPackFile *pPackFile = pSearchPath->GetPackFile(); + + if ( pPackFile && pPackFile->m_bIsMapPath ) + { + pPackFile->m_mutex.Lock(); + pPackFile->m_nOpenFiles--; + if ( pPackFile->m_nOpenFiles == 0 ) + { + if ( pPackFile->m_hPackFileHandleFS ) + { + Trace_FClose( pPackFile->m_hPackFileHandleFS ); + pPackFile->m_hPackFileHandleFS = NULL; + } + } + pPackFile->m_mutex.Unlock(); + pPackFile->Release(); + } + } + } +} + + +void CBaseFileSystem::PrintSearchPaths( void ) +{ + Msg( "---------------\n" ); + Msg( "Paths:\n" ); + + int c = m_SearchPaths.Count(); + for( int i = 0; i < c; i++ ) + { + CSearchPath *pSearchPath = &m_SearchPaths[i]; + + const char *pszPack = ""; + const char *pszType = ""; + if ( pSearchPath->GetPackFile() && pSearchPath->GetPackFile()->m_bIsMapPath ) + { + pszType = "(map)"; + } + else if ( pSearchPath->GetPackFile() ) + { + pszType = "(pack) "; + pszPack = pSearchPath->GetPackFile()->m_ZipName; + } + #ifdef SUPPORT_PACKED_STORE + else if ( pSearchPath->GetPackedStore() ) + { + pszType = "(VPK)"; + pszPack = pSearchPath->GetPackedStore()->FullPathName(); + } + #endif + + Msg( "\"%s\" \"%s\" %s%s\n", pSearchPath->GetPathString(), (const char *)pSearchPath->GetPathIDString(), pszType, pszPack ); + } + + if ( IsX360() && m_ExcludePaths.Count() ) + { + // dump current list + Msg( "\nExclude:\n" ); + char szPath[MAX_PATH]; + for ( int i = 0; i < m_ExcludePaths.Count(); i++ ) + { + if ( String( m_ExcludePaths[i], szPath, sizeof( szPath ) ) ) + { + Msg( "\"%s\"\n", szPath ); + } + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: This is where search paths are created. map files are created at head of list (they occur after +// file system paths have already been set ) so they get highest priority. Otherwise, we add the disk (non-packfile) +// path and then the paks if they exist for the path +// Input : *pPath - +//----------------------------------------------------------------------------- +void CBaseFileSystem::AddSearchPathInternal( const char *pPath, const char *pathID, SearchPathAdd_t addType, bool bAddPackFiles ) +{ + AsyncFinishAll(); + + Assert( ThreadInMainThread() ); + + // Map pak files have their own handler + if ( V_stristr( pPath, ".bsp" ) ) + { + AddMapPackFile( pPath, pathID, addType ); + return; + } + + // So do VPK files + if ( V_stristr( pPath, ".vpk" ) ) + { + AddVPKFile( pPath, pathID, addType ); + return; + } + + // Clean up the name + char newPath[ MAX_FILEPATH ]; + if ( pPath[0] == 0 ) + { + newPath[0] = newPath[1] = 0; + } + else + { + if ( IsX360() || Q_IsAbsolutePath( pPath ) ) + { + Q_strncpy( newPath, pPath, sizeof( newPath ) ); + } + else + { + Q_MakeAbsolutePath( newPath, sizeof(newPath), pPath ); + } +#ifdef _WIN32 + Q_strlower( newPath ); +#endif + AddSeperatorAndFixPath( newPath ); + } + + // Make sure that it doesn't already exist + CUtlSymbol pathSym, pathIDSym; + pathSym = g_PathIDTable.AddString( newPath ); + pathIDSym = g_PathIDTable.AddString( pathID ); + int i; + int c = m_SearchPaths.Count(); + int id = 0; + for ( i = 0; i < c; i++ ) + { + CSearchPath *pSearchPath = &m_SearchPaths[i]; + if ( pSearchPath->GetPath() == pathSym && pSearchPath->GetPathID() == pathIDSym ) + { + if ( ( addType == PATH_ADD_TO_HEAD && i == 0 ) || ( addType == PATH_ADD_TO_TAIL ) ) + { + return; // this entry is already at the head + } + else + { + m_SearchPaths.Remove(i); // remove it from its current position so we can add it back to the head + i--; + c--; + } + } + if ( !id && pSearchPath->GetPath() == pathSym ) + { + // get first found - all reference the same store + id = pSearchPath->m_storeId; + } + } + + if (!id) + { + id = g_iNextSearchPathID++; + } + + if ( IsX360() && bAddPackFiles && ( !Q_stricmp( pathID, "DEFAULT_WRITE_PATH" ) || !Q_stricmp( pathID, "LOGDIR" ) ) ) + { + // xbox can be assured that no zips would ever be loaded on its write path + // otherwise xbox reloads zips because of mirrored drive mappings + bAddPackFiles = false; + } + + // Add to list + bool bAdded = false; + int nIndex = m_SearchPaths.Count(); + if ( bAddPackFiles ) + { + // Add pack files for this path next + AddPackFiles( newPath, pathIDSym, addType ); + bAdded = m_SearchPaths.Count() != nIndex; + } + if ( addType == PATH_ADD_TO_HEAD ) + { + nIndex = m_SearchPaths.Count() - nIndex; + Assert( nIndex >= 0 ); + } + + if ( IsPC() || !bAddPackFiles || !bAdded ) + { + // Grab last entry and set the path + m_SearchPaths.InsertBefore( nIndex ); + } + else if ( IsX360() && bAddPackFiles && bAdded ) + { + // 360 needs to find files (for the preload hit) in the zip first for fast loading + // 360 always adds the non-pack search path *after* the pack file but respects the overall list ordering + if ( addType == PATH_ADD_TO_HEAD ) + { + m_SearchPaths.InsertBefore( nIndex ); + } + else + { + nIndex = m_SearchPaths.Count() - 1; + m_SearchPaths.InsertAfter( nIndex ); + nIndex++; + } + } + + CSearchPath *sp = &m_SearchPaths[ nIndex ]; + + sp->SetPath( pathSym ); + sp->m_pPathIDInfo = FindOrAddPathIDInfo( pathIDSym, -1 ); + + // all matching paths have a reference to the same store + sp->m_storeId = id; + if ( IsX360() && !V_strnicmp( newPath, "net:", 4 ) ) + { + sp->m_bIsRemotePath = true; + } +} + +//----------------------------------------------------------------------------- +CBaseFileSystem::CSearchPath *CBaseFileSystem::FindSearchPathByStoreId( int storeId ) +{ + FOR_EACH_VEC( m_SearchPaths, i ) + { + if ( m_SearchPaths[i].m_storeId == storeId ) + return &m_SearchPaths[i]; + } + + return NULL; +} + +//----------------------------------------------------------------------------- +// Create the search path. +//----------------------------------------------------------------------------- +void CBaseFileSystem::AddSearchPath( const char *pPath, const char *pathID, SearchPathAdd_t addType ) +{ + int currCount = m_SearchPaths.Count(); + + AddSearchPathInternal( pPath, pathID, addType, true ); + + if ( IsX360() && m_DVDMode == DVDMODE_DEV ) + { + // dvd development mode clones a search path based on the remote path for fall through + const char *pRemotePath = CommandLine()->ParmValue( "-remote" ); + const char *pBasePath = CommandLine()->ParmValue( "-basedir" ); + if ( pRemotePath && pBasePath && !V_stristr( pPath, ".bsp" ) ) + { + // isolate the search path from the base path + if ( !V_strnicmp( pPath, pBasePath, strlen( pBasePath ) ) ) + { + // substitue the remote path + char szRemotePath[MAX_PATH]; + V_strncpy( szRemotePath, pRemotePath, sizeof( szRemotePath ) ); + V_strncat( szRemotePath, pPath + strlen( pBasePath ), sizeof( szRemotePath ) ); + + // no pack files are allowed on the fall through remote path + AddSearchPathInternal( szRemotePath, pathID, addType, false ); + } + } + } + + if ( currCount != m_SearchPaths.Count() ) + { +#if !defined( DEDICATED ) + if ( IsDebug() ) + { + // spew updated search paths + // PrintSearchPaths(); + } +#endif + } +} + +//----------------------------------------------------------------------------- +// Returns the search path, each path is separated by ;s. Returns the length of the string returned +// Pack search paths include the pack name, so that callers can still form absolute paths +// and that absolute path can be sent to the filesystem, and mounted as a file inside a pack. +//----------------------------------------------------------------------------- +int CBaseFileSystem::GetSearchPath( const char *pathID, bool bGetPackFiles, OUT_Z_CAP(maxLenInChars) char *pDest, int maxLenInChars ) +{ + AUTO_LOCK( m_SearchPathsMutex ); + + if ( maxLenInChars ) + { + pDest[0] = 0; + } + + // Build up result into string object + CUtlString sResult; + CSearchPathsIterator iter( this, pathID, bGetPackFiles ? FILTER_NONE : FILTER_CULLPACK ); + for ( CSearchPath *pSearchPath = iter.GetFirst(); pSearchPath != NULL; pSearchPath = iter.GetNext() ) + { + if ( !sResult.IsEmpty() ) + sResult += ";"; + CUtlString sName; + if ( pSearchPath->GetPackFile() ) + { + sResult += pSearchPath->GetPackFile()->m_ZipName.String(); + sResult += CORRECT_PATH_SEPARATOR_S; + } + #ifdef SUPPORT_PACKED_STORE + else if ( pSearchPath->GetPackedStore() ) + { + sResult += pSearchPath->GetPackedStore()->FullPathName(); + } + #endif + else + { + sResult += pSearchPath->GetPathString(); + } + } + + // Copy into user's buffer, possibly truncating + if ( maxLenInChars ) + { + V_strncpy( pDest, sResult.String(), maxLenInChars ); + } + + // Return 1 extra for the NULL terminator + return sResult.Length()+1; +} + + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *pPath - +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CBaseFileSystem::RemoveSearchPath( const char *pPath, const char *pathID ) +{ + AsyncFinishAll(); + + char newPath[ MAX_FILEPATH ]; + newPath[ 0 ] = 0; + + if ( pPath ) + { + // +2 for '\0' and potential slash added at end. + Q_strncpy( newPath, pPath, sizeof( newPath ) ); +#ifdef _WIN32 // don't do this on linux! + Q_strlower( newPath ); +#endif + + if ( V_stristr( newPath, ".bsp" ) ) + { + Q_FixSlashes( newPath ); + } + else if ( V_stristr( newPath, ".vpk" ) ) + { + return RemoveVPKFile( newPath, pathID ); + } + else + { + AddSeperatorAndFixPath( newPath ); + } + } + + CUtlSymbol lookup = g_PathIDTable.AddString( newPath ); + CUtlSymbol id = g_PathIDTable.AddString( pathID ); + + bool bret = false; + + // Count backward since we're possibly deleting one or more pack files, too + int i; + int c = m_SearchPaths.Count(); + for( i = c - 1; i >= 0; i-- ) + { + if ( newPath[0] && m_SearchPaths[i].GetPath() != lookup ) + continue; + + if ( FilterByPathID( &m_SearchPaths[i], id ) ) + continue; + + m_SearchPaths.Remove( i ); + bret = true; + } + return bret; +} + + +//----------------------------------------------------------------------------- +// Purpose: Removes all search paths for a given pathID, such as all "GAME" paths. +// Input : pathID - +//----------------------------------------------------------------------------- +void CBaseFileSystem::RemoveSearchPaths( const char *pathID ) +{ + AsyncFinishAll(); + + int nCount = m_SearchPaths.Count(); + for (int i = nCount - 1; i >= 0; i--) + { + if (!Q_stricmp(m_SearchPaths.Element(i).GetPathIDString(), pathID)) + { + m_SearchPaths.FastRemove(i); + } + } +} + + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +CBaseFileSystem::CSearchPath *CBaseFileSystem::FindWritePath( const char *pFilename, const char *pathID ) +{ + CUtlSymbol lookup = g_PathIDTable.AddString( pathID ); + + AUTO_LOCK( m_SearchPathsMutex ); + + // a pathID has been specified, find the first match in the path list + int c = m_SearchPaths.Count(); + for ( int i = 0; i < c; i++ ) + { + // pak files are not allowed to be written to... + CSearchPath *pSearchPath = &m_SearchPaths[i]; + if ( pSearchPath->GetPackFile() || pSearchPath->GetPackedStore() ) + { + continue; + } + + if ( IsX360() && ( m_DVDMode == DVDMODE_DEV ) && pFilename && !pSearchPath->m_bIsRemotePath ) + { + bool bIgnorePath = false; + char szExcludePath[MAX_PATH]; + char szFilename[MAX_PATH]; + V_ComposeFileName( pSearchPath->GetPathString(), pFilename, szFilename, sizeof( szFilename ) ); + for ( int j = 0; j < m_ExcludePaths.Count(); j++ ) + { + if ( g_pFullFileSystem->String( m_ExcludePaths[j], szExcludePath, sizeof( szExcludePath ) ) ) + { + if ( !V_strnicmp( szFilename, szExcludePath, strlen( szExcludePath ) ) ) + { + bIgnorePath = true; + break; + } + } + } + if ( bIgnorePath ) + { + // filename matches exclusion path, skip it + // favoring the next path which should be the path fall through hit + continue; + } + } + + if ( !pathID || ( pSearchPath->GetPathID() == lookup ) ) + { + return pSearchPath; + } + } + + return NULL; +} + + +//----------------------------------------------------------------------------- +// Purpose: Finds a search path that should be used for writing to, given a pathID. +//----------------------------------------------------------------------------- +const char *CBaseFileSystem::GetWritePath( const char *pFilename, const char *pathID ) +{ + CSearchPath *pSearchPath = NULL; + if ( pathID && pathID[ 0 ] != '\0' ) + { + + // Check for "game_write" and "mod_write" + if ( V_stricmp( pathID, "game" ) == 0 ) + pSearchPath = FindWritePath( pFilename, "game_write" ); + else if ( V_stricmp( pathID, "mod" ) == 0 ) + pSearchPath = FindWritePath( pFilename, "mod_write" ); + + if ( pSearchPath == NULL ) + pSearchPath = FindWritePath( pFilename, pathID ); + + if ( pSearchPath ) + { + return pSearchPath->GetPathString(); + } + + Warning( FILESYSTEM_WARNING, "Requested non-existent write path %s!\n", pathID ); + } + + pSearchPath = FindWritePath( pFilename, "DEFAULT_WRITE_PATH" ); + if ( pSearchPath ) + { + return pSearchPath->GetPathString(); + } + + pSearchPath = FindWritePath( pFilename, NULL ); // okay, just return the first search path added! + if ( pSearchPath ) + { + return pSearchPath->GetPathString(); + } + + // Hope this is reasonable!! + return ".\\"; +} + +//----------------------------------------------------------------------------- +// Reads/writes files to utlbuffers. Attempts alignment fixups for optimal read +//----------------------------------------------------------------------------- +CThreadLocal<char *> g_pszReadFilename; +bool CBaseFileSystem::ReadToBuffer( FileHandle_t fp, CUtlBuffer &buf, int nMaxBytes, FSAllocFunc_t pfnAlloc ) +{ + SetBufferSize( fp, 0 ); // TODO: what if it's a pack file? restore buffer size? + + int nBytesToRead = Size( fp ); + if ( nBytesToRead == 0 ) + { + // no data in file + return true; + } + + if ( nMaxBytes > 0 ) + { + // can't read more than file has + nBytesToRead = min( nMaxBytes, nBytesToRead ); + } + + int nBytesRead = 0; + int nBytesOffset = 0; + + int iStartPos = Tell( fp ); + + if ( nBytesToRead != 0 ) + { + int nBytesDestBuffer = nBytesToRead; + unsigned nSizeAlign = 0, nBufferAlign = 0, nOffsetAlign = 0; + + bool bBinary = !( buf.IsText() && !buf.ContainsCRLF() ); + + if ( bBinary && !IsLinux() && !buf.IsExternallyAllocated() && !pfnAlloc && + ( buf.TellPut() == 0 ) && ( buf.TellGet() == 0 ) && ( iStartPos % 4 == 0 ) && + GetOptimalIOConstraints( fp, &nOffsetAlign, &nSizeAlign, &nBufferAlign ) ) + { + // correct conditions to allow an optimal read + if ( iStartPos % nOffsetAlign != 0 ) + { + // move starting position back to nearest alignment + nBytesOffset = ( iStartPos % nOffsetAlign ); + Assert ( ( iStartPos - nBytesOffset ) % nOffsetAlign == 0 ); + Seek( fp, -nBytesOffset, FILESYSTEM_SEEK_CURRENT ); + + // going to read from aligned start, increase target buffer size by offset alignment + nBytesDestBuffer += nBytesOffset; + } + + // snap target buffer size to its size alignment + // add additional alignment slop for target pointer adjustment + nBytesDestBuffer = AlignValue( nBytesDestBuffer, nSizeAlign ) + nBufferAlign; + } + + if ( !pfnAlloc ) + { + buf.EnsureCapacity( nBytesDestBuffer + buf.TellPut() ); + } + else + { + // caller provided allocator + void *pMemory = (*pfnAlloc)( g_pszReadFilename.Get(), nBytesDestBuffer ); + buf.SetExternalBuffer( pMemory, nBytesDestBuffer, 0, buf.GetFlags() & ~CUtlBuffer::EXTERNAL_GROWABLE ); + } + + int seekGet = -1; + if ( nBytesDestBuffer != nBytesToRead ) + { + // doing optimal read, align target pointer + int nAlignedBase = AlignValue( (byte *)buf.Base(), nBufferAlign ) - (byte *)buf.Base(); + buf.SeekPut( CUtlBuffer::SEEK_HEAD, nAlignedBase ); + + // the buffer read position is slid forward to ignore the addtional + // starting offset alignment + seekGet = nAlignedBase + nBytesOffset; + } + + nBytesRead = ReadEx( buf.PeekPut(), buf.Size() - buf.TellPut(), nBytesToRead + nBytesOffset, fp ); + buf.SeekPut( CUtlBuffer::SEEK_CURRENT, nBytesRead ); + + if ( seekGet != -1 ) + { + // can only seek the get after data has been put, otherwise buffer sets overflow error + buf.SeekGet( CUtlBuffer::SEEK_HEAD, seekGet ); + } + + Seek( fp, iStartPos + ( nBytesRead - nBytesOffset ), FILESYSTEM_SEEK_HEAD ); + } + + return (nBytesRead != 0); +} + +//----------------------------------------------------------------------------- +// Reads/writes files to utlbuffers +// NOTE NOTE!! +// If you change this implementation, copy it into CBaseVMPIFileSystem::ReadFile +// in vmpi_filesystem.cpp +//----------------------------------------------------------------------------- +bool CBaseFileSystem::ReadFile( const char *pFileName, const char *pPath, CUtlBuffer &buf, int nMaxBytes, int nStartingByte, FSAllocFunc_t pfnAlloc ) +{ + CHECK_DOUBLE_SLASHES( pFileName ); + + bool bBinary = !( buf.IsText() && !buf.ContainsCRLF() ); + + FileHandle_t fp = Open( pFileName, ( bBinary ) ? "rb" : "rt", pPath ); + if ( !fp ) + return false; + + if ( nStartingByte != 0 ) + { + Seek( fp, nStartingByte, FILESYSTEM_SEEK_HEAD ); + } + + if ( pfnAlloc ) + { + g_pszReadFilename.Set( (char *)pFileName ); + } + + bool bSuccess = ReadToBuffer( fp, buf, nMaxBytes, pfnAlloc ); + + Close( fp ); + + return bSuccess; +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +int CBaseFileSystem::ReadFileEx( const char *pFileName, const char *pPath, void **ppBuf, bool bNullTerminate, bool bOptimalAlloc, int nMaxBytes, int nStartingByte, FSAllocFunc_t pfnAlloc ) +{ + FileHandle_t fp = Open( pFileName, "rb", pPath ); + if ( !fp ) + { + return 0; + } + + if ( IsX360() ) + { + // callers are sloppy, always want optimal + bOptimalAlloc = true; + } + + SetBufferSize( fp, 0 ); // TODO: what if it's a pack file? restore buffer size? + + int nBytesToRead = Size( fp ); + int nBytesRead = 0; + if ( nMaxBytes > 0 ) + { + nBytesToRead = min( nMaxBytes, nBytesToRead ); + if ( bNullTerminate ) + { + nBytesToRead--; + } + } + + if ( nBytesToRead != 0 ) + { + int nBytesBuf; + if ( !*ppBuf ) + { + nBytesBuf = nBytesToRead + ( ( bNullTerminate ) ? 1 : 0 ); + + if ( !pfnAlloc && !bOptimalAlloc ) + { + // AllocOptimalReadBuffer does malloc... + *ppBuf = malloc( nBytesBuf ); + } + else if ( !pfnAlloc ) + { + *ppBuf = AllocOptimalReadBuffer( fp, nBytesBuf, 0 ); + nBytesBuf = GetOptimalReadSize( fp, nBytesBuf ); + } + else + { + *ppBuf = (*pfnAlloc)( pFileName, nBytesBuf ); + } + } + else + { + nBytesBuf = nMaxBytes; + } + + if ( nStartingByte != 0 ) + { + Seek( fp, nStartingByte, FILESYSTEM_SEEK_HEAD ); + } + + nBytesRead = ReadEx( *ppBuf, nBytesBuf, nBytesToRead, fp ); + + if ( bNullTerminate ) + { + ((byte *)(*ppBuf))[nBytesToRead] = 0; + } + } + + Close( fp ); + return nBytesRead; +} + + +//----------------------------------------------------------------------------- +// NOTE NOTE!! +// If you change this implementation, copy it into CBaseVMPIFileSystem::WriteFile +// in vmpi_filesystem.cpp +//----------------------------------------------------------------------------- +bool CBaseFileSystem::WriteFile( const char *pFileName, const char *pPath, CUtlBuffer &buf ) +{ + CHECK_DOUBLE_SLASHES( pFileName ); + + const char *pWriteFlags = "wb"; + if ( buf.IsText() && !buf.ContainsCRLF() ) + { + pWriteFlags = "wt"; + } + + FileHandle_t fp = Open( pFileName, pWriteFlags, pPath ); + if ( !fp ) + return false; + + int nBytesWritten = Write( buf.Base(), buf.TellPut(), fp ); + + Close( fp ); + return (nBytesWritten == buf.TellPut()); +} + + +bool CBaseFileSystem::UnzipFile( const char *pFileName, const char *pPath, const char *pDestination ) +{ + IZip *pZip = IZip::CreateZip( NULL, true ); + + HANDLE hZipFile = pZip->ParseFromDisk( pFileName ); + if ( !hZipFile ) + { + Msg( "Bad or missing zip file, failed to open '%s'\n", pFileName ); + return false; + } + + int iZipIndex = -1; + int iFileSize; + char szFileName[MAX_PATH]; + + // Create Directories + CreateDirHierarchy( pDestination, pPath ); + + while ( 1 ) + { + // Get the next file in the zip + szFileName[0] = '\0'; + iFileSize = 0; + iZipIndex = pZip->GetNextFilename( iZipIndex, szFileName, sizeof( szFileName ), iFileSize ); + + // If there aren't any more files then break out of this while + if ( iZipIndex == -1 ) + break; + + int iFileNameLength = Q_strlen( szFileName ); + if ( szFileName[ iFileNameLength - 1 ] == '/' ) + { + // Its a directory, so create it + szFileName[ iFileNameLength - 1 ] = '\0'; + + char szFinalName[ MAX_PATH ]; + Q_snprintf( szFinalName, sizeof( szFinalName ), "%s%c%s", pDestination, CORRECT_PATH_SEPARATOR, szFileName ); + CreateDirHierarchy( szFinalName, pPath ); + } + } + + // Write Files + while ( 1 ) + { + szFileName[0] = '\0'; + iFileSize = 0; + iZipIndex = pZip->GetNextFilename( iZipIndex, szFileName, sizeof( szFileName ), iFileSize ); + + // If there aren't any more files then break out of this while + if ( iZipIndex == -1 ) + break; + + int iFileNameLength = Q_strlen( szFileName ); + if ( szFileName[ iFileNameLength - 1 ] != '/' ) + { + // It's not a directory, so write the file + CUtlBuffer fileBuffer; + fileBuffer.Purge(); + + if ( pZip->ReadFileFromZip( hZipFile, szFileName, false, fileBuffer ) ) + { + char szFinalName[ MAX_PATH ]; + Q_snprintf( szFinalName, sizeof( szFinalName ), "%s%c%s", pDestination, CORRECT_PATH_SEPARATOR, szFileName ); + + // Make sure the directory actually exists in case the ZIP doesn't list it (our zip utils create zips like this) + char szFilePath[ MAX_PATH ]; + Q_strncpy( szFilePath, szFinalName, sizeof(szFilePath) ); + Q_StripFilename( szFilePath ); + CreateDirHierarchy( szFilePath, pPath ); + + WriteFile( szFinalName, pPath, fileBuffer ); + } + } + } + +#ifdef WIN32 + ::CloseHandle( hZipFile ); +#else + fclose( (FILE *)hZipFile ); +#endif + + IZip::ReleaseZip( pZip ); + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void CBaseFileSystem::RemoveAllSearchPaths( void ) +{ + AUTO_LOCK( m_SearchPathsMutex ); + m_SearchPaths.Purge(); + //m_PackFileHandles.Purge(); +} + + +void CBaseFileSystem::LogFileAccess( const char *pFullFileName ) +{ + if( !m_pLogFile ) + { + return; + } + char buf[1024]; +#if BSPOUTPUT + Q_snprintf( buf, sizeof( buf ), "%s\n%s\n", pShortFileName, pFullFileName); + fprintf( m_pLogFile, "%s", buf ); // STEAM OK +#else + char cwd[MAX_FILEPATH]; + getcwd( cwd, MAX_FILEPATH-1 ); + Q_strcat( cwd, "\\", sizeof( cwd ) ); + if( Q_strnicmp( cwd, pFullFileName, strlen( cwd ) ) == 0 ) + { + const char *pFileNameWithoutExeDir = pFullFileName + strlen( cwd ); + char targetPath[ MAX_FILEPATH ]; + strcpy( targetPath, "%fs_target%\\" ); + strcat( targetPath, pFileNameWithoutExeDir ); + Q_snprintf( buf, sizeof( buf ), "copy \"%s\" \"%s\"\n", pFullFileName, targetPath ); + char tmp[ MAX_FILEPATH ]; + Q_strncpy( tmp, targetPath, sizeof( tmp ) ); + Q_StripFilename( tmp ); + fprintf( m_pLogFile, "mkdir \"%s\"\n", tmp ); // STEAM OK + fprintf( m_pLogFile, "%s", buf ); // STEAM OK + } + else + { + Assert( 0 ); + } +#endif +} + +class CPackedStore; + +class CFileOpenInfo +{ +public: + CFileOpenInfo( CBaseFileSystem *pFileSystem, const char *pFileName, const CBaseFileSystem::CSearchPath *path, const char *pOptions, int flags, char **ppszResolvedFilename ) : + m_pFileSystem( pFileSystem ), m_pFileName( pFileName ), m_pSearchPath( path ), m_pOptions( pOptions ), m_Flags( flags ), m_ppszResolvedFilename( ppszResolvedFilename ) + { + m_pFileHandle = NULL; + m_ePureFileClass = ePureServerFileClass_Any; + if ( m_pFileSystem->m_pPureServerWhitelist ) + { + m_ePureFileClass = m_pFileSystem->m_pPureServerWhitelist->GetFileClass( pFileName ); + } + + if ( m_ppszResolvedFilename ) + *m_ppszResolvedFilename = NULL; + m_pPackFile = NULL; + m_pVPKFile = NULL; + m_AbsolutePath[0] = '\0'; + } + + ~CFileOpenInfo() + { + if ( IsX360() ) + { + return; + } + } + + void SetAbsolutePath( const char *pFormat, ... ) + { + va_list marker; + va_start( marker, pFormat ); + V_vsnprintf( m_AbsolutePath, sizeof( m_AbsolutePath ), pFormat, marker ); + va_end( marker ); + + V_FixSlashes( m_AbsolutePath ); + } + + void SetResolvedFilename( const char *pStr ) + { + if ( m_ppszResolvedFilename ) + { + Assert( !( *m_ppszResolvedFilename ) ); + *m_ppszResolvedFilename = strdup( pStr ); + } + } + + // Handles telling CFileTracker about the file we just opened so it can remember + // where the file came from, and possibly calculate a CRC if necessary. + void HandleFileCRCTracking( const char *pRelativeFileName ) + { + if ( IsX360() ) + { + return; + } + + if ( m_pFileSystem->m_WhitelistFileTrackingEnabled == 0 ) + return; + + if ( m_pFileHandle && m_pSearchPath ) + { + FILE *fp = m_pFileHandle->m_pFile; + int64 nLength = m_pFileHandle->m_nLength; +#ifdef SUPPORT_PACKED_STORE + if ( m_pVPKFile ) + { + m_pFileSystem->m_FileTracker2.NotePackFileAccess( pRelativeFileName, m_pSearchPath->GetPathIDString(), m_pSearchPath->m_storeId, m_pFileHandle->m_VPKHandle ); + return; + } +#endif + // we always record hashes of everything we load. we may filter later. + m_pFileSystem->m_FileTracker2.NoteFileLoadedFromDisk( pRelativeFileName, m_pSearchPath->GetPathIDString(), m_pSearchPath->m_storeId, fp, nLength ); + } + } + +#ifdef SUPPORT_PACKED_STORE + void SetFromPackedStoredFileHandle( const CPackedStoreFileHandle &fHandle, CBaseFileSystem *pFileSystem ) + { + Assert( fHandle ); + Assert( m_pFileHandle == NULL ); + m_pFileHandle = new CFileHandle(pFileSystem); + m_pFileHandle->m_VPKHandle = fHandle; + m_pFileHandle->m_type = FT_NORMAL; + m_pFileHandle->m_nLength = fHandle.m_nFileSize; + } +#endif + +public: + CBaseFileSystem *m_pFileSystem; + + // These are output parameters. + CFileHandle *m_pFileHandle; + char **m_ppszResolvedFilename; + + CPackFile *m_pPackFile; + CPackedStore *m_pVPKFile; + + const char *m_pFileName; + const CBaseFileSystem::CSearchPath *m_pSearchPath; + const char *m_pOptions; + int m_Flags; + + EPureServerFileClass m_ePureFileClass; + + char m_AbsolutePath[MAX_FILEPATH]; // This is set +}; + + +void CBaseFileSystem::HandleOpenRegularFile( CFileOpenInfo &openInfo, bool bIsAbsolutePath ) +{ + openInfo.m_pFileHandle = NULL; + + int64 size; + FILE *fp = Trace_FOpen( openInfo.m_AbsolutePath, openInfo.m_pOptions, openInfo.m_Flags, &size ); + if ( fp ) + { + if ( m_pLogFile ) + { + LogFileAccess( openInfo.m_AbsolutePath ); + } + + if ( m_bOutputDebugString ) + { +#ifdef _WIN32 + Plat_DebugString( "fs_debug: " ); + Plat_DebugString( openInfo.m_AbsolutePath ); + Plat_DebugString( "\n" ); +#elif POSIX + fprintf(stderr, "fs_debug: %s\n", openInfo.m_AbsolutePath ); +#endif + } + + openInfo.m_pFileHandle = new CFileHandle(this); + openInfo.m_pFileHandle->m_pFile = fp; + openInfo.m_pFileHandle->m_type = FT_NORMAL; + openInfo.m_pFileHandle->m_nLength = size; + + openInfo.SetResolvedFilename( openInfo.m_AbsolutePath ); + + LogFileOpen( "Loose", openInfo.m_pFileName, openInfo.m_AbsolutePath ); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: The base file search goes through here +// Input : *path - +// *pFileName - +// *pOptions - +// packfile - +// *filetime - +// Output : FileHandle_t +//----------------------------------------------------------------------------- +FileHandle_t CBaseFileSystem::FindFileInSearchPath( CFileOpenInfo &openInfo ) +{ + VPROF( "CBaseFileSystem::FindFile" ); + + Assert( openInfo.m_pSearchPath ); + openInfo.m_pFileHandle = NULL; + + // Loading from pack file? + CPackFile *pPackFile = openInfo.m_pSearchPath->GetPackFile(); + if ( pPackFile ) + { + openInfo.m_pFileHandle = pPackFile->OpenFile( openInfo.m_pFileName, openInfo.m_pOptions ); + openInfo.m_pPackFile = pPackFile; + if ( openInfo.m_pFileHandle ) + { + char tempStr[MAX_PATH*2+2]; + V_snprintf( tempStr, sizeof( tempStr ), "%s%c%s", pPackFile->m_ZipName.String(), CORRECT_PATH_SEPARATOR, openInfo.m_pFileName ); + openInfo.SetResolvedFilename( tempStr ); + } + + // If it's a BSP file, then the BSP file got CRC'd elsewhere so no need to verify stuff in there. + return (FileHandle_t)openInfo.m_pFileHandle; + } + + // Loading from VPK? + #ifdef SUPPORT_PACKED_STORE + CPackedStore *pVPK = openInfo.m_pSearchPath->GetPackedStore(); + if ( pVPK ) + { + CPackedStoreFileHandle fHandle = pVPK->OpenFile( openInfo.m_pFileName ); + if ( fHandle ) + { + openInfo.SetFromPackedStoredFileHandle( fHandle, this ); + openInfo.SetResolvedFilename( openInfo.m_pFileName ); + + openInfo.m_pVPKFile = pVPK; + LogFileOpen( "VPK", openInfo.m_pFileName, pVPK->BaseName() ); + openInfo.HandleFileCRCTracking( openInfo.m_pFileName ); + return ( FileHandle_t ) openInfo.m_pFileHandle; + } + return NULL; + } + #endif + + // Load loose file specified as relative filename + // Convert filename to lowercase. All files in the + // game logical filesystem must be accessed by lowercase name + char szLowercaseFilename[ MAX_PATH ]; + V_strcpy_safe( szLowercaseFilename, openInfo.m_pFileName ); + V_strlower( szLowercaseFilename ); + + openInfo.SetAbsolutePath( "%s%s", openInfo.m_pSearchPath->GetPathString(), szLowercaseFilename ); + + // now have an absolute name + HandleOpenRegularFile( openInfo, false ); + return (FileHandle_t)openInfo.m_pFileHandle; +} + + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +FileHandle_t CBaseFileSystem::OpenForRead( const char *pFileNameT, const char *pOptions, unsigned flags, const char *pathID, char **ppszResolvedFilename ) +{ + VPROF( "CBaseFileSystem::OpenForRead" ); + + char pFileNameBuff[MAX_PATH]; + const char *pFileName = pFileNameBuff; + + FixUpPath ( pFileNameT, pFileNameBuff, sizeof( pFileNameBuff ) ); + + // Try the memory cache for un-restricted searches or "GAME" items. + if ( !pathID || Q_stricmp( pathID, "GAME" ) == 0 ) + { + CMemoryFileBacking* pBacking = NULL; + { + AUTO_LOCK( m_MemoryFileMutex ); + CUtlHashtable< const char*, CMemoryFileBacking* >& table = m_MemoryFileHash; + UtlHashHandle_t idx = table.Find( pFileName ); + if ( idx != table.InvalidHandle() ) + { + pBacking = table[idx]; + pBacking->AddRef(); + } + } + if ( pBacking ) + { + if ( pBacking->m_nLength != -1 ) + { + CFileHandle* pFile = new CMemoryFileHandle( this, pBacking ); + pFile->m_type = strstr( pOptions, "b" ) ? FT_MEMORY_BINARY : FT_MEMORY_TEXT; + return ( FileHandle_t )pFile; + } + else + { + // length -1 == cached failure to read + return ( FileHandle_t )NULL; + } + } + else if ( ThreadInMainThread() && fs_report_sync_opens.GetInt() > 0 ) + { + DevWarning("blocking load %s\n", pFileName); + } + } + + CFileOpenInfo openInfo( this, pFileName, NULL, pOptions, flags, ppszResolvedFilename ); + + // Already have an absolute path? + // If so, don't bother iterating search paths. + if ( V_IsAbsolutePath( pFileName ) ) + { + openInfo.SetAbsolutePath( "%s", pFileName ); + + // Check if it's of the form C:/a/b/c/blah.zip/materials/blah.vtf + // an absolute path can encode a zip pack file (i.e. caller wants to open the file from within the pack file) + // format must strictly be ????.zip\???? + // assuming a reasonable restriction that the zip must be a pre-existing search path zip + char *pZipExt = V_stristr( openInfo.m_AbsolutePath, ".zip" CORRECT_PATH_SEPARATOR_S ); + if ( !pZipExt ) + pZipExt = V_stristr( openInfo.m_AbsolutePath, ".bsp" CORRECT_PATH_SEPARATOR_S ); + #if defined( SUPPORT_PACKED_STORE ) + if ( !pZipExt ) + pZipExt = V_stristr( openInfo.m_AbsolutePath, ".vpk" CORRECT_PATH_SEPARATOR_S ); + #endif + + if ( pZipExt && pZipExt[5] ) + { + // Cut string at the slash + char *pSlash = pZipExt + 4; + Assert( *pSlash == CORRECT_PATH_SEPARATOR ); + *pSlash = '\0'; + + // want relative portion only, everything after the slash + char *pRelativeFileName = pSlash + 1; + + // Find the zip or VPK in the search paths + for ( int i = 0; i < m_SearchPaths.Count(); i++ ) + { + + // In VPK? + #if defined( SUPPORT_PACKED_STORE ) + CPackedStore *pVPK = m_SearchPaths[i].GetPackedStore(); + if ( pVPK ) + { + if ( V_stricmp( pVPK->FullPathName(), openInfo.m_AbsolutePath ) == 0 ) + { + CPackedStoreFileHandle fHandle = pVPK->OpenFile( pRelativeFileName ); + if ( fHandle ) + { + openInfo.m_pSearchPath = &m_SearchPaths[i]; + openInfo.SetFromPackedStoredFileHandle( fHandle, this ); + } + break; + } + continue; + } + #endif + + // In .zip? + CPackFile *pPackFile = m_SearchPaths[i].GetPackFile(); + if ( pPackFile ) + { + if ( Q_stricmp( pPackFile->m_ZipName.Get(), openInfo.m_AbsolutePath ) == 0 ) + { + openInfo.m_pSearchPath = &m_SearchPaths[i]; + openInfo.m_pFileHandle = pPackFile->OpenFile( pRelativeFileName, openInfo.m_pOptions ); + openInfo.m_pPackFile = pPackFile; + break; + } + } + } + + if ( openInfo.m_pFileHandle ) + { + openInfo.SetResolvedFilename( openInfo.m_pFileName ); + openInfo.HandleFileCRCTracking( pRelativeFileName ); + } + return (FileHandle_t)openInfo.m_pFileHandle; + } + + // Otherwise, it must be a regular file, specified by absolute filename + HandleOpenRegularFile( openInfo, true ); + + // !FIXME! We probably need to deal with CRC tracking, right? + + return (FileHandle_t)openInfo.m_pFileHandle; + } + + // Run through all the search paths. + PathTypeFilter_t pathFilter = FILTER_NONE; + if ( IsX360() ) + { + if ( flags & FSOPEN_NEVERINPACK ) + { + pathFilter = FILTER_CULLPACK; + } + else if ( m_DVDMode == DVDMODE_STRICT ) + { + // most all files on the dvd are expected to be in the pack + // don't allow disk paths to be searched, which is very expensive on the dvd + pathFilter = FILTER_CULLNONPACK; + } + } + + CSearchPathsIterator iter( this, &pFileName, pathID, pathFilter ); + for ( openInfo.m_pSearchPath = iter.GetFirst(); openInfo.m_pSearchPath != NULL; openInfo.m_pSearchPath = iter.GetNext() ) + { + FileHandle_t filehandle = FindFileInSearchPath( openInfo ); + if ( filehandle ) + { + // Check if search path is excluded due to pure server white list, + // then we should make a note of this fact, and keep searching + if ( !openInfo.m_pSearchPath->m_bIsTrustedForPureServer && openInfo.m_ePureFileClass == ePureServerFileClass_AnyTrusted ) + { + #ifdef PURE_SERVER_DEBUG_SPEW + Msg( "Ignoring %s from %s for pure server operation\n", openInfo.m_pFileName, openInfo.m_pSearchPath->GetDebugString() ); + #endif + + m_FileTracker2.NoteFileIgnoredForPureServer( openInfo.m_pFileName, pathID, openInfo.m_pSearchPath->m_storeId ); + Close( filehandle ); + openInfo.m_pFileHandle = NULL; + if ( ppszResolvedFilename && *ppszResolvedFilename ) + { + free( *ppszResolvedFilename ); + *ppszResolvedFilename = NULL; + } + continue; + } + + // + openInfo.HandleFileCRCTracking( openInfo.m_pFileName ); + return filehandle; + } + } + + LogFileOpen( "[Failed]", pFileName, "" ); + return ( FileHandle_t )0; +} + + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +FileHandle_t CBaseFileSystem::OpenForWrite( const char *pFileName, const char *pOptions, const char *pathID ) +{ + char tempPathID[MAX_PATH]; + ParsePathID( pFileName, pathID, tempPathID ); + + if ( ThreadInMainThread() && fs_report_sync_opens.GetInt() ) + { + DevWarning("blocking write %s\n", pFileName); + } + + // Opening for write or append uses the write path + // Unless an absolute path is specified... + const char *pTmpFileName; + char szScratchFileName[MAX_PATH]; + if ( Q_IsAbsolutePath( pFileName ) ) + { + pTmpFileName = pFileName; + } + else + { + ComputeFullWritePath( szScratchFileName, sizeof( szScratchFileName ), pFileName, pathID ); + pTmpFileName = szScratchFileName; + } + + int64 size; + FILE *fp = Trace_FOpen( pTmpFileName, pOptions, 0, &size ); + if ( !fp ) + { + return ( FileHandle_t )0; + } + + CFileHandle *fh = new CFileHandle( this ); + fh->m_nLength = size; + fh->m_type = FT_NORMAL; + fh->m_pFile = fp; + + return ( FileHandle_t )fh; +} + + +// This looks for UNC-type filename specifiers, which should be used instead of +// passing in path ID. So if it finds //mod/cfg/config.cfg, it translates +// pFilename to "cfg/config.cfg" and pPathID to "mod" (mod is placed in tempPathID). +void CBaseFileSystem::ParsePathID( const char* &pFilename, const char* &pPathID, char tempPathID[MAX_PATH] ) +{ + tempPathID[0] = 0; + + if ( !pFilename || pFilename[0] == 0 ) + return; + + // FIXME: Pain! Backslashes are used to denote network drives, forward to denote path ids + // HOORAY! We call FixSlashes everywhere. That will definitely not work + // I'm not changing it yet though because I don't know how painful the bugs would be that would be generated + bool bIsForwardSlash = ( pFilename[0] == '/' && pFilename[1] == '/' ); +// bool bIsBackwardSlash = ( pFilename[0] == '\\' && pFilename[1] == '\\' ); + if ( !bIsForwardSlash ) //&& !bIsBackwardSlash ) + return; + + // They're specifying two path IDs. Ignore the one passed-in. + if ( pPathID ) + { + Warning( FILESYSTEM_WARNING, "FS: Specified two path IDs (%s, %s).\n", pFilename, pPathID ); + } + + // Parse out the path ID. + const char *pIn = &pFilename[2]; + char *pOut = tempPathID; + while ( *pIn && !PATHSEPARATOR( *pIn ) && (pOut - tempPathID) < (MAX_PATH-1) ) + { + *pOut++ = *pIn++; + } + + *pOut = 0; + + if ( tempPathID[0] == '*' ) + { + // * means NULL. + pPathID = NULL; + } + else + { + pPathID = tempPathID; + } + + // Move pFilename up past the part with the path ID. + if ( *pIn == 0 ) + pFilename = pIn; + else + pFilename = pIn + 1; +} + + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +FileHandle_t CBaseFileSystem::Open( const char *pFileName, const char *pOptions, const char *pathID ) +{ + return OpenEx( pFileName, pOptions, 0, pathID ); +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +FileHandle_t CBaseFileSystem::OpenEx( const char *pFileName, const char *pOptions, unsigned flags, const char *pathID, char **ppszResolvedFilename ) +{ + tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s(%s, %s, %u %s )", __FUNCTION__, tmDynamicString( TELEMETRY_LEVEL0, pFileName ), tmDynamicString( TELEMETRY_LEVEL0, pOptions ), flags, tmDynamicString( TELEMETRY_LEVEL0, pathID ) ); + + VPROF_BUDGET( "CBaseFileSystem::Open", VPROF_BUDGETGROUP_OTHER_FILESYSTEM ); + + if ( !pFileName ) + return (FileHandle_t)0; + + CHECK_DOUBLE_SLASHES( pFileName ); + + if ( ThreadInMainThread() && fs_report_sync_opens.GetInt() > 1 ) + { + ::Warning( "Open( %s )\n", pFileName ); + } + + // Allow for UNC-type syntax to specify the path ID. + char tempPathID[MAX_PATH]; + ParsePathID( pFileName, pathID, tempPathID ); + + + // Try each of the search paths in succession + // FIXME: call createdirhierarchy upon opening for write. + if ( strstr( pOptions, "r" ) && !strstr( pOptions, "+" ) ) + { + return OpenForRead( pFileName, pOptions, flags, pathID, ppszResolvedFilename ); + } + + return OpenForWrite( pFileName, pOptions, pathID ); +} + + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void CBaseFileSystem::Close( FileHandle_t file ) +{ + VPROF_BUDGET( "CBaseFileSystem::Close", VPROF_BUDGETGROUP_OTHER_FILESYSTEM ); + if ( !file ) + { + Warning( FILESYSTEM_WARNING, "FS: Tried to Close NULL file handle!\n" ); + return; + } + + delete (CFileHandle*)file; +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void CBaseFileSystem::Seek( FileHandle_t file, int pos, FileSystemSeek_t whence ) +{ + tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s (pos=%d, whence=%d)", __FUNCTION__, pos, whence ); + + VPROF_BUDGET( "CBaseFileSystem::Seek", VPROF_BUDGETGROUP_OTHER_FILESYSTEM ); + CFileHandle *fh = ( CFileHandle *)file; + if ( !fh ) + { + Warning( FILESYSTEM_WARNING, "Tried to Seek NULL file handle!\n" ); + return; + } + + fh->Seek( pos, whence ); +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : file - +// Output : unsigned int +//----------------------------------------------------------------------------- +unsigned int CBaseFileSystem::Tell( FileHandle_t file ) +{ + VPROF_BUDGET( "CBaseFileSystem::Tell", VPROF_BUDGETGROUP_OTHER_FILESYSTEM ); + + if ( !file ) + { + Warning( FILESYSTEM_WARNING, "FS: Tried to Tell NULL file handle!\n" ); + return 0; + } + + + // Pack files are relative + return (( CFileHandle *)file)->Tell(); +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : file - +// Output : unsigned int +//----------------------------------------------------------------------------- +unsigned int CBaseFileSystem::Size( FileHandle_t file ) +{ + tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s", __FUNCTION__ ); + + VPROF_BUDGET( "CBaseFileSystem::Size", VPROF_BUDGETGROUP_OTHER_FILESYSTEM ); + if ( !file ) + { + Warning( FILESYSTEM_WARNING, "FS: Tried to Size NULL file handle!\n" ); + return 0; + } + + return ((CFileHandle *)file)->Size(); +} + + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : file - +// Output : unsigned int +//----------------------------------------------------------------------------- +unsigned int CBaseFileSystem::Size( const char* pFileName, const char *pPathID ) +{ + VPROF_BUDGET( "CBaseFileSystem::Size", VPROF_BUDGETGROUP_OTHER_FILESYSTEM ); + CHECK_DOUBLE_SLASHES( pFileName ); + + // handle the case where no name passed... + if ( !pFileName || !pFileName[0] ) + { + Warning( FILESYSTEM_WARNING, "FS: Tried to Size NULL filename!\n" ); + return 0; + } + + // Ok, fall through to the fast path. + unsigned result = 0; + FileHandle_t h = Open( pFileName, "rb", pPathID ); + if ( h ) + { + result = Size( h ); + Close(h); + } + + return result; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *path - +// *pFileName - +// Output : long +//----------------------------------------------------------------------------- +long CBaseFileSystem::FastFileTime( const CSearchPath *path, const char *pFileName ) +{ + struct _stat buf; + + if ( path->GetPackFile() ) + { + // If we found the file: + if ( path->GetPackFile()->ContainsFile( pFileName ) ) + { + return (path->GetPackFile()->m_lPackFileTime); + } + } +#ifdef SUPPORT_PACKED_STORE + else if ( path->GetPackedStore() ) + { + // Hm, should we support this in some way? + return 0L; + } +#endif + else + { + // Is it an absolute path? + char pTmpFileName[ MAX_FILEPATH ]; + + if ( Q_IsAbsolutePath( pFileName ) ) + { + Q_strncpy( pTmpFileName, pFileName, sizeof( pTmpFileName ) ); + } + else + { + Q_snprintf( pTmpFileName, sizeof( pTmpFileName ), "%s%s", path->GetPathString(), pFileName ); + } + + Q_FixSlashes( pTmpFileName ); + + if( FS_stat( pTmpFileName, &buf ) != -1 ) + { + return buf.st_mtime; + } +#ifdef LINUX + char caseFixedName[ MAX_PATH ]; + bool found = findFileInDirCaseInsensitive_safe( pTmpFileName, caseFixedName ); + if ( found && FS_stat( caseFixedName, &buf ) != -1 ) + { + return buf.st_mtime; + } +#endif + } + + return ( 0L ); +} + + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +bool CBaseFileSystem::EndOfFile( FileHandle_t file ) +{ + if ( !file ) + { + Warning( FILESYSTEM_WARNING, "FS: Tried to EndOfFile NULL file handle!\n" ); + return true; + } + + return ((CFileHandle *)file)->EndOfFile(); +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +int CBaseFileSystem::Read( void *pOutput, int size, FileHandle_t file ) +{ + return ReadEx( pOutput, size, size, file ); +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +int CBaseFileSystem::ReadEx( void *pOutput, int destSize, int size, FileHandle_t file ) +{ + tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s (%d bytes)", __FUNCTION__, size ); + + VPROF_BUDGET( "CBaseFileSystem::Read", VPROF_BUDGETGROUP_OTHER_FILESYSTEM ); + if ( !file ) + { + Warning( FILESYSTEM_WARNING, "FS: Tried to Read NULL file handle!\n" ); + return 0; + } + if ( size < 0 ) + { + return 0; + } + return ((CFileHandle*)file)->Read(pOutput, destSize, size ); +} + +//----------------------------------------------------------------------------- +// Purpose: Blow away current readers +// Input : - +//----------------------------------------------------------------------------- +void CBaseFileSystem::UnloadCompiledKeyValues() +{ +#ifndef DEDICATED + for ( int i = 0; i < IFileSystem::NUM_PRELOAD_TYPES; ++i ) + { + delete m_PreloadData[ i ].m_pReader; + m_PreloadData[ i ].m_pReader = NULL; + } +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: Put data file into list of at specific slot, will be loaded when ::SetupPreloadData() gets called +// Input : type - +// *archiveFile - +//----------------------------------------------------------------------------- +void CBaseFileSystem::LoadCompiledKeyValues( KeyValuesPreloadType_t type, char const *archiveFile ) +{ + // Just add to list for appropriate loader + Assert( type >= 0 && type < IFileSystem::NUM_PRELOAD_TYPES ); + CompiledKeyValuesPreloaders_t& loader = m_PreloadData[ type ]; + Assert( loader.m_CacheFile == 0 ); + loader.m_CacheFile = FindOrAddFileName( archiveFile ); +} + +//----------------------------------------------------------------------------- +// Purpose: Takes a passed in KeyValues& head and fills in the precompiled keyvalues data into it. +// Input : head - +// type - +// *filename - +// *pPathID - +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CBaseFileSystem::LoadKeyValues( KeyValues& head, KeyValuesPreloadType_t type, char const *filename, char const *pPathID /*= 0*/ ) +{ + bool bret = true; + +#ifndef DEDICATED + char tempPathID[MAX_PATH]; + ParsePathID( filename, pPathID, tempPathID ); + + // FIXME: THIS STUFF DOESN'T TRACK pPathID AT ALL RIGHT NOW!!!!! + if ( !m_PreloadData[ type ].m_pReader || !m_PreloadData[ type ].m_pReader->InstanceInPlace( head, filename ) ) + { + bret = head.LoadFromFile( this, filename, pPathID ); + } + return bret; +#else + bret = head.LoadFromFile( this, filename, pPathID ); + return bret; +#endif +} + + +//----------------------------------------------------------------------------- +// Purpose: If the "PreloadedData" hasn't been purged, then this'll try and instance the KeyValues using the fast path of +/// compiled keyvalues loaded during startup. +// Otherwise, it'll just fall through to the regular KeyValues loading routines +// Input : type - +// *filename - +// *pPathID - +// Output : KeyValues +//----------------------------------------------------------------------------- +KeyValues *CBaseFileSystem::LoadKeyValues( KeyValuesPreloadType_t type, char const *filename, char const *pPathID /*= 0*/ ) +{ + KeyValues *kv = NULL; + + if ( !m_PreloadData[ type ].m_pReader ) + { + kv = new KeyValues( filename ); + if ( kv ) + { + kv->LoadFromFile( this, filename, pPathID ); + } + } + else + { +#ifndef DEDICATED + // FIXME: THIS STUFF DOESN'T TRACK pPathID AT ALL RIGHT NOW!!!!! + kv = m_PreloadData[ type ].m_pReader->Instance( filename ); + if ( !kv ) + { + kv = new KeyValues( filename ); + if ( kv ) + { + kv->LoadFromFile( this, filename, pPathID ); + } + } +#endif + } + return kv; +} + +//----------------------------------------------------------------------------- +// Purpose: This is the fallback method of reading the name of the first key in the file +// Input : *filename - +// *pPathID - +// *rootName - +// bufsize - +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CBaseFileSystem::LookupKeyValuesRootKeyName( char const *filename, char const *pPathID, char *rootName, size_t bufsize ) +{ + if ( FileExists( filename, pPathID ) ) + { + // open file and get shader name + FileHandle_t hFile = Open( filename, "r", pPathID ); + if ( hFile == FILESYSTEM_INVALID_HANDLE ) + { + return false; + } + + char buf[ 128 ]; + ReadLine( buf, sizeof( buf ), hFile ); + Close( hFile ); + + // The name will possibly come in as "foo"\n + + // So we need to strip the starting " character + char *pStart = buf; + if ( *pStart == '\"' ) + { + ++pStart; + } + // Then copy the rest of the string + Q_strncpy( rootName, pStart, bufsize ); + + // And then strip off the \n and the " character at the end, in that order + int len = Q_strlen( pStart ); + while ( len > 0 && rootName[ len - 1 ] == '\n' ) + { + rootName[ len - 1 ] = 0; + --len; + } + while ( len > 0 && rootName[ len - 1 ] == '\"' ) + { + rootName[ len - 1 ] = 0; + --len; + } + } + else + { + return false; + } + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: Tries to look up the name of the first key in the file from the compiled data +// Input : type - +// *outbuf - +// bufsize - +// *filename - +// *pPathID - +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CBaseFileSystem::ExtractRootKeyName( KeyValuesPreloadType_t type, char *outbuf, size_t bufsize, char const *filename, char const *pPathID /*= 0*/ ) +{ + char tempPathID[MAX_PATH]; + ParsePathID( filename, pPathID, tempPathID ); + + bool bret = true; + + if ( !m_PreloadData[ type ].m_pReader ) + { + // Use fallback + bret = LookupKeyValuesRootKeyName( filename, pPathID, outbuf, bufsize ); + } + else + { +#ifndef DEDICATED + // Try to use cache + bret = m_PreloadData[ type ].m_pReader->LookupKeyValuesRootKeyName( filename, outbuf, bufsize ); + if ( !bret ) + { + // Not in cache, use fallback + bret = LookupKeyValuesRootKeyName( filename, pPathID, outbuf, bufsize ); + } +#endif + } + return bret; +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void CBaseFileSystem::SetupPreloadData() +{ + int i; + + for ( i = 0; i < m_SearchPaths.Count(); i++ ) + { + CPackFile* pf = m_SearchPaths[i].GetPackFile(); + if ( pf ) + { + pf->SetupPreloadData(); + } + } + +#ifndef DEDICATED + if ( !CommandLine()->FindParm( "-fs_nopreloaddata" ) ) + { + // Loads in the precompiled keyvalues data for each type + for ( i = 0; i < NUM_PRELOAD_TYPES; ++i ) + { + CompiledKeyValuesPreloaders_t& preloader = m_PreloadData[ i ]; + Assert( !preloader.m_pReader ); + + char fn[MAX_PATH]; + if ( preloader.m_CacheFile != 0 && + String( preloader.m_CacheFile, fn, sizeof( fn ) ) ) + { + preloader.m_pReader = new CCompiledKeyValuesReader(); + preloader.m_pReader->LoadFile( fn ); + } + } + } +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void CBaseFileSystem::DiscardPreloadData() +{ + int i; + for( i = 0; i < m_SearchPaths.Count(); i++ ) + { + CPackFile* pf = m_SearchPaths[i].GetPackFile(); + if ( pf ) + { + pf->DiscardPreloadData(); + } + } + + UnloadCompiledKeyValues(); +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +int CBaseFileSystem::Write( void const* pInput, int size, FileHandle_t file ) +{ + VPROF_BUDGET( "CBaseFileSystem::Write", VPROF_BUDGETGROUP_OTHER_FILESYSTEM ); + + AUTOBLOCKREPORTER_FH( Write, this, true, file, FILESYSTEM_BLOCKING_SYNCHRONOUS, FileBlockingItem::FB_ACCESS_WRITE ); + + CFileHandle *fh = ( CFileHandle *)file; + if ( !fh ) + { + Warning( FILESYSTEM_WARNING, "FS: Tried to Write NULL file handle!\n" ); + return 0; + } + return fh->Write( pInput, size ); + +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +int CBaseFileSystem::FPrintf( FileHandle_t file, const char *pFormat, ... ) +{ + va_list args; + va_start( args, pFormat ); + VPROF_BUDGET( "CBaseFileSystem::FPrintf", VPROF_BUDGETGROUP_OTHER_FILESYSTEM ); + CFileHandle *fh = ( CFileHandle *)file; + if ( !fh ) + { + Warning( FILESYSTEM_WARNING, "FS: Tried to FPrintf NULL file handle!\n" ); + return 0; + } +/* + if ( !fh->GetFileHandle() ) + { + Warning( FILESYSTEM_WARNING, "FS: Tried to FPrintf NULL file pointer inside valid file handle!\n" ); + return 0; + } + */ + + + char buffer[65535]; + int len = vsnprintf( buffer, sizeof( buffer), pFormat, args ); + len = fh->Write( buffer, len ); + //int len = FS_vfprintf( fh->GetFileHandle() , pFormat, args ); + va_end( args ); + + + return len; +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void CBaseFileSystem::SetBufferSize( FileHandle_t file, unsigned nBytes ) +{ + CFileHandle *fh = ( CFileHandle *)file; + if ( !fh ) + { + Warning( FILESYSTEM_WARNING, "FS: Tried to SetBufferSize NULL file handle!\n" ); + return; + } + fh->SetBufferSize( nBytes ); +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +bool CBaseFileSystem::IsOk( FileHandle_t file ) +{ + CFileHandle *fh = ( CFileHandle *)file; + if ( !fh ) + { + Warning( FILESYSTEM_WARNING, "FS: Tried to IsOk NULL file handle!\n" ); + return false; + } + + return fh->IsOK(); +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void CBaseFileSystem::Flush( FileHandle_t file ) +{ + VPROF_BUDGET( "CBaseFileSystem::Flush", VPROF_BUDGETGROUP_OTHER_FILESYSTEM ); + CFileHandle *fh = ( CFileHandle *)file; + if ( !fh ) + { + Warning( FILESYSTEM_WARNING, "FS: Tried to Flush NULL file handle!\n" ); + return; + } + + fh->Flush(); + +} + +bool CBaseFileSystem::Precache( const char *pFileName, const char *pPathID) +{ + CHECK_DOUBLE_SLASHES( pFileName ); + + // Allow for UNC-type syntax to specify the path ID. + char tempPathID[MAX_PATH]; + ParsePathID( pFileName, pPathID, tempPathID ); + Assert( pPathID ); + + // Really simple, just open, the file, read it all in and close it. + // We probably want to use file mapping to do this eventually. + FileHandle_t f = Open( pFileName, "rb", pPathID ); + if ( !f ) + return false; + + // not for consoles, the read discard is a negative benefit, slow and clobbers small drive caches + if ( IsPC() ) + { + char buffer[16384]; + while( sizeof(buffer) == Read(buffer,sizeof(buffer),f) ); + } + + Close( f ); + + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +char *CBaseFileSystem::ReadLine( char *pOutput, int maxChars, FileHandle_t file ) +{ + VPROF_BUDGET( "CBaseFileSystem::ReadLine", VPROF_BUDGETGROUP_OTHER_FILESYSTEM ); + CFileHandle *fh = ( CFileHandle *)file; + if ( !fh ) + { + Warning( FILESYSTEM_WARNING, "FS: Tried to ReadLine NULL file handle!\n" ); + return NULL; + } + m_Stats.nReads++; + + int nRead = 0; + + // Read up to maxchars: + while( nRead < ( maxChars - 1 ) ) + { + // Are we at the end of the file? + if( 1 != fh->Read( pOutput + nRead, 1 ) ) + break; + + // Translate for text mode files: + if( ( fh->m_type == FT_PACK_TEXT || fh->m_type == FT_MEMORY_TEXT ) && pOutput[nRead] == '\r' ) + { + // Ignore \r + continue; + } + + // We're done when we hit a '\n' + if( pOutput[nRead] == '\n' ) + { + nRead++; + break; + } + + // Get outta here if we find a NULL. + if( pOutput[nRead] == '\0' ) + { + pOutput[nRead] = '\n'; + nRead++; + break; + } + + nRead++; + } + + if( nRead < maxChars ) + pOutput[nRead] = '\0'; + + + m_Stats.nBytesRead += nRead; + return ( nRead ) ? pOutput : NULL; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *pFileName - +// Output : long +//----------------------------------------------------------------------------- +long CBaseFileSystem::GetFileTime( const char *pFileName, const char *pPathID ) +{ + VPROF_BUDGET( "CBaseFileSystem::GetFileTime", VPROF_BUDGETGROUP_OTHER_FILESYSTEM ); + + CHECK_DOUBLE_SLASHES( pFileName ); + + CSearchPathsIterator iter( this, &pFileName, pPathID ); + + char tempFileName[MAX_PATH]; + Q_strncpy( tempFileName, pFileName, sizeof(tempFileName) ); + Q_FixSlashes( tempFileName ); +#ifdef _WIN32 + Q_strlower( tempFileName ); +#endif + + for ( CSearchPath *pSearchPath = iter.GetFirst(); pSearchPath != NULL; pSearchPath = iter.GetNext() ) + { + long ft = FastFileTime( pSearchPath, tempFileName ); + if ( ft != 0L ) + { + if ( !pSearchPath->GetPackFile() && m_LogFuncs.Count() ) + { + char pTmpFileName[ MAX_FILEPATH ]; + if ( strchr( tempFileName, ':' ) ) + { + Q_strncpy( pTmpFileName, tempFileName, sizeof( pTmpFileName ) ); + } + else + { + Q_snprintf( pTmpFileName, sizeof( pTmpFileName ), "%s%s", pSearchPath->GetPathString(), tempFileName ); + } + + Q_FixSlashes( tempFileName ); + + LogAccessToFile( "filetime", pTmpFileName, "" ); + } + + return ft; + } + } + return 0L; +} + +long CBaseFileSystem::GetPathTime( const char *pFileName, const char *pPathID ) +{ + VPROF_BUDGET( "CBaseFileSystem::GetFileTime", VPROF_BUDGETGROUP_OTHER_FILESYSTEM ); + + CSearchPathsIterator iter( this, &pFileName, pPathID ); + + char tempFileName[MAX_PATH]; + Q_strncpy( tempFileName, pFileName, sizeof(tempFileName) ); + Q_FixSlashes( tempFileName ); +#ifdef _WIN32 + Q_strlower( tempFileName ); +#endif + + long pathTime = 0L; + for ( CSearchPath *pSearchPath = iter.GetFirst(); pSearchPath != NULL; pSearchPath = iter.GetNext() ) + { + long ft = FastFileTime( pSearchPath, tempFileName ); + if ( ft > pathTime ) + pathTime = ft; + if ( ft != 0L ) + { + if ( !pSearchPath->GetPackFile() && m_LogFuncs.Count() ) + { + char pTmpFileName[ MAX_FILEPATH ]; + if ( strchr( tempFileName, ':' ) ) + { + Q_strncpy( pTmpFileName, tempFileName, sizeof( pTmpFileName ) ); + } + else + { + Q_snprintf( pTmpFileName, sizeof( pTmpFileName ), "%s%s", pSearchPath->GetPathString(), tempFileName ); + } + + Q_FixSlashes( tempFileName ); + + LogAccessToFile( "filetime", pTmpFileName, "" ); + } + } + } + return pathTime; +} + +void CBaseFileSystem::MarkAllCRCsUnverified() +{ + if ( IsX360() ) + { + return; + } + + m_FileTracker2.MarkAllCRCsUnverified(); +} + + +void CBaseFileSystem::CacheFileCRCs( const char *pPathname, ECacheCRCType eType, IFileList *pFilter ) +{ + if ( IsX360() ) + { + return; + } +} + +EFileCRCStatus CBaseFileSystem::CheckCachedFileHash( const char *pPathID, const char *pRelativeFilename, int nFileFraction, FileHash_t *pFileHash ) +{ + return m_FileTracker2.CheckCachedFileHash( pPathID, pRelativeFilename, nFileFraction, pFileHash ); +} + + +void CBaseFileSystem::EnableWhitelistFileTracking( bool bEnable, bool bCacheAllVPKHashes, bool bRecalculateAndCheckHashes ) +{ + if ( IsX360() ) + { + m_WhitelistFileTrackingEnabled = false; + return; + } + + if ( m_WhitelistFileTrackingEnabled != -1 ) + { + Error( "CBaseFileSystem::EnableWhitelistFileTracking called more than once." ); + } + + m_WhitelistFileTrackingEnabled = bEnable; + if ( m_WhitelistFileTrackingEnabled && bCacheAllVPKHashes ) + { + CacheAllVPKFileHashes( bCacheAllVPKHashes, bRecalculateAndCheckHashes ); + } +} + + +void CBaseFileSystem::CacheAllVPKFileHashes( bool bCacheAllVPKHashes, bool bRecalculateAndCheckHashes ) +{ +#ifdef SUPPORT_PACKED_STORE + for ( int i = 0; i < m_SearchPaths.Count(); i++ ) + { + CPackedStore *pVPK = m_SearchPaths[i].GetPackedStore(); + if ( pVPK == NULL ) + continue; + if ( !pVPK->BTestDirectoryHash() ) + { + Msg( "VPK dir file hash does not match. File corrupted or modified.\n" ); + } + if ( !pVPK->BTestMasterChunkHash() ) + { + Msg( "VPK chunk hash hash does not match. File corrupted or modified.\n" ); + } + + CUtlVector<ChunkHashFraction_t> &vecChunkHash = pVPK->AccessPackFileHashes(); + CPackedStoreFileHandle fhandle = pVPK->GetHandleForHashingFiles(); + CUtlVector<ChunkHashFraction_t> vecChunkHashFractionCopy; + if ( bRecalculateAndCheckHashes ) + { + CUtlString sPackFileErrors; + pVPK->GetPackFileLoadErrorSummary( sPackFileErrors ); + + if ( sPackFileErrors.Length() ) + { + Msg( "Errors occured loading files.\n" ); + Msg( "%s", sPackFileErrors.String() ); + Msg( "Verify integrity of your game files, perform memory and disk diagnostics on your system.\n" ); + } + else + Msg( "No VPK Errors occured loading files.\n" ); + + Msg( "Recomputing all VPK file hashes.\n" ); + vecChunkHashFractionCopy.Swap( vecChunkHash ); + } + int cFailures = 0; + if ( vecChunkHash.Count() == 0 ) + { + if ( vecChunkHashFractionCopy.Count() == 0 ) + Msg( "File hash information not found: Hashing all VPK files for pure server operation.\n" ); + pVPK->HashAllChunkFiles(); + if ( vecChunkHashFractionCopy.Count() != 0 ) + { + if ( vecChunkHash.Count() != vecChunkHashFractionCopy.Count() ) + { + Msg( "VPK hash count does not match. VPK content may be corrupt.\n" ); + } + else if ( Q_memcmp( vecChunkHash.Base(), vecChunkHashFractionCopy.Base(), vecChunkHash.Count()*sizeof(vecChunkHash[0])) != 0 ) + { + Msg( "VPK hashes do not match. VPK content may be corrupt.\n" ); + // find the actual mismatch + FOR_EACH_VEC( vecChunkHashFractionCopy, iHash ) + { + if ( Q_memcmp( vecChunkHashFractionCopy[iHash].m_md5contents.bits, vecChunkHash[iHash].m_md5contents.bits, sizeof( vecChunkHashFractionCopy[iHash].m_md5contents.bits ) ) != 0 ) + { + Msg( "VPK hash for file %d failure at offset %x.\n", vecChunkHashFractionCopy[iHash].m_nPackFileNumber, vecChunkHashFractionCopy[iHash].m_nFileFraction ); + cFailures++; + } + } + } + } + } + if ( bCacheAllVPKHashes ) + { + Msg( "Loaded %d VPK file hashes from %s for pure server operation.\n", vecChunkHash.Count(), pVPK->FullPathName() ); + FOR_EACH_VEC( vecChunkHash, i ) + { + m_FileTracker2.AddFileHashForVPKFile( vecChunkHash[i].m_nPackFileNumber, vecChunkHash[i].m_nFileFraction, vecChunkHash[i].m_cbChunkLen, vecChunkHash[i].m_md5contents, fhandle ); + } + } + else + { + if ( cFailures == 0 && vecChunkHash.Count() == vecChunkHashFractionCopy.Count() ) + Msg( "File hashes checked. %d matches. no failures.\n", vecChunkHash.Count() ); + else + Msg( "File hashes checked. %d matches. %d failures.\n", vecChunkHash.Count(), cFailures ); + } + } +#endif +} + + +bool CBaseFileSystem::CheckVPKFileHash( int PackFileID, int nPackFileNumber, int nFileFraction, MD5Value_t &md5Value ) +{ +#ifdef SUPPORT_PACKED_STORE + for ( int i = 0; i < m_SearchPaths.Count(); i++ ) + { + CPackedStore *pVPK = m_SearchPaths[i].GetPackedStore(); + if ( pVPK == NULL || pVPK->m_PackFileID != PackFileID ) + continue; + ChunkHashFraction_t fileHashFraction; + if ( pVPK->FindFileHashFraction( nPackFileNumber, nFileFraction, fileHashFraction ) ) + { + CPackedStoreFileHandle fhandle = pVPK->GetHandleForHashingFiles(); + fhandle.m_nFileNumber = nPackFileNumber; + char szFileName[MAX_PATH]; + + pVPK->GetPackFileName( fhandle, szFileName, sizeof(szFileName) ); + + char hex[ 34 ]; + Q_memset( hex, 0, sizeof( hex ) ); + Q_binarytohex( (const byte *)md5Value.bits, sizeof( md5Value.bits ), hex, sizeof( hex ) ); + + char hex2[ 34 ]; + Q_memset( hex2, 0, sizeof( hex2 ) ); + Q_binarytohex( (const byte *)fileHashFraction.m_md5contents.bits, sizeof( fileHashFraction.m_md5contents.bits ), hex2, sizeof( hex2 ) ); + + if ( Q_memcmp( fileHashFraction.m_md5contents.bits, md5Value.bits, sizeof(md5Value.bits) ) != 0 ) + { + Msg( "File %s offset %x hash %s does not match ( should be %s ) \n", szFileName, nFileFraction, hex, hex2 ); + return false; + } + else + { + return true; + } + } + } +#else + Error("CBaseFileSystem::CheckVPKFileHash should not be called, SUPPORT_PACKED_STORE not defined" ); +#endif + return false; +} + +void CBaseFileSystem::RegisterFileWhitelist( IPureServerWhitelist *pWhiteList, IFileList **pFilesToReload ) +{ + if ( pFilesToReload ) + *pFilesToReload = NULL; + + if ( IsX360() ) + { + return; + } + + if ( m_pPureServerWhitelist ) + { + m_pPureServerWhitelist->Release(); + m_pPureServerWhitelist = NULL; + } + if ( pWhiteList ) + { + pWhiteList->AddRef(); + m_pPureServerWhitelist = pWhiteList; + } + + // update which search paths are considered trusted + FOR_EACH_VEC( m_SearchPaths, i ) + { + SetSearchPathIsTrustedSource( &m_SearchPaths[i] ); + } + + // See if we need to reload any files + if ( pFilesToReload ) + *pFilesToReload = m_FileTracker2.GetFilesToUnloadForWhitelistChange( pWhiteList ); +} + +void CBaseFileSystem::NotifyFileUnloaded( const char *pszFilename, const char *pPathId ) +{ + m_FileTracker2.NoteFileUnloaded( pszFilename, pPathId ); +} + +void CBaseFileSystem::SetSearchPathIsTrustedSource( CSearchPath *pSearchPath ) +{ + // Most paths are not considered trusted + pSearchPath->m_bIsTrustedForPureServer = false; + + // If we don't have a pure server whitelist, we cannot say that any + // particular files are trusted. (But then again, all files will be + // accepted, so it won't really matter.) + if ( m_pPureServerWhitelist == NULL ) + return; + + // Treat map packs as trusted, because we will send the CRC of the map pack to the server + if ( pSearchPath->GetPackFile() && pSearchPath->GetPackFile()->m_bIsMapPath ) + { + #ifdef PURE_SERVER_DEBUG_SPEW + Msg( "Setting map pack search path %s as trusted\n", pSearchPath->GetDebugString() ); + #endif + pSearchPath->m_bIsTrustedForPureServer = true; + return; + } + + #ifdef SUPPORT_PACKED_STORE + // Only signed VPK's can be trusted + CPackedStoreRefCount *pVPK = pSearchPath->GetPackedStore(); + if ( pVPK == NULL ) + { + #ifdef PURE_SERVER_DEBUG_SPEW + Msg( "Setting %s as untrusted (loose files)\n", pSearchPath->GetDebugString() ); + #endif + return; + } + if ( !pVPK->m_bSignatureValid ) + { + #ifdef PURE_SERVER_DEBUG_SPEW + Msg( "Setting %s as untrusted (unsigned VPK)\n", pSearchPath->GetDebugString() ); + #endif + return; + } + const CUtlVector<uint8> &key = pVPK->GetSignaturePublicKey(); + for ( int iKeyIndex = 0 ; iKeyIndex < m_pPureServerWhitelist->GetTrustedKeyCount() ; ++iKeyIndex ) + { + int nKeySz = 0; + const byte *pbKey = m_pPureServerWhitelist->GetTrustedKey( iKeyIndex, &nKeySz ); + Assert( pbKey != NULL && nKeySz > 0 ); + if ( key.Count() == nKeySz && V_memcmp( pbKey, key.Base(), nKeySz ) == 0 ) + { + #ifdef PURE_SERVER_DEBUG_SPEW + Msg( "Setting %s as untrusted\n", pSearchPath->GetDebugString() ); + #endif + pSearchPath->m_bIsTrustedForPureServer = true; + return; + } + } + + #ifdef PURE_SERVER_DEBUG_SPEW + Msg( "Setting %s as untrusted. (Key not in trusted key list)\n", pSearchPath->GetDebugString() ); + #endif + #endif +} + + +int CBaseFileSystem::GetUnverifiedFileHashes( CUnverifiedFileHash *pFiles, int nMaxFiles ) +{ + return m_FileTracker2.GetUnverifiedFileHashes( pFiles, nMaxFiles ); +} + + + +int CBaseFileSystem::GetWhitelistSpewFlags() +{ + return m_WhitelistSpewFlags; +} + + +void CBaseFileSystem::SetWhitelistSpewFlags( int flags ) +{ + m_WhitelistSpewFlags = flags; +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *pString - +// maxCharsIncludingTerminator - +// fileTime - +//----------------------------------------------------------------------------- +void CBaseFileSystem::FileTimeToString( char *pString, int maxCharsIncludingTerminator, long fileTime ) +{ + if ( IsX360() ) + { + char szTemp[ 256 ]; + + time_t time = fileTime; + V_strncpy( szTemp, ctime( &time ), sizeof( szTemp ) ); + char *pFinalColon = Q_strrchr( szTemp, ':' ); + if ( pFinalColon ) + *pFinalColon = '\0'; + + // Clip off the day of the week + V_strncpy( pString, szTemp + 4, maxCharsIncludingTerminator ); + } + else + { + time_t time = fileTime; + V_strncpy( pString, ctime( &time ), maxCharsIncludingTerminator ); + + // We see a linefeed at the end of these strings...if there is one, gobble it up + int len = V_strlen( pString ); + if ( pString[ len - 1 ] == '\n' ) + { + pString[ len - 1 ] = '\0'; + } + + pString[maxCharsIncludingTerminator-1] = '\0'; + } +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *pFileName - +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CBaseFileSystem::FileExists( const char *pFileName, const char *pPathID ) +{ + VPROF_BUDGET( "CBaseFileSystem::FileExists", VPROF_BUDGETGROUP_OTHER_FILESYSTEM ); + + CHECK_DOUBLE_SLASHES( pFileName ); + + FileHandle_t h = Open( pFileName, "rb", pPathID ); + if ( h ) + { + Close(h); + return true; + } + + return false; +} + +bool CBaseFileSystem::IsFileWritable( char const *pFileName, char const *pPathID /*=0*/ ) +{ + CHECK_DOUBLE_SLASHES( pFileName ); + + struct _stat buf; + + char tempPathID[MAX_PATH]; + ParsePathID( pFileName, pPathID, tempPathID ); + + if ( Q_IsAbsolutePath( pFileName ) ) + { + if( FS_stat( pFileName, &buf ) != -1 ) + { +#ifdef WIN32 + if( buf.st_mode & _S_IWRITE ) +#elif LINUX + if( buf.st_mode & S_IWRITE ) +#else + if( buf.st_mode & S_IWRITE ) +#endif + { + return true; + } + } + return false; + } + + CSearchPathsIterator iter( this, &pFileName, pPathID, FILTER_CULLPACK ); + for ( CSearchPath *pSearchPath = iter.GetFirst(); pSearchPath != NULL; pSearchPath = iter.GetNext() ) + { + char pTmpFileName[ MAX_FILEPATH ]; + Q_snprintf( pTmpFileName, sizeof( pTmpFileName ), "%s%s", pSearchPath->GetPathString(), pFileName ); + Q_FixSlashes( pTmpFileName ); + if ( FS_stat( pTmpFileName, &buf ) != -1 ) + { +#ifdef WIN32 + if ( buf.st_mode & _S_IWRITE ) +#elif LINUX + if ( buf.st_mode & S_IWRITE ) +#else + if ( buf.st_mode & S_IWRITE ) +#endif + { + return true; + } + } + } + return false; +} + + +bool CBaseFileSystem::SetFileWritable( char const *pFileName, bool writable, const char *pPathID /*= 0*/ ) +{ + CHECK_DOUBLE_SLASHES( pFileName ); + +#ifdef _WIN32 + int pmode = writable ? ( _S_IWRITE | _S_IREAD ) : ( _S_IREAD ); +#else + int pmode = writable ? ( S_IWRITE | S_IREAD ) : ( S_IREAD ); +#endif + + char tempPathID[MAX_PATH]; + ParsePathID( pFileName, pPathID, tempPathID ); + + if ( Q_IsAbsolutePath( pFileName ) ) + { + return ( FS_chmod( pFileName, pmode ) == 0 ); + } + + CSearchPathsIterator iter( this, &pFileName, pPathID, FILTER_CULLPACK ); + for ( CSearchPath *pSearchPath = iter.GetFirst(); pSearchPath != NULL; pSearchPath = iter.GetNext() ) + { + char pTmpFileName[ MAX_FILEPATH ]; + Q_snprintf( pTmpFileName, sizeof( pTmpFileName ), "%s%s", pSearchPath->GetPathString(), pFileName ); + Q_FixSlashes( pTmpFileName ); + + if ( FS_chmod( pTmpFileName, pmode ) == 0 ) + { + return true; + } + } + + // Failure + return false; +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *pFileName - +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CBaseFileSystem::IsDirectory( const char *pFileName, const char *pathID ) +{ + CHECK_DOUBLE_SLASHES( pFileName ); + + // Allow for UNC-type syntax to specify the path ID. + struct _stat buf; + + char pTempBuf[MAX_PATH]; + Q_strncpy( pTempBuf, pFileName, sizeof(pTempBuf) ); + Q_StripTrailingSlash( pTempBuf ); + pFileName = pTempBuf; + + char tempPathID[MAX_PATH]; + ParsePathID( pFileName, pathID, tempPathID ); + if ( Q_IsAbsolutePath( pFileName ) ) + { + if ( FS_stat( pFileName, &buf ) != -1 ) + { + if ( buf.st_mode & _S_IFDIR ) + return true; + } + return false; + } + + CSearchPathsIterator iter( this, &pFileName, pathID, FILTER_CULLPACK ); + for ( CSearchPath *pSearchPath = iter.GetFirst(); pSearchPath != NULL; pSearchPath = iter.GetNext() ) + { +#ifdef SUPPORT_PACKED_STORE + if ( pSearchPath->GetPackedStore() ) + { + CUtlStringList outDir, outFile; + pSearchPath->GetPackedStore()->GetFileAndDirLists( outDir, outFile, false ); + FOR_EACH_VEC( outDir, i ) + { + if ( !Q_stricmp( outDir[i], pFileName ) ) + return true; + } + + } + else +#endif // SUPPORT_PACKED_STORE + { + char pTmpFileName[ MAX_FILEPATH ]; + Q_snprintf( pTmpFileName, sizeof( pTmpFileName ), "%s%s", pSearchPath->GetPathString(), pFileName ); + Q_FixSlashes( pTmpFileName ); + if ( FS_stat( pTmpFileName, &buf ) != -1 ) + { + if ( buf.st_mode & _S_IFDIR ) + return true; + } + } + } + return false; +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *path - +//----------------------------------------------------------------------------- +void CBaseFileSystem::CreateDirHierarchy( const char *pRelativePathT, const char *pathID ) +{ + // Allow for UNC-type syntax to specify the path ID. + char tempPathID[MAX_PATH]; + ParsePathID(pRelativePathT, pathID, tempPathID); // use the original path param to preserve "//" + + char pRelativePathBuff[ MAX_PATH ]; + const char *pRelativePath = pRelativePathBuff; + + FixUpPath ( pRelativePathT, pRelativePathBuff, sizeof( pRelativePathBuff ) ); + + CHECK_DOUBLE_SLASHES( pRelativePath ); + + char szScratchFileName[MAX_PATH]; + if ( !Q_IsAbsolutePath( pRelativePath ) ) + { + Assert( pathID ); + + + ComputeFullWritePath( szScratchFileName, sizeof( szScratchFileName ), pRelativePath, pathID ); + } + else + { + Q_strncpy( szScratchFileName, pRelativePath, sizeof(szScratchFileName) ); + } + + int len = strlen( szScratchFileName ) + 1; + char *end = szScratchFileName + len; + char *s = szScratchFileName; + while( s < end ) + { + if( *s == CORRECT_PATH_SEPARATOR && s != szScratchFileName && ( IsLinux() || *( s - 1 ) != ':' ) ) + { + *s = '\0'; +#if defined( _WIN32 ) + _mkdir( szScratchFileName ); +#elif defined( POSIX ) + mkdir( szScratchFileName, S_IRWXU | S_IRGRP | S_IROTH );// owner has rwx, rest have r +#endif + *s = CORRECT_PATH_SEPARATOR; + } + s++; + } + +#if defined( _WIN32 ) + _mkdir( szScratchFileName ); +#elif defined( POSIX ) + mkdir( szScratchFileName, S_IRWXU | S_IRGRP | S_IROTH ); +#endif +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *pWildCard - +// *pHandle - +// Output : const char +//----------------------------------------------------------------------------- +const char *CBaseFileSystem::FindFirstEx( const char *pWildCard, const char *pPathID, FileFindHandle_t *pHandle ) +{ + CHECK_DOUBLE_SLASHES( pWildCard ); + + return FindFirstHelper( pWildCard, pPathID, pHandle, NULL ); +} + + +const char *CBaseFileSystem::FindFirstHelper( const char *pWildCardT, const char *pPathID, FileFindHandle_t *pHandle, int *pFoundStoreID ) +{ + VPROF_BUDGET( "CBaseFileSystem::FindFirst", VPROF_BUDGETGROUP_OTHER_FILESYSTEM ); + Assert(pWildCardT); + Assert(pHandle); + + FileFindHandle_t hTmpHandle = m_FindData.AddToTail(); + FindData_t *pFindData = &m_FindData[hTmpHandle]; + Assert( pFindData ); + if ( pPathID ) + { + pFindData->m_FilterPathID = g_PathIDTable.AddString( pPathID ); + } + + char pWildCard[ MAX_PATH ]; + + FixUpPath ( pWildCardT, pWildCard, sizeof( pWildCard ) ); + + int maxlen = strlen( pWildCard ) + 1; + pFindData->wildCardString.AddMultipleToTail( maxlen ); + Q_strncpy( pFindData->wildCardString.Base(), pWildCard, maxlen ); + pFindData->findHandle = INVALID_HANDLE_VALUE; + + if ( Q_IsAbsolutePath( pWildCard ) ) + { + // Absolute path, cannot be VPK or Pak + pFindData->findHandle = FS_FindFirstFile( pWildCard, &pFindData->findData ); + pFindData->currentSearchPathID = -1; + } + else + { + int c = m_SearchPaths.Count(); + for( pFindData->currentSearchPathID = 0; + pFindData->currentSearchPathID < c; + pFindData->currentSearchPathID++ ) + { + CSearchPath *pSearchPath = &m_SearchPaths[pFindData->currentSearchPathID]; + + if ( FilterByPathID( pSearchPath, pFindData->m_FilterPathID ) ) + continue; + + // already visited this path? + if ( pFindData->m_VisitedSearchPaths.MarkVisit( *pSearchPath ) ) + continue; + + // If this is a VPK or Pak file, build list of matches now and use helper + bool bIsVPKOrPak = false; + if ( pSearchPath->GetPackFile() ) + { + // XXX(johns) This support didn't exist for a long time, and I'm now worried about various things + // looking for misc files suddenly finding them in the (untrusted) BSP and causing security + // nightmares. For now, restricting FindFirst() support to BSPs only when the BSP search path + // is explicitly requested, but this would otherwise work fine. + if ( !pPathID || V_strcmp( pPathID, "BSP" ) != 0 ) + { + continue; + } + + Assert( pFindData->m_dirMatchesFromVPKOrPak.Count() == 0 ); + Assert( pFindData->m_fileMatchesFromVPKOrPak.Count() == 0 ); + pSearchPath->GetPackFile()->GetFileAndDirLists( pWildCard, pFindData->m_dirMatchesFromVPKOrPak, pFindData->m_fileMatchesFromVPKOrPak, true ); + bIsVPKOrPak = true; + } + + #ifdef SUPPORT_PACKED_STORE + if ( pSearchPath->GetPackedStore() ) + { + Assert( pFindData->m_dirMatchesFromVPKOrPak.Count() == 0 ); + Assert( pFindData->m_fileMatchesFromVPKOrPak.Count() == 0 ); + pSearchPath->GetPackedStore()->GetFileAndDirLists( pWildCard, pFindData->m_dirMatchesFromVPKOrPak, pFindData->m_fileMatchesFromVPKOrPak, true ); + bIsVPKOrPak = true; + } + #endif + + if ( bIsVPKOrPak ) + { + if ( FindNextFileInVPKOrPakHelper( pFindData ) ) + { + // Remember that we visited this file already. + pFindData->m_VisitedFiles.Insert( pFindData->findData.cFileName, 0 ); + *pHandle = hTmpHandle; + return pFindData->findData.cFileName; + } + continue; + } + + // Otherwise, raw FS find for relative path + char pTmpFileName[ MAX_FILEPATH ]; + Q_snprintf( pTmpFileName, sizeof( pTmpFileName ), "%s%s", pSearchPath->GetPathString(), pFindData->wildCardString.Base() ); + Q_FixSlashes( pTmpFileName ); + pFindData->findHandle = FS_FindFirstFile( pTmpFileName, &pFindData->findData ); + pFindData->m_CurrentStoreID = pSearchPath->m_storeId; + + if( pFindData->findHandle != INVALID_HANDLE_VALUE ) + break; + } + } + + // We have a result from the filesystem + if( pFindData->findHandle != INVALID_HANDLE_VALUE ) + { + // Remember that we visited this file already. + pFindData->m_VisitedFiles.Insert( pFindData->findData.cFileName, 0 ); + + if ( pFoundStoreID ) + *pFoundStoreID = pFindData->m_CurrentStoreID; + + *pHandle = hTmpHandle; + return pFindData->findData.cFileName; + } + + // Handle failure here + pFindData = 0; + m_FindData.Remove(hTmpHandle); + *pHandle = -1; + + return NULL; +} + +const char *CBaseFileSystem::FindFirst( const char *pWildCard, FileFindHandle_t *pHandle ) +{ + return FindFirstEx( pWildCard, NULL, pHandle ); +} + + +// Get the next file, trucking through the path. . don't check for duplicates. +bool CBaseFileSystem::FindNextFileHelper( FindData_t *pFindData, int *pFoundStoreID ) +{ + // Try the same search path that we were already searching on. + if( FS_FindNextFile( pFindData->findHandle, &pFindData->findData ) ) + { + if ( pFoundStoreID ) + *pFoundStoreID = pFindData->m_CurrentStoreID; + + return true; + } + + if ( FindNextFileInVPKOrPakHelper( pFindData ) ) + return true; + + // This happens when we searched a full path + if ( pFindData->currentSearchPathID < 0 ) + return false; + + pFindData->currentSearchPathID++; + + if ( pFindData->findHandle != INVALID_HANDLE_VALUE ) + { + FS_FindClose( pFindData->findHandle ); + } + pFindData->findHandle = INVALID_HANDLE_VALUE; + + int c = m_SearchPaths.Count(); + for( ; pFindData->currentSearchPathID < c; ++pFindData->currentSearchPathID ) + { + CSearchPath *pSearchPath = &m_SearchPaths[pFindData->currentSearchPathID]; + + if ( FilterByPathID( pSearchPath, pFindData->m_FilterPathID ) ) + continue; + + // already visited this path + if ( pFindData->m_VisitedSearchPaths.MarkVisit( *pSearchPath ) ) + continue; + + if ( pSearchPath->GetPackFile() ) + { + // XXX(johns) This support didn't exist for a long time, and I'm now worried about various things + // looking for misc files suddenly finding them in the (untrusted) BSP and causing security + // nightmares. For now, restricting FindFirst() support to BSPs only when the BSP search path + // is explicitly requested, but this would otherwise work fine. + if ( !pFindData->m_FilterPathID || V_strcmp( g_PathIDTable.String( pFindData->m_FilterPathID ), "BSP" ) != 0 ) + { + continue; + } + Assert( pFindData->m_dirMatchesFromVPKOrPak.Count() == 0 ); + Assert( pFindData->m_fileMatchesFromVPKOrPak.Count() == 0 ); + pSearchPath->GetPackFile()->GetFileAndDirLists( pFindData->wildCardString.Base(), pFindData->m_dirMatchesFromVPKOrPak, pFindData->m_fileMatchesFromVPKOrPak, true ); + if ( FindNextFileInVPKOrPakHelper( pFindData ) ) + return true; + continue; + } + + #ifdef SUPPORT_PACKED_STORE + if ( pSearchPath->GetPackedStore() ) + { + Assert( pFindData->m_dirMatchesFromVPKOrPak.Count() == 0 ); + Assert( pFindData->m_fileMatchesFromVPKOrPak.Count() == 0 ); + pSearchPath->GetPackedStore()->GetFileAndDirLists( pFindData->wildCardString.Base(), pFindData->m_dirMatchesFromVPKOrPak, pFindData->m_fileMatchesFromVPKOrPak, true ); + if ( FindNextFileInVPKOrPakHelper( pFindData ) ) + return true; + continue; + } + #endif + + char pTmpFileName[ MAX_FILEPATH ]; + Q_snprintf( pTmpFileName, sizeof( pTmpFileName ), "%s%s", pSearchPath->GetPathString(), pFindData->wildCardString.Base() ); + Q_FixSlashes( pTmpFileName ); + pFindData->findHandle = FS_FindFirstFile( pTmpFileName, &pFindData->findData ); + pFindData->m_CurrentStoreID = pSearchPath->m_storeId; + if( pFindData->findHandle != INVALID_HANDLE_VALUE ) + { + if ( pFoundStoreID ) + *pFoundStoreID = pFindData->m_CurrentStoreID; + + return true; + } + } + + return false; +} + +bool CBaseFileSystem::FindNextFileInVPKOrPakHelper( FindData_t *pFindData ) +{ + // Return the next one from the list of VPK matches if there is one + if ( pFindData->m_fileMatchesFromVPKOrPak.Count() > 0 ) + { + V_strncpy( pFindData->findData.cFileName, V_UnqualifiedFileName( pFindData->m_fileMatchesFromVPKOrPak[0] ), sizeof( pFindData->findData.cFileName ) ); + pFindData->findData.dwFileAttributes = 0; + delete pFindData->m_fileMatchesFromVPKOrPak.Head(); + pFindData->m_fileMatchesFromVPKOrPak.RemoveMultipleFromHead( 1 ); + + return true; + } + + // Return the next one from the list of VPK matches if there is one + if ( pFindData->m_dirMatchesFromVPKOrPak.Count() > 0 ) + { + V_strncpy( pFindData->findData.cFileName, V_UnqualifiedFileName( pFindData->m_dirMatchesFromVPKOrPak[0] ), sizeof( pFindData->findData.cFileName ) ); + pFindData->findData.dwFileAttributes = FILE_ATTRIBUTE_DIRECTORY; + delete pFindData->m_dirMatchesFromVPKOrPak.Head(); + pFindData->m_dirMatchesFromVPKOrPak.RemoveMultipleFromHead( 1 ); + + return true; + } + + return false; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : handle - +// Output : const char +//----------------------------------------------------------------------------- +const char *CBaseFileSystem::FindNext( FileFindHandle_t handle ) +{ + VPROF_BUDGET( "CBaseFileSystem::FindNext", VPROF_BUDGETGROUP_OTHER_FILESYSTEM ); + FindData_t *pFindData = &m_FindData[handle]; + + while( 1 ) + { + if( FindNextFileHelper( pFindData, NULL ) ) + { + if ( pFindData->m_VisitedFiles.Find( pFindData->findData.cFileName ) == -1 ) + { + pFindData->m_VisitedFiles.Insert( pFindData->findData.cFileName, 0 ); + return pFindData->findData.cFileName; + } + } + else + { + return NULL; + } + } +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : handle - +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CBaseFileSystem::FindIsDirectory( FileFindHandle_t handle ) +{ + FindData_t *pFindData = &m_FindData[handle]; + return !!( pFindData->findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ); +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : handle - +//----------------------------------------------------------------------------- +void CBaseFileSystem::FindClose( FileFindHandle_t handle ) +{ + if ( ( handle < 0 ) || ( !m_FindData.IsInList( handle ) ) ) + return; + + FindData_t *pFindData = &m_FindData[handle]; + Assert(pFindData); + + if ( pFindData->findHandle != INVALID_HANDLE_VALUE) + { + FS_FindClose( pFindData->findHandle ); + } + pFindData->findHandle = INVALID_HANDLE_VALUE; + + pFindData->wildCardString.Purge(); + pFindData->m_fileMatchesFromVPKOrPak.PurgeAndDeleteElements(); + pFindData->m_dirMatchesFromVPKOrPak.PurgeAndDeleteElements(); + m_FindData.Remove( handle ); +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *pFileName - +//----------------------------------------------------------------------------- +void CBaseFileSystem::GetLocalCopy( const char *pFileName ) +{ + // do nothing. . everything is local. +} + +//----------------------------------------------------------------------------- +// Purpose: Fixes up Path names. Will fix up platform specific slashes, remove +// any ../ or ./, fix //s, and lowercase anything under the directory +// that the game is installed to. We expect all files to be lower cased +// there - especially on Linux (where case sensitivity is the norm). +// +// Input : *pFileName - Original name to convert +// *pFixedUpFileName - a buffer to put the converted filename into +// sizeFixedUpFileName - the size of the above buffer in chars +//----------------------------------------------------------------------------- +bool CBaseFileSystem::FixUpPath( const char *pFileName, char *pFixedUpFileName, int sizeFixedUpFileName ) +{ + // If appropriate fixes up the filename to ensure that it's handled properly by the system. + // + V_strncpy( pFixedUpFileName, pFileName, sizeFixedUpFileName ); + V_FixSlashes( pFixedUpFileName, CORRECT_PATH_SEPARATOR ); +// V_RemoveDotSlashes( pFixedUpFileName, CORRECT_PATH_SEPARATOR, true ); + V_FixDoubleSlashes( pFixedUpFileName ); + + if ( !V_IsAbsolutePath( pFixedUpFileName ) ) + { + V_strlower( pFixedUpFileName ); + } + else + { + // Get the BASE_PATH, skip past - if necessary, and lowercase the rest + // Not just yet... + + + int iBaseLength = 0; + char pBaseDir[MAX_PATH]; + + // Need to get "BASE_PATH" from the filesystem paths, and then check this name against it. + // + iBaseLength = GetSearchPath( "BASE_PATH", true, pBaseDir, sizeof( pBaseDir ) ); + if ( iBaseLength ) + { + // If the first part of the pFixedUpFilename is pBaseDir + // then lowercase the part after that. + if ( *pBaseDir && (iBaseLength+1 < V_strlen( pFixedUpFileName ) ) && (0 != V_strncmp( pBaseDir, pFixedUpFileName, iBaseLength ) ) ) + { + V_strlower( &pFixedUpFileName[iBaseLength-1] ); + } + } + + } + +// Msg("CBaseFileSystem::FixUpPath: Converted %s to %s\n", pFileName, pFixedUpFileName); // too noisy + +#ifdef NEVER // Useful if you're trying to see why your file may not be found (if you have a mixed case file) + if (strncmp(pFixedUpFileName, pFileName, 256)) + { + printf("FixUpPath->Converting %s to %s\n",pFileName, pFixedUpFileName); + } +#endif // NEVER + return true; +} + + +//----------------------------------------------------------------------------- +// Converts a partial path into a full path +// Relative paths that are pack based are returned as an absolute path .../zip?.zip/foo +// A pack absolute path can be sent back in for opening, and the file will be properly +// detected as pack based and mounted inside the pack. +//----------------------------------------------------------------------------- +const char *CBaseFileSystem::RelativePathToFullPath( const char *pFileName, const char *pPathID, OUT_Z_CAP(maxLenInChars) char *pDest, int maxLenInChars, PathTypeFilter_t pathFilter, PathTypeQuery_t *pPathType ) +{ + CHECK_DOUBLE_SLASHES( pFileName ); + + struct _stat buf; + + if ( pPathType ) + { + *pPathType = PATH_IS_NORMAL; + } + + // Convert filename to lowercase. All files in the + // game logical filesystem must be accessed by lowercase name + char szLowercaseFilename[ MAX_PATH ]; + FixUpPath( pFileName, szLowercaseFilename, sizeof( szLowercaseFilename ) ); + pFileName = szLowercaseFilename; + + // Fill in the default in case it's not found... + V_strncpy( pDest, pFileName, maxLenInChars ); + +// @FD This is arbitrary and seems broken. If the caller needs this filter, they should +// request it with the flag themselves. As it is, I cannot search all the file paths +// for a file using this function because there is no option that says, "No, really, I +// mean ALL SEARCH PATHS." The current problem I'm trying to fix is that sounds are not +// working if they are in the BSP. I wrote code that assumed that I could just ask for +// the absolute path of a file, since we are able to open files with these absolute +// filenames, and that each particular filesystem call wouldn't have its own individual +// quirks. +// if ( IsPC() && pathFilter == FILTER_NONE ) +// { +// // X360TBD: PC legacy behavior never returned pack paths +// // do legacy behavior to ensure naive callers don't break +// pathFilter = FILTER_CULLPACK; +// } + + + CSearchPathsIterator iter( this, &pFileName, pPathID, pathFilter ); + for ( CSearchPath *pSearchPath = iter.GetFirst(); pSearchPath != NULL; pSearchPath = iter.GetNext() ) + { + + CPackFile *pPack = pSearchPath->GetPackFile(); + if ( pPack ) + { + if ( pPack->ContainsFile( pFileName ) ) + { + if ( pPathType ) + { + if ( pPack->m_bIsMapPath ) + { + *pPathType |= PATH_IS_MAPPACKFILE; + } + else + { + *pPathType |= PATH_IS_PACKFILE; + } + if ( pSearchPath->m_bIsRemotePath ) + { + *pPathType |= PATH_IS_REMOTE; + } + } + + // form an encoded absolute path that can be decoded by our FS as pak based + const char *pszPackName = pPack->m_ZipName.String(); + int len = V_strlen( pszPackName ); + int nTotalLen = len + 1 + V_strlen( pFileName ); + if ( nTotalLen >= maxLenInChars ) + { + ::Warning( "File %s was found in %s, but resulting abs filename won't fit in callers buffer of %d bytes\n", + pFileName, pszPackName, maxLenInChars ); + Assert( false ); + return NULL; + } + + V_strncpy( pDest, pszPackName, maxLenInChars ); + V_AppendSlash( pDest, maxLenInChars ); + V_strncat( pDest, pFileName, maxLenInChars ); + Assert( V_strlen( pDest ) == nTotalLen ); + return pDest; + } + + continue; + } + + // Found in VPK? + #ifdef SUPPORT_PACKED_STORE + CPackedStore *pVPK = pSearchPath->GetPackedStore(); + if ( pVPK ) + { + CPackedStoreFileHandle vpkHandle = pVPK->OpenFile( pFileName ); + if ( vpkHandle ) + { + const char *pszVpkName = vpkHandle.m_pOwner->FullPathName(); + Assert( V_GetFileExtension( pszVpkName ) != NULL ); + + int len = V_strlen( pszVpkName ); + int nTotalLen = len + 1 + V_strlen( pFileName ); + if ( nTotalLen >= maxLenInChars ) + { + ::Warning( "File %s was found in %s, but resulting abs filename won't fit in callers buffer of %d bytes\n", + pFileName, pszVpkName, maxLenInChars ); + Assert( false ); + return NULL; + } + + V_strncpy( pDest, pszVpkName, maxLenInChars ); + V_AppendSlash( pDest, maxLenInChars ); + V_strncat( pDest, pFileName, maxLenInChars ); + V_FixSlashes( pDest ); + return pDest; + } + continue; + } + #endif + + char pTmpFileName[ MAX_FILEPATH ]; + V_sprintf_safe( pTmpFileName, "%s%s", pSearchPath->GetPathString(), pFileName ); + V_FixSlashes( pTmpFileName ); + if ( FS_stat( pTmpFileName, &buf ) != -1 ) + { + V_strncpy( pDest, pTmpFileName, maxLenInChars ); + if ( pPathType && pSearchPath->m_bIsRemotePath ) + { + *pPathType |= PATH_IS_REMOTE; + } + return pDest; + } + } + + // not found + return NULL; +} + +const char *CBaseFileSystem::GetLocalPath( const char *pFileName, OUT_Z_CAP(maxLenInChars) char *pDest, int maxLenInChars ) +{ + CHECK_DOUBLE_SLASHES( pFileName ); + + return RelativePathToFullPath( pFileName, NULL, pDest, maxLenInChars ); +} + + +//----------------------------------------------------------------------------- +// Returns true on success, otherwise false if it can't be resolved +//----------------------------------------------------------------------------- +bool CBaseFileSystem::FullPathToRelativePathEx( const char *pFullPath, const char *pPathId, OUT_Z_CAP(maxLenInChars) char *pDest, int maxLenInChars ) +{ + CHECK_DOUBLE_SLASHES( pFullPath ); + + int nInlen = V_strlen( pFullPath ); + if ( nInlen <= 0 ) + { + pDest[ 0 ] = 0; + return false; + } + + V_strncpy( pDest, pFullPath, maxLenInChars ); + + char pInPath[ MAX_FILEPATH ]; + V_strcpy_safe( pInPath, pFullPath ); +#ifdef _WIN32 + V_strlower( pInPath ); +#endif + V_FixSlashes( pInPath ); + + CUtlSymbol lookup; + if ( pPathId ) + { + lookup = g_PathIDTable.AddString( pPathId ); + } + + int c = m_SearchPaths.Count(); + for( int i = 0; i < c; i++ ) + { + // FIXME: Should this work with embedded pak files? + if ( m_SearchPaths[i].GetPackFile() && m_SearchPaths[i].GetPackFile()->m_bIsMapPath ) + continue; + + // Skip paths that are not on the specified search path + if ( FilterByPathID( &m_SearchPaths[i], lookup ) ) + continue; + + char pSearchBase[ MAX_FILEPATH ]; + V_strncpy( pSearchBase, m_SearchPaths[i].GetPathString(), sizeof( pSearchBase ) ); +#ifdef _WIN32 + V_strlower( pSearchBase ); +#endif + V_FixSlashes( pSearchBase ); + int nSearchLen = V_strlen( pSearchBase ); + if ( V_strnicmp( pSearchBase, pInPath, nSearchLen ) ) + continue; + + V_strncpy( pDest, &pInPath[ nSearchLen ], maxLenInChars ); + return true; + } + + return false; +} + + +//----------------------------------------------------------------------------- +// Obsolete version +//----------------------------------------------------------------------------- +bool CBaseFileSystem::FullPathToRelativePath( const char *pFullPath, OUT_Z_CAP(maxLenInChars) char *pDest, int maxLenInChars ) +{ + return FullPathToRelativePathEx( pFullPath, NULL, pDest, maxLenInChars ); +} + + +//----------------------------------------------------------------------------- +// Returns true on successfully retrieve case-sensitive full path, otherwise false +//----------------------------------------------------------------------------- +bool CBaseFileSystem::GetCaseCorrectFullPath_Ptr( const char *pFullPath, OUT_Z_CAP(maxLenInChars) char *pDest, int maxLenInChars ) +{ + CHECK_DOUBLE_SLASHES( pFullPath ); + +#ifndef _WIN32 + + V_strncpy( pDest, pFullPath, maxLenInChars ); + return true; + +#else + + char szCurrentDir[MAX_PATH]; + V_strcpy_safe( szCurrentDir, pFullPath ); + V_StripLastDir( szCurrentDir, sizeof( szCurrentDir ) ); + + CUtlString strSearchName = pFullPath; + strSearchName.StripTrailingSlash(); + strSearchName = strSearchName.Slice( V_strlen( szCurrentDir ), strSearchName.Length() ); + + CUtlString strSearchPath = szCurrentDir; + strSearchPath += "*"; + + CUtlString strFoundCaseCorrectName; + FileFindHandle_t findHandle; + const char *pszCaseCorrectName = FindFirst( strSearchPath.Get(), &findHandle ); + while ( pszCaseCorrectName ) + { + if ( V_stricmp( strSearchName.String(), pszCaseCorrectName ) == 0 ) + { + strFoundCaseCorrectName = pszCaseCorrectName; + break; + } + pszCaseCorrectName = FindNext( findHandle ); + } + FindClose( findHandle ); + + // Not found + if ( strFoundCaseCorrectName.IsEmpty() ) + { + V_strncpy( pDest, pFullPath, maxLenInChars ); + return false; + } + + // If drive path, no need to recurse anymore. + bool bResult = false; + char szDir[MAX_PATH]; + if ( !IsDirectory( szCurrentDir, NULL ) ) + { + V_strupr( szCurrentDir ); + V_strncpy( szDir, szCurrentDir, sizeof( szDir ) ); + bResult = true; + } + else + { + bResult = GetCaseCorrectFullPath( szCurrentDir, szDir ); + } + + // connect the current path with the case-correct dir/file name + V_MakeAbsolutePath( pDest, maxLenInChars, strFoundCaseCorrectName.String(), szDir ); + + return bResult; +#endif // _WIN32 +} + + + +//----------------------------------------------------------------------------- +// Deletes a file +//----------------------------------------------------------------------------- +void CBaseFileSystem::RemoveFile( char const* pRelativePath, const char *pathID ) +{ + CHECK_DOUBLE_SLASHES( pRelativePath ); + + // Allow for UNC-type syntax to specify the path ID. + char tempPathID[MAX_PATH]; + ParsePathID( pRelativePath, pathID, tempPathID ); + + Assert( pathID || !IsX360() ); + + // Opening for write or append uses Write Path + char szScratchFileName[MAX_PATH]; + if ( Q_IsAbsolutePath( pRelativePath ) ) + { + Q_strncpy( szScratchFileName, pRelativePath, sizeof( szScratchFileName ) ); + } + else + { + ComputeFullWritePath( szScratchFileName, sizeof( szScratchFileName ), pRelativePath, pathID ); + } + int fail = unlink( szScratchFileName ); + if ( fail != 0 ) + { + Warning( FILESYSTEM_WARNING, "Unable to remove %s!\n", szScratchFileName ); + } +} + + +//----------------------------------------------------------------------------- +// Renames a file +//----------------------------------------------------------------------------- +bool CBaseFileSystem::RenameFile( char const *pOldPath, char const *pNewPath, const char *pathID ) +{ + Assert( pOldPath && pNewPath ); + + CHECK_DOUBLE_SLASHES( pOldPath ); + CHECK_DOUBLE_SLASHES( pNewPath ); + + // Allow for UNC-type syntax to specify the path ID. + char pPathIdCopy[MAX_PATH]; + const char *pOldPathId = pathID; + if ( pathID ) + { + Q_strncpy( pPathIdCopy, pathID, sizeof( pPathIdCopy ) ); + pOldPathId = pPathIdCopy; + } + + char tempOldPathID[MAX_PATH]; + ParsePathID( pOldPath, pOldPathId, tempOldPathID ); + Assert( pOldPathId ); + + // Allow for UNC-type syntax to specify the path ID. + char tempNewPathID[MAX_PATH]; + ParsePathID( pNewPath, pathID, tempNewPathID ); + Assert( pathID ); + + char pNewFileName[ MAX_PATH ]; + char szScratchFileName[MAX_PATH]; + + // The source file may be in a fallback directory, so just resolve the actual path, don't assume pathid... + RelativePathToFullPath( pOldPath, pOldPathId, szScratchFileName, sizeof( szScratchFileName ) ); + + // Figure out the dest path + if ( !Q_IsAbsolutePath( pNewPath ) ) + { + ComputeFullWritePath( pNewFileName, sizeof( pNewFileName ), pNewPath, pathID ); + } + else + { + Q_strncpy( pNewFileName, pNewPath, sizeof(pNewFileName) ); + } + + // Make sure the directory exitsts, too + char pPathOnly[ MAX_PATH ]; + Q_strncpy( pPathOnly, pNewFileName, sizeof( pPathOnly ) ); + Q_StripFilename( pPathOnly ); + CreateDirHierarchy( pPathOnly, pathID ); + + // Now copy the file over + int fail = rename( szScratchFileName, pNewFileName ); + if (fail != 0) + { + Warning( FILESYSTEM_WARNING, "Unable to rename %s to %s!\n", szScratchFileName, pNewFileName ); + return false; + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : **ppdir - +//----------------------------------------------------------------------------- +bool CBaseFileSystem::GetCurrentDirectory( char* pDirectory, int maxlen ) +{ +#if defined( _WIN32 ) && !defined( _X360 ) + if ( !::GetCurrentDirectoryA( maxlen, pDirectory ) ) +#elif defined( POSIX ) || defined( _X360 ) + if ( !getcwd( pDirectory, maxlen ) ) +#endif + return false; + + Q_FixSlashes(pDirectory); + + // Strip the last slash + int len = strlen(pDirectory); + if ( pDirectory[ len-1 ] == CORRECT_PATH_SEPARATOR ) + { + pDirectory[ len-1 ] = 0; + } + + return true; +} + + +//----------------------------------------------------------------------------- +// Purpose: +// Input : pfnWarning - warning function callback +//----------------------------------------------------------------------------- +void CBaseFileSystem::SetWarningFunc( void (*pfnWarning)( const char *fmt, ... ) ) +{ + m_pfnWarning = pfnWarning; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : level - +//----------------------------------------------------------------------------- +void CBaseFileSystem::SetWarningLevel( FileWarningLevel_t level ) +{ + m_fwLevel = level; +} + +const FileSystemStatistics *CBaseFileSystem::GetFilesystemStatistics() +{ + return &m_Stats; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : level - +// *fmt - +// ... - +//----------------------------------------------------------------------------- +void CBaseFileSystem::Warning( FileWarningLevel_t level, const char *fmt, ... ) +{ + if ( level > m_fwLevel ) + return; + + if ( ( fs_warning_mode.GetInt() == 1 && !ThreadInMainThread() ) || ( fs_warning_mode.GetInt() == 2 && ThreadInMainThread() ) ) + return; + + va_list argptr; + char warningtext[ 4096 ]; + + va_start( argptr, fmt ); + Q_vsnprintf( warningtext, sizeof( warningtext ), fmt, argptr ); + va_end( argptr ); + + // Dump to stdio + printf( "%s", warningtext ); + if ( m_pfnWarning ) + { + (*m_pfnWarning)( warningtext ); + } + else + { +#ifdef _WIN32 + Plat_DebugString( warningtext ); +#endif + } +} + + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +CBaseFileSystem::COpenedFile::COpenedFile( void ) +{ + m_pFile = NULL; + m_pName = NULL; +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +CBaseFileSystem::COpenedFile::~COpenedFile( void ) +{ + delete[] m_pName; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : src - +//----------------------------------------------------------------------------- +CBaseFileSystem::COpenedFile::COpenedFile( const COpenedFile& src ) +{ + m_pFile = src.m_pFile; + if ( src.m_pName ) + { + int len = strlen( src.m_pName ) + 1; + m_pName = new char[ len ]; + Q_strncpy( m_pName, src.m_pName, len ); + } + else + { + m_pName = NULL; + } +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : src - +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CBaseFileSystem::COpenedFile::operator==( const CBaseFileSystem::COpenedFile& src ) const +{ + return src.m_pFile == m_pFile; +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *name - +//----------------------------------------------------------------------------- +void CBaseFileSystem::COpenedFile::SetName( char const *name ) +{ + delete[] m_pName; + int len = strlen( name ) + 1; + m_pName = new char[ len ]; + Q_strncpy( m_pName, name, len ); +} + +//----------------------------------------------------------------------------- +// Purpose: +// Output : char +//----------------------------------------------------------------------------- +char const *CBaseFileSystem::COpenedFile::GetName( void ) +{ + return m_pName ? m_pName : "???"; +} + + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +CBaseFileSystem::CSearchPath::CSearchPath( void ) +{ + m_Path = g_PathIDTable.AddString( "" ); + m_pDebugPath = ""; + + m_storeId = 0; + m_pPackFile = NULL; + m_pPathIDInfo = NULL; + m_bIsRemotePath = false; + m_pPackedStore = NULL; + m_bIsTrustedForPureServer = false; +} + +const char *CBaseFileSystem::CSearchPath::GetDebugString() const +{ + if ( GetPackFile() ) + { + return GetPackFile()->m_ZipName; + } + #ifdef SUPPORT_PACKED_STORE + if ( GetPackedStore() ) + { + return GetPackedStore()->FullPathName(); + } + #endif + return GetPathString(); +} + +bool CBaseFileSystem::CSearchPath::IsMapPath() const +{ + return GetPackFile()->m_bIsMapPath; +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +CBaseFileSystem::CSearchPath::~CSearchPath( void ) +{ + if ( m_pPackFile ) + { + m_pPackFile->Release(); + } + if ( m_pPackedStore ) + { + m_pPackedStore->Release(); + } +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +CBaseFileSystem::CSearchPath *CBaseFileSystem::CSearchPathsIterator::GetFirst() +{ + if ( m_SearchPaths.Count() ) + { + m_visits.Reset(); + m_iCurrent = -1; + return GetNext(); + } + return &m_EmptySearchPath; +} + + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +CBaseFileSystem::CSearchPath *CBaseFileSystem::CSearchPathsIterator::GetNext() +{ + CSearchPath *pSearchPath = NULL; + + for ( m_iCurrent++; m_iCurrent < m_SearchPaths.Count(); m_iCurrent++ ) + { + pSearchPath = &m_SearchPaths[m_iCurrent]; + + if ( m_PathTypeFilter == FILTER_CULLPACK && pSearchPath->GetPackFile() ) + continue; + + if ( m_PathTypeFilter == FILTER_CULLNONPACK && !pSearchPath->GetPackFile() ) + continue; + + if ( CBaseFileSystem::FilterByPathID( pSearchPath, m_pathID ) ) + continue; + + // 360 can optionally ignore a local search path in dvddev mode + // ignoring a local search path falls through to its cloned remote path + // map paths are exempt from this exclusion logic + if ( IsX360() && ( m_DVDMode == DVDMODE_DEV ) && m_Filename[0] && !pSearchPath->m_bIsRemotePath ) + { + bool bIsMapPath = pSearchPath->GetPackFile() && pSearchPath->GetPackFile()->m_bIsMapPath; + if ( !bIsMapPath ) + { + bool bIgnorePath = false; + char szExcludePath[MAX_PATH]; + char szFilename[MAX_PATH]; + V_ComposeFileName( pSearchPath->GetPathString(), m_Filename, szFilename, sizeof( szFilename ) ); + for ( int i = 0; i < m_ExcludePaths.Count(); i++ ) + { + if ( g_pFullFileSystem->String( m_ExcludePaths[i], szExcludePath, sizeof( szExcludePath ) ) ) + { + if ( !V_strnicmp( szFilename, szExcludePath, strlen( szExcludePath ) ) ) + { + bIgnorePath = true; + break; + } + } + } + if ( bIgnorePath ) + { + // filename matches exclusion path, skip it + continue; + } + } + } + + if ( !m_visits.MarkVisit( *pSearchPath ) ) + break; + } + + if ( m_iCurrent < m_SearchPaths.Count() ) + { + return pSearchPath; + } + + return NULL; +} + +void CBaseFileSystem::CSearchPathsIterator::CopySearchPaths( const CUtlVector<CSearchPath> &searchPaths ) +{ + m_SearchPaths = searchPaths; + for ( int i = 0; i < m_SearchPaths.Count(); i++ ) + { + if ( m_SearchPaths[i].GetPackFile() ) + { + m_SearchPaths[i].GetPackFile()->AddRef(); + } + else if ( m_SearchPaths[i].GetPackedStore() ) + { + m_SearchPaths[i].GetPackedStore()->AddRef(); + } + } +} + +//----------------------------------------------------------------------------- +// Purpose: Load/unload a DLL +//----------------------------------------------------------------------------- +CSysModule *CBaseFileSystem::LoadModule( const char *pFileName, const char *pPathID, bool bValidatedDllOnly ) +{ + CHECK_DOUBLE_SLASHES( pFileName ); + + LogFileAccess( pFileName ); + if ( !pPathID ) + { + pPathID = "EXECUTABLE_PATH"; // default to the bin dir + } + + char tempPathID[ MAX_PATH ]; + ParsePathID( pFileName, pPathID, tempPathID ); + + CUtlSymbol lookup = g_PathIDTable.AddString( pPathID ); + + // a pathID has been specified, find the first match in the path list + int c = m_SearchPaths.Count(); + for ( int i = 0; i < c; i++ ) + { + // pak files don't have modules + if ( m_SearchPaths[i].GetPackFile() ) + continue; + + if ( FilterByPathID( &m_SearchPaths[i], lookup ) ) + continue; + + Q_snprintf( tempPathID, sizeof(tempPathID), "%s%s", m_SearchPaths[i].GetPathString(), pFileName ); // append the path to this dir. + CSysModule *pModule = Sys_LoadModule( tempPathID ); + if ( pModule ) + { + // we found the binary in one of our search paths + return pModule; + } + } + + // couldn't load it from any of the search paths, let LoadLibrary try + return Sys_LoadModule( pFileName ); +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void CBaseFileSystem::UnloadModule( CSysModule *pModule ) +{ + Sys_UnloadModule( pModule ); +} + +//----------------------------------------------------------------------------- +// Purpose: Adds a filesystem logging function +//----------------------------------------------------------------------------- +void CBaseFileSystem::AddLoggingFunc( FileSystemLoggingFunc_t logFunc ) +{ + Assert(!m_LogFuncs.IsValidIndex(m_LogFuncs.Find(logFunc))); + m_LogFuncs.AddToTail(logFunc); +} + +//----------------------------------------------------------------------------- +// Purpose: Removes a filesystem logging function +//----------------------------------------------------------------------------- +void CBaseFileSystem::RemoveLoggingFunc( FileSystemLoggingFunc_t logFunc ) +{ + m_LogFuncs.FindAndRemove(logFunc); +} + +//----------------------------------------------------------------------------- +// Make sure that slashes are of the right kind and that there is a slash at the +// end of the filename. +// WARNING!!: assumes that you have an extra byte allocated in the case that you need +// a slash at the end. +//----------------------------------------------------------------------------- +static void AddSeperatorAndFixPath( char *str ) +{ + char *lastChar = &str[strlen( str ) - 1]; + if( *lastChar != CORRECT_PATH_SEPARATOR && *lastChar != INCORRECT_PATH_SEPARATOR ) + { + lastChar[1] = CORRECT_PATH_SEPARATOR; + lastChar[2] = '\0'; + } + Q_FixSlashes( str ); + + if ( IsX360() ) + { + // 360 FS won't resolve any path with ../ + V_RemoveDotSlashes( str ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *pFileName - +// Output : FileNameHandle_t +//----------------------------------------------------------------------------- +FileNameHandle_t CBaseFileSystem::FindOrAddFileName( char const *pFileName ) +{ + return m_FileNames.FindOrAddFileName( pFileName ); +} + +FileNameHandle_t CBaseFileSystem::FindFileName( char const *pFileName ) +{ + return m_FileNames.FindFileName( pFileName ); +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : handle - +// Output : char const +//----------------------------------------------------------------------------- +bool CBaseFileSystem::String( const FileNameHandle_t& handle, char *buf, int buflen ) +{ + return m_FileNames.String( handle, buf, buflen ); +} + +int CBaseFileSystem::GetPathIndex( const FileNameHandle_t &handle ) +{ + return m_FileNames.PathIndex(handle); +} + +CBaseFileSystem::CPathIDInfo* CBaseFileSystem::FindOrAddPathIDInfo( const CUtlSymbol &id, int bByRequestOnly ) +{ + for ( int i=0; i < m_PathIDInfos.Count(); i++ ) + { + CBaseFileSystem::CPathIDInfo *pInfo = m_PathIDInfos[i]; + if ( pInfo->GetPathID() == id ) + { + if ( bByRequestOnly != -1 ) + { + pInfo->m_bByRequestOnly = (bByRequestOnly != 0); + } + return pInfo; + } + } + + // Add a new one. + CBaseFileSystem::CPathIDInfo *pInfo = new CBaseFileSystem::CPathIDInfo; + m_PathIDInfos.AddToTail( pInfo ); + pInfo->SetPathID( id ); + pInfo->m_bByRequestOnly = (bByRequestOnly == 1); + return pInfo; +} + + +void CBaseFileSystem::MarkPathIDByRequestOnly( const char *pPathID, bool bRequestOnly ) +{ + FindOrAddPathIDInfo( g_PathIDTable.AddString( pPathID ), bRequestOnly ); +} + +#if defined( TRACK_BLOCKING_IO ) + +void CBaseFileSystem::EnableBlockingFileAccessTracking( bool state ) +{ + m_bBlockingFileAccessReportingEnabled = state; +} + +bool CBaseFileSystem::IsBlockingFileAccessEnabled() const +{ + return m_bBlockingFileAccessReportingEnabled; +} + +IBlockingFileItemList *CBaseFileSystem::RetrieveBlockingFileAccessInfo() +{ + Assert( m_pBlockingItems ); + return m_pBlockingItems; +} + +void CBaseFileSystem::RecordBlockingFileAccess( bool synchronous, const FileBlockingItem& item ) +{ + AUTO_LOCK( m_BlockingFileMutex ); + + // Not tracking anything + if ( !m_bBlockingFileAccessReportingEnabled ) + return; + + if ( synchronous && !m_bAllowSynchronousLogging && ( item.m_ItemType == FILESYSTEM_BLOCKING_SYNCHRONOUS ) ) + return; + + // Track it + m_pBlockingItems->Add( item ); +} + +bool CBaseFileSystem::SetAllowSynchronousLogging( bool state ) +{ + bool oldState = m_bAllowSynchronousLogging; + m_bAllowSynchronousLogging = state; + return oldState; +} + +void CBaseFileSystem::BlockingFileAccess_EnterCriticalSection() +{ + m_BlockingFileMutex.Lock(); +} + +void CBaseFileSystem::BlockingFileAccess_LeaveCriticalSection() +{ + m_BlockingFileMutex.Unlock(); +} + +#endif // TRACK_BLOCKING_IO + +bool CBaseFileSystem::GetFileTypeForFullPath( char const *pFullPath, wchar_t *buf, size_t bufSizeInBytes ) +{ +#if !defined( _X360 ) && !defined( POSIX ) + wchar_t wcharpath[512]; + ::MultiByteToWideChar( CP_UTF8, 0, pFullPath, -1, wcharpath, sizeof( wcharpath ) / sizeof(wchar_t) ); + wcharpath[(sizeof( wcharpath ) / sizeof(wchar_t)) - 1] = L'\0'; + + SHFILEINFOW info = { 0 }; + DWORD_PTR dwResult = SHGetFileInfoW( + wcharpath, + 0, + &info, + sizeof( info ), + SHGFI_TYPENAME + ); + if ( dwResult ) + { + wcsncpy( buf, info.szTypeName, ( bufSizeInBytes / sizeof( wchar_t ) ) ); + buf[( bufSizeInBytes / sizeof( wchar_t ) ) - 1] = L'\0'; + return true; + } + else +#endif + { + char ext[32]; + Q_ExtractFileExtension( pFullPath, ext, sizeof( ext ) ); +#ifdef POSIX + _snwprintf( buf, ( bufSizeInBytes / sizeof( wchar_t ) ) - 1, L"%s File", V_strupr( ext ) ); // Matches what Windows does +#else + _snwprintf( buf, ( bufSizeInBytes / sizeof( wchar_t ) ) - 1, L".%S", ext ); +#endif + buf[( bufSizeInBytes / sizeof( wchar_t ) ) - 1] = L'\0'; + } + return false; +} + + +bool CBaseFileSystem::GetOptimalIOConstraints( FileHandle_t hFile, unsigned *pOffsetAlign, unsigned *pSizeAlign, unsigned *pBufferAlign ) +{ + if ( pOffsetAlign ) + *pOffsetAlign = 1; + if ( pSizeAlign ) + *pSizeAlign = 1; + if ( pBufferAlign ) + *pBufferAlign = 1; + return false; +} + +//----------------------------------------------------------------------------- + +FileCacheHandle_t CBaseFileSystem::CreateFileCache( ) +{ + return new CFileCacheObject( this ); +} + +//----------------------------------------------------------------------------- + +void CBaseFileSystem::AddFilesToFileCache( FileCacheHandle_t cacheId, const char **ppFileNames, int nFileNames, const char *pPathID ) +{ + // For now, assuming that we're only used with GAME. + Assert( pPathID && Q_strcasecmp( pPathID, "GAME" ) == 0 ); + return static_cast< CFileCacheObject * >( cacheId )->AddFiles( ppFileNames, nFileNames ); +} + +//----------------------------------------------------------------------------- + +bool CBaseFileSystem::IsFileCacheLoaded( FileCacheHandle_t cacheId ) +{ + return static_cast< CFileCacheObject * >( cacheId )->IsReady(); +} + +//----------------------------------------------------------------------------- + +void CBaseFileSystem::DestroyFileCache( FileCacheHandle_t cacheId ) +{ + delete static_cast< CFileCacheObject * >( cacheId ); +} + +//----------------------------------------------------------------------------- + +bool CBaseFileSystem::IsFileCacheFileLoaded( FileCacheHandle_t cacheId, const char* pFileName ) +{ +#ifdef _DEBUG + CFileCacheObject* pFileCache = static_cast< CFileCacheObject * >( cacheId ); + bool bFileIsHeldByCache = false; + { + AUTO_LOCK( pFileCache->m_InfosMutex ); + for ( int i = 0; i < pFileCache->m_Infos.Count(); ++i ) + { + if ( pFileCache->m_Infos[i]->pFileName && Q_strcmp( pFileCache->m_Infos[i]->pFileName, pFileName ) ) + { + bFileIsHeldByCache = true; + break; + } + } + } + Assert( bFileIsHeldByCache ); +#endif + + AUTO_LOCK( m_MemoryFileMutex ); + return m_MemoryFileHash.Find( pFileName ) != m_MemoryFileHash.InvalidHandle(); +} + +//----------------------------------------------------------------------------- + +bool CBaseFileSystem::RegisterMemoryFile( CMemoryFileBacking *pFile, CMemoryFileBacking **ppExistingFileWithRef ) +{ + Assert( pFile->m_pFS == static_cast< IFileSystem* >( this ) ); + Assert( pFile->m_pFileName && pFile->m_pFileName[0] ); + AUTO_LOCK( m_MemoryFileMutex ); + + CMemoryFileBacking *pInTable = m_MemoryFileHash[ m_MemoryFileHash.Insert( pFile->m_pFileName, pFile ) ]; + pInTable->m_nRegistered++; + pInTable->AddRef(); // either for table or for ppExistingFileWithRef return + + if ( pFile == pInTable ) + { + Assert( pInTable->m_nRegistered == 1 ); + *ppExistingFileWithRef = NULL; + return true; + } + else + { + Assert( pInTable->m_nRegistered > 1 ); + *ppExistingFileWithRef = pInTable; + return false; + } +} + +//----------------------------------------------------------------------------- + +void CBaseFileSystem::UnregisterMemoryFile( CMemoryFileBacking *pFile ) +{ + Assert( pFile->m_pFS == static_cast< IFileSystem* >( this ) && pFile->m_nRegistered > 0 ); + bool bRelease = false; + { + AUTO_LOCK( m_MemoryFileMutex ); + pFile->m_nRegistered--; + if ( pFile->m_nRegistered == 0 ) + { + m_MemoryFileHash.Remove( pFile->m_pFileName ); + bRelease = true; + } + } + // Release potentially a complex op, prefer to perform it outside of mutex. + if (bRelease) + { + pFile->Release(); + } +} + + +//----------------------------------------------------------------------------- +// Purpose: Constructs a file handle +// Input : base file system handle +// Output : +//----------------------------------------------------------------------------- +CFileHandle::CFileHandle( CBaseFileSystem* fs ) +{ + Init( fs ); +} + +CFileHandle::~CFileHandle() +{ + Assert( IsValid() ); +#if !defined( _RETAIL ) + delete[] m_pszTrueFileName; +#endif + + if ( m_pPackFileHandle ) + { + delete m_pPackFileHandle; + m_pPackFileHandle = NULL; + } + + if ( m_pFile ) + { + m_fs->Trace_FClose( m_pFile ); + m_pFile = NULL; + } + + m_nMagic = FREE_MAGIC; +} + +void CFileHandle::Init( CBaseFileSystem *fs ) +{ + m_nMagic = MAGIC; + m_pFile = NULL; + m_nLength = 0; + m_type = FT_NORMAL; + m_pPackFileHandle = NULL; + + m_fs = fs; + +#if !defined( _RETAIL ) + m_pszTrueFileName = 0; +#endif +} + +bool CFileHandle::IsValid() +{ + return ( m_nMagic == MAGIC ); +} + +int CFileHandle::GetSectorSize() +{ + Assert( IsValid() ); + + if ( m_pFile ) + { + return m_fs->FS_GetSectorSize( m_pFile ); + } + else if ( m_pPackFileHandle ) + { + return m_pPackFileHandle->GetSectorSize(); + } + else if ( m_type == FT_MEMORY_BINARY || m_type == FT_MEMORY_TEXT ) + { + return 1; + } + else + { + return -1; + } +} + +bool CFileHandle::IsOK() +{ +#if defined( SUPPORT_PACKED_STORE ) + if ( m_VPKHandle ) + { + return true; + } +#endif + if ( m_pFile ) + { + return ( IsValid() && m_fs->FS_ferror( m_pFile ) == 0 ); + } + else if ( m_pPackFileHandle || m_type == FT_MEMORY_BINARY || m_type == FT_MEMORY_TEXT ) + { + return IsValid(); + } + + m_fs->Warning( FILESYSTEM_WARNING, "FS: Tried to IsOk NULL file pointer inside valid file handle!\n" ); + return false; +} + +void CFileHandle::Flush() +{ + Assert( IsValid() ); + + if ( m_pFile ) + { + m_fs->FS_fflush( m_pFile ); + } +} + +void CFileHandle::SetBufferSize( int nBytes ) +{ + Assert( IsValid() ); + + if ( m_pFile ) + { + m_fs->FS_setbufsize( m_pFile, nBytes ); + } + else if ( m_pPackFileHandle ) + { + m_pPackFileHandle->SetBufferSize( nBytes ); + } +} + +int CFileHandle::Read( void* pBuffer, int nLength ) +{ + Assert( IsValid() ); + return Read( pBuffer, -1, nLength ); +} + +int CFileHandle::Read( void* pBuffer, int nDestSize, int nLength ) +{ + Assert( IsValid() ); + +#if defined( SUPPORT_PACKED_STORE ) + if ( m_VPKHandle ) + { + if ( nDestSize >= 0 ) + nLength = MIN( nLength, nDestSize ); + return m_VPKHandle.Read( pBuffer, nLength ); + } +#endif + // Is this a regular file or a pack file? + if ( m_pFile ) + { + return m_fs->FS_fread( pBuffer, nDestSize, nLength, m_pFile ); + } + else if ( m_pPackFileHandle ) + { + // Pack file handle handles clamping all the reads: + return m_pPackFileHandle->Read( pBuffer, nDestSize, nLength ); + } + else if ( m_type == FT_MEMORY_BINARY || m_type == FT_MEMORY_TEXT ) + { + return static_cast< CMemoryFileHandle* >( this )->Read( pBuffer, nDestSize, nLength ); + } + + return 0; +} + +int CFileHandle::Write( const void* pBuffer, int nLength ) +{ + tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s", __FUNCTION__ ); + if( ThreadInMainThread() ) + { + tmPlotI32( TELEMETRY_LEVEL0, TMPT_MEMORY, 0, nLength, "FileBytesWrite" ); + } + + Assert( IsValid() ); + + if ( !m_pFile ) + { + m_fs->Warning( FILESYSTEM_WARNING, "FS: Tried to Write NULL file pointer inside valid file handle!\n" ); + return 0; + } + + size_t nBytesWritten = m_fs->FS_fwrite( (void*)pBuffer, nLength, m_pFile ); + + m_fs->Trace_FWrite(nBytesWritten,m_pFile); + + return nBytesWritten; +} + +int CFileHandle::Seek( int64 nOffset, int nWhence ) +{ + Assert( IsValid() ); + +#if defined( SUPPORT_PACKED_STORE ) + if ( m_VPKHandle ) + { + return m_VPKHandle.Seek( nOffset, nWhence ); + } +#endif + if ( m_pFile ) + { + m_fs->FS_fseek( m_pFile, nOffset, nWhence ); + // TODO - FS_fseek should return the resultant offset + return 0; + } + else if ( m_pPackFileHandle ) + { + return m_pPackFileHandle->Seek( nOffset, nWhence ); + } + else if ( m_type == FT_MEMORY_BINARY || m_type == FT_MEMORY_TEXT ) + { + return static_cast< CMemoryFileHandle* >( this )->Seek( nOffset, nWhence ); + } + + return -1; +} + +int CFileHandle::Tell() +{ + Assert( IsValid() ); + +#if defined( SUPPORT_PACKED_STORE ) + if ( m_VPKHandle ) + { + return m_VPKHandle.Tell(); + } +#endif + if ( m_pFile ) + { + return m_fs->FS_ftell( m_pFile ); + } + else if ( m_pPackFileHandle ) + { + return m_pPackFileHandle->Tell(); + } + else if ( m_type == FT_MEMORY_BINARY || m_type == FT_MEMORY_TEXT ) + { + return static_cast< CMemoryFileHandle* >( this )->Tell(); + } + + return -1; +} + +int CFileHandle::Size() +{ + Assert( IsValid() ); + + int nReturnedSize = -1; + +#if defined( SUPPORT_PACKED_STORE ) + if ( m_VPKHandle ) + { + return m_VPKHandle.m_nFileSize; + } +#endif + if ( m_pFile ) + { + nReturnedSize = m_nLength; + } + else if ( m_pPackFileHandle ) + { + nReturnedSize = m_pPackFileHandle->Size(); + } + else if ( m_type == FT_MEMORY_BINARY || m_type == FT_MEMORY_TEXT ) + { + return static_cast< CMemoryFileHandle* >( this )->Size(); + } + + return nReturnedSize; +} + +int64 CFileHandle::AbsoluteBaseOffset() +{ + Assert( IsValid() ); + + if ( !m_pFile && m_pPackFileHandle ) + { + return m_pPackFileHandle->AbsoluteBaseOffset(); + } + else + { + return 0; + } +} + +bool CFileHandle::EndOfFile() +{ + Assert( IsValid() ); + + return ( Tell() >= Size() ); +} + +//----------------------------------------------------------------------------- + +int CMemoryFileHandle::Read( void* pBuffer, int nDestSize, int nLength ) +{ + nLength = min( nLength, (int) m_nLength - m_nPosition ); + if ( nLength > 0 ) + { + Assert( m_nPosition >= 0 ); + memcpy( pBuffer, m_pBacking->m_pData + m_nPosition, nLength ); + m_nPosition += nLength; + return nLength; + } + if ( m_nPosition >= (int) m_nLength ) + { + return -1; + } + return 0; +} + +int CMemoryFileHandle::Seek( int64 nOffset64, int nWhence ) +{ + if ( nWhence == SEEK_SET ) + { + m_nPosition = (int) clamp( nOffset64, 0ll, m_nLength ); + } + else if ( nWhence == SEEK_CUR ) + { + m_nPosition += (int) clamp( nOffset64, -(int64)m_nPosition, m_nLength - m_nPosition ); + } + else if ( nWhence == SEEK_END ) + { + m_nPosition = (int) m_nLength + (int) clamp( nOffset64, -m_nLength, 0ll ); + } + return m_nPosition; +} + +//----------------------------------------------------------------------------- + +CBaseFileSystem::CFileCacheObject::CFileCacheObject( CBaseFileSystem* pFS ) +: m_pFS( pFS ) +{ +} + +void CBaseFileSystem::CFileCacheObject::AddFiles( const char **ppFileNames, int nFileNames ) +{ + CUtlVector< Info_t* > infos; + infos.SetCount( nFileNames ); + for ( int i = 0; i < nFileNames; ++i ) + { + infos[i] = new Info_t; + Info_t &info = *infos[i]; + info.pFileName = strdup( ppFileNames[i] ); + V_FixSlashes( (char*) info.pFileName ); +#ifdef _WIN32 + Q_strlower( (char*) info.pFileName ); +#endif + info.hIOAsync = NULL; + info.pBacking = NULL; + info.pOwner = NULL; + } + + AUTO_LOCK( m_InfosMutex ); + int offset = m_Infos.AddMultipleToTail( nFileNames, infos.Base() ); + + m_nPending += nFileNames; + ProcessNewEntries(offset); +} + +void CBaseFileSystem::CFileCacheObject::ProcessNewEntries( int start ) +{ + for ( int i = start; i < m_Infos.Count(); ++i ) + { + Info_t &info = *m_Infos[i]; + if ( !info.pOwner ) + { + info.pOwner = this; + + // NOTE: currently only caching files with GAME pathID + FileAsyncRequest_t request; + request.pszFilename = info.pFileName; + request.pszPathID = "GAME"; + request.flags = FSASYNC_FLAGS_ALLOCNOFREE; + request.pfnCallback = &IOCallback; + request.pContext = &info; + if ( m_pFS->AsyncRead( request, &info.hIOAsync ) != FSASYNC_OK ) + { + --m_nPending; + } + } + } +} + +void CBaseFileSystem::CFileCacheObject::IOCallback( const FileAsyncRequest_t &request, int nBytesRead, FSAsyncStatus_t err ) +{ + Assert( request.pContext ); + Info_t &info = *(Info_t *)request.pContext; + + CMemoryFileBacking *pBacking = new CMemoryFileBacking( info.pOwner->m_pFS ); + pBacking->m_pData = NULL; + pBacking->m_pFileName = info.pFileName; + pBacking->m_nLength = ( err == FSASYNC_OK && nBytesRead == 0 ) ? 0 : -1; + + if ( request.pData ) + { + if ( err == FSASYNC_OK && nBytesRead > 0 ) + { + pBacking->m_pData = (const char*) request.pData; // transfer data ownership + pBacking->m_nLength = nBytesRead; + } + else + { + info.pOwner->m_pFS->FreeOptimalReadBuffer( request.pData ); + } + } + + CMemoryFileBacking *pExistingBacking = NULL; + if ( !info.pOwner->m_pFS->RegisterMemoryFile( pBacking, &pExistingBacking ) ) + { + // Someone already registered this file + info.pFileName = pExistingBacking->m_pFileName; + pBacking->Release(); + pBacking = pExistingBacking; + } + + //DevWarning("preload %s %d\n", request.pszFilename, pBacking->m_nLength); + + info.pBacking = pBacking; + info.pOwner->m_nPending--; +} + +CBaseFileSystem::CFileCacheObject::~CFileCacheObject() +{ + AUTO_LOCK( m_InfosMutex ); + for ( int i = 0; i < m_Infos.Count(); ++i ) + { + Info_t& info = *m_Infos[i]; + if ( info.hIOAsync ) + { + // job is guaranteed to not be running after abort + m_pFS->AsyncAbort( info.hIOAsync ); + m_pFS->AsyncRelease( info.hIOAsync ); + } + + if ( info.pBacking ) + { + m_pFS->UnregisterMemoryFile( info.pBacking ); + info.pBacking->Release(); + } + else + { + free( (char*) info.pFileName ); + } + + delete m_Infos[i]; + } + Assert( m_nPending == 0 ); +} diff --git a/filesystem/basefilesystem.h b/filesystem/basefilesystem.h new file mode 100644 index 0000000..e413db7 --- /dev/null +++ b/filesystem/basefilesystem.h @@ -0,0 +1,1001 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//===========================================================================// + +#ifndef BASEFILESYSTEM_H +#define BASEFILESYSTEM_H + +#ifdef _WIN32 +#pragma once +#endif + +#if defined( _WIN32 ) + +#if !defined( _X360 ) + #include <io.h> + #include <direct.h> + #define WIN32_LEAN_AND_MEAN + #include <windows.h> +#endif +#undef GetCurrentDirectory +#undef GetJob +#undef AddJob + +#include "tier0/threadtools.h" +#include <stdio.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <malloc.h> +#include <string.h> +#include "tier1/utldict.h" + +#elif defined(POSIX) + #include <unistd.h> // unlink + #include "linux_support.h" + #define INVALID_HANDLE_VALUE (void *)-1 + + // undo the prepended "_" 's + #define _chmod chmod + #define _stat stat + #define _alloca alloca + #define _S_IFDIR S_IFDIR +#endif + +#include <time.h> +#include "refcount.h" +#include "filesystem.h" +#include "tier1/utlvector.h" +#include <stdarg.h> +#include "tier1/utlhashtable.h" +#include "tier1/utlrbtree.h" +#include "tier1/utlsymbol.h" +#include "tier1/utllinkedlist.h" +#include "tier1/utlstring.h" +#include "tier1/UtlSortVector.h" +#include "bspfile.h" +#include "tier1/utldict.h" +#include "tier1/tier1.h" +#include "byteswap.h" +#include "threadsaferefcountedobject.h" +#include "filetracker.h" +// #include "filesystem_init.h" + +#if defined( SUPPORT_PACKED_STORE ) +#include "vpklib/packedstore.h" +#endif + +#include "tier0/memdbgon.h" + +#ifdef _WIN32 +#define CORRECT_PATH_SEPARATOR '\\' +#define INCORRECT_PATH_SEPARATOR '/' +#elif defined(POSIX) +#define CORRECT_PATH_SEPARATOR '/' +#define INCORRECT_PATH_SEPARATOR '\\' +#endif + +#ifdef _WIN32 +#define PATHSEPARATOR(c) ((c) == '\\' || (c) == '/') +#elif defined(POSIX) +#define PATHSEPARATOR(c) ((c) == '/') +#endif //_WIN32 + +#define MAX_FILEPATH 512 + +extern CUtlSymbolTableMT g_PathIDTable; + +enum FileMode_t +{ + FM_BINARY, + FM_TEXT +}; + +enum FileType_t +{ + FT_NORMAL, + FT_PACK_BINARY, + FT_PACK_TEXT, + FT_MEMORY_BINARY, + FT_MEMORY_TEXT +}; + +class IThreadPool; +class CBlockingFileItemList; +class KeyValues; +class CCompiledKeyValuesReader; +class CBaseFileSystem; +class CPackFileHandle; +class CPackFile; +class IFileList; +class CFileOpenInfo; +class CFileAsyncReadJob; + +//----------------------------------------------------------------------------- + +class CFileHandle +{ +public: + CFileHandle( CBaseFileSystem* fs ); + virtual ~CFileHandle(); + + void Init( CBaseFileSystem* fs ); + + int GetSectorSize(); + bool IsOK(); + void Flush(); + void SetBufferSize( int nBytes ); + + int Read( void* pBuffer, int nLength ); + int Read( void* pBuffer, int nDestSize, int nLength ); + + int Write( const void* pBuffer, int nLength ); + int Seek( int64 nOffset, int nWhence ); + int Tell(); + int Size(); + + int64 AbsoluteBaseOffset(); + bool EndOfFile(); + +#if !defined( _RETAIL ) + char *m_pszTrueFileName; + char const *Name() const { return m_pszTrueFileName ? m_pszTrueFileName : ""; } + + void SetName( char const *pName ) + { + Assert( pName ); + Assert( !m_pszTrueFileName ); + int len = Q_strlen( pName ); + m_pszTrueFileName = new char[len + 1]; + memcpy( m_pszTrueFileName, pName, len + 1 ); + } +#endif + + CPackFileHandle *m_pPackFileHandle; +#if defined( SUPPORT_PACKED_STORE ) + CPackedStoreFileHandle m_VPKHandle; +#endif + int64 m_nLength; + FileType_t m_type; + FILE *m_pFile; + +protected: + CBaseFileSystem *m_fs; + + enum + { + MAGIC = 0x43464861, // 'CFHa', + FREE_MAGIC = 0x4672654d // 'FreM' + }; + unsigned int m_nMagic; + + bool IsValid(); +}; + +class CMemoryFileHandle : public CFileHandle +{ +public: + CMemoryFileHandle( CBaseFileSystem* pFS, CMemoryFileBacking* pBacking ) + : CFileHandle( pFS ), m_pBacking( pBacking ), m_nPosition( 0 ) { m_nLength = pBacking->m_nLength; } + + ~CMemoryFileHandle() { m_pBacking->Release(); } + + int Read( void* pBuffer, int nDestSize, int nLength ); + int Seek( int64 nOffset, int nWhence ); + int Tell() { return m_nPosition; } + int Size() { return (int) m_nLength; } + + CMemoryFileBacking *m_pBacking; + int m_nPosition; + +private: + CMemoryFileHandle( const CMemoryFileHandle& ); // not defined + CMemoryFileHandle& operator=( const CMemoryFileHandle& ); // not defined +}; + + +//----------------------------------------------------------------------------- + +#ifdef AsyncRead +#undef AsyncRead +#undef AsyncReadMutiple +#endif + +#ifdef SUPPORT_PACKED_STORE +class CPackedStoreRefCount : public CPackedStore, public CRefCounted<CRefCountServiceMT> +{ +public: + CPackedStoreRefCount( char const *pFileBasename, char *pszFName, IBaseFileSystem *pFS ); + + bool m_bSignatureValid; +}; +#else +class CPackedStoreRefCount : public CRefCounted<CRefCountServiceMT> +{ +}; +#endif + +//----------------------------------------------------------------------------- + +abstract_class CBaseFileSystem : public CTier1AppSystem< IFileSystem > +{ + friend class CPackFileHandle; + friend class CZipPackFileHandle; + friend class CPackFile; + friend class CZipPackFile; + friend class CFileHandle; + friend class CFileTracker; + friend class CFileTracker2; + friend class CFileOpenInfo; + + typedef CTier1AppSystem< IFileSystem > BaseClass; + +public: + CBaseFileSystem(); + ~CBaseFileSystem(); + + // Methods of IAppSystem + virtual void *QueryInterface( const char *pInterfaceName ); + virtual InitReturnVal_t Init(); + virtual void Shutdown(); + + void InitAsync(); + void ShutdownAsync(); + + void ParsePathID( const char* &pFilename, const char* &pPathID, char tempPathID[MAX_PATH] ); + + // file handling + virtual FileHandle_t Open( const char *pFileName, const char *pOptions, const char *pathID ); + virtual FileHandle_t OpenEx( const char *pFileName, const char *pOptions, unsigned flags = 0, const char *pathID = 0, char **ppszResolvedFilename = NULL ); + virtual void Close( FileHandle_t ); + virtual void Seek( FileHandle_t file, int pos, FileSystemSeek_t method ); + virtual unsigned int Tell( FileHandle_t file ); + virtual unsigned int Size( FileHandle_t file ); + virtual unsigned int Size( const char *pFileName, const char *pPathID ); + + virtual void SetBufferSize( FileHandle_t file, unsigned nBytes ); + virtual bool IsOk( FileHandle_t file ); + virtual void Flush( FileHandle_t file ); + virtual bool Precache( const char *pFileName, const char *pPathID ); + virtual bool EndOfFile( FileHandle_t file ); + + virtual int Read( void *pOutput, int size, FileHandle_t file ); + virtual int ReadEx( void* pOutput, int sizeDest, int size, FileHandle_t file ); + virtual int Write( void const* pInput, int size, FileHandle_t file ); + virtual char *ReadLine( char *pOutput, int maxChars, FileHandle_t file ); + virtual int FPrintf( FileHandle_t file, PRINTF_FORMAT_STRING const char *pFormat, ... ) FMTFUNCTION( 3, 4 ); + + // Reads/writes files to utlbuffers + virtual bool ReadFile( const char *pFileName, const char *pPath, CUtlBuffer &buf, int nMaxBytes, int nStartingByte, FSAllocFunc_t pfnAlloc = NULL ); + virtual bool WriteFile( const char *pFileName, const char *pPath, CUtlBuffer &buf ); + virtual bool UnzipFile( const char *pFileName, const char *pPath, const char *pDestination ); + virtual int ReadFileEx( const char *pFileName, const char *pPath, void **ppBuf, bool bNullTerminate, bool bOptimalAlloc, int nMaxBytes = 0, int nStartingByte = 0, FSAllocFunc_t pfnAlloc = NULL ); + virtual bool ReadToBuffer( FileHandle_t hFile, CUtlBuffer &buf, int nMaxBytes = 0, FSAllocFunc_t pfnAlloc = NULL ); + + // Optimal buffer + bool GetOptimalIOConstraints( FileHandle_t hFile, unsigned *pOffsetAlign, unsigned *pSizeAlign, unsigned *pBufferAlign ); + void *AllocOptimalReadBuffer( FileHandle_t hFile, unsigned nSize, unsigned nOffset ) { return malloc( nSize ); } + void FreeOptimalReadBuffer( void *p ) { free( p ); } + + // Gets the current working directory + virtual bool GetCurrentDirectory( char* pDirectory, int maxlen ); + + // this isn't implementable on STEAM as is. + virtual void CreateDirHierarchy( const char *path, const char *pathID ); + + // returns true if the file is a directory + virtual bool IsDirectory( const char *pFileName, const char *pathID ); + + // path info + virtual const char *GetLocalPath( const char *pFileName, OUT_Z_CAP(maxLenInChars) char *pDest, int maxLenInChars ); + virtual bool FullPathToRelativePath( const char *pFullpath, OUT_Z_CAP(maxLenInChars) char *pDest, int maxLenInChars ); + virtual bool GetCaseCorrectFullPath_Ptr( const char *pFullPath, OUT_Z_CAP(maxLenInChars) char *pDest, int maxLenInChars ); + + // removes a file from disk + virtual void RemoveFile( char const* pRelativePath, const char *pathID ); + + // Remove all search paths (including write path?) + virtual void RemoveAllSearchPaths( void ); + + // Purpose: Removes all search paths for a given pathID, such as all "GAME" paths. + virtual void RemoveSearchPaths( const char *pathID ); + + // STUFF FROM IFileSystem + // Add paths in priority order (mod dir, game dir, ....) + // Can also add pak files (errr, NOT YET!) + virtual void AddSearchPath( const char *pPath, const char *pathID, SearchPathAdd_t addType ); + virtual bool RemoveSearchPath( const char *pPath, const char *pathID ); + virtual void PrintSearchPaths( void ); + + virtual void MarkPathIDByRequestOnly( const char *pPathID, bool bRequestOnly ); + + virtual bool FileExists( const char *pFileName, const char *pPathID = NULL ); + virtual long GetFileTime( const char *pFileName, const char *pPathID = NULL ); + virtual bool IsFileWritable( char const *pFileName, const char *pPathID = NULL ); + virtual bool SetFileWritable( char const *pFileName, bool writable, const char *pPathID = 0 ); + virtual void FileTimeToString( char *pString, int maxChars, long fileTime ); + + virtual const char *FindFirst( const char *pWildCard, FileFindHandle_t *pHandle ); + virtual const char *FindFirstEx( const char *pWildCard, const char *pPathID, FileFindHandle_t *pHandle ); + virtual const char *FindNext( FileFindHandle_t handle ); + virtual bool FindIsDirectory( FileFindHandle_t handle ); + virtual void FindClose( FileFindHandle_t handle ); + + virtual void PrintOpenedFiles( void ); + virtual void SetWarningFunc( void (*pfnWarning)( PRINTF_FORMAT_STRING const char *fmt, ... ) ); + virtual void SetWarningLevel( FileWarningLevel_t level ); + virtual void AddLoggingFunc( FileSystemLoggingFunc_t logFunc ); + virtual void RemoveLoggingFunc( FileSystemLoggingFunc_t logFunc ); + virtual bool RenameFile( char const *pOldPath, char const *pNewPath, const char *pathID ); + + virtual void GetLocalCopy( const char *pFileName ); + + virtual bool FixUpPath( const char *pFileName, char *pFixedUpFileName, int sizeFixedUpFileName ); + + virtual FileNameHandle_t FindOrAddFileName( char const *pFileName ); + virtual FileNameHandle_t FindFileName( char const *pFileName ); + virtual bool String( const FileNameHandle_t& handle, char *buf, int buflen ); + virtual int GetPathIndex( const FileNameHandle_t &handle ); + long GetPathTime( const char *pFileName, const char *pPathID ); + + virtual void EnableWhitelistFileTracking( bool bEnable, bool bCacheAllVPKHashes, bool bRecalculateAndCheckHashes ); + virtual void RegisterFileWhitelist( IPureServerWhitelist *pWhiteList, IFileList **ppFilesToReload ) OVERRIDE; + virtual void MarkAllCRCsUnverified(); + virtual void CacheFileCRCs( const char *pPathname, ECacheCRCType eType, IFileList *pFilter ); + //void CacheFileCRCs_R( const char *pPathname, ECacheCRCType eType, IFileList *pFilter, CUtlDict<int,int> &searchPathNames ); + virtual EFileCRCStatus CheckCachedFileHash( const char *pPathID, const char *pRelativeFilename, int nFileFraction, FileHash_t *pFileHash ); + virtual int GetUnverifiedFileHashes( CUnverifiedFileHash *pFiles, int nMaxFiles ); + virtual int GetWhitelistSpewFlags(); + virtual void SetWhitelistSpewFlags( int flags ); + virtual void InstallDirtyDiskReportFunc( FSDirtyDiskReportFunc_t func ); + + // Low-level file caching + virtual FileCacheHandle_t CreateFileCache(); + virtual void AddFilesToFileCache( FileCacheHandle_t cacheId, const char **ppFileNames, int nFileNames, const char *pPathID ); + virtual bool IsFileCacheFileLoaded( FileCacheHandle_t cacheId, const char* pFileName ); + virtual bool IsFileCacheLoaded( FileCacheHandle_t cacheId ); + virtual void DestroyFileCache( FileCacheHandle_t cacheId ); + + virtual void CacheAllVPKFileHashes( bool bCacheAllVPKHashes, bool bRecalculateAndCheckHashes ); + virtual bool CheckVPKFileHash( int PackFileID, int nPackFileNumber, int nFileFraction, MD5Value_t &md5Value ); + virtual void NotifyFileUnloaded( const char *pszFilename, const char *pPathId ) OVERRIDE; + + // Returns the file system statistics retreived by the implementation. Returns NULL if not supported. + virtual const FileSystemStatistics *GetFilesystemStatistics(); + + // Load dlls + virtual CSysModule *LoadModule( const char *pFileName, const char *pPathID, bool bValidatedDllOnly ); + virtual void UnloadModule( CSysModule *pModule ); + + //-------------------------------------------------------- + // asynchronous file loading + //-------------------------------------------------------- + virtual FSAsyncStatus_t AsyncReadMultiple( const FileAsyncRequest_t *pRequests, int nRequests, FSAsyncControl_t *pControls ); + virtual FSAsyncStatus_t AsyncReadMultipleCreditAlloc( const FileAsyncRequest_t *pRequests, int nRequests, const char *pszFile, int line, FSAsyncControl_t *phControls = NULL ); + virtual FSAsyncStatus_t AsyncFinish( FSAsyncControl_t hControl, bool wait ); + virtual FSAsyncStatus_t AsyncGetResult( FSAsyncControl_t hControl, void **ppData, int *pSize ); + virtual FSAsyncStatus_t AsyncAbort( FSAsyncControl_t hControl ); + virtual FSAsyncStatus_t AsyncStatus( FSAsyncControl_t hControl ); + virtual FSAsyncStatus_t AsyncSetPriority(FSAsyncControl_t hControl, int newPriority); + virtual FSAsyncStatus_t AsyncFlush(); + virtual FSAsyncStatus_t AsyncAppend(const char *pFileName, const void *pSrc, int nSrcBytes, bool bFreeMemory, FSAsyncControl_t *pControl) { return AsyncWrite( pFileName, pSrc, nSrcBytes, bFreeMemory, true, pControl); } + virtual FSAsyncStatus_t AsyncWrite(const char *pFileName, const void *pSrc, int nSrcBytes, bool bFreeMemory, bool bAppend, FSAsyncControl_t *pControl); + virtual FSAsyncStatus_t AsyncWriteFile(const char *pFileName, const CUtlBuffer *pSrc, int nSrcBytes, bool bFreeMemory, bool bAppend, FSAsyncControl_t *pControl); + virtual FSAsyncStatus_t AsyncAppendFile(const char *pDestFileName, const char *pSrcFileName, FSAsyncControl_t *pControl); + virtual void AsyncFinishAll( int iToPriority = INT_MIN ); + virtual void AsyncFinishAllWrites(); + virtual bool AsyncSuspend(); + virtual bool AsyncResume(); + + virtual void AsyncAddRef( FSAsyncControl_t hControl ); + virtual void AsyncRelease( FSAsyncControl_t hControl ); + virtual FSAsyncStatus_t AsyncBeginRead( const char *pszFile, FSAsyncFile_t *phFile ); + virtual FSAsyncStatus_t AsyncEndRead( FSAsyncFile_t hFile ); + virtual void AsyncAddFetcher( IAsyncFileFetch *pFetcher ); + virtual void AsyncRemoveFetcher( IAsyncFileFetch *pFetcher ); + + //-------------------------------------------------------- + // pack files + //-------------------------------------------------------- + bool AddPackFile( const char *pFileName, const char *pathID ); + bool AddPackFileFromPath( const char *pPath, const char *pakfile, bool bCheckForAppendedPack, const char *pathID ); + + // converts a partial path into a full path + // can be filtered to restrict path types and can provide info about resolved path + virtual const char *RelativePathToFullPath( const char *pFileName, const char *pPathID, OUT_Z_CAP(maxLenInChars) char *pDest, int maxLenInChars, PathTypeFilter_t pathFilter = FILTER_NONE, PathTypeQuery_t *pPathType = NULL ); + + // Returns the search path, each path is separated by ;s. Returns the length of the string returned + virtual int GetSearchPath( const char *pathID, bool bGetPackFiles, OUT_Z_CAP(maxLenInChars) char *pDest, int maxLenInChars ); + +#if defined( TRACK_BLOCKING_IO ) + virtual void EnableBlockingFileAccessTracking( bool state ); + virtual bool IsBlockingFileAccessEnabled() const; + virtual IBlockingFileItemList *RetrieveBlockingFileAccessInfo(); + + virtual void RecordBlockingFileAccess( bool synchronous, const FileBlockingItem& item ); + + virtual bool SetAllowSynchronousLogging( bool state ); +#endif + + virtual bool GetFileTypeForFullPath( char const *pFullPath, wchar_t *buf, size_t bufSizeInBytes ); + + virtual void BeginMapAccess(); + virtual void EndMapAccess(); + virtual bool FullPathToRelativePathEx( const char *pFullpath, const char *pPathId, OUT_Z_CAP(maxLenInChars) char *pDest, int maxLenInChars ); + + FSAsyncStatus_t SyncRead( const FileAsyncRequest_t &request ); + FSAsyncStatus_t SyncWrite(const char *pszFilename, const void *pSrc, int nSrcBytes, bool bFreeMemory, bool bAppend ); + FSAsyncStatus_t SyncAppendFile(const char *pAppendToFileName, const char *pAppendFromFileName ); + FSAsyncStatus_t SyncGetFileSize( const FileAsyncRequest_t &request ); + void DoAsyncCallback( const FileAsyncRequest_t &request, void *pData, int nBytesRead, FSAsyncStatus_t result ); + + void SetupPreloadData(); + void DiscardPreloadData(); + + virtual void LoadCompiledKeyValues( KeyValuesPreloadType_t type, char const *archiveFile ); + + // If the "PreloadedData" hasn't been purged, then this'll try and instance the KeyValues using the fast path of compiled keyvalues loaded during startup. + // Otherwise, it'll just fall through to the regular KeyValues loading routines + virtual KeyValues *LoadKeyValues( KeyValuesPreloadType_t type, char const *filename, char const *pPathID = 0 ); + virtual bool LoadKeyValues( KeyValues& head, KeyValuesPreloadType_t type, char const *filename, char const *pPathID = 0 ); + virtual bool ExtractRootKeyName( KeyValuesPreloadType_t type, char *outbuf, size_t bufsize, char const *filename, char const *pPathID = 0 ); + + virtual DVDMode_t GetDVDMode() { return m_DVDMode; } + + FSDirtyDiskReportFunc_t GetDirtyDiskReportFunc() { return m_DirtyDiskReportFunc; } + + //----------------------------------------------------------------------------- + // MemoryFile cache implementation + //----------------------------------------------------------------------------- + class CFileCacheObject; + + // XXX For now, we assume that all path IDs are "GAME", never cache files + // outside of the game search path, and preferentially return those files + // whenever anyone searches for a match even if an on-disk file in another + // folder would have been found first in a traditional search. extending + // the memory cache to cover non-game files isn't necessary right now, but + // should just be a matter of defining a more complex key type. (henryg) + + // Register a CMemoryFileBacking; must balance with UnregisterMemoryFile. + // Returns false and outputs an ref-bumped pointer to the existing entry + // if the same file has already been registered by someone else; this must + // be Unregistered to maintain the balance. + virtual bool RegisterMemoryFile( CMemoryFileBacking *pFile, CMemoryFileBacking **ppExistingFileWithRef ); + + // Unregister a CMemoryFileBacking; must balance with RegisterMemoryFile. + virtual void UnregisterMemoryFile( CMemoryFileBacking *pFile ); + + //------------------------------------ + // Synchronous path for file operations + //------------------------------------ + class CPathIDInfo + { + public: + const CUtlSymbol& GetPathID() const; + const char* GetPathIDString() const; + void SetPathID( CUtlSymbol id ); + + public: + // See MarkPathIDByRequestOnly. + bool m_bByRequestOnly; + + private: + CUtlSymbol m_PathID; + const char *m_pDebugPathID; + }; + + //////////////////////////////////////////////// + // IMPLEMENTATION DETAILS FOR CBaseFileSystem // + //////////////////////////////////////////////// + + class CSearchPath + { + public: + CSearchPath( void ); + ~CSearchPath( void ); + + const char* GetPathString() const; + const char* GetDebugString() const; + + // Path ID ("game", "mod", "gamebin") accessors. + const CUtlSymbol& GetPathID() const; + const char* GetPathIDString() const; + + // Search path (c:\hl2\hl2) accessors. + void SetPath( CUtlSymbol id ); + const CUtlSymbol& GetPath() const; + + void SetPackFile(CPackFile *pPackFile) { m_pPackFile = pPackFile; } + CPackFile *GetPackFile() const { return m_pPackFile; } + + #ifdef SUPPORT_PACKED_STORE + void SetPackedStore( CPackedStoreRefCount *pPackedStore ) { m_pPackedStore = pPackedStore; } + #endif + CPackedStoreRefCount *GetPackedStore() const { return m_pPackedStore; } + + bool IsMapPath() const; + + int m_storeId; + + // Used to track if its search + CPathIDInfo *m_pPathIDInfo; + + bool m_bIsRemotePath; + + bool m_bIsTrustedForPureServer; + + private: + CUtlSymbol m_Path; + const char *m_pDebugPath; + CPackFile *m_pPackFile; + CPackedStoreRefCount *m_pPackedStore; + }; + + class CSearchPathsVisits + { + public: + void Reset() + { + m_Visits.RemoveAll(); + } + + bool MarkVisit( const CSearchPath &searchPath ) + { + if ( m_Visits.Find( searchPath.m_storeId ) == m_Visits.InvalidIndex() ) + { + MEM_ALLOC_CREDIT(); + m_Visits.AddToTail( searchPath.m_storeId ); + return false; + } + return true; + } + + private: + CUtlVector<int> m_Visits; // This is a copy of IDs for the search paths we've visited, so + }; + + class CSearchPathsIterator + { + public: + CSearchPathsIterator( CBaseFileSystem *pFileSystem, const char **ppszFilename, const char *pszPathID, PathTypeFilter_t pathTypeFilter = FILTER_NONE ) + : m_iCurrent( -1 ), + m_PathTypeFilter( pathTypeFilter ) + { + char tempPathID[MAX_PATH]; + if ( *ppszFilename && (*ppszFilename)[0] == '/' && (*ppszFilename)[1] == '/' ) // ONLY '//' (and not '\\') for our special format + { + // Allow for UNC-type syntax to specify the path ID. + pFileSystem->ParsePathID( *ppszFilename, pszPathID, tempPathID ); + } + if ( pszPathID ) + { + m_pathID = g_PathIDTable.AddString( pszPathID ); + } + else + { + m_pathID = UTL_INVAL_SYMBOL; + } + + if ( *ppszFilename && !Q_IsAbsolutePath( *ppszFilename ) ) + { + // Copy paths to minimize mutex lock time + pFileSystem->m_SearchPathsMutex.Lock(); + CopySearchPaths( pFileSystem->m_SearchPaths ); + pFileSystem->m_SearchPathsMutex.Unlock(); + + pFileSystem->FixUpPath ( *ppszFilename, m_Filename, sizeof( m_Filename ) ); + } + else + { + // If it's an absolute path, it isn't worth using the paths at all. Simplify + // client logic by pretending there's a search path of 1 + m_EmptyPathIDInfo.m_bByRequestOnly = false; + m_EmptySearchPath.m_pPathIDInfo = &m_EmptyPathIDInfo; + m_EmptySearchPath.SetPath( m_pathID ); + m_EmptySearchPath.m_storeId = -1; + m_Filename[0] = '\0'; + } + } + + CSearchPathsIterator( CBaseFileSystem *pFileSystem, const char *pszPathID, PathTypeFilter_t pathTypeFilter = FILTER_NONE ) + : m_iCurrent( -1 ), + m_PathTypeFilter( pathTypeFilter ) + { + if ( pszPathID ) + { + m_pathID = g_PathIDTable.AddString( pszPathID ); + } + else + { + m_pathID = UTL_INVAL_SYMBOL; + } + // Copy paths to minimize mutex lock time + pFileSystem->m_SearchPathsMutex.Lock(); + CopySearchPaths( pFileSystem->m_SearchPaths ); + pFileSystem->m_SearchPathsMutex.Unlock(); + m_Filename[0] = '\0'; + } + + CSearchPath *GetFirst(); + CSearchPath *GetNext(); + + private: + CSearchPathsIterator( const CSearchPathsIterator & ); + void operator=(const CSearchPathsIterator &); + void CopySearchPaths( const CUtlVector<CSearchPath> &searchPaths ); + + int m_iCurrent; + CUtlSymbol m_pathID; + CUtlVector<CSearchPath> m_SearchPaths; + CSearchPathsVisits m_visits; + CSearchPath m_EmptySearchPath; + CPathIDInfo m_EmptyPathIDInfo; + PathTypeFilter_t m_PathTypeFilter; + char m_Filename[MAX_PATH]; // set for relative names only + }; + + friend class CSearchPathsIterator; + + struct FindData_t + { + WIN32_FIND_DATA findData; + int currentSearchPathID; + CUtlVector<char> wildCardString; + HANDLE findHandle; + CSearchPathsVisits m_VisitedSearchPaths; // This is a copy of IDs for the search paths we've visited, so avoids searching duplicate paths. + int m_CurrentStoreID; // CSearchPath::m_storeId of the current search path. + + CUtlSymbol m_FilterPathID; // What path ID are we looking at? Ignore all others. (Only set by FindFirstEx). + + CUtlDict<int,int> m_VisitedFiles; // We go through the search paths in priority order, and we use this to make sure + // that we don't return the same file more than once. + CUtlStringList m_fileMatchesFromVPKOrPak; + CUtlStringList m_dirMatchesFromVPKOrPak; + }; + + friend class CSearchPath; + + IPureServerWhitelist *m_pPureServerWhitelist; + int m_WhitelistSpewFlags; // Combination of WHITELIST_SPEW_ flags. + + // logging functions + CUtlVector< FileSystemLoggingFunc_t > m_LogFuncs; + + CThreadMutex m_SearchPathsMutex; + CUtlVector< CSearchPath > m_SearchPaths; + CUtlVector<CPathIDInfo*> m_PathIDInfos; + CUtlLinkedList<FindData_t> m_FindData; + + CSearchPath *FindSearchPathByStoreId( int storeId ); + + int m_iMapLoad; + + // Global list of pack file handles + CUtlVector<CPackFile *> m_ZipFiles; + + FILE *m_pLogFile; + bool m_bOutputDebugString; + + IThreadPool * m_pThreadPool; + CThreadFastMutex m_AsyncCallbackMutex; + + // Statistics: + FileSystemStatistics m_Stats; + +#if defined( TRACK_BLOCKING_IO ) + CBlockingFileItemList *m_pBlockingItems; + bool m_bBlockingFileAccessReportingEnabled; + bool m_bAllowSynchronousLogging; + + friend class CBlockingFileItemList; + friend class CAutoBlockReporter; +#endif + + CFileTracker2 m_FileTracker2; + +protected: + //---------------------------------------------------------------------------- + // Purpose: Functions implementing basic file system behavior. + //---------------------------------------------------------------------------- + virtual FILE *FS_fopen( const char *filename, const char *options, unsigned flags, int64 *size ) = 0; + virtual void FS_setbufsize( FILE *fp, unsigned nBytes ) = 0; + virtual void FS_fclose( FILE *fp ) = 0; + virtual void FS_fseek( FILE *fp, int64 pos, int seekType ) = 0; + virtual long FS_ftell( FILE *fp ) = 0; + virtual int FS_feof( FILE *fp ) = 0; + size_t FS_fread( void *dest, size_t size, FILE *fp ) { return FS_fread( dest, (size_t)-1, size, fp ); } + virtual size_t FS_fread( void *dest, size_t destSize, size_t size, FILE *fp ) = 0; + virtual size_t FS_fwrite( const void *src, size_t size, FILE *fp ) = 0; + virtual bool FS_setmode( FILE *fp, FileMode_t mode ) { return false; } + virtual size_t FS_vfprintf( FILE *fp, const char *fmt, va_list list ) = 0; + virtual int FS_ferror( FILE *fp ) = 0; + virtual int FS_fflush( FILE *fp ) = 0; + virtual char *FS_fgets( char *dest, int destSize, FILE *fp ) = 0; + virtual int FS_stat( const char *path, struct _stat *buf, bool *pbLoadedFromSteamCache=NULL ) = 0; + virtual int FS_chmod( const char *path, int pmode ) = 0; + virtual HANDLE FS_FindFirstFile( const char *findname, WIN32_FIND_DATA *dat) = 0; + virtual bool FS_FindNextFile(HANDLE handle, WIN32_FIND_DATA *dat) = 0; + virtual bool FS_FindClose(HANDLE handle) = 0; + virtual int FS_GetSectorSize( FILE * ) { return 1; } + +#if defined( TRACK_BLOCKING_IO ) + void BlockingFileAccess_EnterCriticalSection(); + void BlockingFileAccess_LeaveCriticalSection(); + + CThreadMutex m_BlockingFileMutex; + +#endif + + void GetFileNameForHandle( FileHandle_t handle, char *buf, size_t buflen ); + +protected: + //----------------------------------------------------------------------------- + // Purpose: For tracking unclosed files + // NOTE: The symbol table could take up memory that we don't want to eat here. + // In that case, we shouldn't store them in a table, or we should store them as locally allocates stings + // so we can control the size + //----------------------------------------------------------------------------- + class COpenedFile + { + public: + COpenedFile( void ); + ~COpenedFile( void ); + + COpenedFile( const COpenedFile& src ); + + bool operator==( const COpenedFile& src ) const; + + void SetName( char const *name ); + char const *GetName( void ); + + FILE *m_pFile; + char *m_pName; + }; + + CThreadFastMutex m_MemoryFileMutex; + CUtlHashtable< const char*, CMemoryFileBacking* > m_MemoryFileHash; + + + //CUtlRBTree< COpenedFile, int > m_OpenedFiles; + CThreadMutex m_OpenedFilesMutex; + CUtlVector <COpenedFile> m_OpenedFiles; + + static bool OpenedFileLessFunc( COpenedFile const& src1, COpenedFile const& src2 ); + + FileWarningLevel_t m_fwLevel; + void (*m_pfnWarning)( PRINTF_FORMAT_STRING const char *fmt, ... ); + + FILE *Trace_FOpen( const char *filename, const char *options, unsigned flags, int64 *size ); + void Trace_FClose( FILE *fp ); + void Trace_FRead( int size, FILE* file ); + void Trace_FWrite( int size, FILE* file ); + + void Trace_DumpUnclosedFiles( void ); + +public: + void LogAccessToFile( char const *accesstype, char const *fullpath, char const *options ); + void Warning( FileWarningLevel_t level, PRINTF_FORMAT_STRING const char *fmt, ... ); + +protected: + // Note: if pFoundStoreID is passed in, then it will set that to the CSearchPath::m_storeId value of the search path it found the file in. + const char* FindFirstHelper( const char *pWildCard, const char *pPathID, FileFindHandle_t *pHandle, int *pFoundStoreID ); + bool FindNextFileHelper( FindData_t *pFindData, int *pFoundStoreID ); + bool FindNextFileInVPKOrPakHelper( FindData_t *pFindData ); + + void RemoveAllMapSearchPaths( void ); + void AddMapPackFile( const char *pPath, const char *pPathID, SearchPathAdd_t addType ); + void AddPackFiles( const char *pPath, const CUtlSymbol &pathID, SearchPathAdd_t addType ); + bool PreparePackFile( CPackFile &packfile, int offsetofpackinmetafile, int64 filelen ); + void AddVPKFile( const char *pPath, const char *pPathID, SearchPathAdd_t addType ); + bool RemoveVPKFile( const char *pPath, const char *pPathID ); + + void HandleOpenRegularFile( CFileOpenInfo &openInfo, bool bIsAbsolutePath ); + + FileHandle_t FindFileInSearchPath( CFileOpenInfo &openInfo ); + long FastFileTime( const CSearchPath *path, const char *pFileName ); + + const char *GetWritePath( const char *pFilename, const char *pathID ); + + // Computes a full write path + void ComputeFullWritePath( char* pDest, int maxlen, const char *pWritePathID, char const *pRelativePath ); + + void AddSearchPathInternal( const char *pPath, const char *pathID, SearchPathAdd_t addType, bool bAddPackFiles ); + + // Opens a file for read or write + FileHandle_t OpenForRead( const char *pFileName, const char *pOptions, unsigned flags, const char *pathID, char **ppszResolvedFilename = NULL ); + FileHandle_t OpenForWrite( const char *pFileName, const char *pOptions, const char *pathID ); + CSearchPath *FindWritePath( const char *pFilename, const char *pathID ); + + // Helper function for fs_log file logging + void LogFileAccess( const char *pFullFileName ); + bool LookupKeyValuesRootKeyName( char const *filename, char const *pPathID, char *rootName, size_t bufsize ); + void UnloadCompiledKeyValues(); + + // If bByRequestOnly is -1, then it will default to false if it doesn't already exist, and it + // won't change it if it does already exist. Otherwise, it will be set to the value of bByRequestOnly. + CPathIDInfo* FindOrAddPathIDInfo( const CUtlSymbol &id, int bByRequestOnly ); + static bool FilterByPathID( const CSearchPath *pSearchPath, const CUtlSymbol &pathID ); + + // Global/shared filename/path table + CUtlFilenameSymbolTable m_FileNames; + + int m_WhitelistFileTrackingEnabled; // -1 if unset, 0 if disabled (single player), 1 if enabled (multiplayer). + FSDirtyDiskReportFunc_t m_DirtyDiskReportFunc; + + void SetSearchPathIsTrustedSource( CSearchPath *pPath ); + + struct CompiledKeyValuesPreloaders_t + { + CompiledKeyValuesPreloaders_t() : + m_CacheFile( 0 ), + m_pReader( 0 ) + { + } + FileNameHandle_t m_CacheFile; + CCompiledKeyValuesReader *m_pReader; + }; + + CompiledKeyValuesPreloaders_t m_PreloadData[ NUM_PRELOAD_TYPES ]; + + static CUtlSymbol m_GamePathID; + static CUtlSymbol m_BSPPathID; + + static DVDMode_t m_DVDMode; + + // Pack exclude paths are strictly for 360 to allow holes in search paths and pack files + // which fall through to support new or dynamic data on the host pc. + static CUtlVector< FileNameHandle_t > m_ExcludePaths; + + /// List of installed hooks to intercept async file operations + CUtlVector< IAsyncFileFetch * > m_vecAsyncFetchers; + + /// List of active async jobs being serviced by customer fetchers + CUtlVector< CFileAsyncReadJob * > m_vecAsyncCustomFetchJobs; + + /// Remove a custom fetch job from the list (and release our reference) + friend class CFileAsyncReadJob; + void RemoveAsyncCustomFetchJob( CFileAsyncReadJob *pJob ); +}; + +inline const CUtlSymbol& CBaseFileSystem::CPathIDInfo::GetPathID() const +{ + return m_PathID; +} + + +inline const char* CBaseFileSystem::CPathIDInfo::GetPathIDString() const +{ + return g_PathIDTable.String( m_PathID ); +} + + +inline const char* CBaseFileSystem::CSearchPath::GetPathString() const +{ + return g_PathIDTable.String( m_Path ); +} + + +inline void CBaseFileSystem::CPathIDInfo::SetPathID( CUtlSymbol sym ) +{ + m_PathID = sym; + m_pDebugPathID = GetPathIDString(); +} + + +inline const CUtlSymbol& CBaseFileSystem::CSearchPath::GetPathID() const +{ + return m_pPathIDInfo->GetPathID(); +} + + +inline const char* CBaseFileSystem::CSearchPath::GetPathIDString() const +{ + return m_pPathIDInfo->GetPathIDString(); +} + + +inline void CBaseFileSystem::CSearchPath::SetPath( CUtlSymbol id ) +{ + m_Path = id; + m_pDebugPath = g_PathIDTable.String( m_Path ); +} + + +inline const CUtlSymbol& CBaseFileSystem::CSearchPath::GetPath() const +{ + return m_Path; +} + + +inline bool CBaseFileSystem::FilterByPathID( const CSearchPath *pSearchPath, const CUtlSymbol &pathID ) +{ + if ( (UtlSymId_t)pathID == UTL_INVAL_SYMBOL ) + { + // They didn't specify a specific search path, so if this search path's path ID is by + // request only, then ignore it. + return pSearchPath->m_pPathIDInfo->m_bByRequestOnly; + } + else + { + // Bit of a hack, but specifying "BSP" as the search path will search in "GAME" for only the map/.bsp pack file path + if ( pathID == m_BSPPathID ) + { + if ( pSearchPath->GetPathID() != m_GamePathID ) + return true; + + if ( !pSearchPath->GetPackFile() ) + return true; + + if ( !pSearchPath->IsMapPath() ) + return true; + + return false; + } + else + { + return (pSearchPath->GetPathID() != pathID); + } + } +} + +#if defined( TRACK_BLOCKING_IO ) + +class CAutoBlockReporter +{ +public: + + CAutoBlockReporter( CBaseFileSystem *fs, bool synchronous, char const *filename, int eBlockType, int nTypeOfAccess ) : + m_pFS( fs ), + m_Item( eBlockType, filename, 0.0f, nTypeOfAccess ), + m_bSynchronous( synchronous ) + { + Assert( m_pFS ); + m_Timer.Start(); + } + + CAutoBlockReporter( CBaseFileSystem *fs, bool synchronous, FileHandle_t handle, int eBlockType, int nTypeOfAccess ) : + m_pFS( fs ), + m_Item( eBlockType, NULL, 0.0f, nTypeOfAccess ), + m_bSynchronous( synchronous ) + { + Assert( m_pFS ); + char name[ 512 ]; + m_pFS->GetFileNameForHandle( handle, name, sizeof( name ) ); + m_Item.SetFileName( name ); + m_Timer.Start(); + } + + ~CAutoBlockReporter() + { + m_Timer.End(); + m_Item.m_flElapsed = m_Timer.GetDuration().GetSeconds(); + m_pFS->RecordBlockingFileAccess( m_bSynchronous, m_Item ); + } + +private: + + CBaseFileSystem *m_pFS; + + CFastTimer m_Timer; + FileBlockingItem m_Item; + bool m_bSynchronous; +}; + +#define AUTOBLOCKREPORTER_FN( name, fs, sync, filename, blockType, accessType ) CAutoBlockReporter block##name( fs, sync, filename, blockType, accessType ); +#define AUTOBLOCKREPORTER_FH( name, fs, sync, handle, blockType, accessType ) CAutoBlockReporter block##name( fs, sync, handle, blockType, accessType ); + +#else + +#define AUTOBLOCKREPORTER_FN( name, fs, sync, filename, blockType, accessType ) // Nothing +#define AUTOBLOCKREPORTER_FH( name, fs, sync, handle , blockType, accessType ) // Nothing + +#endif + +// singleton accessor +CBaseFileSystem *BaseFileSystem(); + +#include "tier0/memdbgoff.h" + +#endif // BASEFILESYSTEM_H diff --git a/filesystem/filesystem_async.cpp b/filesystem/filesystem_async.cpp new file mode 100644 index 0000000..4196e09 --- /dev/null +++ b/filesystem/filesystem_async.cpp @@ -0,0 +1,1538 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: CBaseFileSystem Async Operation +// +// The CBaseFileSystem methods implement the IFileSystem +// asynchronous entry points. The model for reads currently is a +// callback model where the callback can take place either in the +// context of the main thread or the worker thread. It would be +// easy to do a polled model later. Async operations return a +// handle which is used to refer to the operation later. The +// handle is actually a pointer to a reference counted "job" +// object that holds all the context, status, and results of an +// operation. +// +//============================================================================= + +#include <limits.h> +#if defined( _WIN32 ) && !defined( _X360 ) +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#endif +#include "tier0/vcrmode.h" +#include "tier1/convar.h" +#include "vstdlib/jobthread.h" +#include "tier1/utlmap.h" +#include "tier1/utlbuffer.h" +#include "tier0/icommandline.h" +#include "vstdlib/random.h" +#include "basefilesystem.h" + +// VCR mode for now is handled by not running async. This is primarily for +// performance reasons. VCR mode would preclude the use of a lock-free job +// retrieval. Can change if need in future, but it's best to do so if needed, +// and to make it a deliberate compile time choice to keep the fast path. +#undef WaitForSingleObject + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +ConVar async_mode( "async_mode", "0", 0, "Set the async filesystem mode (0 = async, 1 = synchronous)" ); +#define GetAsyncMode() ( (FSAsyncMode_t)( async_mode.GetInt() ) ) + +#ifndef DISABLE_ASYNC + +#ifndef _RETAIL + +ConVar async_simulate_delay( "async_simulate_delay", "0", 0, "Simulate a delay of up to a set msec per file operation" ); +ConVar async_allow_held_files( "async_allow_held_files", "1", 0, "Allow AsyncBegin/EndRead()" ); + +#define SimulateDelay() if ( async_simulate_delay.GetInt() == 0 || ThreadInMainThread() ) ; else ThreadSleep( RandomInt( 1, async_simulate_delay.GetInt() ) ) +#define AsyncAllowHeldFiles() async_allow_held_files.GetBool() + +CON_COMMAND( async_suspend, "" ) +{ + BaseFileSystem()->AsyncSuspend(); +} + +CON_COMMAND( async_resume, "" ) +{ + BaseFileSystem()->AsyncResume(); +} + +#else + +#define SimulateDelay() ((void)0) +#define AsyncAllowHeldFiles() true + +#endif + +#else + +#define SimulateDelay() ((void)0) +#define GetAsyncMode() FSAM_SYNC +#define AsyncAllowHeldFiles() false + +#endif + +//----------------------------------------------------------------------------- +// Need to support old external. New implementation has less granular priority for efficiency +//----------------------------------------------------------------------------- + +inline JobPriority_t ConvertPriority( int iFilesystemPriority ) +{ + if ( iFilesystemPriority == 0 ) + { + return JP_NORMAL; + } + else if ( iFilesystemPriority > 0 ) + { + return JP_HIGH; + } + return JP_LOW; +} + +//----------------------------------------------------------------------------- +// +// Support for holding files open +// +//----------------------------------------------------------------------------- + +struct AsyncOpenedFile_t : CRefCounted<CRefCountServiceNoDelete> // no mutex needed, under control of CAsyncOpenedFiles +{ + AsyncOpenedFile_t() : hFile(FILESYSTEM_INVALID_HANDLE) {} + FileHandle_t hFile; +}; + +class CAsyncOpenedFiles +{ +public: + CAsyncOpenedFiles() + { + m_map.SetLessFunc( CaselessStringLessThan ); + } + + FSAsyncFile_t FindOrAdd( const char *pszFilename ) + { + char szFixedName[MAX_FILEPATH]; + Q_strncpy( szFixedName, pszFilename, sizeof( szFixedName ) ); + Q_FixSlashes( szFixedName ); + + Assert( (int)FS_INVALID_ASYNC_FILE == m_map.InvalidIndex() ); + + AUTO_LOCK( m_mutex ); + + int iEntry = m_map.Find( szFixedName ); + if ( iEntry == m_map.InvalidIndex() ) + { + iEntry = m_map.Insert( strdup( szFixedName ), new AsyncOpenedFile_t ); + } + else + { + m_map[iEntry]->AddRef(); + } + return (FSAsyncFile_t)iEntry; + } + + FSAsyncFile_t Find( const char *pszFilename ) + { + char szFixedName[MAX_FILEPATH]; + Q_strncpy( szFixedName, pszFilename, sizeof( szFixedName ) ); + Q_FixSlashes( szFixedName ); + + AUTO_LOCK( m_mutex ); + + int iEntry = m_map.Find( szFixedName ); + if ( iEntry != m_map.InvalidIndex() ) + { + m_map[iEntry]->AddRef(); + } + + return (FSAsyncFile_t)iEntry; + } + + AsyncOpenedFile_t *Get( FSAsyncFile_t item ) + { + if ( item == FS_INVALID_ASYNC_FILE) + { + return NULL; + } + + AUTO_LOCK( m_mutex ); + + int iEntry = (CUtlMap<CUtlString, AsyncOpenedFile_t>::IndexType_t)(int)item; + Assert( m_map.IsValidIndex( iEntry ) ); + m_map[iEntry]->AddRef(); + return m_map[iEntry]; + } + + void AddRef( FSAsyncFile_t item ) + { + if ( item == FS_INVALID_ASYNC_FILE) + { + return; + } + + AUTO_LOCK( m_mutex ); + + int iEntry = (CUtlMap<CUtlString, AsyncOpenedFile_t>::IndexType_t)(int)item; + Assert( m_map.IsValidIndex( iEntry ) ); + m_map[iEntry]->AddRef(); + } + + void Release( FSAsyncFile_t item ) + { + if ( item == FS_INVALID_ASYNC_FILE) + { + return; + } + + AUTO_LOCK( m_mutex ); + + int iEntry = (CUtlMap<CUtlString, AsyncOpenedFile_t>::IndexType_t)(int)item; + Assert( m_map.IsValidIndex( iEntry ) ); + if ( m_map[iEntry]->Release() == 0 ) + { + if ( m_map[iEntry]->hFile != FILESYSTEM_INVALID_HANDLE ) + { + BaseFileSystem()->Close( m_map[iEntry]->hFile ); + } + delete m_map[iEntry]; + delete m_map.Key( iEntry ); + m_map.RemoveAt( iEntry ); + } + } + +private: + CThreadFastMutex m_mutex; + + CUtlMap<const char *, AsyncOpenedFile_t *> m_map; +}; + +CAsyncOpenedFiles g_AsyncOpenedFiles; + + +//----------------------------------------------------------------------------- +// Async Modes +//----------------------------------------------------------------------------- +enum FSAsyncMode_t +{ + FSAM_ASYNC, + FSAM_SYNC, +}; + +#define FSASYNC_WRITE_PRIORITY JP_LOW + +//--------------------------------------------------------- + +// Cast to int in order to indicate that we are intentionally comparing different +// enum types, to suppress gcc warnings. +ASSERT_INVARIANT( FSASYNC_OK == (int)JOB_OK ); +ASSERT_INVARIANT( FSASYNC_STATUS_PENDING == (int)JOB_STATUS_PENDING ); +ASSERT_INVARIANT( FSASYNC_STATUS_INPROGRESS == (int)JOB_STATUS_INPROGRESS ); +ASSERT_INVARIANT( FSASYNC_STATUS_ABORTED == (int)JOB_STATUS_ABORTED ); +ASSERT_INVARIANT( FSASYNC_STATUS_UNSERVICED == (int)JOB_STATUS_UNSERVICED ); + +//--------------------------------------------------------- +// A standard filesystem job +//--------------------------------------------------------- +class CFileAsyncJob : public CJob +{ +public: + CFileAsyncJob( JobPriority_t priority = JP_NORMAL ) + : CJob( priority ) + { + SetFlags( GetFlags() | JF_IO ); + } + + virtual JobStatus_t GetResult( void **ppData, int *pSize ) { *ppData = NULL; *pSize = 0; return GetStatus(); } + virtual bool IsWrite() const { return false; } + CFileAsyncReadJob *AsReadJob() { return NULL; } +}; + +//--------------------------------------------------------- +// A standard filesystem read job +//--------------------------------------------------------- +class CFileAsyncReadJob : public CFileAsyncJob, + protected FileAsyncRequest_t +{ +public: + CFileAsyncReadJob( const FileAsyncRequest_t &fromRequest, CBaseFileSystem *pOwnerFileSystem ) + : CFileAsyncJob( ConvertPriority( fromRequest.priority ) ), + FileAsyncRequest_t( fromRequest ), + m_pResultData( NULL ), + m_nResultSize( 0 ), + m_pRealContext( fromRequest.pContext ), + m_pfnRealCallback( fromRequest.pfnCallback ), + m_pCustomFetcher(NULL), + m_hCustomFetcherHandle(NULL), + m_pOwnerFileSystem(pOwnerFileSystem) + { +#if defined( TRACK_BLOCKING_IO ) + m_Timer.Start(); +#endif + pszFilename = strdup( fromRequest.pszFilename ); + Q_FixSlashes( const_cast<char*>( pszFilename ) ); + + pContext = this; + pfnCallback = InterceptCallback; + + if ( hSpecificAsyncFile != FS_INVALID_ASYNC_FILE ) + { + g_AsyncOpenedFiles.AddRef( hSpecificAsyncFile ); + } +#if (defined(_DEBUG) || defined(USE_MEM_DEBUG)) + m_pszAllocCreditFile = NULL; + m_nAllocCreditLine = 0; +#endif + } + + ~CFileAsyncReadJob() + { + if ( hSpecificAsyncFile != FS_INVALID_ASYNC_FILE ) + { + g_AsyncOpenedFiles.Release( hSpecificAsyncFile ); + } + + if ( pszFilename ) + free( (void *)pszFilename ); + } + + CFileAsyncReadJob *AsReadJob() { return this; } + + virtual char const *Describe() + { + return pszFilename; + } + + const FileAsyncRequest_t *GetRequest() const + { + return this; + } + + virtual JobStatus_t DoExecute() + { + SimulateDelay(); +#if defined( TRACK_BLOCKING_IO ) + bool oldState = BaseFileSystem()->SetAllowSynchronousLogging( false ); +#endif + +#if (defined(_DEBUG) || defined(USE_MEM_DEBUG)) + if ( m_pszAllocCreditFile ) + MemAlloc_PushAllocDbgInfo( m_pszAllocCreditFile, m_nAllocCreditLine ); +#endif + + JobStatus_t retval; + if ( m_pCustomFetcher ) + { + Assert( GetRefCount() > 1 ); // This will produce self-destruction. No-can-do + if ( m_pCustomFetcher->FinishSynchronous( m_hCustomFetcherHandle ) == FSASYNC_OK ) + { + retval = JOB_OK; + } + else + { + retval = -1; // generic failure code...? + } + } + else + { + int iPrevPriority = ThreadGetPriority(); + ThreadSetPriority( 2 ); + retval = BaseFileSystem()->SyncRead( *this ); + ThreadSetPriority( iPrevPriority ); + } + +#if (defined(_DEBUG) || defined(USE_MEM_DEBUG)) + if ( m_pszAllocCreditFile ) + MemAlloc_PopAllocDbgInfo(); +#endif + +#if defined( TRACK_BLOCKING_IO ) + m_Timer.End(); + FileBlockingItem item( FILESYSTEM_BLOCKING_ASYNCHRONOUS, Describe(), m_Timer.GetDuration().GetSeconds(), FileBlockingItem::FB_ACCESS_READ ); + BaseFileSystem()->RecordBlockingFileAccess( false, item ); + BaseFileSystem()->SetAllowSynchronousLogging( oldState ); +#endif + return retval; + } + + virtual JobStatus_t GetResult( void **ppData, int *pSize ) + { + if ( m_pResultData ) + { + *ppData = m_pResultData; + *pSize = m_nResultSize; + } + return GetStatus(); + } + + static void InterceptCallback( const FileAsyncRequest_t &request, int nBytesRead, FSAsyncStatus_t result ) + { + CFileAsyncReadJob *pJob = (CFileAsyncReadJob *)request.pContext; + if ( result == FSASYNC_OK && !( request.flags & FSASYNC_FLAGS_FREEDATAPTR ) ) + { + pJob->m_pResultData = request.pData; + pJob->m_nResultSize = nBytesRead; + } + + if ( pJob->m_pfnRealCallback ) + { + // Going to slam the values. Not used after this point. Make temps if that changes + FileAsyncRequest_t &temp = const_cast<FileAsyncRequest_t &>(request); + temp.pfnCallback = pJob->m_pfnRealCallback; + temp.pContext = pJob->m_pRealContext; + + (*pJob->m_pfnRealCallback)( temp, nBytesRead, result ); + } + + // Check if we a called by a custom fetcher, then we aren't owned by + // a thread pool, so we should clean up. + if ( pJob->m_pCustomFetcher ) + { + + // The job is finished + pJob->SlamStatus( (JobStatus_t)result ); + + // The fetcher is going to destroy this, so make sure we clear our handle, + // remembering thatwe've been deleted + pJob->m_hCustomFetcherHandle = 0; + + // Remove us from the list of active jobs. This will + // also decrement our ref count, which might delete us! + pJob->m_pOwnerFileSystem->RemoveAsyncCustomFetchJob( pJob ); + } + } + + void SetAllocCredit( const char *pszFile, int line ) + { +#if (defined(_DEBUG) || defined(USE_MEM_DEBUG)) + m_pszAllocCreditFile = pszFile; + m_nAllocCreditLine = line; +#endif + } + + IAsyncFileFetch * m_pCustomFetcher; + IAsyncFileFetch::Handle m_hCustomFetcherHandle; + CBaseFileSystem * m_pOwnerFileSystem; +private: + void * m_pResultData; + int m_nResultSize; + void * m_pRealContext; + FSAsyncCallbackFunc_t m_pfnRealCallback; +#if defined( TRACK_BLOCKING_IO ) + CFastTimer m_Timer; +#endif +#if (defined(_DEBUG) || defined(USE_MEM_DEBUG)) + const char * m_pszAllocCreditFile; + int m_nAllocCreditLine; +#endif +}; + +//--------------------------------------------------------- +// Append to a file +//--------------------------------------------------------- +static int g_nAsyncWriteJobs; + +class CFileAsyncWriteJob : public CFileAsyncJob +{ +public: + CFileAsyncWriteJob( const char *pszFilename, const void *pData, unsigned nBytes, bool bFreeMemory, bool bAppend ) + : CFileAsyncJob( FSASYNC_WRITE_PRIORITY ), + m_pData( pData ), + m_nBytes( nBytes ), + m_bFreeMemory( bFreeMemory ), + m_bAppend( bAppend ) + { +#if defined( TRACK_BLOCKING_IO ) + m_Timer.Start(); +#endif + m_pszFilename = strdup( pszFilename ); + g_nAsyncWriteJobs++; + + SetFlags( GetFlags() | JF_SERIAL ); + } + + ~CFileAsyncWriteJob() + { + g_nAsyncWriteJobs--; + free( (void *)m_pszFilename ); + } + + virtual char const *Describe() { return m_pszFilename; } + + virtual bool IsWrite() const { return true; } + + virtual JobStatus_t DoExecute() + { + SimulateDelay(); +#if defined( TRACK_BLOCKING_IO ) + bool oldState = BaseFileSystem()->SetAllowSynchronousLogging( false ); +#endif + JobStatus_t retval = BaseFileSystem()->SyncWrite( m_pszFilename, m_pData, m_nBytes, false, m_bAppend ); + +#if defined( TRACK_BLOCKING_IO ) + m_Timer.End(); + FileBlockingItem item( FILESYSTEM_BLOCKING_ASYNCHRONOUS, Describe(), m_Timer.GetDuration().GetSeconds(), FileBlockingItem::FB_ACCESS_WRITE ); + BaseFileSystem()->RecordBlockingFileAccess( false, item ); + BaseFileSystem()->SetAllowSynchronousLogging( oldState ); +#endif + return retval; + } + + virtual void DoCleanup() + { + if ( m_pData && m_bFreeMemory ) + { + free( (void*) m_pData ); + } + } + +protected: + bool m_bFreeMemory; +private: + const char *m_pszFilename; + const void *m_pData; + int m_nBytes; + bool m_bAppend; +#if defined( TRACK_BLOCKING_IO ) + CFastTimer m_Timer; +#endif +}; + +class CFileAsyncWriteFileJob : public CFileAsyncWriteJob +{ +public: + CFileAsyncWriteFileJob( const char *pszFilename, const CUtlBuffer *pData, unsigned nBytes, bool bFreeMemory, bool bAppend ) + : CFileAsyncWriteJob( pszFilename, pData->Base(), nBytes, bFreeMemory, bAppend ), + m_pBuffer( pData ) + { + } + + virtual void DoCleanup() + { + if ( m_pBuffer && m_bFreeMemory ) + { + delete m_pBuffer; + } + } + +private: + const CUtlBuffer *m_pBuffer; +}; + +//--------------------------------------------------------- +// Append two files +//--------------------------------------------------------- +class CFileAsyncAppendFileJob : public CFileAsyncJob +{ +public: + CFileAsyncAppendFileJob( const char *pszAppendTo, const char *pszAppendFrom ) + : CFileAsyncJob( FSASYNC_WRITE_PRIORITY ) + { +#if defined( TRACK_BLOCKING_IO ) + m_Timer.Start(); +#endif + m_pszAppendTo = strdup( pszAppendTo ); + m_pszAppendFrom = strdup( pszAppendFrom ); + Q_FixSlashes( const_cast<char*>( m_pszAppendTo ) ); + Q_FixSlashes( const_cast<char*>( m_pszAppendFrom ) ); + g_nAsyncWriteJobs++; + + SetFlags( GetFlags() | JF_SERIAL ); + } + + ~CFileAsyncAppendFileJob() + { + g_nAsyncWriteJobs--; + } + + virtual char const *Describe() { return m_pszAppendTo; } + + virtual bool IsWrite() const { return true; } + + virtual JobStatus_t DoExecute() + { + SimulateDelay(); +#if defined( TRACK_BLOCKING_IO ) + bool oldState = BaseFileSystem()->SetAllowSynchronousLogging( false ); +#endif + JobStatus_t retval = BaseFileSystem()->SyncAppendFile( m_pszAppendTo, m_pszAppendFrom ); + +#if defined( TRACK_BLOCKING_IO ) + m_Timer.End(); + FileBlockingItem item( FILESYSTEM_BLOCKING_ASYNCHRONOUS, Describe(), m_Timer.GetDuration().GetSeconds(), FileBlockingItem::FB_ACCESS_APPEND ); + BaseFileSystem()->RecordBlockingFileAccess( false, item ); + BaseFileSystem()->SetAllowSynchronousLogging( oldState ); +#endif + + return retval; + } + +private: + const char *m_pszAppendTo; + const char *m_pszAppendFrom; +#if defined( TRACK_BLOCKING_IO ) + CFastTimer m_Timer; +#endif +}; + +//--------------------------------------------------------- +// Job to find out file size +//--------------------------------------------------------- + +class CFileAsyncFileSizeJob : public CFileAsyncReadJob +{ +public: + CFileAsyncFileSizeJob( const FileAsyncRequest_t &fromRequest, CBaseFileSystem *pOwnerFileSystem ) + : CFileAsyncReadJob( fromRequest, pOwnerFileSystem ) + { +#if defined( TRACK_BLOCKING_IO ) + m_Timer.Start(); +#endif + } + + virtual JobStatus_t DoExecute() + { + SimulateDelay(); +#if defined( TRACK_BLOCKING_IO ) + bool oldState = BaseFileSystem()->SetAllowSynchronousLogging( false ); +#endif + JobStatus_t retval = BaseFileSystem()->SyncGetFileSize( *this ); +#if defined( TRACK_BLOCKING_IO ) + m_Timer.End(); + FileBlockingItem item( FILESYSTEM_BLOCKING_ASYNCHRONOUS, Describe(), m_Timer.GetDuration().GetSeconds(), FileBlockingItem::FB_ACCESS_SIZE ); + BaseFileSystem()->RecordBlockingFileAccess( false, item ); + BaseFileSystem()->SetAllowSynchronousLogging( oldState ); +#endif + return retval; + } +#if defined( TRACK_BLOCKING_IO ) +private: + CFastTimer m_Timer; +#endif +}; + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +void CBaseFileSystem::InitAsync() +{ + Assert( !m_pThreadPool ); + if ( m_pThreadPool ) + return; + + if ( IsX360() && !IsRetail() && Plat_IsInDebugSession() ) + { + class CBreakThread : public CThread + { + virtual int Run() + { + for (;;) + { + Sleep(1000); + static int wakeCount; + wakeCount++; + volatile static int bForceResume = false; + if ( bForceResume ) + { + bForceResume = false; + BaseFileSystem()->AsyncResume(); + } + } + // Unreachable. + return 0; + } + }; + + static CBreakThread breakThread; + breakThread.SetName( "DebugBreakThread" ); + breakThread.Start( 1024 ); + } + + if ( CommandLine()->FindParm( "-noasync" ) ) + { + Msg( "Async I/O disabled from command line\n" ); + return; + } + + if ( VCRGetMode() == VCR_Disabled ) + { + // create the i/o thread pool + m_pThreadPool = CreateThreadPool(); + + ThreadPoolStartParams_t params; + params.iThreadPriority = 0; + params.bIOThreads = true; + params.nThreadsMax = 4; // Limit count of IO threads to a maximum of 4. + if ( IsX360() ) + { + // override defaults + // 360 has a single i/o thread on the farthest proc + params.nThreads = 1; + params.fDistribute = TRS_TRUE; + params.bUseAffinityTable = true; + params.iAffinityTable[0] = XBOX_PROCESSOR_3; + } + else if( IsPC() ) + { + // override defaults + // maximum # of async I/O thread on PC is 2 + params.nThreads = 1; + } + + if ( !m_pThreadPool->Start( params, "IOJob" ) ) + { + SafeRelease( m_pThreadPool ); + } + } +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +void CBaseFileSystem::ShutdownAsync() +{ + if ( m_pThreadPool ) + { + AsyncFlush(); + m_pThreadPool->Stop(); + SafeRelease( m_pThreadPool ); + } +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +void CBaseFileSystem::AsyncAddFetcher( IAsyncFileFetch *pFetcher ) +{ + m_vecAsyncFetchers.AddToTail( pFetcher ); +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +void CBaseFileSystem::AsyncRemoveFetcher( IAsyncFileFetch *pFetcher ) +{ + + // Abort any active jobs + int i = 0; + while ( i < m_vecAsyncCustomFetchJobs.Count() ) + { + if ( m_vecAsyncCustomFetchJobs[i]->m_pCustomFetcher == pFetcher ) + { + AsyncAbort( (FSAsyncControl_t)m_vecAsyncCustomFetchJobs[i] ); + } + else + { + ++i; + } + } + + // Remove it from the hook list + i = 0; + while ( i < m_vecAsyncFetchers.Count() ) + { + if ( m_vecAsyncFetchers[i] == pFetcher ) + { + m_vecAsyncFetchers.Remove( i ); + } + else + { + ++i; + } + } +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +void CBaseFileSystem::RemoveAsyncCustomFetchJob( CFileAsyncReadJob *pJob ) +{ + Assert( pJob ); + Assert( pJob->m_pOwnerFileSystem == this ); + Assert( pJob->m_pCustomFetcher ); + Assert( !pJob->m_hCustomFetcherHandle ); + + // Linear search, This list is usually very small, and completion doesn't + // happen often, anyway + int i = 0; + bool bFound = false; + while ( i < m_vecAsyncCustomFetchJobs.Count() ) + { + if ( m_vecAsyncCustomFetchJobs[i] == pJob ) + { + m_vecAsyncCustomFetchJobs.Remove( i ); + Assert( !bFound ); + bFound = true; + } + else + { + ++i; + } + } + + // Release our reference. + Assert( bFound ); + if ( bFound ) + { + pJob->Release(); + } +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +FSAsyncStatus_t CBaseFileSystem::AsyncReadMultiple( const FileAsyncRequest_t *pRequests, int nRequests, FSAsyncControl_t *phControls ) +{ + return AsyncReadMultipleCreditAlloc( pRequests, nRequests, NULL, 0, phControls ); +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +FSAsyncStatus_t CBaseFileSystem::AsyncReadMultipleCreditAlloc( const FileAsyncRequest_t *pRequests, int nRequests, const char *pszFile, int line, FSAsyncControl_t *phControls ) +{ + bool bAsyncMode = ( GetAsyncMode() == FSAM_ASYNC ); + bool bSynchronous = ( !bAsyncMode || ( pRequests[0].flags & FSASYNC_FLAGS_SYNC ) || !m_pThreadPool ); + + if ( !bAsyncMode ) + { + AsyncFinishAll(); + } + + CFileAsyncReadJob *pJob; + + for ( int i = 0; i < nRequests; i++ ) + { + if ( pRequests[i].nBytes >= 0 ) + { + pJob = new CFileAsyncReadJob( pRequests[i], this ); + } + else + { + pJob = new CFileAsyncFileSizeJob( pRequests[i], this ); + } + +#if (defined(_DEBUG) || defined(USE_MEM_DEBUG)) + pJob->SetAllocCredit( pszFile, line ); +#endif + + // Search list of application custom fetchers and see if any of them want it + for ( int j = 0; j < m_vecAsyncFetchers.Count(); j++ ) + { + IAsyncFileFetch::Handle handle; + FSAsyncStatus_t status = m_vecAsyncFetchers[j]->Start( *pJob->GetRequest(), &handle, m_pThreadPool ); + if ( status == FSASYNC_OK ) + { + pJob->m_pCustomFetcher = m_vecAsyncFetchers[j]; + pJob->m_hCustomFetcherHandle = handle; + break; + } + + // !KLUDGE! For now, this is the only other acceptable failure + Assert ( status == FSASYNC_ERR_NOT_MINE ); + } + + // Found custom fetcher? + if ( pJob->m_pCustomFetcher != NULL ) + { + m_vecAsyncCustomFetchJobs.AddToTail( pJob ); // this counts as a reference + + // Give them back the control handle, if they wanted it + if ( phControls ) + { + phControls[i] = (FSAsyncControl_t)pJob; + pJob->AddRef(); + } + + // Execute job synchronously, if requested + if ( bSynchronous ) + { + pJob->Execute(); + } + else + { + // We need to manually slam the job status to + // in progress, in case they poll it + pJob->SlamStatus( JOB_STATUS_INPROGRESS ); + } + + // We'll deal with it in our callback. DO NOT + // put it in the thread pool. If other, regular async jobs + // come in, we want those to be processed immediately. + // We don't have any reason to think that we need to wait + // on the custom fetcher job in order to do local disk access. + // (Even if there is, we don't have anough knowledge at this level + // to properly deal with it.) + continue; + } + + if ( !bSynchronous ) + { + // async mode, queue request + m_pThreadPool->AddJob( pJob ); + } + else + { + // synchronous mode, execute now + pJob->Execute(); + } + + if ( phControls ) + { + phControls[i] = (FSAsyncControl_t)pJob; + } + else + { + pJob->Release(); + } + } + + return FSASYNC_OK; +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +FSAsyncStatus_t CBaseFileSystem::AsyncWrite(const char *pFileName, const void *pSrc, int nSrcBytes, bool bFreeMemory, bool bAppend, FSAsyncControl_t *pControl ) +{ + bool bAsyncMode = ( GetAsyncMode() == FSAM_ASYNC ); + bool bSynchronous = ( !bAsyncMode || !m_pThreadPool ); + + if ( !bAsyncMode ) + { + AsyncFinishAll(); + } + + CJob *pJob = new CFileAsyncWriteJob( pFileName, pSrc, nSrcBytes, bFreeMemory, bAppend ); + + if ( !bSynchronous ) + { + m_pThreadPool->AddJob( pJob ); + } + else + { + pJob->Execute(); + } + + if ( pControl ) + { + *pControl = (FSAsyncControl_t)pJob; + } + else + { + pJob->Release(); + } + + return FSASYNC_OK; +} + + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +FSAsyncStatus_t CBaseFileSystem::AsyncWriteFile(const char *pFileName, const CUtlBuffer *pBuff, int nSrcBytes, bool bFreeMemory, bool bAppend, FSAsyncControl_t *pControl ) +{ + bool bAsyncMode = ( GetAsyncMode() == FSAM_ASYNC ); + bool bSynchronous = ( !bAsyncMode || !m_pThreadPool ); + + if ( !bAsyncMode ) + { + AsyncFinishAll(); + } + + CJob *pJob = new CFileAsyncWriteFileJob( pFileName, pBuff, nSrcBytes, bFreeMemory, bAppend ); + + if ( !bSynchronous ) + { + m_pThreadPool->AddJob( pJob ); + } + else + { + pJob->Execute(); + } + + if ( pControl ) + { + *pControl = (FSAsyncControl_t)pJob; + } + else + { + pJob->Release(); + } + + return FSASYNC_OK; +} + + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +FSAsyncStatus_t CBaseFileSystem::AsyncAppendFile(const char *pAppendToFileName, const char *pAppendFromFileName, FSAsyncControl_t *pControl ) +{ + bool bAsyncMode = ( GetAsyncMode() == FSAM_ASYNC ); + bool bSynchronous = ( !bAsyncMode || !m_pThreadPool ); + + if ( !bAsyncMode ) + { + AsyncFinishAll(); + } + + CJob *pJob = new CFileAsyncAppendFileJob( pAppendToFileName, pAppendFromFileName ); + + if ( !bSynchronous ) + { + m_pThreadPool->AddJob( pJob ); + } + else + { + pJob->Execute(); + } + + if ( pControl ) + { + *pControl = (FSAsyncControl_t)pJob; + } + else + { + pJob->Release(); + } + + return FSASYNC_OK; +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +CThreadMutex g_AsyncFinishMutex; +void CBaseFileSystem::AsyncFinishAll( int iToPriority ) +{ + if ( m_pThreadPool) + { + AUTO_LOCK( g_AsyncFinishMutex ); + m_pThreadPool->ExecuteToPriority( ConvertPriority( iToPriority ) ); + } +} + + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- + +static bool AsyncWriteJobFilter( CJob *pJob ) +{ + CFileAsyncJob *pFileJob = dynamic_cast<CFileAsyncJob *>(pJob); + return ( pFileJob && pFileJob->IsWrite() ); +} + +void CBaseFileSystem::AsyncFinishAllWrites() +{ + if ( m_pThreadPool && g_nAsyncWriteJobs ) + { + AUTO_LOCK( g_AsyncFinishMutex ); + m_pThreadPool->ExecuteAll( AsyncWriteJobFilter ); + } +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +bool CBaseFileSystem::AsyncSuspend() +{ + if ( m_pThreadPool ) + { + m_pThreadPool->SuspendExecution(); + } + + return true; +} + + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +bool CBaseFileSystem::AsyncResume() +{ + if ( m_pThreadPool ) + { + m_pThreadPool->ResumeExecution(); + } + + return true; +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +FSAsyncStatus_t CBaseFileSystem::AsyncBeginRead( const char *pszFile, FSAsyncFile_t *phFile ) +{ +#if !(defined(FILESYSTEM_STEAM) || defined(DEDICATED)) + if ( AsyncAllowHeldFiles() ) + { + *phFile = g_AsyncOpenedFiles.FindOrAdd( pszFile ); + return FSASYNC_OK; + } +#endif + *phFile = FS_INVALID_ASYNC_FILE; + return FSASYNC_OK; +} + + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +FSAsyncStatus_t CBaseFileSystem::AsyncEndRead( FSAsyncFile_t hFile ) +{ +#if !(defined(FILESYSTEM_STEAM) || defined(DEDICATED)) + if ( hFile != FS_INVALID_ASYNC_FILE ) + g_AsyncOpenedFiles.Release( hFile ); +#endif + + return FSASYNC_OK; +} + + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +FSAsyncStatus_t CBaseFileSystem::AsyncFinish( FSAsyncControl_t hControl, bool wait ) +{ + if ( wait ) + { + CJob *pJob = (CJob *)hControl; + if ( !pJob ) + { + return FSASYNC_ERR_FAILURE; + } + +#if defined( TRACK_BLOCKING_IO ) + CFastTimer timer; + timer.Start(); + BaseFileSystem()->SetAllowSynchronousLogging( false ); +#endif + FSAsyncStatus_t retval = (FSAsyncStatus_t)pJob->Execute(); + +#if defined( TRACK_BLOCKING_IO ) + timer.End(); + FileBlockingItem item( FILESYSTEM_BLOCKING_ASYNCHRONOUS_BLOCK, pJob->Describe(), timer.GetDuration().GetSeconds(), FileBlockingItem::FB_ACCESS_READ ); + BaseFileSystem()->RecordBlockingFileAccess( false, item ); + BaseFileSystem()->SetAllowSynchronousLogging( true ); +#endif + return retval; + } + + AsyncSetPriority( hControl, INT_MAX ); + return FSASYNC_OK; +} + + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +FSAsyncStatus_t CBaseFileSystem::AsyncGetResult( FSAsyncControl_t hControl, void **ppData, int *pSize ) +{ + if ( ppData ) + { + *ppData = NULL; + } + if ( pSize ) + { + *pSize = 0; + } + + CFileAsyncJob *pJob = (CFileAsyncJob *)hControl; + if ( !pJob ) + { + return FSASYNC_ERR_FAILURE; + } + if ( pJob->IsFinished() ) + { + return (FSAsyncStatus_t)pJob->GetResult( ppData, pSize ); + } + return FSASYNC_STATUS_PENDING; +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +FSAsyncStatus_t CBaseFileSystem::AsyncAbort( FSAsyncControl_t hControl ) +{ + CFileAsyncJob *pJob = (CFileAsyncJob *)hControl; + if ( !pJob ) + { + return FSASYNC_ERR_FAILURE; + } + + // Custom job doesn't have a job manager, needs to be handled specially + CFileAsyncReadJob *pReadJob = pJob->AsReadJob(); + if ( pReadJob && pReadJob->m_pCustomFetcher ) + { + Assert( pReadJob->m_pOwnerFileSystem == this ); + + FSAsyncStatus_t status = (FSAsyncStatus_t)pReadJob->GetStatus(); + if ( status == (FSAsyncStatus_t)JOB_STATUS_INPROGRESS) { + + // Slam the status. The default behaviour doesn't change the status + // if the task is in progess for some reason + status = (FSAsyncStatus_t)JOB_STATUS_ABORTED; + pReadJob->SlamStatus( status ); + + // Tell fetcher to abort job + Assert( pReadJob->m_hCustomFetcherHandle ); + pReadJob->m_pCustomFetcher->Abort( pReadJob->m_hCustomFetcherHandle ); + pReadJob->m_hCustomFetcherHandle = NULL; + + // Remove us from the list of active jobs. This will + // also decrement our ref count, which might delete us! + RemoveAsyncCustomFetchJob( pReadJob ); + } + + return status; + } + + return (FSAsyncStatus_t)pJob->Abort(); +} + + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +FSAsyncStatus_t CBaseFileSystem::AsyncStatus( FSAsyncControl_t hControl ) +{ + CJob *pJob = (CJob *)hControl; + if ( !pJob ) + { + return FSASYNC_ERR_FAILURE; + } + return (FSAsyncStatus_t)pJob->GetStatus(); +} + + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +FSAsyncStatus_t CBaseFileSystem::AsyncFlush() +{ + if ( m_pThreadPool ) + { + m_pThreadPool->AbortAll(); + } + + // Abort all custom jobs + while ( m_vecAsyncCustomFetchJobs.Count() > 0 ) + { + CFileAsyncReadJob *pJob = m_vecAsyncCustomFetchJobs[0]; + Assert( pJob->m_pCustomFetcher ); + AsyncAbort( (FSAsyncControl_t)pJob ); + } + + return FSASYNC_OK; +} + + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +FSAsyncStatus_t CBaseFileSystem::AsyncSetPriority(FSAsyncControl_t hControl, int newPriority) +{ + if ( m_pThreadPool ) + { + CJob *pJob = (CJob *)hControl; + + if ( !pJob ) + { + return FSASYNC_ERR_FAILURE; + } + + JobPriority_t internalPriority = ConvertPriority( newPriority ); + if ( internalPriority != pJob->GetPriority() ) + { + m_pThreadPool->ChangePriority( pJob, internalPriority ); + } + + } + return FSASYNC_OK; +} + + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +void CBaseFileSystem::AsyncAddRef( FSAsyncControl_t hControl ) +{ + CJob *pJob = (CJob *)hControl; + if ( pJob ) + { + pJob->AddRef(); + } +} + + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +void CBaseFileSystem::AsyncRelease( FSAsyncControl_t hControl ) +{ + CJob *pJob = (CJob *)hControl; + if ( pJob ) + { + pJob->Release(); + } +} + + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- + +FSAsyncStatus_t CBaseFileSystem::SyncRead( const FileAsyncRequest_t &request ) +{ + Assert( request.nBytes >=0 ); + + if ( request.nBytes < 0 || request.nOffset < 0 ) + { + Msg( "Invalid async read of %s\n", request.pszFilename ); + DoAsyncCallback( request, NULL, 0, FSASYNC_ERR_FILEOPEN ); + return FSASYNC_ERR_FILEOPEN; + } + + FSAsyncStatus_t result; + + AsyncOpenedFile_t *pHeldFile = ( request.hSpecificAsyncFile != FS_INVALID_ASYNC_FILE ) ? g_AsyncOpenedFiles.Get( request.hSpecificAsyncFile ) : NULL; + + FileHandle_t hFile; + + if ( !pHeldFile || pHeldFile->hFile == FILESYSTEM_INVALID_HANDLE ) + { + hFile = OpenEx( request.pszFilename, "rb", 0, request.pszPathID ); + if ( pHeldFile ) + { + pHeldFile->hFile = hFile; + } + } + else + { + hFile = pHeldFile->hFile; + } + + if ( hFile ) + { + // ------------------------------------------------------ + int nBytesToRead = ( request.nBytes ) ? request.nBytes : Size( hFile ) - request.nOffset; + int nBytesBuffer; + void *pDest; + + if ( nBytesToRead < 0 ) + { + nBytesToRead = 0; // bad offset? + } + + if ( request.pData ) + { + // caller provided buffer + Assert( !( request.flags & FSASYNC_FLAGS_NULLTERMINATE ) ); + pDest = request.pData; + nBytesBuffer = nBytesToRead; + } + else + { + // allocate an optimal buffer + unsigned nOffsetAlign; + nBytesBuffer = nBytesToRead + ( ( request.flags & FSASYNC_FLAGS_NULLTERMINATE ) ? 1 : 0 ); + if ( GetOptimalIOConstraints( hFile, &nOffsetAlign, NULL, NULL) && ( request.nOffset % nOffsetAlign == 0 ) ) + { + nBytesBuffer = GetOptimalReadSize( hFile, nBytesBuffer ); + } + + if ( !request.pfnAlloc ) + { + pDest = AllocOptimalReadBuffer( hFile, nBytesBuffer, request.nOffset ); + } + else + { + pDest = (*request.pfnAlloc)( request.pszFilename, nBytesBuffer ); + } + } + + SetBufferSize( hFile, 0 ); // TODO: what if it's a pack file? restore buffer size? + + if ( request.nOffset > 0 ) + { + Seek( hFile, request.nOffset, FILESYSTEM_SEEK_HEAD ); + } + + // perform the read operation + int nBytesRead = ReadEx( pDest, nBytesBuffer, nBytesToRead, hFile ); + + if ( !pHeldFile ) + { + Close( hFile ); + } + + if ( request.flags & FSASYNC_FLAGS_NULLTERMINATE ) + { + ((char *)pDest)[nBytesRead] = 0; + } + + result = ( ( nBytesRead == 0 ) && ( nBytesToRead != 0 ) ) ? FSASYNC_ERR_READING : FSASYNC_OK; + DoAsyncCallback( request, pDest, min( nBytesRead, nBytesToRead ), result ); + } + else + { + DoAsyncCallback( request, NULL, 0, FSASYNC_ERR_FILEOPEN ); + result = FSASYNC_ERR_FILEOPEN; + } + + if ( pHeldFile ) + { + g_AsyncOpenedFiles.Release( request.hSpecificAsyncFile ); + } + + if ( m_fwLevel >= FILESYSTEM_WARNING_REPORTALLACCESSES_ASYNC ) + { + LogAccessToFile( "async", request.pszFilename, "" ); + } + + return result; +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +FSAsyncStatus_t CBaseFileSystem::SyncGetFileSize( const FileAsyncRequest_t &request ) +{ + int size = Size( request.pszFilename, request.pszPathID ); + + DoAsyncCallback( request, NULL, size, ( size ) ? FSASYNC_OK : FSASYNC_ERR_FILEOPEN ); + + return FSASYNC_OK; +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +FSAsyncStatus_t CBaseFileSystem::SyncWrite(const char *pszFilename, const void *pSrc, int nSrcBytes, bool bFreeMemory, bool bAppend ) +{ + FileHandle_t hFile = OpenEx( pszFilename, ( bAppend ) ? "ab+" : "wb", IsX360() ? FSOPEN_NEVERINPACK : 0, NULL ); + if ( hFile ) + { + SetBufferSize( hFile, 0 ); + Write( pSrc, nSrcBytes, hFile ); + Close( hFile ); + if ( bFreeMemory ) + { + free( (void*)pSrc ); + } + + if ( m_fwLevel >= FILESYSTEM_WARNING_REPORTALLACCESSES_ASYNC ) + { + LogAccessToFile( "asyncwrite", pszFilename, "" ); + } + + return FSASYNC_OK; + } + + return FSASYNC_ERR_FILEOPEN; +} + + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +FSAsyncStatus_t CBaseFileSystem::SyncAppendFile(const char *pAppendToFileName, const char *pAppendFromFileName ) +{ + FileHandle_t hDestFile = OpenEx( pAppendToFileName, "ab+", IsX360() ? FSOPEN_NEVERINPACK : 0, NULL ); + FSAsyncStatus_t result = FSASYNC_ERR_FAILURE; + if ( hDestFile ) + { + SetBufferSize( hDestFile, 0 ); + FileHandle_t hSourceFile = OpenEx( pAppendFromFileName, "rb", IsX360() ? FSOPEN_NEVERINPACK : 0, NULL ); + if ( hSourceFile ) + { + SetBufferSize( hSourceFile, 0 ); + const int BUFSIZE = 128 * 1024; + int fileSize = Size( hSourceFile ); + char *buf = (char *)malloc( BUFSIZE ); + int size; + + while ( fileSize > 0 ) + { + if ( fileSize > BUFSIZE ) + size = BUFSIZE; + else + size = fileSize; + Read( buf, size, hSourceFile ); + Write( buf, size, hDestFile ); + + fileSize -= size; + } + + free(buf); + Close( hSourceFile ); + result = FSASYNC_OK; + } + Close( hDestFile ); + } + + if ( m_fwLevel >= FILESYSTEM_WARNING_REPORTALLACCESSES_ASYNC ) + { + LogAccessToFile( "asyncappend", pAppendToFileName, "" ); + } + + return result; +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +void CBaseFileSystem::DoAsyncCallback( const FileAsyncRequest_t &request, void *pData, int nBytesRead, FSAsyncStatus_t result ) +{ + void *pDataToFree = NULL; + + if ( request.pfnCallback ) + { + AUTO_LOCK( m_AsyncCallbackMutex ); + if ( pData && request.pData != pData ) + { + // Allocated the data here + FileAsyncRequest_t temp = request; + temp.pData = pData; + { + AUTOBLOCKREPORTER_FN( DoAsyncCallback, this, false, temp.pszFilename, FILESYSTEM_BLOCKING_CALLBACKTIMING, FileBlockingItem::FB_ACCESS_READ ); + (*request.pfnCallback)( temp, nBytesRead, result ); + } + if ( !( request.flags & FSASYNC_FLAGS_ALLOCNOFREE ) ) + { + pDataToFree = pData; + } + } + else + { + { + AUTOBLOCKREPORTER_FN( DoAsyncCallback, this, false, request.pszFilename, FILESYSTEM_BLOCKING_CALLBACKTIMING, FileBlockingItem::FB_ACCESS_READ ); + (*request.pfnCallback)( request, nBytesRead, result ); + } + if ( ( request.flags & FSASYNC_FLAGS_FREEDATAPTR ) ) + { + pDataToFree = request.pData; + } + } + } + + if ( pDataToFree ) + { + Assert( !request.pfnAlloc ); +#if defined( OSX ) || defined( LINUX ) + // The ugly delete[] (void*) method generates a compile warning on osx, as it should. + free( pDataToFree ); +#else + delete [] pDataToFree; +#endif + } +} + diff --git a/filesystem/filesystem_stdio.cpp b/filesystem/filesystem_stdio.cpp new file mode 100644 index 0000000..43916fe --- /dev/null +++ b/filesystem/filesystem_stdio.cpp @@ -0,0 +1,1606 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//=============================================================================// +#ifdef _WIN32 +#include <io.h> +#include <fcntl.h> +#endif + +#include "basefilesystem.h" +#include "packfile.h" +#include "tier0/dbg.h" +#include "tier0/threadtools.h" +#ifdef _WIN32 +#include "tier0/tslist.h" +#elif defined(POSIX) +#include <fcntl.h> +#ifdef LINUX +#include <sys/file.h> +#endif +#endif +#include "tier1/convar.h" +#include "tier0/vcrmode.h" +#include "tier0/vprof.h" +#include "tier1/fmtstr.h" +#include "tier1/utlrbtree.h" +#include "vstdlib/osversion.h" + +#ifdef _X360 +#undef WaitForSingleObject +#endif + +// memdbgon must be the last include file in a .cpp file!!! +#include "tier0/memdbgon.h" + +ASSERT_INVARIANT( SEEK_CUR == FILESYSTEM_SEEK_CURRENT ); +ASSERT_INVARIANT( SEEK_SET == FILESYSTEM_SEEK_HEAD ); +ASSERT_INVARIANT( SEEK_END == FILESYSTEM_SEEK_TAIL ); + +//----------------------------------------------------------------------------- + +class CFileSystem_Stdio : public CBaseFileSystem +{ +public: + CFileSystem_Stdio(); + ~CFileSystem_Stdio(); + + // Used to get at older versions + void *QueryInterface( const char *pInterfaceName ); + + // Higher level filesystem methods requiring specific behavior + virtual void GetLocalCopy( const char *pFileName ); + virtual int HintResourceNeed( const char *hintlist, int forgetEverything ); + virtual bool IsFileImmediatelyAvailable(const char *pFileName); + virtual WaitForResourcesHandle_t WaitForResources( const char *resourcelist ); + virtual bool GetWaitForResourcesProgress( WaitForResourcesHandle_t handle, float *progress /* out */ , bool *complete /* out */ ); + virtual void CancelWaitForResources( WaitForResourcesHandle_t handle ); + virtual bool IsSteam() const { return false; } + virtual FilesystemMountRetval_t MountSteamContent( int nExtraAppId = -1 ) { return FILESYSTEM_MOUNT_OK; } + + bool GetOptimalIOConstraints( FileHandle_t hFile, unsigned *pOffsetAlign, unsigned *pSizeAlign, unsigned *pBufferAlign ); + void *AllocOptimalReadBuffer( FileHandle_t hFile, unsigned nSize, unsigned nOffset ); + void FreeOptimalReadBuffer( void *p ); + +protected: + // implementation of CBaseFileSystem virtual functions + virtual FILE *FS_fopen( const char *filename, const char *options, unsigned flags, int64 *size ); + virtual void FS_setbufsize( FILE *fp, unsigned nBytes ); + virtual void FS_fclose( FILE *fp ); + virtual void FS_fseek( FILE *fp, int64 pos, int seekType ); + virtual long FS_ftell( FILE *fp ); + virtual int FS_feof( FILE *fp ); + virtual size_t FS_fread( void *dest, size_t destSize, size_t size, FILE *fp ); + virtual size_t FS_fwrite( const void *src, size_t size, FILE *fp ); + virtual bool FS_setmode( FILE *fp, FileMode_t mode ); + virtual size_t FS_vfprintf( FILE *fp, const char *fmt, va_list list ); + virtual int FS_ferror( FILE *fp ); + virtual int FS_fflush( FILE *fp ); + virtual char *FS_fgets( char *dest, int destSize, FILE *fp ); + virtual int FS_stat( const char *path, struct _stat *buf, bool *pbLoadedFromSteamCache=NULL ); + virtual int FS_chmod( const char *path, int pmode ); + virtual HANDLE FS_FindFirstFile(const char *findname, WIN32_FIND_DATA *dat); + virtual bool FS_FindNextFile(HANDLE handle, WIN32_FIND_DATA *dat); + virtual bool FS_FindClose(HANDLE handle); + virtual int FS_GetSectorSize( FILE * ); + +private: + bool CanAsync() const + { + return m_bCanAsync; + } + + bool m_bMounted; + bool m_bCanAsync; +}; + + +//----------------------------------------------------------------------------- +// Per-file worker classes +//----------------------------------------------------------------------------- +abstract_class CStdFilesystemFile +{ +public: + virtual ~CStdFilesystemFile() {} + virtual void FS_setbufsize( unsigned nBytes ) = 0; + virtual void FS_fclose() = 0; + virtual void FS_fseek( int64 pos, int seekType ) = 0; + virtual long FS_ftell() = 0; + virtual int FS_feof() = 0; + virtual size_t FS_fread( void *dest, size_t destSize, size_t size ) = 0; + virtual size_t FS_fwrite( const void *src, size_t size ) = 0; + virtual bool FS_setmode( FileMode_t mode ) = 0; + virtual size_t FS_vfprintf( const char *fmt, va_list list ) = 0; + virtual int FS_ferror() = 0; + virtual int FS_fflush() = 0; + virtual char *FS_fgets( char *dest, int destSize ) = 0; + virtual int FS_GetSectorSize() { return 1; } +}; + +//--------------------------------------------------------- + +class CStdioFile : public CStdFilesystemFile +{ +public: + static CStdioFile *FS_fopen( const char *filename, const char *options, int64 *size ); + + virtual void FS_setbufsize( unsigned nBytes ); + virtual void FS_fclose(); + virtual void FS_fseek( int64 pos, int seekType ); + virtual long FS_ftell(); + virtual int FS_feof(); + virtual size_t FS_fread( void *dest, size_t destSize, size_t size); + virtual size_t FS_fwrite( const void *src, size_t size ); + virtual bool FS_setmode( FileMode_t mode ); + virtual size_t FS_vfprintf( const char *fmt, va_list list ); + virtual int FS_ferror(); + virtual int FS_fflush(); + virtual char *FS_fgets( char *dest, int destSize ); + +#ifdef POSIX + static CUtlMap< ino_t, CThreadMutex * > m_LockedFDMap; + static CThreadMutex m_MutexLockedFD; +#endif +private: + CStdioFile( FILE *pFile, bool bWriteable ) + : m_pFile( pFile ), m_bWriteable( bWriteable ) + { + } + + FILE *m_pFile; + bool m_bWriteable; +}; + +#ifdef POSIX +CUtlMap< ino_t, CThreadMutex * > CStdioFile::m_LockedFDMap; +CThreadMutex CStdioFile::m_MutexLockedFD; +#endif + +//----------------------------------------------------------------------------- + +#ifdef _WIN32 +class CWin32ReadOnlyFile : public CStdFilesystemFile +{ +public: + static bool CanOpen( const char *filename, const char *options ); + static CWin32ReadOnlyFile *FS_fopen( const char *filename, const char *options, int64 *size ); + + virtual void FS_setbufsize( unsigned nBytes ) {} + virtual void FS_fclose(); + virtual void FS_fseek( int64 pos, int seekType ); + virtual long FS_ftell(); + virtual int FS_feof(); + virtual size_t FS_fread( void *dest, size_t destSize, size_t size); + virtual size_t FS_fwrite( const void *src, size_t size ) { return 0; } + virtual bool FS_setmode( FileMode_t mode ) { Error( "Can't set mode, open a second file in right mode\n" ); return false; } + virtual size_t FS_vfprintf( const char *fmt, va_list list ) { return 0; } + virtual int FS_ferror() { return 0; } + virtual int FS_fflush() { return 0; } + virtual char *FS_fgets( char *dest, int destSize ); + virtual int FS_GetSectorSize() { return m_SectorSize; } + +private: + CWin32ReadOnlyFile( HANDLE hFileUnbuffered, HANDLE hFileBuffered, int sectorSize, int64 fileSize, bool bOverlapped ) + : m_hFileUnbuffered( hFileUnbuffered ), + m_hFileBuffered( hFileBuffered ), + m_ReadPos( 0 ), + m_Size( fileSize ), + m_SectorSize( sectorSize ), + m_bOverlapped( bOverlapped ) + { + } + + int64 m_ReadPos; + int64 m_Size; + HANDLE m_hFileUnbuffered; + HANDLE m_hFileBuffered; + CThreadFastMutex m_Mutex; + int m_SectorSize; + bool m_bOverlapped; +}; + +#endif + +//----------------------------------------------------------------------------- +// singleton +//----------------------------------------------------------------------------- +CFileSystem_Stdio g_FileSystem_Stdio; +#if defined(_WIN32) && defined(DEDICATED) +CBaseFileSystem *BaseFileSystem_Stdio( void ) +{ + return &g_FileSystem_Stdio; +} +#endif + +#ifdef DEDICATED // "hack" to allow us to not export a stdio version of the FILESYSTEM_INTERFACE_VERSION anywhere + +IFileSystem *g_pFileSystem = &g_FileSystem_Stdio; +IBaseFileSystem *g_pBaseFileSystem = &g_FileSystem_Stdio; + +#else + +EXPOSE_SINGLE_INTERFACE_GLOBALVAR( CFileSystem_Stdio, IFileSystem, FILESYSTEM_INTERFACE_VERSION, g_FileSystem_Stdio ); +EXPOSE_SINGLE_INTERFACE_GLOBALVAR( CFileSystem_Stdio, IBaseFileSystem, BASEFILESYSTEM_INTERFACE_VERSION, g_FileSystem_Stdio ); + +#endif + +//----------------------------------------------------------------------------- + +#ifndef _RETAIL +bool UseOptimalBufferAllocation() +{ + static bool bUseOptimalBufferAllocation = ( IsX360() || ( !IsLinux() && Q_stristr( Plat_GetCommandLine(), "-unbuffered_io" ) != NULL ) ); + return bUseOptimalBufferAllocation; +} +ConVar filesystem_unbuffered_io( "filesystem_unbuffered_io", "1", 0, "" ); +#define UseUnbufferedIO() ( UseOptimalBufferAllocation() && filesystem_unbuffered_io.GetBool() ) +#else +#define UseUnbufferedIO() true +#endif + +ConVar filesystem_native( "filesystem_native", "1", 0, "Use native FS or STDIO" ); +ConVar filesystem_max_stdio_read( "filesystem_max_stdio_read", IsX360() ? "64" : "16", 0, "" ); +ConVar filesystem_report_buffered_io( "filesystem_report_buffered_io", "0" ); + +//----------------------------------------------------------------------------- +// constructor +//----------------------------------------------------------------------------- +CFileSystem_Stdio::CFileSystem_Stdio() +{ + m_bMounted = false; + m_bCanAsync = true; +#ifdef POSIX + SetDefLessFunc( CStdioFile::m_LockedFDMap ); +#endif +} + + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +CFileSystem_Stdio::~CFileSystem_Stdio() +{ +#ifdef POSIX + FOR_EACH_MAP_FAST( CStdioFile::m_LockedFDMap, i ) + { + Assert( CStdioFile::m_LockedFDMap[ i ] ); + delete CStdioFile::m_LockedFDMap[ i ]; + } + CStdioFile::m_LockedFDMap.RemoveAll(); +#endif + + Assert(!m_bMounted); +} + + +//----------------------------------------------------------------------------- +// QueryInterface: +//----------------------------------------------------------------------------- +void *CFileSystem_Stdio::QueryInterface( const char *pInterfaceName ) +{ + // We also implement the IMatSystemSurface interface + if (!Q_strncmp( pInterfaceName, FILESYSTEM_INTERFACE_VERSION, Q_strlen(FILESYSTEM_INTERFACE_VERSION) + 1)) + return (IFileSystem*)this; + + return CBaseFileSystem::QueryInterface( pInterfaceName ); +} + + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +bool CFileSystem_Stdio::GetOptimalIOConstraints( FileHandle_t hFile, unsigned *pOffsetAlign, unsigned *pSizeAlign, unsigned *pBufferAlign ) +{ + unsigned sectorSize; + + if ( hFile && UseOptimalBufferAllocation() ) + { + CFileHandle *fh = ( CFileHandle *)hFile; + sectorSize = fh->GetSectorSize(); + + if ( !sectorSize || ( fh->m_pPackFileHandle && ( fh->m_pPackFileHandle->AbsoluteBaseOffset() % sectorSize ) ) ) + { + sectorSize = 1; + } + } + else + { + sectorSize = 1; + } + + if ( pOffsetAlign ) + { + *pOffsetAlign = sectorSize; + } + + if ( pSizeAlign ) + { + *pSizeAlign = sectorSize; + } + + if ( pBufferAlign ) + { + if ( IsX360() ) + { + *pBufferAlign = 4; + } + else + { + *pBufferAlign = sectorSize; + } + } + + return ( sectorSize > 1 ); +} + + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +void *CFileSystem_Stdio::AllocOptimalReadBuffer( FileHandle_t hFile, unsigned nSize, unsigned nOffset ) +{ + if ( !UseOptimalBufferAllocation() ) + { + return CBaseFileSystem::AllocOptimalReadBuffer( hFile, nSize, nOffset ); + } + + unsigned sectorSize; + if ( hFile != FILESYSTEM_INVALID_HANDLE ) + { + CFileHandle *fh = ( CFileHandle *)hFile; + sectorSize = fh->GetSectorSize(); + + if ( !nSize ) + { + nSize = fh->Size(); + } + + if ( fh->m_pPackFileHandle ) + { + nOffset += fh->m_pPackFileHandle->AbsoluteBaseOffset(); + } + } + else + { + // an invalid handle gets a fake "optimal" but valid buffer + // this path is for a caller that isn't doing i/o, + // but needs an "optimal" buffer that can end up passed to FreeOptimalReadBuffer() + sectorSize = 4; + } + + bool bOffsetIsAligned = ( nOffset % sectorSize == 0 ); + unsigned nAllocSize = ( bOffsetIsAligned ) ? AlignValue( nSize, sectorSize ) : nSize; + + if ( IsX360() ) + { + return malloc( nAllocSize ); + } + else + { + unsigned nAllocAlignment = ( bOffsetIsAligned ) ? sectorSize : 4; + return _aligned_malloc( nAllocSize, nAllocAlignment ); + } +} + + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +void CFileSystem_Stdio::FreeOptimalReadBuffer( void *p ) +{ + if ( !UseOptimalBufferAllocation() ) + { + CBaseFileSystem::FreeOptimalReadBuffer( p ); + return; + } + + if ( p ) + { + if ( IsX360() ) + { + free( p ); + } + else + { + _aligned_free( p ); + } + } +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +FILE *CFileSystem_Stdio::FS_fopen( const char *filenameT, const char *options, unsigned flags, int64 *size ) +{ + CStdFilesystemFile *pFile = NULL; + + char filename[ MAX_PATH ]; + + CBaseFileSystem::FixUpPath ( filenameT, filename, sizeof( filename ) ); + +#ifdef _WIN32 + if ( CWin32ReadOnlyFile::CanOpen( filename, options ) ) + { + pFile = CWin32ReadOnlyFile::FS_fopen( filename, options, size ); + if ( pFile ) + { + return (FILE *)pFile; + } + } +#endif + + pFile = CStdioFile::FS_fopen( filename, options, size ); + + return (FILE *)pFile; +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +void CFileSystem_Stdio::FS_setbufsize( FILE *fp, unsigned nBytes ) +{ + CStdFilesystemFile *pFile = ((CStdFilesystemFile *)fp); + pFile->FS_setbufsize( nBytes ); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +void CFileSystem_Stdio::FS_fclose( FILE *fp ) +{ + CStdFilesystemFile *pFile = ((CStdFilesystemFile *)fp); + + pFile->FS_fclose(); + delete pFile; +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +void CFileSystem_Stdio::FS_fseek( FILE *fp, int64 pos, int seekType ) +{ + CStdFilesystemFile *pFile = ((CStdFilesystemFile *)fp); + + pFile->FS_fseek( pos, seekType ); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +long CFileSystem_Stdio::FS_ftell( FILE *fp ) +{ + CStdFilesystemFile *pFile = ((CStdFilesystemFile *)fp); + return pFile->FS_ftell(); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +int CFileSystem_Stdio::FS_feof( FILE *fp ) +{ + CStdFilesystemFile *pFile = ((CStdFilesystemFile *)fp); + return pFile->FS_feof(); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +size_t CFileSystem_Stdio::FS_fread( void *dest, size_t destSize, size_t size, FILE *fp ) +{ + tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s", __FUNCTION__ ); + if( ThreadInMainThread() ) + { + tmPlotI32( TELEMETRY_LEVEL0, TMPT_MEMORY, 0, size, "FileBytesRead" ); + } + + CStdFilesystemFile *pFile = ((CStdFilesystemFile *)fp); + size_t nBytesRead = pFile->FS_fread( dest, destSize, size); + + Trace_FRead( nBytesRead, fp ); + + return nBytesRead; +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +size_t CFileSystem_Stdio::FS_fwrite( const void *src, size_t size, FILE *fp ) +{ + tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s %t", __FUNCTION__, tmSendCallStack( TELEMETRY_LEVEL0, 0 ) ); + if( ThreadInMainThread() ) + { + tmPlotI32( TELEMETRY_LEVEL0, TMPT_MEMORY, 0, size, "FileBytesWrite" ); + } + + CStdFilesystemFile *pFile = ((CStdFilesystemFile *)fp); + + size_t nBytesWritten = pFile->FS_fwrite(src, size); + + return nBytesWritten; +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +bool CFileSystem_Stdio::FS_setmode( FILE *fp, FileMode_t mode ) +{ + CStdFilesystemFile *pFile = ((CStdFilesystemFile *)fp); + return pFile->FS_setmode( mode ); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +size_t CFileSystem_Stdio::FS_vfprintf( FILE *fp, const char *fmt, va_list list ) +{ + CStdFilesystemFile *pFile = ((CStdFilesystemFile *)fp); + return pFile->FS_vfprintf(fmt, list); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +int CFileSystem_Stdio::FS_ferror( FILE *fp ) +{ + CStdFilesystemFile *pFile = ((CStdFilesystemFile *)fp); + return pFile->FS_ferror(); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +int CFileSystem_Stdio::FS_fflush( FILE *fp ) +{ + CStdFilesystemFile *pFile = ((CStdFilesystemFile *)fp); + return pFile->FS_fflush(); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +char *CFileSystem_Stdio::FS_fgets( char *dest, int destSize, FILE *fp ) +{ + CStdFilesystemFile *pFile = ((CStdFilesystemFile *)fp); + return pFile->FS_fgets(dest, destSize); +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : *path - +// pmode - +// Output : int +//----------------------------------------------------------------------------- +int CFileSystem_Stdio::FS_chmod( const char *pathT, int pmode ) +{ + if ( !pathT ) + return -1; + + char path[ MAX_PATH ]; + + CBaseFileSystem::FixUpPath ( pathT, path, sizeof( path ) ); + + int rt = _chmod( path, pmode ); +#if defined(LINUX) + if (rt==-1) + { + char caseFixedName[ MAX_PATH ]; + const bool found = findFileInDirCaseInsensitive_safe( path, caseFixedName ); + if ( found ) + { + rt=_chmod( caseFixedName, pmode ); + } + } +#endif + return rt; +} + + +//----------------------------------------------------------------------------- +// Purpose: A replacement for _stat() backed by GetFileAttributesEx for XP users +// +// Workaround for: +// https://connect.microsoft.com/VisualStudio/feedback/details/1600505/stat-not-working-on-windows-xp-using-v14-xp-platform-toolset-vs2015 +// +// This is not well tested or meant to be a proper implementation of stat(), but rather a band-aid for XP users only +// until microsoft pushes a runtime update to fix above issue :-/ +//----------------------------------------------------------------------------- +#if defined(_WIN32) && defined(FILESYSTEM_MSVC2015_STAT_BUG_WORKAROUND) +static int WindowsXPStatShim( const char *pathT, struct _stat *buf ) +{ + WIN32_FILE_ATTRIBUTE_DATA fileAttributes; + if ( !GetFileAttributesEx(pathT, GetFileExInfoStandard, &fileAttributes) ) + { + *_errno() = ENOENT; + return -1; + } + + memset( buf, 0, sizeof(struct _stat) ); + + // Mode + unsigned short permBits = _S_IREAD; // If GetFileAttributes let us see it, we can read it. I think. + if ( fileAttributes.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ) + { + buf->st_mode |= _S_IFDIR; + permBits |= _S_IEXEC; + } + else + { + buf->st_mode |= _S_IFREG; + + const char *pExt = V_GetFileExtension( pathT ); + if ( V_strcasecmp( pExt, "exe" ) == 0 || + V_strcasecmp( pExt, "bat" ) == 0 || + V_strcasecmp( pExt, "com" ) == 0 || + V_strcasecmp( pExt, "cmd" ) == 0 ) + { + // Windows stat seems to set this flag for these extensions + permBits |= _S_IEXEC; + } + } + + if ( fileAttributes.dwFileAttributes & FILE_ATTRIBUTE_READONLY ) + { + permBits |= S_IWRITE; + } + + // Duplicate permission bits to user/group/world (windows stat doesn't care) + buf->st_mode |= permBits | permBits >> 3 | permBits >> 6; + + // Device is just drive-letter-index according to msdn + char driveLetter = tolower( pathT[ 0 ] ); + if ( driveLetter >= 'a' && driveLetter <= 'z' && pathT[1] == ':' ) + { + unsigned char driveIdx = driveLetter - 'a'; + buf->st_dev = driveIdx; + buf->st_rdev = driveIdx; + } + else + { + buf->st_dev = _getdrive(); + buf->st_rdev = _getdrive(); + } + + buf->st_nlink = 1; + buf->st_size = (uint64_t)fileAttributes.nFileSizeHigh << 32 | fileAttributes.nFileSizeLow; + // The 90s was a hell of a time I guess. + uint64_t actualAccessTime = (uint64_t)fileAttributes.ftLastAccessTime.dwHighDateTime << 32 | (uint64_t)fileAttributes.ftLastAccessTime.dwLowDateTime; + uint64_t actualModTime = (uint64_t)fileAttributes.ftLastWriteTime.dwHighDateTime << 32 | (uint64_t)fileAttributes.ftLastWriteTime.dwLowDateTime; + uint64_t actualCreationTime = (uint64_t)fileAttributes.ftCreationTime.dwHighDateTime << 32 | (uint64_t)fileAttributes.ftCreationTime.dwLowDateTime; + uint64_t ullMSUniverseToEveryoneElseUniverse = (369 * 365 + 89) * 86400ull; // okay + buf->st_atime = actualAccessTime / 10000000ull - ullMSUniverseToEveryoneElseUniverse; + buf->st_mtime = actualModTime / 10000000ull - ullMSUniverseToEveryoneElseUniverse; + buf->st_ctime = actualCreationTime / 10000000ull - ullMSUniverseToEveryoneElseUniverse; + + // st_uid/st_gid always 0 on windows + + return 0; +} +#endif // defined(_WIN32) && defined(FILESYSTEM_MSVC2015_STAT_BUG_WORKAROUND) + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +int CFileSystem_Stdio::FS_stat( const char *pathT, struct _stat *buf, bool *pbLoadedFromSteamCache ) +{ + if ( pbLoadedFromSteamCache ) + *pbLoadedFromSteamCache = false; + + if ( !pathT ) + { + return -1; + } + + char path[ MAX_PATH ]; + + CBaseFileSystem::FixUpPath ( pathT, path, sizeof( path ) ); + + int rt = _stat( path, buf ); + + // Workaround bug wherein stat() randomly fails on Windows XP. See comment on function. +#if defined(_WIN32) && defined(FILESYSTEM_MSVC2015_STAT_BUG_WORKAROUND) + if ( rt == -1 ) + { + EOSType eOSType = GetOSType(); + if ( eOSType == k_eWin2000 || eOSType == k_eWinXP || eOSType == k_eWin2003 ) + { + rt = WindowsXPStatShim( path, buf ); + } + } +#endif // defined(_WIN32) && defined(FILESYSTEM_MSVC2015_STAT_BUG_WORKAROUND) + +#if defined(LINUX) + if ( rt == -1 ) + { + char caseFixedName[ MAX_PATH ]; + bool found = findFileInDirCaseInsensitive_safe( path, caseFixedName ); + if ( found ) + { + rt = _stat( caseFixedName, buf ); + } + } +#endif + return rt; +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +HANDLE CFileSystem_Stdio::FS_FindFirstFile(const char *findnameT, WIN32_FIND_DATA *dat) +{ + char findname[ MAX_PATH ]; + + CBaseFileSystem::FixUpPath ( findnameT, findname, sizeof( findname ) ); + + return ::FindFirstFile(findname, dat); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +bool CFileSystem_Stdio::FS_FindNextFile(HANDLE handle, WIN32_FIND_DATA *dat) +{ + + if (INVALID_HANDLE_VALUE == handle) // invalid handle should return false + return false; + + return (::FindNextFile(handle, dat) != 0); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +bool CFileSystem_Stdio::FS_FindClose(HANDLE handle) +{ + return (::FindClose(handle) != 0); +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +int CFileSystem_Stdio::FS_GetSectorSize( FILE *fp ) +{ + CStdFilesystemFile *pFile = ((CStdFilesystemFile *)fp); + return pFile->FS_GetSectorSize(); +} + +//----------------------------------------------------------------------------- +// Purpose: files are always immediately available on disk +//----------------------------------------------------------------------------- +bool CFileSystem_Stdio::IsFileImmediatelyAvailable(const char *pFileName) +{ + return true; +} + +// enable this if you want the stdio filesystem to pretend it's steam, and make people wait for resources +//#define DEBUG_WAIT_FOR_RESOURCES_API + +#if defined(DEBUG_WAIT_FOR_RESOURCES_API) +static float g_flDebugProgress = 0.0f; +#endif + +//----------------------------------------------------------------------------- +// Purpose: steam call, unnecessary in stdio +//----------------------------------------------------------------------------- +WaitForResourcesHandle_t CFileSystem_Stdio::WaitForResources( const char *resourcelist ) +{ +#if defined(DEBUG_WAIT_FOR_RESOURCES_API) + g_flDebugProgress = 0.0f; +#endif + + return 1; +} + +//----------------------------------------------------------------------------- +// Purpose: steam call, unnecessary in stdio +//----------------------------------------------------------------------------- +bool CFileSystem_Stdio::GetWaitForResourcesProgress( WaitForResourcesHandle_t handle, float *progress /* out */ , bool *complete /* out */ ) +{ +#if defined(DEBUG_WAIT_FOR_RESOURCES_API) + g_flDebugProgress += 0.002f; + if (g_flDebugProgress < 1.0f) + { + *progress = g_flDebugProgress; + *complete = false; + return true; + } +#endif + + // always return that we're complete + *progress = 0.0f; + *complete = true; + return true; +} + +//----------------------------------------------------------------------------- +// Purpose: steam call, unnecessary in stdio +//----------------------------------------------------------------------------- +void CFileSystem_Stdio::CancelWaitForResources( WaitForResourcesHandle_t handle ) +{ +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void CFileSystem_Stdio::GetLocalCopy( const char *pFileName ) +{ + // do nothing. . everything is local. +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +int CFileSystem_Stdio::HintResourceNeed( const char *hintlist, int forgetEverything ) +{ + // do nothing. . everything is local. + return 0; +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +CStdioFile *CStdioFile::FS_fopen( const char *filenameT, const char *options, int64 *size ) +{ + FILE *pFile = NULL; + char *p = NULL; + char filename[MAX_PATH]; + struct _stat buf; + + V_strncpy( filename, filenameT, sizeof(filename) ); + + // stop newline characters at end of filename + p = strchr( filename, '\n' ); + if ( p ) + *p = '\0'; + p = strchr( filename, '\r' ); + if ( p ) + *p = '\0'; + + + pFile = fopen(filename, options); + if (pFile && size) + { + // todo: replace with filelength()? + int rt = _stat( filename, &buf ); + if (rt == 0) + { + *size = buf.st_size; + } + } + +#if defined(LINUX) + if(!pFile && !strchr(options,'w') && !strchr(options,'+') ) // try opening the lower cased version + { + char caseFixedName[ MAX_PATH ]; + bool found = findFileInDirCaseInsensitive_safe( filename, caseFixedName ); + if ( found ) + { + pFile = fopen( caseFixedName, options ); + + if (pFile && size) + { + // todo: replace with filelength()? + struct _stat buf; + int rt = _stat( caseFixedName, &buf ); + if (rt == 0) + { + *size = buf.st_size; + } + } + } + } +#endif + + if ( pFile ) + { + bool bWriteable = false; + if ( strchr(options,'w') || strchr(options,'a') ) + bWriteable = true; + +#if defined POSIX + if ( bWriteable ) + { + CThreadMutex *pMutex = NULL; + + { + AUTO_LOCK( m_MutexLockedFD ); + // Win32 has an undocumented feature that is serialized ALL writes to a file across threads (i.e only 1 thread can open a file at a time) + // so add a lock here to mimic that behavior + + int iLockID = m_LockedFDMap.Find( buf.st_ino ); + if ( iLockID != m_LockedFDMap.InvalidIndex() ) + { + pMutex = m_LockedFDMap[iLockID]; + } + else + { + CThreadMutex *newMutex = new CThreadMutex; + pMutex = m_LockedFDMap[m_LockedFDMap.Insert( buf.st_ino, newMutex )]; + } + } + // grab the lock once we have UNLOCKED m_MutexLockedFD so we don't deadlock on a close + pMutex->Lock(); + + rewind( pFile ); + + // we need to get the file size again after the lock returns + if (pFile && size) + { + int rt = _stat( filename, &buf ); + if (rt == 0) + { + *size = buf.st_size; + } + } + + } +#endif + return new CStdioFile( pFile, bWriteable ); + } + + return NULL; +} + + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +void CStdioFile::FS_setbufsize( unsigned nBytes ) +{ +#ifdef _WIN32 + if ( nBytes ) + { + setvbuf( m_pFile, NULL, _IOFBF, 32768 ); + } + else + { + setvbuf( m_pFile, NULL, _IONBF, 0 ); +#if defined(_MSC_VER) && ( _MSC_VER < 1900 ) + // hack to make microsoft stdio not always read one stray byte on odd sized files + m_pFile->_bufsiz = 1; +#endif + } +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +void CStdioFile::FS_fclose() +{ +#ifdef POSIX + if ( m_bWriteable ) + { + AUTO_LOCK( m_MutexLockedFD ); + + struct _stat buf; + int fd = fileno_unlocked( m_pFile ); + fstat( fd, &buf ); + + fflush( m_pFile ); + int iLockID = m_LockedFDMap.Find( buf.st_ino ); + if ( iLockID != m_LockedFDMap.InvalidIndex() ) + { + m_LockedFDMap[iLockID]->Unlock(); + } + } +#endif + fclose(m_pFile); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +void CStdioFile::FS_fseek( int64 pos, int seekType ) +{ + fseek( m_pFile, pos, seekType ); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +long CStdioFile::FS_ftell() +{ + return ftell(m_pFile); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +int CStdioFile::FS_feof() +{ + return feof(m_pFile); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +size_t CStdioFile::FS_fread( void *dest, size_t destSize, size_t size ) +{ + tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s %t", __FUNCTION__, tmSendCallStack( TELEMETRY_LEVEL0, 0 ) ); + if( ThreadInMainThread() ) + { + tmPlotI32( TELEMETRY_LEVEL0, TMPT_MEMORY, 0, size, "FileBytesRead" ); + } + + // read (size) of bytes to ensure truncated reads returns bytes read and not 0 + return fread( dest, 1, size, m_pFile ); +} + + +#define WRITE_CHUNK (256 * 1024) + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +// +// This routine breaks data into chunks if the amount to be written is beyond WRITE_CHUNK (256kb) +// Windows can fail on monolithic writes of ~12MB or more, so we work around that here +//----------------------------------------------------------------------------- +size_t CStdioFile::FS_fwrite( const void *src, size_t size ) +{ + tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s %t", __FUNCTION__, tmSendCallStack( TELEMETRY_LEVEL0, 0 ) ); + if( ThreadInMainThread() ) + { + tmPlotI32( TELEMETRY_LEVEL0, TMPT_MEMORY, 0, size, "FileBytesWrite" ); + } + + if ( size > WRITE_CHUNK ) + { + size_t remaining = size; + const byte* current = (const byte *) src; + size_t total = 0; + + while ( remaining > 0 ) + { + size_t bytesToCopy = min(remaining, (size_t)WRITE_CHUNK); + + total += fwrite(current, 1, bytesToCopy, m_pFile); + + remaining -= bytesToCopy; + current += bytesToCopy; + } + + Assert( total == size ); + return total; + } + + return fwrite(src, 1, size, m_pFile);// return number of bytes written (because we have size = 1, count = bytes, so it return bytes) +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +bool CStdioFile::FS_setmode( FileMode_t mode ) +{ +#ifdef _WIN32 + int fd = _fileno( m_pFile ); + int newMode = ( mode == FM_BINARY ) ? _O_BINARY : _O_TEXT; + return ( _setmode( fd, newMode) != -1 ); +#else + return false; +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +size_t CStdioFile::FS_vfprintf( const char *fmt, va_list list ) +{ + return vfprintf(m_pFile, fmt, list); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +int CStdioFile::FS_ferror() +{ + return ferror(m_pFile); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +int CStdioFile::FS_fflush() +{ + return fflush(m_pFile); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +char *CStdioFile::FS_fgets( char *dest, int destSize ) +{ + return fgets(dest, destSize, m_pFile); +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +#ifdef _WIN32 + +ConVar filesystem_use_overlapped_io( "filesystem_use_overlapped_io", "1", 0, "" ); +#define UseOverlappedIO() filesystem_use_overlapped_io.GetBool() + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +int GetSectorSize( const char *pszFilename ) +{ + if ( ( !pszFilename[0] || !pszFilename[1] ) || + ( pszFilename[0] == '\\' && pszFilename[1] == '\\' ) || + ( pszFilename[0] == '/' && pszFilename[1] == '/' ) ) + { + // Cannot determine sector size with a UNC path (need volume identifier) + return 0; + } + + if ( IsX360() ) + { + // purposely dvd centric, which is also the worst case + return XBOX_DVD_SECTORSIZE; + } + +#if defined( _WIN32 ) && !defined( FILESYSTEM_STEAM ) && !defined( _X360 ) + char szAbsoluteFilename[MAX_FILEPATH]; + if ( pszFilename[1] != ':' ) + { + Q_MakeAbsolutePath( szAbsoluteFilename, sizeof(szAbsoluteFilename), pszFilename ); + pszFilename = szAbsoluteFilename; + } + + DWORD sectorSize = 1; + + struct DriveSectorSize_t + { + char volume; + DWORD sectorSize; + }; + + static DriveSectorSize_t cachedSizes[4]; + + char volume = tolower( *pszFilename ); + + int i; + for ( i = 0; i < ARRAYSIZE(cachedSizes) && cachedSizes[i].volume; i++ ) + { + if ( cachedSizes[i].volume == volume ) + { + sectorSize = cachedSizes[i].sectorSize; + break; + } + } + + if ( sectorSize == 1 ) + { + char root[4] = "X:\\"; + root[0] = *pszFilename; + + DWORD ignored; + if ( !GetDiskFreeSpace( root, &ignored, §orSize, &ignored, &ignored ) ) + { + sectorSize = 0; + } + + if ( i < ARRAYSIZE(cachedSizes) ) + { + cachedSizes[i].volume = volume; + cachedSizes[i].sectorSize = sectorSize; + } + } + + return sectorSize; +#else + return 0; +#endif +} + + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- + +class CThreadIOEventPool +{ +public: + ~CThreadIOEventPool() + { + CThreadEvent *pEvent; + + while ( m_Events.PopItem( &pEvent ) ) + { + delete pEvent; + } + } + + CThreadEvent *GetEvent() + { + CThreadEvent *pEvent; + + if ( m_Events.PopItem( &pEvent ) ) + { + return pEvent; + } + + return new CThreadEvent; + } + + void ReleaseEvent( CThreadEvent *pEvent ) + { + m_Events.PushItem( pEvent ); + } + +private: + CTSList<CThreadEvent *> m_Events; +}; + + +CThreadIOEventPool g_ThreadIOEvents; + + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +bool CWin32ReadOnlyFile::CanOpen( const char *filename, const char *options ) +{ + return ( options[0] == 'r' && options[1] == 'b' && options[2] == 0 && filesystem_native.GetBool() ); +} + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- + +static HANDLE OpenWin32File( const char *filename, bool bOverlapped, bool bUnbuffered, int64 *pFileSize ) +{ + HANDLE hFile; + + DWORD createFlags = FILE_ATTRIBUTE_NORMAL; + + if ( bOverlapped ) + { + createFlags |= FILE_FLAG_OVERLAPPED; + } + + if ( bUnbuffered ) + { + createFlags |= FILE_FLAG_NO_BUFFERING; + } + + hFile = ::CreateFile( filename, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, createFlags, NULL ); + if ( hFile != INVALID_HANDLE_VALUE && !*pFileSize ) + { + LARGE_INTEGER fileSize; + if ( !GetFileSizeEx( hFile, &fileSize ) ) + { + CloseHandle( hFile ); + hFile = INVALID_HANDLE_VALUE; + } + *pFileSize = fileSize.QuadPart; + } + return hFile; +} + +CWin32ReadOnlyFile *CWin32ReadOnlyFile::FS_fopen( const char *filename, const char *options, int64 *size ) +{ + Assert( CanOpen( filename, options ) ); + + int sectorSize = 0; + bool bTryUnbuffered = ( UseUnbufferedIO() && ( sectorSize = GetSectorSize( filename ) ) != 0 ); + bool bOverlapped = UseOverlappedIO(); + + HANDLE hFileUnbuffered = INVALID_HANDLE_VALUE; + int64 fileSize = 0; + + if ( bTryUnbuffered ) + { + hFileUnbuffered = OpenWin32File( filename, bOverlapped, true, &fileSize ); + if ( hFileUnbuffered == INVALID_HANDLE_VALUE ) + { + return NULL; + } + } + + HANDLE hFileBuffered = OpenWin32File( filename, bOverlapped, false, &fileSize ); + if ( hFileBuffered == INVALID_HANDLE_VALUE ) + { + if ( hFileUnbuffered != INVALID_HANDLE_VALUE ) + { + CloseHandle( hFileUnbuffered ); + } + return NULL; + } + + if ( size ) + { + *size = fileSize; + } + + return new CWin32ReadOnlyFile( hFileUnbuffered, hFileBuffered, ( sectorSize ) ? sectorSize : 1, fileSize, bOverlapped ); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +void CWin32ReadOnlyFile::FS_fclose() +{ + if ( m_hFileUnbuffered != INVALID_HANDLE_VALUE ) + { + CloseHandle( m_hFileUnbuffered ); + } + + if ( m_hFileBuffered != INVALID_HANDLE_VALUE ) + { + CloseHandle( m_hFileBuffered ); + } +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +void CWin32ReadOnlyFile::FS_fseek( int64 pos, int seekType ) +{ + switch ( seekType ) + { + case SEEK_SET: + m_ReadPos = pos; + break; + + case SEEK_CUR: + m_ReadPos += pos; + break; + + case SEEK_END: + m_ReadPos = m_Size - pos; + break; + } +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +long CWin32ReadOnlyFile::FS_ftell() +{ + return m_ReadPos; +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +int CWin32ReadOnlyFile::FS_feof() +{ + return ( m_ReadPos >= m_Size ); +} + +// ends up on a thread's stack, don't blindly increase without awareness of that implication +// 360 threads have small stacks, using small buffer of the worst case quantum sector size +#if !defined( _X360 ) +#define READ_TEMP_BUFFER ( 32*1024 ) +#else +#define READ_TEMP_BUFFER ( 2*XBOX_DVD_SECTORSIZE ) +#endif + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +size_t CWin32ReadOnlyFile::FS_fread( void *dest, size_t destSize, size_t size ) +{ + VPROF_BUDGET( "CWin32ReadOnlyFile::FS_fread", VPROF_BUDGETGROUP_OTHER_FILESYSTEM ); + tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s %t", __FUNCTION__, tmSendCallStack( TELEMETRY_LEVEL0, 0 ) ); + if( ThreadInMainThread() ) + { + tmPlotI32( TELEMETRY_LEVEL0, TMPT_MEMORY, 0, size, "FileBytesRead" ); + } + + if ( !size || ( m_hFileUnbuffered == INVALID_HANDLE_VALUE && m_hFileBuffered == INVALID_HANDLE_VALUE ) ) + { + return 0; + } + + CThreadEvent *pEvent = NULL; + + if ( destSize == (size_t)-1 ) + { + destSize = size; + } + + byte tempBuffer[READ_TEMP_BUFFER]; + HANDLE hReadFile = m_hFileBuffered; + int nBytesToRead = size; + byte *pDest = (byte *)dest; + int64 offset = m_ReadPos; + + if ( m_hFileUnbuffered != INVALID_HANDLE_VALUE ) + { + const int destBaseAlign = ( IsX360() ) ? 4 : m_SectorSize; + bool bDestBaseIsAligned = ( (DWORD)dest % destBaseAlign == 0 ); + bool bCanReadUnbufferedDirect = ( bDestBaseIsAligned && ( destSize % m_SectorSize == 0 ) && ( m_ReadPos % m_SectorSize == 0 ) ); + + if ( bCanReadUnbufferedDirect ) + { + // fastest path, unbuffered + nBytesToRead = AlignValue( size, m_SectorSize ); + hReadFile = m_hFileUnbuffered; + } + else + { + // not properly aligned, snap to alignments + // attempt to perform single unbuffered operation using stack buffer + int64 alignedOffset = AlignValue( ( m_ReadPos - m_SectorSize ) + 1, m_SectorSize ); + unsigned int alignedBytesToRead = AlignValue( ( m_ReadPos - alignedOffset ) + size, m_SectorSize ); + if ( alignedBytesToRead <= sizeof( tempBuffer ) - destBaseAlign ) + { + // read operation can be performed as unbuffered follwed by a post fixup + nBytesToRead = alignedBytesToRead; + offset = alignedOffset; + pDest = AlignValue( tempBuffer, destBaseAlign ); + hReadFile = m_hFileUnbuffered; + } + } + } + + OVERLAPPED overlapped = { 0 }; + if ( m_bOverlapped ) + { + pEvent = g_ThreadIOEvents.GetEvent(); + overlapped.hEvent = *pEvent; + } + +#ifdef REPORT_BUFFERED_IO + if ( hReadFile == m_hFileBuffered && filesystem_report_buffered_io.GetBool() ) + { + Msg( "Buffered Operation :(\n" ); + } +#endif + + // some disk drivers will fail if read is too large + static int MAX_READ = filesystem_max_stdio_read.GetInt()*1024*1024; + const int MIN_READ = 64*1024; + bool bReadOk = true; + DWORD nBytesRead = 0; + size_t result = 0; + int64 currentOffset = offset; + + while ( bReadOk && nBytesToRead > 0 ) + { + int nCurBytesToRead = min( nBytesToRead, MAX_READ ); + DWORD nCurBytesRead = 0; + + overlapped.Offset = currentOffset & 0xFFFFFFFF; + overlapped.OffsetHigh = ( currentOffset >> 32 ) & 0xFFFFFFFF; + + bReadOk = ( ::ReadFile( hReadFile, pDest + nBytesRead, nCurBytesToRead, &nCurBytesRead, &overlapped ) != 0 ); + if ( !bReadOk ) + { + if ( m_bOverlapped && GetLastError() == ERROR_IO_PENDING ) + { + bReadOk = true; + } + } + + if ( bReadOk ) + { + if ( !m_bOverlapped || GetOverlappedResult( hReadFile, &overlapped, &nCurBytesRead, true ) ) + { + nBytesRead += nCurBytesRead; + nBytesToRead -= nCurBytesRead; + currentOffset += nCurBytesRead; + } + else + { + if ( m_bOverlapped ) + { + if ( GetLastError() == ERROR_HANDLE_EOF ) + { + nBytesToRead = 0; // we have hit the end of the file + } + else + { + bReadOk = false; + } + } + else + { + bReadOk = false; + } + } + + if ( !m_bOverlapped && nCurBytesRead == 0 ) + { + nBytesToRead = 0; // we have hit the end of the file + } + + } + + if ( nBytesToRead > 0 && nCurBytesRead == 0 ) // if you failed to ready anything this time then bail the loop + { + DevMsg( "Got zero length read" ); + bReadOk = false; + } + else if ( !bReadOk ) + { + DWORD dwError = GetLastError(); + + if ( IsX360() ) + { + if ( dwError == ERROR_DISK_CORRUPT || dwError == ERROR_FILE_CORRUPT ) + { + FSDirtyDiskReportFunc_t func = g_FileSystem_Stdio.GetDirtyDiskReportFunc(); + if ( func ) + { + func(); + result = 0; + } + } + } + + if ( dwError == ERROR_NO_SYSTEM_RESOURCES && MAX_READ > MIN_READ ) + { + MAX_READ /= 2; + bReadOk = true; + DevMsg( "ERROR_NO_SYSTEM_RESOURCES: Reducing max read to %d bytes\n", MAX_READ ); + } + else + { + DevMsg( "Unknown read error %d\n", dwError ); + } + } + } + + if ( bReadOk ) + { + if ( nBytesRead && hReadFile == m_hFileUnbuffered && pDest != dest ) + { + int nBytesExtra = ( m_ReadPos - offset ); + nBytesRead -= nBytesExtra; + if ( nBytesRead ) + { + memcpy( dest, (byte *)pDest + nBytesExtra, size ); + } + } + + result = min( (size_t)nBytesRead, size ); + } + + if ( m_bOverlapped ) + { + pEvent->Reset(); + g_ThreadIOEvents.ReleaseEvent( pEvent ); + } + + m_ReadPos += result; + + return result; +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +char *CWin32ReadOnlyFile::FS_fgets( char *dest, int destSize ) +{ + if ( FS_feof() ) + { + return NULL; + } + int nStartPos = m_ReadPos; + int nBytesRead = FS_fread( dest, destSize, destSize ); + if ( !nBytesRead ) + { + return NULL; + } + + dest[min( nBytesRead, destSize - 1)] = 0; + char *pNewline = strchr( dest, '\n' ); + if ( pNewline ) + { + // advance past, leave \n + pNewline++; + *pNewline = 0; + } + else + { + pNewline = &dest[min( nBytesRead, destSize - 1)]; + } + m_ReadPos = nStartPos + ( pNewline - dest ) + 1; + + return dest; +} + + +#endif // _WIN32 diff --git a/filesystem/filesystem_stdio.vpc b/filesystem/filesystem_stdio.vpc new file mode 100644 index 0000000..18e7255 --- /dev/null +++ b/filesystem/filesystem_stdio.vpc @@ -0,0 +1,107 @@ +//----------------------------------------------------------------------------- +// FILESYSTEM_STDIO.VPC +// +// Project Script +//----------------------------------------------------------------------------- + +$macro SRCDIR ".." +$Macro OUTBINDIR "$SRCDIR\..\game\bin" + +$include "$SRCDIR\vpc_scripts\source_dll_base.vpc" +$include "$SRCDIR\vpc_scripts\source_cryptlib_include.vpc" + +$Configuration "Debug" +{ + $General + { + $OutputDirectory "Debug_Stdio" [$WINDOWS] + $IntermediateDirectory "Debug_Stdio" [$WINDOWS] + $OutputDirectory "Debug_Stdio_360" [$X360] + $IntermediateDirectory "Debug_Stdio_360" [$X360] + } +} + +$Configuration "Release" +{ + $General + { + $OutputDirectory "Release_Stdio" [$WINDOWS] + $IntermediateDirectory "Release_Stdio" [$WINDOWS] + $OutputDirectory "Release_Stdio_360" [$X360] + $IntermediateDirectory "Release_Stdio_360" [$X360] + } +} + +$Configuration +{ + $Compiler + { + $PreprocessorDefinitions "$BASE;FILESYSTEM_STDIO_EXPORTS;DONT_PROTECT_FILEIO_FUNCTIONS;PROTECTED_THINGS_ENABLE" [$WIN64] + $PreprocessorDefinitions "$BASE;FILESYSTEM_STDIO_EXPORTS;DONT_PROTECT_FILEIO_FUNCTIONS;PROTECTED_THINGS_ENABLE;_USE_32BIT_TIME_T" [!$WIN64] + + // Enable super-fun workaround for using MSVC2015 to target Windows XP + // https://connect.microsoft.com/VisualStudio/feedback/details/1600505/stat-not-working-on-windows-xp-using-v14-xp-platform-toolset-vs2015 + $PreprocessorDefinitions "$BASE;FILESYSTEM_MSVC2015_STAT_BUG_WORKAROUND" [$VS2015] + + // Add VPK support + $PreprocessorDefinitions "$BASE;SUPPORT_PACKED_STORE" [!$WIN64] + } + $Linker + { + $SystemLibraries "iconv" [$OSXALL] + } +} + +$Project "FileSystem_Stdio" +{ + $Folder "Source Files" + { + $File "basefilesystem.cpp" + $File "packfile.cpp" + $File "filetracker.cpp" + $File "filesystem_async.cpp" + $File "filesystem_stdio.cpp" + $File "$SRCDIR\public\kevvaluescompiler.cpp" + $File "$SRCDIR\public\zip_utils.cpp" + $File "QueuedLoader.cpp" + $File "linux_support.cpp" [$POSIX] + } + + + $Folder "Header Files" + { + $File "basefilesystem.h" + $File "packfile.h" + $File "filetracker.h" + $File "threadsaferefcountedobject.h" + $File "$SRCDIR\public\tier0\basetypes.h" + $File "$SRCDIR\public\bspfile.h" + $File "$SRCDIR\public\bspflags.h" + $File "$SRCDIR\public\mathlib\bumpvects.h" + $File "$SRCDIR\public\tier1\characterset.h" + $File "$SRCDIR\public\tier0\dbg.h" + $File "$SRCDIR\public\tier0\fasttimer.h" + $File "$SRCDIR\public\filesystem.h" + $File "$SRCDIR\public\ifilelist.h" + $File "$SRCDIR\public\appframework\IAppSystem.h" + $File "$SRCDIR\public\tier1\interface.h" + $File "$SRCDIR\public\mathlib\mathlib.h" + $File "$SRCDIR\public\tier0\platform.h" + $File "$SRCDIR\public\tier1\strtools.h" + $File "$SRCDIR\public\tier1\utlmemory.h" + $File "$SRCDIR\public\tier1\utlrbtree.h" + $File "$SRCDIR\public\tier1\utlsymbol.h" + $File "$SRCDIR\public\tier1\utlvector.h" + $File "$SRCDIR\public\mathlib\vector.h" + $File "$SRCDIR\public\mathlib\vector2d.h" + $File "$SRCDIR\public\mathlib\vector4d.h" + $File "$SRCDIR\public\vstdlib\vstdlib.h" + $File "$SRCDIR\public\keyvaluescompiler.h" + $File "$SRCDIR\public\filesystem\IQueuedLoader.h" + } + + $Folder "Link Libraries" + { + $Lib tier2 + } +} diff --git a/filesystem/filesystem_stdio/ThreadSafeRefCountedObject.h b/filesystem/filesystem_stdio/ThreadSafeRefCountedObject.h new file mode 100644 index 0000000..89164db --- /dev/null +++ b/filesystem/filesystem_stdio/ThreadSafeRefCountedObject.h @@ -0,0 +1,85 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +//============================================================================= + +#ifndef THREADSAFEREFCOUNTEDOBJECT_H +#define THREADSAFEREFCOUNTEDOBJECT_H +#ifdef _WIN32 +#pragma once +#endif + + +// This class can be used for fast access to an object from multiple threads, +// and the main thread can wait until the threads are done using the object before it frees the object. +template< class T > +class CThreadSafeRefCountedObject +{ +public: + CThreadSafeRefCountedObject( T initVal ) + { + m_RefCount = 0; + m_pObject = initVal; + m_RefCount = 0; + } + + void Init( T pObj ) + { + Assert( ThreadInMainThread() ); + Assert( !m_pObject ); + m_RefCount = 0; + m_pObject = pObj; + m_RefCount = 1; + } + + // Threads that access the object need to use AddRef/Release to access it. + T AddRef() + { + if ( ++m_RefCount > 1 ) + { + return m_pObject; + } + else + { + // If the refcount was 0 when we called this, then the whitelist is about to be freed. + --m_RefCount; + return NULL; + } + } + void ReleaseRef( T pObj ) + { + if ( --m_RefCount >= 1 ) + { + Assert( m_pObject == pObj ); + } + } + + // The main thread can use this to access the object, since only it can Init() and Free() the object. + T GetInMainThread() + { + Assert( ThreadInMainThread() ); + return m_pObject; + } + + // The main thread calls this after it has released its last reference to the object. + void ResetWhenNoRemainingReferences( T newValue ) + { + Assert( ThreadInMainThread() ); + + // Wait until we can free it. + while ( m_RefCount > 0 ) + { + CThread::Sleep( 20 ); + } + + m_pObject = newValue; + } + +private: + CInterlockedIntT<long> m_RefCount; + T m_pObject; +}; + + +#endif // THREADSAFEREFCOUNTEDOBJECT_H diff --git a/filesystem/filesystem_steam.cpp b/filesystem/filesystem_steam.cpp new file mode 100644 index 0000000..8456f85 --- /dev/null +++ b/filesystem/filesystem_steam.cpp @@ -0,0 +1,1536 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//=============================================================================// + +#include "basefilesystem.h" +#include "steamcommon.h" +#include "SteamInterface.h" +#include "tier0/dbg.h" +#include "tier0/icommandline.h" +#include "steam/steam_api.h" +#ifdef POSIX +#include <fcntl.h> +#ifdef LINUX +#include <sys/file.h> +#endif +#include <dlfcn.h> +#define _S_IWRITE S_IWRITE +#define _S_IWRITE S_IWRITE +#define _S_IFREG S_IFREG +#define FILE_ATTRIBUTE_OFFLINE 0x1000 +#endif + +#ifdef _WIN32 + extern "C" + { + __declspec(dllimport) int __stdcall IsDebuggerPresent(); + } +#endif + +ISteamInterface *steam = NULL; +static SteamHandle_t g_pLastErrorFile; +static TSteamError g_tLastError; +static TSteamError g_tLastErrorNoFile; + +void CheckError( SteamHandle_t fp, TSteamError & steamError) +{ + if (steamError.eSteamError == eSteamErrorContentServerConnect) + { + // fatal error +#ifdef WIN32 + // kill the current window so the user can see the error + HWND hwnd = GetForegroundWindow(); + if (hwnd) + { + DestroyWindow(hwnd); + } + + // show the error + MessageBox(NULL, "Could not acquire necessary game files because the connection to Steam servers was lost.", "Source - Fatal Error", MB_OK | MB_ICONEXCLAMATION); + + // get out of here immediately + TerminateProcess(GetCurrentProcess(), 0); +#else + fprintf( stderr, "Could not acquire necessary game files because the connection to Steam servers was lost." ); + exit(-1); +#endif + return; + } + + if (fp) + { + if (steamError.eSteamError != eSteamErrorNone || g_tLastError.eSteamError != eSteamErrorNone) + { + g_pLastErrorFile = fp; + g_tLastError = steamError; + } + } + else + { + // write to the NULL error checker + if (steamError.eSteamError != eSteamErrorNone || g_tLastErrorNoFile.eSteamError != eSteamErrorNone) + { + g_tLastErrorNoFile = steamError; + } + } +} + + +#ifdef POSIX +class CSteamFile +{ +public: + explicit CSteamFile( SteamHandle_t file, bool bWriteable, const char *pchName ) : m_File( file ), m_bWriteable( bWriteable ), m_FileName(pchName) {} + ~CSteamFile() {} + SteamHandle_t Handle() { return m_File; } + bool BWriteable() { return m_bWriteable; } + CUtlSymbol GetFileName() { return m_FileName; } +private: + SteamHandle_t m_File; + bool m_bWriteable; + CUtlSymbol m_FileName; +}; +#endif + + +class CFileSystem_Steam : public CBaseFileSystem +{ +public: + CFileSystem_Steam(); + ~CFileSystem_Steam(); + + // Methods of IAppSystem + virtual InitReturnVal_t Init(); + virtual void Shutdown(); + virtual void * QueryInterface( const char *pInterfaceName ); + + // Higher level filesystem methods requiring specific behavior + virtual void GetLocalCopy( const char *pFileName ); + virtual int HintResourceNeed( const char *hintlist, int forgetEverything ); + virtual CSysModule * LoadModule( const char *pFileName, const char *pPathID, bool bValidatedDllOnly ); + virtual bool IsFileImmediatelyAvailable(const char *pFileName); + + // resource waiting + virtual WaitForResourcesHandle_t WaitForResources( const char *resourcelist ); + virtual bool GetWaitForResourcesProgress( WaitForResourcesHandle_t handle, float *progress /* out */ , bool *complete /* out */ ); + virtual void CancelWaitForResources( WaitForResourcesHandle_t handle ); + virtual bool IsSteam() const { return true; } + virtual FilesystemMountRetval_t MountSteamContent( int nExtraAppId = -1 ); + +protected: + // implementation of CBaseFileSystem virtual functions + virtual FILE *FS_fopen( const char *filename, const char *options, unsigned flags, int64 *size, CFileLoadInfo *pInfo ); + virtual void FS_setbufsize( FILE *fp, unsigned nBytes ); + virtual void FS_fclose( FILE *fp ); + virtual void FS_fseek( FILE *fp, int64 pos, int seekType ); + virtual long FS_ftell( FILE *fp ); + virtual int FS_feof( FILE *fp ); + virtual size_t FS_fread( void *dest, size_t destSize, size_t size, FILE *fp ); + virtual size_t FS_fwrite( const void *src, size_t size, FILE *fp ); + virtual size_t FS_vfprintf( FILE *fp, const char *fmt, va_list list ); + virtual int FS_ferror( FILE *fp ); + virtual int FS_fflush( FILE *fp ); + virtual char *FS_fgets( char *dest, int destSize, FILE *fp ); + virtual int FS_stat( const char *path, struct _stat *buf, bool *pbLoadedFromSteamCache=NULL ); + virtual int FS_chmod( const char *path, int pmode ); + virtual HANDLE FS_FindFirstFile(const char *findname, WIN32_FIND_DATA *dat); + virtual bool FS_FindNextFile(HANDLE handle, WIN32_FIND_DATA *dat); + virtual bool FS_FindClose(HANDLE handle); + +private: + bool IsFileInSteamCache( const char *file ); + bool IsFileInSteamCache2( const char *file ); + void ViewSteamCache( const char* szDir, bool bRecurse ); + bool m_bSteamInitialized; + bool m_bCurrentlyLoading; + bool m_bAssertFilesImmediatelyAvailable; + bool m_bCanAsync; + bool m_bSelfMounted; + bool m_bContentLoaded; + bool m_bSDKToolMode; + + SteamCallHandle_t m_hWaitForResourcesCallHandle; + int m_iCurrentReturnedCallHandle; + HMODULE m_hSteamDLL; + void LoadAndStartSteam(); +#ifdef POSIX + static CUtlMap< int, CInterlockedInt > m_LockedFDMap; +#endif +}; + +#ifdef POSIX +CUtlMap< int, CInterlockedInt> CFileSystem_Steam::m_LockedFDMap; +#endif + + +//----------------------------------------------------------------------------- +// singleton +//----------------------------------------------------------------------------- +static CFileSystem_Steam g_FileSystem_Steam; +#if defined(DEDICATED) +CBaseFileSystem *BaseFileSystem_Steam( void ) +{ + return &g_FileSystem_Steam; +} +#endif + +#ifdef DEDICATED // "hack" to allow us to not export a stdio version of the FILESYSTEM_INTERFACE_VERSION anywhere + +IFileSystem *g_pFileSystemSteam = &g_FileSystem_Steam; +IBaseFileSystem *g_pBaseFileSystemSteam = &g_FileSystem_Steam; + +#else + +EXPOSE_SINGLE_INTERFACE_GLOBALVAR( CFileSystem_Steam, IFileSystem, FILESYSTEM_INTERFACE_VERSION, g_FileSystem_Steam ); +EXPOSE_SINGLE_INTERFACE_GLOBALVAR( CFileSystem_Steam, IBaseFileSystem, BASEFILESYSTEM_INTERFACE_VERSION, g_FileSystem_Steam ); + +#endif + + +//----------------------------------------------------------------------------- +// constructor +//----------------------------------------------------------------------------- +CFileSystem_Steam::CFileSystem_Steam() +{ + m_bSteamInitialized = false; + m_bCurrentlyLoading = false; + m_bAssertFilesImmediatelyAvailable = false; + m_bCanAsync = true; + m_bContentLoaded = false; + m_hWaitForResourcesCallHandle = STEAM_INVALID_CALL_HANDLE; + m_iCurrentReturnedCallHandle = 1; + m_hSteamDLL = NULL; + m_bSDKToolMode = false; +#ifdef POSIX + SetDefLessFunc( m_LockedFDMap ); +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +CFileSystem_Steam::~CFileSystem_Steam() +{ + m_bSteamInitialized = false; +} + +bool CFileSystem_Steam::IsFileInSteamCache2( const char *file ) +{ + if ( !m_bContentLoaded || m_bSDKToolMode) + { + return true; + } + + // see if the file exists + TSteamElemInfo info; + TSteamError error; + + SteamHandle_t h = steam->FindFirst( file, eSteamFindRemoteOnly, &info, &error ); + if ( h == STEAM_INVALID_HANDLE ) + { + return false; + } + else + { + steam->FindClose( h, &error ); + } + + return true; +} + + +void MountDependencies( int iAppId, CUtlVector<unsigned int> &depList ) +{ + TSteamError steamError; + + // Setup the buffers for the TSteamApp structure. + char buffers[4][2048]; + TSteamApp steamApp; + steamApp.szName = buffers[0]; + steamApp.uMaxNameChars = sizeof( buffers[0] ); + steamApp.szLatestVersionLabel = buffers[1]; + steamApp.uMaxLatestVersionLabelChars = sizeof( buffers[1] ); + steamApp.szCurrentVersionLabel = buffers[2]; + steamApp.uMaxCurrentVersionLabelChars = sizeof( buffers[2] ); + steamApp.szInstallDirName = buffers[3]; + steamApp.uMaxInstallDirNameChars = sizeof( buffers[3] ); + + // Ask how many caches depend on this app ID. + steam->EnumerateApp( iAppId, &steamApp, &steamError ); + if ( steamError.eSteamError != eSteamErrorNone ) + Error( "EnumerateApp( %d ) failed: %s", iAppId, steamError.szDesc ); + + // Mount each cache. + for ( int i=0; i < (int)steamApp.uNumDependencies; i++ ) + { + TSteamAppDependencyInfo appDependencyInfo; + steam->EnumerateAppDependency( iAppId, i, &appDependencyInfo, &steamError ); + if ( steamError.eSteamError != eSteamErrorNone ) + Error( "EnumerateAppDependency( %d, %d ) failed: %s", iAppId, i, steamError.szDesc ); + + if ( depList.Find( appDependencyInfo.uAppId ) == -1 ) + { + depList.AddToTail( appDependencyInfo.uAppId ); + + // Make sure that the user owns the app before attempting to mount it + int isSubscribed = false, isPending = false; + steam->IsAppSubscribed( appDependencyInfo.uAppId, &isSubscribed, &isPending, &steamError ); + if ( isSubscribed ) + { + steam->MountFilesystem( appDependencyInfo.uAppId, "", &steamError ); + if ( steamError.eSteamError != eSteamErrorNone && steamError.eSteamError != eSteamErrorNotSubscribed ) + { + Error( "MountFilesystem( %d ) failed: %s", appDependencyInfo.uAppId, steamError.szDesc ); + } + } + } + } +} + + +//----------------------------------------------------------------------------- +// QueryInterface: +//----------------------------------------------------------------------------- +void *CFileSystem_Steam::QueryInterface( const char *pInterfaceName ) +{ + // We also implement the IMatSystemSurface interface + if (!Q_strncmp( pInterfaceName, FILESYSTEM_INTERFACE_VERSION, Q_strlen(FILESYSTEM_INTERFACE_VERSION) + 1)) + return (IFileSystem*)this; + + return CBaseFileSystem::QueryInterface( pInterfaceName ); +} + + +//----------------------------------------------------------------------------- +// Methods of IAppSystem +//----------------------------------------------------------------------------- +InitReturnVal_t CFileSystem_Steam::Init() +{ + m_bSteamInitialized = true; + m_bSelfMounted = false; + + LoadAndStartSteam(); + + return CBaseFileSystem::Init(); +} + +void CFileSystem_Steam::Shutdown() +{ + Assert( m_bSteamInitialized ); + + if ( !steam ) + return; + + + TSteamError steamError; + + // If we're not running Steam in local mode, remove all mount points from the STEAM VFS. + if ( !CommandLine()->CheckParm("-steamlocal") && !m_bSelfMounted && !steam->UnmountAppFilesystem(&steamError) ) + { +#ifdef WIN32 + OutputDebugString(steamError.szDesc); +#endif + Assert(!("STEAM VFS failed to unmount")); + + // just continue on as if nothing happened + // ::MessageBox(NULL, szErrorMsg, "Half-Life FileSystem_Steam Error", MB_OK); + // exit( -1 ); + } + + steam->Cleanup(&steamError); + + if ( m_hSteamDLL ) + { + Sys_UnloadModule( (CSysModule *)m_hSteamDLL ); + m_hSteamDLL = NULL; + } + m_bSteamInitialized = false; +} + + +void CFileSystem_Steam::LoadAndStartSteam() +{ + if ( !m_hSteamDLL ) + { + const char *pchSteamInstallPath = SteamAPI_GetSteamInstallPath(); + if ( pchSteamInstallPath ) + { + char szSteamDLLPath[ MAX_PATH ]; +#ifdef WIN32 + V_ComposeFileName( pchSteamInstallPath, "steam" DLL_EXT_STRING, szSteamDLLPath, Q_ARRAYSIZE(szSteamDLLPath) ); +#elif defined(POSIX) + V_ComposeFileName( pchSteamInstallPath, "libsteam" DLL_EXT_STRING, szSteamDLLPath, Q_ARRAYSIZE(szSteamDLLPath) ); +#else +#error +#endif + // try to load the steam.dll from the running steam process first + m_hSteamDLL = (HMODULE)Sys_LoadModule( szSteamDLLPath ); + } + + if ( !m_hSteamDLL ) +#ifdef WIN32 + m_hSteamDLL = (HMODULE)Sys_LoadModule( "steam" DLL_EXT_STRING ); +#elif defined(POSIX) + m_hSteamDLL = (HMODULE)Sys_LoadModule( "libsteam" DLL_EXT_STRING ); +#else +#error +#endif + } + + if ( m_hSteamDLL ) + { + typedef void *(*PFSteamCreateInterface)( const char *pchSteam ); +#ifdef WIN32 + PFSteamCreateInterface pfnSteamCreateInterface = (PFSteamCreateInterface)GetProcAddress( m_hSteamDLL, "_f" ); +#else + PFSteamCreateInterface pfnSteamCreateInterface = (PFSteamCreateInterface)dlsym( (void *)m_hSteamDLL, "_f" ); +#endif + if ( pfnSteamCreateInterface ) + steam = (ISteamInterface *)pfnSteamCreateInterface( STEAM_INTERFACE_VERSION ); + } + + if ( !steam ) + { + Error("CFileSystem_Steam::Init() failed: failed to find steam interface\n"); +#ifdef WIN32 + ::DestroyWindow( GetForegroundWindow() ); + ::MessageBox(NULL, "CFileSystem_Steam::Init() failed: failed to find steam interface", "Half-Life FileSystem_Steam Error", MB_OK); +#endif + _exit( -1 ); + } + + TSteamError steamError; + if (!steam->Startup(STEAM_USING_FILESYSTEM | STEAM_USING_LOGGING | STEAM_USING_USERID | STEAM_USING_ACCOUNT, &steamError)) + { + Error("SteamStartup() failed: %s\n", steamError.szDesc); +#ifdef WIN32 + ::DestroyWindow( GetForegroundWindow() ); + ::MessageBox(NULL, steamError.szDesc, "Half-Life FileSystem_Steam Error", MB_OK); +#endif + _exit( -1 ); + } +} + + +//----------------------------------------------------------------------------- +// Methods of IAppSystem +//----------------------------------------------------------------------------- +FilesystemMountRetval_t CFileSystem_Steam::MountSteamContent( int nExtraAppId ) +{ + m_bContentLoaded = true; + FilesystemMountRetval_t retval = FILESYSTEM_MOUNT_OK; + + // MWD: This is here because of Hammer's funky startup sequence that requires MountSteamContent() be called in CHammerApp::PreInit(). Once that root problem is addressed this will be removed; + if ( NULL == steam ) + { + LoadAndStartSteam(); + } + + // only mount if we're already logged in + // if we're not logged in, assume the app will login & mount the cache itself + // this enables both the game and the platform to use this same code, even though they mount caches at different times + int loggedIn = 0; + TSteamError steamError; + int result = steam->IsLoggedIn(&loggedIn, &steamError); + if (!result || loggedIn) + { + if ( nExtraAppId != -1 ) + { + m_bSDKToolMode = true; + + CUtlVector<unsigned int> depList; + if ( nExtraAppId < -1 ) + { + // Special way to tell them to mount a specific App ID's depots. + MountDependencies( -nExtraAppId, depList ); + return FILESYSTEM_MOUNT_OK; + } + else + { + const char *pMainAppId = NULL; + + // If they specified extra app IDs they want to mount after the main one, then we mount + // the caches manually here. +#ifdef _WIN32 + // Use GetEnvironmentVariable instead of getenv because getenv doesn't pick up changes + // to the process environment after the DLL was loaded. + char szMainAppId[128]; + if ( GetEnvironmentVariable( "steamappid", szMainAppId, sizeof( szMainAppId ) ) != 0 ) + { + pMainAppId = szMainAppId; + } +#else + // LINUX BUG: see above + pMainAppId = getenv( "SteamAppId" ); +#endif // _WIN32 + + if ( !pMainAppId ) + Error( "Extra App ID set to %d, but no SteamAppId.", nExtraAppId ); + + //swapping this mount order ensures the most current engine binaries are used by tools + MountDependencies( nExtraAppId, depList ); + MountDependencies( atoi( pMainAppId ), depList ); + return FILESYSTEM_MOUNT_OK; + } + } + else if (!steam->MountAppFilesystem(&steamError)) + { + Error("MountAppFilesystem() failed: %s\n", steamError.szDesc); +#ifdef WIN32 + ::DestroyWindow( GetForegroundWindow() ); + ::MessageBox(NULL, steamError.szDesc, "Half-Life FileSystem_Steam Error", MB_OK); +#endif + _exit( -1 ); + } + + } + else + { + m_bSelfMounted = true; + } + + return retval; +} + + + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +FILE *CFileSystem_Steam::FS_fopen( const char *filenameT, const char *options, unsigned flags, int64 *size, CFileLoadInfo *pInfo ) +{ + char filename[MAX_PATH]; + + FixUpPath ( filenameT, filename, sizeof( filename ) ); + + // make sure the file is immediately available + if (m_bAssertFilesImmediatelyAvailable && !m_bCurrentlyLoading) + { + if (!IsFileImmediatelyAvailable(filename)) + { + Msg("Steam FS: '%s' not immediately available when not in loading dialog", filename); + } + } + + if ( !steam ) + { + AssertMsg( 0, "CFileSystem_Steam::FS_fopen used with null steam interface!" ); + return NULL; + } + + CFileLoadInfo dummyInfo; + if ( !pInfo ) + { + dummyInfo.m_bSteamCacheOnly = false; + pInfo = &dummyInfo; + } + + SteamHandle_t f = 0; + +#ifdef POSIX + FILE *pFile = NULL; + bool bWriteable = false; + if ( strchr(options,'w') || strchr(options,'a') ) + bWriteable = true; + + if ( bWriteable ) + { + pFile = fopen( filename, options ); + if (pFile && size) + { + // todo: replace with filelength()? + struct _stat buf; + int rt = _stat( filename, &buf ); + if (rt == 0) + { + *size = buf.st_size; + } + } + if ( pFile ) + { + + // Win32 has an undocumented feature that is serialized ALL writes to a file across threads (i.e only 1 thread can open a file at a time) + // so use flock here to mimic that behavior + + ThreadId_t curThread = ThreadGetCurrentId(); + + { + CThreadFastMutex Locklock; + AUTO_LOCK( Locklock ); + int fd = fileno_unlocked( pFile ); + int iLockID = m_LockedFDMap.Find( fd ); + int ret = flock( fd, LOCK_EX | LOCK_NB ); + if ( ret < 0 ) + { + if ( errno == EWOULDBLOCK ) + { + if ( iLockID != m_LockedFDMap.InvalidIndex() && + m_LockedFDMap[iLockID] != -1 && + curThread != m_LockedFDMap[iLockID] ) + { + ret = flock( fd, LOCK_EX ); + if ( ret < 0 ) + { + fclose( pFile ); + return NULL; + } + } + } + else + { + fclose( pFile ); + return NULL; + } + } + + if ( iLockID != m_LockedFDMap.InvalidIndex() ) + m_LockedFDMap[iLockID] = curThread; + else + m_LockedFDMap.Insert( fd, curThread ); + + } + rewind( pFile ); + } + } + else + { +#endif + + TSteamError steamError; + unsigned int fileSize; + int bLocal = 0; + f = steam->OpenFileEx(filename, options, pInfo->m_bSteamCacheOnly, &fileSize, &bLocal, &steamError); + + pInfo->m_bLoadedFromSteamCache = (bLocal == 0); + if (size) + { + *size = fileSize; + } + + CheckError( f, steamError ); + +#ifdef POSIX + } + + if ( f || pFile ) + { + CSteamFile *steamFile = new CSteamFile( pFile ? (SteamHandle_t)pFile : f, bWriteable, filename ); + f = (SteamHandle_t)steamFile; + } +#endif + return (FILE *)f; +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +void CFileSystem_Steam::FS_setbufsize( FILE *fp, unsigned nBytes ) +{ +} + + +//----------------------------------------------------------------------------- +// Purpose: steam call, unnecessary in stdio +//----------------------------------------------------------------------------- +WaitForResourcesHandle_t CFileSystem_Steam::WaitForResources( const char *resourcelist ) +{ + char szResourceList[MAX_PATH]; + Q_strncpy( szResourceList, resourcelist, sizeof(szResourceList) ); + Q_DefaultExtension( szResourceList, ".lst", sizeof(szResourceList) ); + + // cancel any old call + TSteamError steamError; + m_hWaitForResourcesCallHandle = steam->WaitForResources(szResourceList, &steamError); + if (steamError.eSteamError == eSteamErrorNone) + { + // return a new call handle + return (WaitForResourcesHandle_t)(++m_iCurrentReturnedCallHandle); + } + + Msg("SteamWaitForResources() failed: %s\n", steamError.szDesc); + return (WaitForResourcesHandle_t)FILESYSTEM_INVALID_HANDLE; +} + +//----------------------------------------------------------------------------- +// Purpose: steam call, unnecessary in stdio +//----------------------------------------------------------------------------- +bool CFileSystem_Steam::GetWaitForResourcesProgress( WaitForResourcesHandle_t handle, float *progress /* out */ , bool *complete /* out */ ) +{ + // clear the input + *progress = 0.0f; + *complete = true; + + // check to see if they're using an old handle + if (m_iCurrentReturnedCallHandle != handle) + return false; + if (m_hWaitForResourcesCallHandle == STEAM_INVALID_CALL_HANDLE) + return false; + + // get the progress + TSteamError steamError; + TSteamProgress steamProgress; + int result = steam->ProcessCall(m_hWaitForResourcesCallHandle, &steamProgress, &steamError); + if (result && steamError.eSteamError == eSteamErrorNone) + { + // we've finished successfully + m_hWaitForResourcesCallHandle = STEAM_INVALID_CALL_HANDLE; + *complete = true; + *progress = 1.0f; + return true; + } + else if (steamError.eSteamError != eSteamErrorNotFinishedProcessing) + { + // we have an error, just call it done + m_hWaitForResourcesCallHandle = STEAM_INVALID_CALL_HANDLE; + Msg("SteamProcessCall(SteamWaitForResources()) failed: %s\n", steamError.szDesc); + return false; + } + + // return the progress + if (steamProgress.bValid) + { + *progress = (float)steamProgress.uPercentDone / (100.0f * STEAM_PROGRESS_PERCENT_SCALE); + } + else + { + *progress = 0; + } + *complete = false; + + return (steamProgress.bValid != false); +} + +//----------------------------------------------------------------------------- +// Purpose: steam call, unnecessary in stdio +//----------------------------------------------------------------------------- +void CFileSystem_Steam::CancelWaitForResources( WaitForResourcesHandle_t handle ) +{ + // check to see if they're using an old handle + if (m_iCurrentReturnedCallHandle != handle) + return; + if (m_hWaitForResourcesCallHandle == STEAM_INVALID_CALL_HANDLE) + return; + + TSteamError steamError; + steam->AbortCall(m_hWaitForResourcesCallHandle, &steamError); + m_hWaitForResourcesCallHandle = STEAM_INVALID_CALL_HANDLE; +} + + +//----------------------------------------------------------------------------- +// Purpose: helper for posix file handle wrapper +//----------------------------------------------------------------------------- +#ifdef POSIX +FILE *GetFileHandle( CSteamFile *steamFile ) +{ + if ( !steamFile ) + return NULL; + + return (FILE *)steamFile->Handle(); +} +bool BWriteable( CSteamFile *steamFile ) +{ + return steamFile && steamFile->BWriteable(); +} +#endif + + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +void CFileSystem_Steam::FS_fclose( FILE *fp ) +{ +#ifdef POSIX + CSteamFile *steamFile = (CSteamFile *)fp; + fp = GetFileHandle( steamFile ); + if ( BWriteable( steamFile ) ) + { + int fd = fileno_unlocked( fp ); + fflush( fp ); + flock( fd, LOCK_UN ); + int iLockID = m_LockedFDMap.Find( fd ); + if ( iLockID != m_LockedFDMap.InvalidIndex() ) + m_LockedFDMap[ iLockID ] = -1; + + fclose( fp ); + } + else + { +#endif + TSteamError steamError; + steam->CloseFile((SteamHandle_t)fp, &steamError); + CheckError( (SteamHandle_t)fp, steamError ); +#ifdef POSIX + } +#endif +} + + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +void CFileSystem_Steam::FS_fseek( FILE *fp, int64 pos, int seekType ) +{ +#ifdef POSIX + CSteamFile *steamFile = (CSteamFile *)fp; + fp = GetFileHandle( steamFile ); + if ( BWriteable( steamFile ) ) + { + fseek( fp, pos, seekType ); + } + else + { +#endif + TSteamError steamError; + int result; + result = steam->SeekFile((SteamHandle_t)fp, (int32)pos, (ESteamSeekMethod)seekType, &steamError); + CheckError((SteamHandle_t)fp, steamError); +#ifdef POSIX + } +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +long CFileSystem_Steam::FS_ftell( FILE *fp ) +{ +#ifdef POSIX + CSteamFile *steamFile = (CSteamFile *)fp; + fp = GetFileHandle( steamFile ); + if ( BWriteable( steamFile ) ) + { + return ftell(fp); + } + else + { +#endif + long steam_offset; + TSteamError steamError; + + steam_offset = steam->TellFile((SteamHandle_t)fp, &steamError); + if ( steamError.eSteamError != eSteamErrorNone ) + { + CheckError((SteamHandle_t)fp, steamError); + return -1L; + } + + return steam_offset; +#ifdef POSIX + } +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +int CFileSystem_Steam::FS_feof( FILE *fp ) +{ +#ifdef POSIX + CSteamFile *steamFile = (CSteamFile *)fp; + fp = GetFileHandle( steamFile ); + if ( BWriteable( steamFile ) ) + { + return feof(fp); + } + else + { +#endif + long orig_pos; + + // Figure out where in the file we currently are... + orig_pos = FS_ftell(fp); + + if ( (SteamHandle_t)fp == g_pLastErrorFile && g_tLastError.eSteamError == eSteamErrorEOF ) + return 1; + + if ( g_tLastError.eSteamError != eSteamErrorNone ) + return 0; + + // Jump to the end... + FS_fseek(fp, 0L, SEEK_END); + + // If we were already at the end, return true + if ( orig_pos == FS_ftell(fp) ) + return 1; + + // Otherwise, go back to the original spot and return false. + FS_fseek(fp, orig_pos, SEEK_SET); + return 0; +#ifdef POSIX + } +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +size_t CFileSystem_Steam::FS_fread( void *dest, size_t destSize, size_t size, FILE *fp ) +{ +#ifdef POSIX + CSteamFile *steamFile = (CSteamFile *)fp; + fp = GetFileHandle( steamFile ); + if ( BWriteable( steamFile ) ) + { + return fread( dest, 1, size, fp ); + } + else + { +#endif + TSteamError steamError; + int blocksRead = steam->ReadFile(dest, 1, size, (SteamHandle_t)fp, &steamError); + CheckError((SteamHandle_t)fp, steamError); + return blocksRead; // steam reads in atomic blocks of "size" bytes +#ifdef POSIX + } +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +size_t CFileSystem_Steam::FS_fwrite( const void *src, size_t size, FILE *fp ) +{ +#ifdef POSIX + CSteamFile *steamFile = (CSteamFile *)fp; + fp = GetFileHandle( steamFile ); + if ( BWriteable( steamFile ) ) + { +#define WRITE_CHUNK (256 * 1024) + if ( size > WRITE_CHUNK ) + { + size_t remaining = size; + const byte* current = (const byte *) src; + size_t total = 0; + + while ( remaining > 0 ) + { + size_t bytesToCopy = min(remaining, WRITE_CHUNK); + + total += fwrite(current, 1, bytesToCopy, fp); + + remaining -= bytesToCopy; + current += bytesToCopy; + } + + Assert( total == size ); + return total; + } + + return fwrite(src, 1, size, fp);// return number of bytes written (because we have size = 1, count = bytes, so it return bytes) + } + else + { +#endif + TSteamError steamError; + int result = steam->WriteFile(src, 1, size, (SteamHandle_t)fp, &steamError); + CheckError((SteamHandle_t)fp, steamError); + return result; +#ifdef POSIX + } +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +size_t CFileSystem_Steam::FS_vfprintf( FILE *fp, const char *fmt, va_list list ) +{ +#ifdef POSIX + CSteamFile *steamFile = (CSteamFile *)fp; + fp = GetFileHandle( steamFile ); + if ( BWriteable( steamFile ) ) + { + return vfprintf(fp, fmt, list); + } + else + { +#endif + int blen, plen; + char *buf; + + if ( !fp || !fmt ) + return 0; + + // Open the null device...used by vfprintf to determine the length of + // the formatted string. + FILE *nullDeviceFP = fopen("nul:", "w"); + if ( !nullDeviceFP ) + return 0; + + // Figure out how long the formatted string will be...dump formatted + // string to the bit bucket. + blen = vfprintf(nullDeviceFP, fmt, list); + fclose(nullDeviceFP); + if ( !blen ) + { + return 0; + } + + // Get buffer in which to build the formatted string. + buf = (char *)malloc(blen+1); + if ( !buf ) + { + return 0; + } + + // Build the formatted string. + plen = _vsnprintf(buf, blen, fmt, list); + va_end(list); + if ( plen != blen ) + { + free(buf); + return 0; + } + + buf[ blen ] = 0; + + // Write out the formatted string. + if ( plen != (int)FS_fwrite(buf, plen, fp) ) + { + free(buf); + return 0; + } + + free(buf); + return plen; +#ifdef POSIX + } +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +int CFileSystem_Steam::FS_ferror( FILE *fp ) +{ + if (fp) + { +#ifdef POSIX + CSteamFile *steamFile = (CSteamFile *)fp; + fp = GetFileHandle( steamFile ); + if ( BWriteable( steamFile ) ) + { + return ferror(fp); + } + else + { +#endif + if ((SteamHandle_t)fp != g_pLastErrorFile) + { + // it's asking for an error for a previous file, return no error + return 0; + } + + return ( g_tLastError.eSteamError != eSteamErrorNone ); +#ifdef POSIX + } +#endif + } + return g_tLastErrorNoFile.eSteamError != eSteamErrorNone; +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +int CFileSystem_Steam::FS_fflush( FILE *fp ) +{ +#ifdef POSIX + CSteamFile *steamFile = (CSteamFile *)fp; + fp = GetFileHandle( steamFile ); + if ( BWriteable( steamFile ) ) + { + return fflush(fp); + } + else + { +#endif + TSteamError steamError; + int result = steam->FlushFile((SteamHandle_t)fp, &steamError); + CheckError((SteamHandle_t)fp, steamError); + return result; +#ifdef POSIX + } +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +char *CFileSystem_Steam::FS_fgets( char *dest, int destSize, FILE *fp ) +{ +#ifdef POSIX + CSteamFile *steamFile = (CSteamFile *)fp; + fp = GetFileHandle( steamFile ); + if ( BWriteable( steamFile ) ) + { + return fgets(dest, destSize, fp); + } + else + { +#endif + unsigned char c; + int numCharRead = 0; + + // Read at most n chars from the file or until a newline + *dest = c = '\0'; + while ( (numCharRead < destSize-1) && (c != '\n') ) + { + // Read in the next char... + if ( FS_fread(&c, 1, 1, fp) != 1 ) + { + if ( g_tLastError.eSteamError != eSteamErrorEOF || numCharRead == 0 ) + { + return NULL; // If we hit an error, return NULL. + } + + numCharRead = destSize; // Hit EOF, no more to read, all done... + } + + else + { + *dest++ = c; // add the char to the string and point to the next pos + *dest = '\0'; // append NULL + numCharRead++; // count the char read + } + } + return dest; // Has a NULL termination... +#ifdef POSIX + } +#endif +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +int CFileSystem_Steam::FS_stat( const char *path, struct _stat *buf, bool *pbLoadedFromSteamCache ) +{ + TSteamElemInfo Info; + TSteamError steamError; + + if ( pbLoadedFromSteamCache ) + *pbLoadedFromSteamCache = false; + + if ( !steam ) + { + // The dedicated server gets here once at startup. When setting up the executable path before loading + // base modules like engine.dll, the filesystem looks for zipX.zip but we haven't mounted steam content + // yet so steam is null. +#if !defined( DEDICATED ) + AssertMsg( 0, "CFileSystem_Steam::FS_stat used with null steam interface!" ); +#endif + return -1; + } + + memset(buf, 0, sizeof(struct _stat)); + int returnVal= steam->Stat(path, &Info, &steamError); + if ( returnVal == 0 ) + { + if (Info.bIsDir ) + { + buf->st_mode |= _S_IFDIR; + buf->st_size = 0; + } + else + { + if ( pbLoadedFromSteamCache ) + *pbLoadedFromSteamCache = ( Info.bIsLocal == 0 ); + + // Now we want to know if it's writable or not. First see if there is a local copy. + struct _stat testBuf; + int rt = _stat( path, &testBuf ); + if ( rt == 0 ) + { + // Ok, there's a local copy. Now check if the copy on our HD is writable. + if ( testBuf.st_mode & _S_IWRITE ) + buf->st_mode |= _S_IWRITE; + } + + buf->st_mode |= _S_IFREG; + buf->st_size = Info.uSizeOrCount; + } + + buf->st_atime = Info.lLastAccessTime; + buf->st_mtime = Info.lLastModificationTime; + buf->st_ctime = Info.lCreationTime; + } + + CheckError(NULL, steamError); + return returnVal; +} + +#ifdef _WIN32 +#include <io.h> +#endif + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +int CFileSystem_Steam::FS_chmod( const char *path, int pmode ) +{ + return _chmod( path, pmode ); +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +HANDLE CFileSystem_Steam::FS_FindFirstFile(const char *findname, WIN32_FIND_DATA *dat) +{ + TSteamElemInfo steamFindInfo; + HANDLE hResult = INVALID_HANDLE_VALUE; + SteamHandle_t steamResult; + TSteamError steamError; + + steamResult = steam->FindFirst(findname, eSteamFindAll, &steamFindInfo, &steamError); + CheckError(NULL, steamError); + + if ( steamResult == STEAM_INVALID_HANDLE ) + { + hResult = INVALID_HANDLE_VALUE; + } + else + { + hResult = (HANDLE)steamResult; + strcpy(dat->cFileName, steamFindInfo.cszName); + +// NEED TO DEAL WITH THIS STUFF!!! FORTUNATELY HALF-LIFE DOESN'T USE ANY OF IT +// AND ARCANUM USES _findfirst() etc. +// +// findInfo->ftLastWriteTime = steamFindInfo.lLastModificationTime; +// findInfo->ftCreationTime = steamFindInfo.lCreationTime; +// findInfo->ftLastAccessTime = steamFindInfo.lLastAccessTime; +// findInfo->nFileSizeHigh = ; +// findInfo->nFileSizeLow = ; + + // Determine if the found object is a directory... + if ( steamFindInfo.bIsDir ) + dat->dwFileAttributes |= FILE_ATTRIBUTE_DIRECTORY; + else + dat->dwFileAttributes &= ~FILE_ATTRIBUTE_DIRECTORY; + + // Determine if the found object was local or remote. + // ***NOTE*** we are hijacking the FILE_ATTRIBUTE_OFFLINE bit and using it in a different + // (but similar) manner than the WIN32 documentation indicates ***NOTE*** + if ( steamFindInfo.bIsLocal ) + dat->dwFileAttributes &= ~FILE_ATTRIBUTE_OFFLINE; + else + dat->dwFileAttributes |= FILE_ATTRIBUTE_OFFLINE; + } + + return hResult; +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +bool CFileSystem_Steam::FS_FindNextFile(HANDLE handle, WIN32_FIND_DATA *dat) +{ + TSteamElemInfo steamFindInfo; + bool result; + TSteamError steamError; + + result = (steam->FindNext((SteamHandle_t)handle, &steamFindInfo, &steamError) == 0); + CheckError(NULL, steamError); + + if ( result ) + { + strcpy(dat->cFileName, steamFindInfo.cszName); + if ( steamFindInfo.bIsDir ) + dat->dwFileAttributes |= FILE_ATTRIBUTE_DIRECTORY; + else + dat->dwFileAttributes &= ~FILE_ATTRIBUTE_DIRECTORY; + } + return result; +} + +//----------------------------------------------------------------------------- +// Purpose: low-level filesystem wrapper +//----------------------------------------------------------------------------- +bool CFileSystem_Steam::FS_FindClose(HANDLE handle) +{ + TSteamError steamError; + int result = (steam->FindClose((SteamHandle_t)handle, &steamError) == 0); + CheckError(NULL, steamError); + return result != 0; +} + +//----------------------------------------------------------------------------- +// Purpose: files are always immediately available on disk +//----------------------------------------------------------------------------- +bool CFileSystem_Steam::IsFileImmediatelyAvailable(const char *pFileName) +{ + TSteamError steamError; + return (steam->IsFileImmediatelyAvailable(pFileName, &steamError) != 0); +} + +//----------------------------------------------------------------------------- +// Purpose: +//----------------------------------------------------------------------------- +void CFileSystem_Steam::GetLocalCopy( const char *pFileName ) +{ + // Now try to find the dll under Steam so we can do a GetLocalCopy() on it + TSteamError steamError; + +/* +#ifdef WIN32 + struct _stat StatBuf; + if ( FS_stat(pFileName, &StatBuf) == -1 ) + { + // Use the environment search path to try and find it + char* pPath = getenv("PATH"); + + // Use the .EXE name to determine the root directory + char srchPath[ MAX_PATH ]; +#ifdef WIN32 + HINSTANCE hInstance = ( HINSTANCE )GetModuleHandle( 0 ); + if ( !GetModuleFileName( hInstance, srchPath, MAX_PATH ) ) + { + ::MessageBox( 0, "Failed calling GetModuleFileName", "Half-Life Steam Filesystem Error", MB_OK ); + return; + } +#else + srchPath[0] = '.'; + srchPath[1] = '\0'; +#endif + + // Get the length of the root directory the .exe is in + char* pSeperator = strrchr( srchPath, CORRECT_PATH_SEPARATOR ); + int nBaseLen = 0; + if ( pSeperator ) + { + nBaseLen = pSeperator - srchPath; + } + + // Extract each section of the path + char* pStart = pPath; + char* pEnd = 0; + bool bSearch = true; + while ( bSearch ) + { +#ifdef WIN32 +#define PATH_SEP ";" +#else +#define PATH_SEP ":" +#endif + pEnd = strstr( pStart, PATH_SEP ); + int nSize = pEnd - pStart; + if ( !pEnd ) + { + bSearch = false; + // If pEnd is NULL then nSize will be rubbish, so calculate + // it sensibly. + nSize = strlen( pStart ); + } + + // Is this path even potentially in the base directory? + if ( nSize > nBaseLen ) + { + // Create a new path (relative to the base directory) by stripping off + // nBaseLen characters and therefore doing FS_stat relative to the current + // directory, which should be the base directory. + Assert( sizeof(srchPath) > nBaseLen + strlen(pFileName) + 2 ); + nSize -= nBaseLen+1; + memcpy( srchPath, pStart+nBaseLen+1, nSize ); + memcpy( srchPath+nSize, pFileName, strlen(pFileName)+1 ); + // If the path starts with a directory separator then we won't get a + // relative path, so skip the check. + if ( srchPath[0] != CORRECT_PATH_SEPARATOR ) + { + if ( FS_stat(srchPath, &StatBuf) == 0 ) + { + steam->GetLocalFileCopy(srchPath, &steamError); + break; + } + } + } + pStart = pEnd+1; + } + } + else +#endif +*/ + { + // Convert _srv.so to .so... + const char *pDllStringExtension = V_GetFileExtension( DLL_EXT_STRING ); + const char *pModuleExtension = pDllStringExtension ? ( pDllStringExtension - 1 ) : DLL_EXT_STRING; + + // If we got an extension, and this filename has it, then check if it's loaded. + if( pModuleExtension && V_stristr( pFileName, pModuleExtension ) ) + { + // We can't be copying files over the top of .so files if they're already loaded + // in memory. mmap2( ... MAP_PRIVATE ... ) says "it is unspecified whether changes + // made to the file after the mmap() call are visible in the mapped region." Testing + // and lots of debugging (thanks Pierre-Loup!) has shown that they are, in fact, + // blasted right over your nicely loaded and fixed up object. + CSysModule *module = Sys_LoadModule( pFileName, SYS_NOLOAD ); + + if( module ) + { + // Sys_LoadModule( SYS_NOLOAD ) increments the refcount, so bump that back down. + Sys_UnloadModule( module ); + return; + } + } + + steam->GetLocalFileCopy(pFileName, &steamError); + } +} + +//----------------------------------------------------------------------------- +// Purpose: Load a DLL +// Input : *path +//----------------------------------------------------------------------------- +CSysModule * CFileSystem_Steam::LoadModule( const char *pFileName, const char *pPathID, bool bValidatedDllOnly ) +{ + char szNewPath[ MAX_PATH ]; + CBaseFileSystem::ParsePathID( pFileName, pPathID, szNewPath ); + + // File must end in .dll + char szExtension[] = DLL_EXT_STRING; + Assert( Q_strlen(pFileName) < sizeof(szNewPath) ); + + Q_strncpy( szNewPath, pFileName, sizeof( szNewPath ) ); + if ( !Q_stristr(szNewPath, szExtension) ) + { + Assert( strlen(pFileName) + sizeof(szExtension) < sizeof(szNewPath) ); + Q_strncat( szNewPath, szExtension, sizeof( szNewPath ), COPY_ALL_CHARACTERS ); + } + + LogFileAccess( szNewPath ); + if ( !pPathID ) + { + pPathID = "EXECUTABLE_PATH"; // default to the bin dir + } + + CUtlSymbol lookup = g_PathIDTable.AddString( pPathID ); + + // a pathID has been specified, find the first match in the path list + int c = m_SearchPaths.Count(); + for (int i = 0; i < c; i++) + { + // pak files are not allowed to be written to... + if (m_SearchPaths[i].GetPackFile()) + continue; + + if ( m_SearchPaths[i].GetPathID() == lookup ) + { + char newPathName[MAX_PATH]; + Q_snprintf( newPathName, sizeof(newPathName), "%s%s", m_SearchPaths[i].GetPathString(), szNewPath ); // append the path to this dir. + + // make sure the file exists, and is in the Steam cache + + if ( bValidatedDllOnly && !IsFileInSteamCache(newPathName) ) + continue; + + // Get a local copy from Steam + bool bGetLocalCopy = true; + + if ( m_bSDKToolMode ) + bGetLocalCopy = false; +#ifdef _WIN32 + if ( IsDebuggerPresent() ) + bGetLocalCopy = false; +#endif + if ( bGetLocalCopy ) + GetLocalCopy( newPathName ); + + CSysModule *module = Sys_LoadModule( newPathName ); + if ( module ) // we found the binary in one of our search paths + { + if ( bValidatedDllOnly && !IsFileInSteamCache2(newPathName) ) + { + return NULL; + } + else + { + return module; + } + } + } + } + + if ( bValidatedDllOnly && IsFileInSteamCache(szNewPath) ) + { + // couldn't load it from any of the search paths, let LoadLibrary try + return Sys_LoadModule( szNewPath ); + } + + return NULL; +} + +void CFileSystem_Steam::ViewSteamCache(const char* szDir, bool bRecurse) +{ + TSteamElemInfo info; + TSteamError error; + char szPath[MAX_PATH]; + + V_snprintf( szPath, sizeof(szPath),"%s%c*.*", szDir, CORRECT_PATH_SEPARATOR ); + + SteamHandle_t h = steam->FindFirst( szPath, eSteamFindRemoteOnly, &info, &error ); + int ret = 0; + + if ( h != STEAM_INVALID_HANDLE ) + { + do + { + Msg( "View Steam Cache: '%s%c%s' \n", szDir, CORRECT_PATH_SEPARATOR, info.cszName ); + + if ( bRecurse && info.bIsDir && (0 == V_stristr( info.cszName, "." ) ) ) + { + V_snprintf( szPath, sizeof(szPath),"%s%c%s", szDir, CORRECT_PATH_SEPARATOR, info.cszName ); + ViewSteamCache( szPath, true ); + } + + ret = steam->FindNext( h, &info, &error ); + + } while( 0 == ret ); + + steam->FindClose( h, &error ); + } +} + + +// HACK HACK - to allow IsFileInSteamCache() to use the old C exported interface +extern "C" SteamHandle_t SteamFindFirst( const char *cszPattern, ESteamFindFilter eFilter, TSteamElemInfo *pFindInfo, TSteamError *pError ); +extern "C" int SteamFindClose( SteamHandle_t hDirectory, TSteamError *pError ); + +//----------------------------------------------------------------------------- +// Purpose: returns true if the file exists and is in a mounted Steam cache +//----------------------------------------------------------------------------- +bool CFileSystem_Steam::IsFileInSteamCache( const char *file ) +{ + if ( !m_bContentLoaded || m_bSDKToolMode ) + { + return true; + } + + // see if the file exists + TSteamElemInfo info; + TSteamError error; + + SteamHandle_t h = steam->FindFirst( file, eSteamFindRemoteOnly, &info, &error ); + if ( h == STEAM_INVALID_HANDLE ) + { + return false; + } + else + { + steam->FindClose( h, &error ); + } + + return true; +} + + +int CFileSystem_Steam::HintResourceNeed( const char *hintlist, int forgetEverything ) +{ + TSteamError steamError; + int result = steam->HintResourceNeed( hintlist, forgetEverything, &steamError ); + CheckError(NULL, steamError); + return result; +} + + diff --git a/filesystem/filesystem_steam.vpc b/filesystem/filesystem_steam.vpc new file mode 100644 index 0000000..7f1ecf3 --- /dev/null +++ b/filesystem/filesystem_steam.vpc @@ -0,0 +1,80 @@ +//----------------------------------------------------------------------------- +// FILESYSTEM_STEAM.VPC +// +// Project Script +//----------------------------------------------------------------------------- + +$Macro SRCDIR ".." +$Macro OUTBINDIR "$SRCDIR\..\game\bin" + +$Include "$SRCDIR\vpc_scripts\source_dll_base.vpc" + +$Configuration +{ + $Compiler + { + $AdditionalIncludeDirectories "$SRCDIR\public,$SRCDIR\public\tier1,$SRCDIR\common" + $PreprocessorDefinitions "$BASE;FILESYSTEM_STDIO_EXPORTS;DONT_PROTECT_FILEIO_FUNCTIONS;PROTECTED_THINGS_ENABLE;FILESYSTEM_STEAM" + } + $Linker + { + $SystemLibraries "iconv" [$OSXALL] + } +} + +$Project "FileSystem_Steam" +{ + $Folder "Source Files" + { + $File "basefilesystem.cpp" + $File "packfile.cpp" + $File "filetracker.cpp" + $File "filesystem_async.cpp" + $File "filesystem_steam.cpp" + $File "linux_support.cpp" [$POSIX] + $File "QueuedLoader.cpp" + $File "$SRCDIR\public\kevvaluescompiler.cpp" + $File "$SRCDIR\public\keyvaluescompiler.h" + $File "$SRCDIR\public\zip_utils.cpp" + } + + $Folder "Header Files" + { + $File "basefilesystem.h" + $File "packfile.h" + $File "filetracker.h" + $File "threadsaferefcountedobject.h" + $File "$SRCDIR\public\tier0\basetypes.h" + $File "$SRCDIR\public\bspfile.h" + $File "$SRCDIR\public\bspflags.h" + $File "$SRCDIR\public\mathlib\bumpvects.h" + $File "$SRCDIR\public\tier1\characterset.h" + $File "$SRCDIR\public\tier0\dbg.h" + $File "$SRCDIR\public\tier0\fasttimer.h" + $File "$SRCDIR\public\filesystem.h" + $File "$SRCDIR\public\ifilelist.h" + $File "$SRCDIR\public\appframework\IAppSystem.h" + $File "$SRCDIR\public\tier1\interface.h" + $File "$SRCDIR\public\mathlib\mathlib.h" + $File "$SRCDIR\public\tier0\platform.h" + $File "$SRCDIR\public\tier1\strtools.h" + $File "$SRCDIR\public\tier1\utlmemory.h" + $File "$SRCDIR\public\tier1\utlrbtree.h" + $File "$SRCDIR\public\tier1\utlsymbol.h" + $File "$SRCDIR\public\tier1\utlvector.h" + $File "$SRCDIR\public\mathlib\vector.h" + $File "$SRCDIR\public\mathlib\vector2d.h" + $File "$SRCDIR\public\mathlib\vector4d.h" + $File "$SRCDIR\public\vstdlib\vstdlib.h" + $File "$SRCDIR\public\zip_utils.h" + } + + $Folder "Link Libraries" + { + $Lib tier2 + + $DynamicFile "$SRCDIR\lib\public\steam_api.lib" [$WIN32] + $DynamicFile "$SRCDIR\lib\$PLATFORM\$_IMPLIB_PREFIXsteam$_IMPLIB_EXT" [$POSIX || $LINUX] + $DynamicFile "$SRCDIR\lib\$PLATFORM\$_IMPLIB_PREFIXsteam_api$_IMPLIB_EXT" [$POSIX || $LINUX] + } +} diff --git a/filesystem/filetracker.cpp b/filesystem/filetracker.cpp new file mode 100644 index 0000000..056af15 --- /dev/null +++ b/filesystem/filetracker.cpp @@ -0,0 +1,596 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +//============================================================================= + +#include "basefilesystem.h" +#include "tier0/vprof.h" + +// NOTE: This has to be the last file included! +#include "tier0/memdbgon.h" + +#if !defined( DEDICATED ) + +#ifdef SUPPORT_PACKED_STORE + +unsigned ThreadStubProcessMD5Requests( void *pParam ) +{ + return ((CFileTracker2 *)pParam)->ThreadedProcessMD5Requests(); +} + +//----------------------------------------------------------------------------- +// ThreadedProcessMD5Requests +// Calculate the MD5s of all the blocks submitted to us +//----------------------------------------------------------------------------- +unsigned CFileTracker2::ThreadedProcessMD5Requests() +{ + ThreadSetDebugName( "ProcessMD5Requests" ); + + while ( m_bThreadShouldRun ) + { + StuffToMD5_t stuff; + + while ( m_PendingJobs.PopItem( &stuff ) ) + { + tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s", __FUNCTION__ ); + + MD5Context_t ctx; + memset( &ctx, 0, sizeof(MD5Context_t) ); + MD5Init( &ctx ); + MD5Update( &ctx, stuff.m_pubBuffer, stuff.m_cubBuffer ); + MD5Final( stuff.m_md5Value.bits, &ctx); + + { + // update the FileTracker MD5 database + AUTO_LOCK( m_Mutex ); + + TrackedVPKFile_t &trackedVPKFile = m_treeTrackedVPKFiles[ stuff.m_idxTrackedVPKFile ]; + TrackedFile_t &trackedfile = m_treeAllOpenedFiles[ trackedVPKFile.m_idxAllOpenedFiles ]; + + memcpy( trackedfile.m_filehashFinal.m_md5contents.bits, stuff.m_md5Value.bits, sizeof( trackedfile.m_filehashFinal.m_md5contents.bits ) ); + trackedfile.m_filehashFinal.m_crcIOSequence = stuff.m_cubBuffer; + trackedfile.m_filehashFinal.m_cbFileLen = stuff.m_cubBuffer; + trackedfile.m_filehashFinal.m_eFileHashType = FileHash_t::k_EFileHashTypeEntireFile; + trackedfile.m_filehashFinal.m_nPackFileNumber = trackedVPKFile.m_nPackFileNumber; + trackedfile.m_filehashFinal.m_PackFileID = trackedVPKFile.m_PackFileID; + } + + m_CompletedJobs.PushItem( stuff ); + m_threadEventWorkCompleted.Set(); + } + + { + tmZone( TELEMETRY_LEVEL0, TMZF_IDLE, "m_threadEventWorkToDo" ); + + m_threadEventWorkToDo.Wait( 1000 ); + } + } + + return 0; +} + +//----------------------------------------------------------------------------- +// SubmitThreadedMD5Request +// add pubBuffer,cubBuffer to our queue of stuff to MD5 +// caller promises that the memory will remain valid +// until BlockUntilMD5RequestComplete() is called +// returns: request handle +//----------------------------------------------------------------------------- +int CFileTracker2::SubmitThreadedMD5Request( uint8 *pubBuffer, int cubBuffer, int PackFileID, int nPackFileNumber, int nPackFileFraction ) +{ + int idxList; + StuffToMD5_t stuff; + + { + AUTO_LOCK( m_Mutex ); + + TrackedVPKFile_t trackedVPKFileFind; + trackedVPKFileFind.m_nPackFileNumber = nPackFileNumber; + trackedVPKFileFind.m_PackFileID = PackFileID; + trackedVPKFileFind.m_nFileFraction = nPackFileFraction; + + int idxTrackedVPKFile = m_treeTrackedVPKFiles.Find( trackedVPKFileFind ); + if ( idxTrackedVPKFile != m_treeTrackedVPKFiles.InvalidIndex() ) + { + // dont early out if we have already done the MD5, if the caller wants us + // to do it again - then do it again + m_cDupMD5s++; + } + else + { + // this is an error, we should already know about the file + Assert(0); + return 0; + } + + SubmittedMd5Job_t submittedjob; + submittedjob.m_bFinished = false; + idxList = m_SubmittedJobs.AddToTail( submittedjob ); + + stuff.m_pubBuffer = pubBuffer; + stuff.m_cubBuffer = cubBuffer; + stuff.m_idxTrackedVPKFile = idxTrackedVPKFile; + stuff.m_idxListSubmittedJobs = idxList; + } + + // Start thread if it wasn't already active. Do this down here due to the + // return 0 above us. Ie, don't start the thread unless we actually have work + // to do. + if ( m_hWorkThread == NULL ) + { + Assert( !m_bThreadShouldRun ); + m_bThreadShouldRun = true; + m_hWorkThread = CreateSimpleThread( ThreadStubProcessMD5Requests, this ); + } + + // submit the work + m_PendingJobs.PushItem( stuff ); + m_threadEventWorkToDo.Set(); + + return idxList + 1; +} + +//----------------------------------------------------------------------------- +// IsMD5RequestComplete +// is request identified by iRequest finished? +// ( the caller wants to free the memory, but now must wait until we finish +// calculating the MD5 ) +//----------------------------------------------------------------------------- +bool CFileTracker2::IsMD5RequestComplete( int iRequest, MD5Value_t *pMd5ValueOut ) +{ + AUTO_LOCK( m_Mutex ); + int idxListWaiting = iRequest - 1; + + // deal with all completed jobs + StuffToMD5_t stuff; + while ( m_CompletedJobs.PopItem( &stuff ) ) + { + int idxList = stuff.m_idxListSubmittedJobs; + Q_memcpy( &m_SubmittedJobs[ idxList ].m_md5Value, &stuff.m_md5Value, sizeof( MD5Value_t ) ); + m_SubmittedJobs[ idxList ].m_bFinished = true; + } + + // did the one we wanted finish? + if ( m_SubmittedJobs[ idxListWaiting ].m_bFinished ) + { + Q_memcpy( pMd5ValueOut, &m_SubmittedJobs[ idxListWaiting ].m_md5Value, sizeof( MD5Value_t ) ); + // you can not ask again, we have removed it from the list + m_SubmittedJobs.Remove(idxListWaiting); + return true; + } + + // not done yet + return false; +} + +//----------------------------------------------------------------------------- +// BlockUntilMD5RequestComplete +// block until request identified by iRequest is finished +// ( the caller wants to free the memory, but now must wait until we finish +// calculating the MD5 ) +//----------------------------------------------------------------------------- +bool CFileTracker2::BlockUntilMD5RequestComplete( int iRequest, MD5Value_t *pMd5ValueOut ) +{ + while ( 1 ) + { + if ( IsMD5RequestComplete( iRequest, pMd5ValueOut ) ) + return true; + m_cThreadBlocks++; + m_threadEventWorkCompleted.Wait( 1 ); + } + return false; +} + +#endif // SUPPORT_PACKED_STORE + +CFileTracker2::CFileTracker2( CBaseFileSystem *pFileSystem ): + m_treeAllOpenedFiles( TrackedFile_t::Less ), + m_treeTrackedVPKFiles( TrackedVPKFile_t::Less ) +{ +#if defined( DEDICATED ) + Assert( 0 ); +#endif + + m_pFileSystem = pFileSystem; + + m_cThreadBlocks = 0; + m_cDupMD5s = 0; + +#ifdef SUPPORT_PACKED_STORE + m_bThreadShouldRun = false; + m_hWorkThread = NULL; +#endif +} + +CFileTracker2::~CFileTracker2() +{ +#ifdef SUPPORT_PACKED_STORE + Assert( !m_bThreadShouldRun ); + Assert( m_hWorkThread == NULL ); +#endif +} + +void CFileTracker2::ShutdownAsync() +{ +#ifdef SUPPORT_PACKED_STORE + m_bThreadShouldRun = false; + m_threadEventWorkToDo.Set(); + // wait for it to die + if ( m_hWorkThread ) + { + ThreadJoin( m_hWorkThread ); + ReleaseThreadHandle( m_hWorkThread ); + m_hWorkThread = NULL; + } +#endif +} + +void CFileTracker2::MarkAllCRCsUnverified() +{ + // AUTO_LOCK( m_Mutex ); +} + +int CFileTracker2::GetUnverifiedFileHashes( CUnverifiedFileHash *pFiles, int nMaxFiles ) +{ + return 0; +} + +EFileCRCStatus CFileTracker2::CheckCachedFileHash( const char *pPathID, const char *pRelativeFilename, int nFileFraction, FileHash_t *pFileHash ) +{ + Assert( ThreadInMainThread() ); + AUTO_LOCK( m_Mutex ); + + TrackedFile_t trackedfileFind; + trackedfileFind.RebuildFileName( m_stringPool, pRelativeFilename, pPathID, nFileFraction ); + + int idx = m_treeAllOpenedFiles.Find( trackedfileFind ); + if ( idx != m_treeAllOpenedFiles.InvalidIndex() ) + { + TrackedFile_t &trackedfile = m_treeAllOpenedFiles[ idx ]; + + if ( trackedfile.m_bFileInVPK ) + { + // the FileHash is not meaningful, because the file is in a VPK, we have hashed the entire VPK + // if the user is sending us a hash for this file, it means he has extracted it from the VPK and tricked the client into loading it + // instead of the version in the VPK. + return k_eFileCRCStatus_FileInVPK; + } + + return k_eFileCRCStatus_CantOpenFile; + } + else + { + return k_eFileCRCStatus_CantOpenFile; + } +} + +void TrackedFile_t::RebuildFileName( CStringPool &stringPool, const char *pFilename, const char *pPathID, int nFileFraction ) +{ + char szFixedName[ MAX_PATH ]; + char szPathName[ MAX_PATH ]; + + V_strcpy_safe( szFixedName, pFilename ); + V_RemoveDotSlashes( szFixedName ); + V_FixSlashes( szFixedName ); + V_strlower( szFixedName ); // !KLUDGE! + m_filename = stringPool.Allocate( szFixedName ); + + V_strcpy_safe( szPathName, pPathID ? pPathID : "" ); + V_strupr( szPathName ); // !KLUDGE! + m_path = stringPool.Allocate( szPathName ); + + // CRC32_t crcFilename; + // CRC32_Init( &crcFilename ); + // CRC32_ProcessBuffer( &crcFilename, m_filename, Q_strlen( m_filename ) ); + // CRC32_ProcessBuffer( &crcFilename, m_path, Q_strlen( m_path ) ); + // CRC32_Final( &crcFilename ); + + // m_crcIdentifier = crcFilename; + + m_nFileFraction = nFileFraction; +} + +#ifdef SUPPORT_PACKED_STORE + +void CFileTracker2::NotePackFileAccess( const char *pFilename, const char *pPathID, int iSearchPathStoreId, CPackedStoreFileHandle &VPKHandle ) +{ +#if !defined( _GAMECONSOLE ) && !defined( DEDICATED ) + AUTO_LOCK( m_Mutex ); + Assert( iSearchPathStoreId > 0 ); + + int idxFile = IdxFileFromName( pFilename, pPathID, 0, false ); + TrackedFile_t &trackedfile = m_treeAllOpenedFiles[ idxFile ]; + + // we could use the CRC data from the VPK header - and verify it + // VPKHandle.GetFileCRCFromHeaderData(); + // for now all we are going to do is track that this file came from a VPK + trackedfile.m_PackFileID = VPKHandle.m_pOwner->m_PackFileID; + trackedfile.m_nPackFileNumber = VPKHandle.m_nFileNumber; // this might be useful to send up + trackedfile.m_iLoadedSearchPathStoreId = iSearchPathStoreId; + trackedfile.m_bFileInVPK = true; +#endif // !defined( _GAMECONSOLE ) && !defined( DEDICATED ) +} + +#endif // SUPPORT_PACKED_STORE + +struct FileListToUnloadForWhitelistChange : public IFileList +{ + virtual bool IsFileInList( const char *pFilename ) + { + char szFixedName[ MAX_PATH ]; + GetFixedName( pFilename, szFixedName ); + return m_dictFiles.Find( szFixedName ) >= 0; + } + + virtual void Release() + { + delete this; + } + + void AddFile( const char *pszFilename ) + { + char szFixedName[ MAX_PATH ]; + GetFixedName( pszFilename, szFixedName ); + if ( m_dictFiles.Find( szFixedName ) < 0 ) + m_dictFiles.Insert( szFixedName ); + } + + void GetFixedName( const char *pszFilename, char *pszFixedName ) + { + V_strncpy( pszFixedName, pszFilename, MAX_PATH ); + V_strlower( pszFixedName ); + V_FixSlashes( pszFixedName ); + } + + CUtlDict<int> m_dictFiles; +}; + +IFileList *CFileTracker2::GetFilesToUnloadForWhitelistChange( IPureServerWhitelist *pNewWhiteList ) +{ + FileListToUnloadForWhitelistChange *pResult = new FileListToUnloadForWhitelistChange; + + for ( int i = m_treeAllOpenedFiles.FirstInorder() ; i >= 0 ; i = m_treeAllOpenedFiles.NextInorder( i ) ) + { + TrackedFile_t &f = m_treeAllOpenedFiles[i]; + + // !KLUDGE! If we ignored it at all, just reload it. + // This is more conservative than we need to be, but the set of files we are ignoring is probably + // pretty small so it should be fine. + if ( f.m_bIgnoredForPureServer ) + { + f.m_bIgnoredForPureServer = false; +#ifdef PURE_SERVER_DEBUG_SPEW + Msg( "%s was ignored for pure server purposes. Queuing for reload\n", f.m_filename ); +#endif + pResult->AddFile( f.m_filename ); + continue; + } + + if ( f.m_iLoadedSearchPathStoreId != 0 && pNewWhiteList && pNewWhiteList->GetFileClass( f.m_filename ) == ePureServerFileClass_AnyTrusted ) + { + // Check if we loaded it from a path that no longer exists or is no longer trusted + const CBaseFileSystem::CSearchPath *pSearchPath = m_pFileSystem->FindSearchPathByStoreId( f.m_iLoadedSearchPathStoreId ); + if ( pSearchPath == NULL ) + { +#ifdef PURE_SERVER_DEBUG_SPEW + Msg( "%s was loaded from search path that's no longer mounted. Queuing for reload\n", f.m_filename ); +#endif + pResult->AddFile( f.m_filename ); + } + else if ( !pSearchPath->m_bIsTrustedForPureServer ) + { +#ifdef PURE_SERVER_DEBUG_SPEW + Msg( "%s was loaded from search path that's not currently trusted. Queuing for reload\n", f.m_filename ); +#endif + pResult->AddFile( f.m_filename ); + } + else + { +#if defined( _DEBUG ) && defined( PURE_SERVER_DEBUG_SPEW ) + Msg( "%s is OK. Keeping\n", f.m_filename ); +#endif + } + } + } + + // Do we need to reload anything? + if ( pResult->m_dictFiles.Count() > 0 ) + return pResult; + + // Nothing to reload, return an empty list as an optimization + pResult->Release(); + return NULL; +} + +#ifdef SUPPORT_PACKED_STORE + +void CFileTracker2::AddFileHashForVPKFile( int nPackFileNumber, int nFileFraction, int cbFileLen, MD5Value_t &md5, CPackedStoreFileHandle &VPKHandle ) +{ +#if !defined( DEDICATED ) + AUTO_LOCK( m_Mutex ); + + char szDataFileName[MAX_PATH]; + VPKHandle.m_nFileNumber = nPackFileNumber; + VPKHandle.GetPackFileName( szDataFileName, sizeof(szDataFileName) ); + const char *pszFileName = V_GetFileName( szDataFileName ); + + TrackedVPKFile_t trackedVPKFile; + trackedVPKFile.m_nPackFileNumber = VPKHandle.m_nFileNumber; + trackedVPKFile.m_PackFileID = VPKHandle.m_pOwner->m_PackFileID; + trackedVPKFile.m_nFileFraction = nFileFraction; + trackedVPKFile.m_idxAllOpenedFiles = IdxFileFromName( pszFileName, "GAME", nFileFraction, true ); + + m_treeTrackedVPKFiles.Insert( trackedVPKFile ); + + TrackedFile_t &trackedfile = m_treeAllOpenedFiles[ trackedVPKFile.m_idxAllOpenedFiles ]; + // These set in IdxFileFromName: + // trackedfile.m_crcIdentifier + // trackedfile.m_filename + // trackedfile.m_path + // trackedfile.m_bPackOrVPKFile + // trackedfile.m_nFileFraction + // Not set: + // trackedfile.m_iLoadedSearchPathStoreId + // trackedfile.m_bIgnoredForPureServer + trackedfile.m_bFileInVPK = false; + trackedfile.m_bPackOrVPKFile = true; + trackedfile.m_filehashFinal.m_cbFileLen = cbFileLen; + trackedfile.m_filehashFinal.m_eFileHashType = FileHash_t::k_EFileHashTypeEntireFile; + trackedfile.m_filehashFinal.m_nPackFileNumber = nPackFileNumber; + trackedfile.m_filehashFinal.m_PackFileID = VPKHandle.m_pOwner->m_PackFileID; + trackedfile.m_filehashFinal.m_crcIOSequence = cbFileLen; + Q_memcpy( trackedfile.m_filehashFinal.m_md5contents.bits, md5.bits, sizeof( md5.bits) ); +#endif // !DEDICATED +} + +#endif // SUPPORT_PACKED_STORE + +int CFileTracker2::IdxFileFromName( const char *pFilename, const char *pPathID, int nFileFraction, bool bPackOrVPKFile ) +{ + TrackedFile_t trackedfile; + + trackedfile.RebuildFileName( m_stringPool, pFilename, pPathID, nFileFraction ); + trackedfile.m_bPackOrVPKFile = bPackOrVPKFile; + + int idxFile = m_treeAllOpenedFiles.Find( trackedfile ); + if ( idxFile == m_treeAllOpenedFiles.InvalidIndex() ) + { + idxFile = m_treeAllOpenedFiles.Insert( trackedfile ); + } + + return idxFile; +} + +#ifdef SUPPORT_PACKED_STORE + +int CFileTracker2::NotePackFileOpened( const char *pVPKAbsPath, const char *pPathID, int64 nLength ) +{ +#if !defined( _GAMECONSOLE ) + AUTO_LOCK( m_Mutex ); + + int idxFile = IdxFileFromName( pVPKAbsPath, pPathID, 0, true ); + + TrackedFile_t &trackedfile = m_treeAllOpenedFiles[ idxFile ]; + // we have the real name we want to use. correct the name + trackedfile.m_bPackOrVPKFile = true; + trackedfile.m_PackFileID = idxFile + 1; + trackedfile.m_filehashFinal.m_PackFileID = trackedfile.m_PackFileID; + trackedfile.m_filehashFinal.m_nPackFileNumber = -1; + m_treeAllOpenedFiles.Reinsert( idxFile ); + return idxFile + 1; +#else + return 0; +#endif +} + +#endif // SUPPORT_PACKED_STORE + +void CFileTracker2::NoteFileIgnoredForPureServer( const char *pFilename, const char *pPathID, int iSearchPathStoreId ) +{ +#if !defined( _GAMECONSOLE ) + AUTO_LOCK( m_Mutex ); + + int idxFile = IdxFileFromName( pFilename, pPathID, 0, false ); + m_treeAllOpenedFiles[ idxFile ].m_bIgnoredForPureServer = true; +#endif +} + +void CFileTracker2::NoteFileLoadedFromDisk( const char *pFilename, const char *pPathID, int iSearchPathStoreId, FILE *fp, int64 nLength ) +{ +#if !defined( _GAMECONSOLE ) && !defined( DEDICATED ) + AUTO_LOCK( m_Mutex ); + + Assert( iSearchPathStoreId != 0 ); + + int idxFile = IdxFileFromName( pFilename, pPathID, 0, false ); + TrackedFile_t &trackedfile = m_treeAllOpenedFiles[ idxFile ]; + trackedfile.m_iLoadedSearchPathStoreId = iSearchPathStoreId; +#endif +} + +void CFileTracker2::NoteFileUnloaded( const char *pFilename, const char *pPathID ) +{ +#if !defined( _GAMECONSOLE ) + AUTO_LOCK( m_Mutex ); + + // Locate bookeeping entry, if any + TrackedFile_t trackedfile; + trackedfile.RebuildFileName( m_stringPool, pFilename, pPathID, 0 ); + + int idxFile = m_treeAllOpenedFiles.Find( trackedfile ); + if ( idxFile >= 0 ) + { + // Clear state + TrackedFile_t &trackedfile = m_treeAllOpenedFiles[ idxFile ]; + trackedfile.m_iLoadedSearchPathStoreId = 0; + trackedfile.m_bIgnoredForPureServer = false; + } +#endif +} + +int CFileTracker2::ListOpenedFiles( bool bAllOpened, const char *pchFilenameFind ) +{ + AUTO_LOCK( m_Mutex ); + + int i; + int InvalidIndex; + + if ( bAllOpened ) + { + i = m_treeAllOpenedFiles.FirstInorder(); + InvalidIndex = m_treeAllOpenedFiles.InvalidIndex(); + } + else + { + i = m_treeTrackedVPKFiles.FirstInorder(); + InvalidIndex = m_treeTrackedVPKFiles.InvalidIndex(); + } + + Msg( "#, Path, FileName, (PackFileID, PackFileNumber), FileLen, FileFraction\n" ); + + int count = 0; + int cPackFiles = 0; + while ( i != InvalidIndex ) + { + int index = bAllOpened ? i : m_treeTrackedVPKFiles[ i ].m_idxAllOpenedFiles; + TrackedFile_t &file = m_treeAllOpenedFiles[ index ]; + + if ( file.m_PackFileID ) + cPackFiles++; + if ( !pchFilenameFind || + Q_strstr( file.m_filename, pchFilenameFind ) || + Q_strstr( file.m_path, pchFilenameFind ) ) + { + Msg( "%d %s %s ( %d, %d ) %d %d%s%s\n", + count, file.m_path, file.m_filename, file.m_PackFileID, file.m_nPackFileNumber, + file.m_filehashFinal.m_cbFileLen, file.m_nFileFraction /*, file.m_crcIdentifier*/, + file.m_bFileInVPK ? " (invpk)" : "", + file.m_bPackOrVPKFile ? " (vpk)" : ""); + } + + i = bAllOpened ? m_treeAllOpenedFiles.NextInorder( i ) : m_treeTrackedVPKFiles.NextInorder( i ); + count++; + } + + Msg( "cThreadedBlocks:%d cDupMD5s:%d\n", m_cThreadBlocks, m_cDupMD5s ); + Msg( "TrackedVPKFiles:%d AllOpenedFiles:%d files VPKfiles:%d StringPoolCount:%d\n", + m_treeTrackedVPKFiles.Count(), m_treeAllOpenedFiles.Count(), cPackFiles, m_stringPool.Count() ); + return m_treeAllOpenedFiles.Count(); +} + +static void CC_TrackerListAllFiles( const CCommand &args ) +{ + const char *pchFilenameFind = ( args.ArgC() >= 2 ) ? args[1] : NULL; + BaseFileSystem()->m_FileTracker2.ListOpenedFiles( true, pchFilenameFind ); +} +static ConCommand trackerlistallfiles( "trackerlistallfiles", CC_TrackerListAllFiles, "TrackerListAllFiles" ); + +static void CC_TrackerListVPKFiles( const CCommand &args ) +{ + const char *pchFilenameFind = ( args.ArgC() >= 2 ) ? args[1] : NULL; + BaseFileSystem()->m_FileTracker2.ListOpenedFiles( false, pchFilenameFind ); +} +static ConCommand trackerlistvpkfiles( "trackerlistvpkfiles", CC_TrackerListVPKFiles, "TrackerListVPKFiles" ); + +#endif // !DEDICATED diff --git a/filesystem/filetracker.h b/filesystem/filetracker.h new file mode 100644 index 0000000..5bc5a4b --- /dev/null +++ b/filesystem/filetracker.h @@ -0,0 +1,232 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +//============================================================================= + +#ifndef FILETRACKER_H +#define FILETRACKER_H +#ifdef _WIN32 +#pragma once +#endif + +class CBaseFileSystem; +class CPackedStoreFileHandle; + +#if !defined( DEDICATED ) + +// Comments from Fletcher: +// +// TF isn’t sending any hashes to the server. (That’s probably what CSGO is doing, +// but not TF.) Comparing hashes doesn’t work when there are optional updates. (We +// release a new client without updating the server.) Also on TF, we don’t ship +// any textures or audio to the dedicated server. +// +// On TF, the client just confirms that the files were loaded from a trusted source. +// +// When a client connects to a “pure” server, the client is supposed to limit +// which modified files are allowed.) + +#include "ifilelist.h" +#include "tier1/utldict.h" +#include "tier0/tslist.h" +#include "tier1/stringpool.h" + +struct TrackedFile_t +{ + TrackedFile_t() + { + m_nFileFraction = 0; + m_PackFileID = 0; + m_nPackFileNumber = 0; + m_bPackOrVPKFile = false; + m_bFileInVPK = false; + m_iLoadedSearchPathStoreId = 0; + m_bIgnoredForPureServer = false; + } + + FileHash_t m_filehashFinal; + + const char *m_filename; + const char *m_path; + int m_nFileFraction; + int m_iLoadedSearchPathStoreId; // ID of the search path that we loaded from. Zero if we are not currently loaded. + + int m_PackFileID; + int m_nPackFileNumber; + bool m_bPackOrVPKFile; + bool m_bFileInVPK; + bool m_bIgnoredForPureServer; // Did we ignore a file by this name as a result of pure server rules, the last time it was opened? + + // The crcIdentifier is a CRC32 of the filename and path. It could be used for quick comparisons of + // path+filename, but since we don't do that at the moment we don't need this. + // CRC32_t m_crcIdentifier; + + void RebuildFileName( CStringPool &stringPool, const char *pFilename, const char *pPathID, int nFileFraction ); + + static bool Less( const TrackedFile_t& lhs, const TrackedFile_t& rhs ) + { + int nCmp = Q_strcmp( lhs.m_path, rhs.m_path ); + if ( nCmp < 0 ) + return true; + else if ( nCmp > 0 ) + return false; + + nCmp = Q_strcmp( lhs.m_filename, rhs.m_filename ); + if ( nCmp < 0 ) + return true; + + return false; + } +}; + +struct TrackedVPKFile_t +{ + TrackedVPKFile_t() + { + m_PackFileID = 0; + m_nPackFileNumber = 0; + m_nFileFraction = 0; + } + int m_PackFileID; + int m_nPackFileNumber; + int m_nFileFraction; + int m_idxAllOpenedFiles; // Index into m_treeAllOpenedFiles + + static bool Less( const TrackedVPKFile_t& lhs, const TrackedVPKFile_t& rhs ) + { + if ( lhs.m_nPackFileNumber < rhs.m_nPackFileNumber ) + return true; + else if ( lhs.m_nPackFileNumber > rhs.m_nPackFileNumber ) + return false; + + if ( lhs.m_nFileFraction < rhs.m_nFileFraction ) + return true; + else if ( lhs.m_nFileFraction > rhs.m_nFileFraction ) + return false; + + if ( lhs.m_PackFileID < rhs.m_PackFileID ) + return true; + + return false; + } +}; + +class StuffToMD5_t +{ +public: + uint8 *m_pubBuffer; + int m_cubBuffer; + MD5Value_t m_md5Value; + int m_idxTrackedVPKFile; + int m_idxListSubmittedJobs; +}; + +class SubmittedMd5Job_t +{ +public: + bool m_bFinished; + MD5Value_t m_md5Value; +}; + +class CFileTracker2 +#ifdef SUPPORT_PACKED_STORE + : IThreadedFileMD5Processor +#endif +{ +public: + CFileTracker2( CBaseFileSystem *pFileSystem ); + ~CFileTracker2(); + + void ShutdownAsync(); + + void MarkAllCRCsUnverified(); + int GetUnverifiedFileHashes( CUnverifiedFileHash *pFiles, int nMaxFiles ); + EFileCRCStatus CheckCachedFileHash( const char *pPathID, const char *pRelativeFilename, int nFileFraction, FileHash_t *pFileHash ); + +#ifdef SUPPORT_PACKED_STORE + unsigned ThreadedProcessMD5Requests(); + virtual int SubmitThreadedMD5Request( uint8 *pubBuffer, int cubBuffer, int PackFileID, int nPackFileNumber, int nPackFileFraction ); + virtual bool BlockUntilMD5RequestComplete( int iRequest, MD5Value_t *pMd5ValueOut ); + virtual bool IsMD5RequestComplete( int iRequest, MD5Value_t *pMd5ValueOut ); + + int NotePackFileOpened( const char *pVPKAbsPath, const char *pPathID, int64 nLength ); + void NotePackFileAccess( const char *pFilename, const char *pPathID, int iSearchPathStoreId, CPackedStoreFileHandle &VPKHandle ); + void AddFileHashForVPKFile( int nPackFileNumber, int nFileFraction, int cbFileLen, MD5Value_t &md5, CPackedStoreFileHandle &fhandle ); +#endif + + void NoteFileIgnoredForPureServer( const char *pFilename, const char *pPathID, int iSearchPathStoreId ); + void NoteFileLoadedFromDisk( const char *pFilename, const char *pPathID, int iSearchPathStoreId, FILE *fp, int64 nLength ); + void NoteFileUnloaded( const char *pFilename, const char *pPathID ); + int ListOpenedFiles( bool bAllOpened, const char *pchFilenameFind ); + + IFileList *GetFilesToUnloadForWhitelistChange( IPureServerWhitelist *pNewWhiteList ); + +private: + int IdxFileFromName( const char *pFilename, const char *pPathID, int nFileFraction, bool bPackOrVPKFile ); + + CStringPool m_stringPool; + CUtlRBTree< TrackedFile_t, int > m_treeAllOpenedFiles; + CUtlRBTree< TrackedVPKFile_t, int > m_treeTrackedVPKFiles; + + CBaseFileSystem *m_pFileSystem; + CThreadMutex m_Mutex; // Threads call into here, so we need to be safe. + +#ifdef SUPPORT_PACKED_STORE + CThreadEvent m_threadEventWorkToDo; + CThreadEvent m_threadEventWorkCompleted; + volatile bool m_bThreadShouldRun; + ThreadHandle_t m_hWorkThread; + + CTSQueue< StuffToMD5_t > m_PendingJobs; + CTSQueue< StuffToMD5_t > m_CompletedJobs; + CUtlLinkedList< SubmittedMd5Job_t > m_SubmittedJobs; +#endif // SUPPORT_PACKED_STORE + + // Stats + int m_cThreadBlocks; + int m_cDupMD5s; +}; + +#else + +// +// Dedicated server NULL filetracker. Pretty much does nothing. +// +class CFileTracker2 +#ifdef SUPPORT_PACKED_STORE + : IThreadedFileMD5Processor +#endif +{ +public: + CFileTracker2( CBaseFileSystem *pFileSystem ) {} + ~CFileTracker2() {} + + void ShutdownAsync() {} + + void MarkAllCRCsUnverified() {} + int GetUnverifiedFileHashes( CUnverifiedFileHash *pFiles, int nMaxFiles ) { return 0; } + EFileCRCStatus CheckCachedFileHash( const char *pPathID, const char *pRelativeFilename, int nFileFraction, FileHash_t *pFileHash ) + { return k_eFileCRCStatus_CantOpenFile; } + +#ifdef SUPPORT_PACKED_STORE + virtual int SubmitThreadedMD5Request( uint8 *pubBuffer, int cubBuffer, int PackFileID, int nPackFileNumber, int nPackFileFraction ) + { return 0; } + virtual bool BlockUntilMD5RequestComplete( int iRequest, MD5Value_t *pMd5ValueOut ) { Assert(0); return true; } + virtual bool IsMD5RequestComplete( int iRequest, MD5Value_t *pMd5ValueOut ) { Assert(0); return true; } + + int NotePackFileOpened( const char *pVPKAbsPath, const char *pPathID, int64 nLength ) { return 0; } + void NotePackFileAccess( const char *pFilename, const char *pPathID, int iSearchPathStoreId, CPackedStoreFileHandle &VPKHandle ) {} + void AddFileHashForVPKFile( int nPackFileNumber, int nFileFraction, int cbFileLen, MD5Value_t &md5, CPackedStoreFileHandle &fhandle ) {} +#endif + + void NoteFileIgnoredForPureServer( const char *pFilename, const char *pPathID, int iSearchPathStoreId ) {} + void NoteFileLoadedFromDisk( const char *pFilename, const char *pPathID, int iSearchPathStoreId, FILE *fp, int64 nLength ) {} + void NoteFileUnloaded( const char *pFilename, const char *pPathID ) {} + + IFileList *GetFilesToUnloadForWhitelistChange( IPureServerWhitelist *pNewWhiteList ) { return NULL; } +}; + +#endif // DEDICATED + +#endif // FILETRACKER_H diff --git a/filesystem/linux_support.cpp b/filesystem/linux_support.cpp new file mode 100644 index 0000000..c68967c --- /dev/null +++ b/filesystem/linux_support.cpp @@ -0,0 +1,266 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//=============================================================================// + +#include "linux_support.h" +#include "tier0/threadtools.h" // For ThreadInMainThread() +#include "tier1/strtools.h" + +char selectBuf[PATH_MAX]; + +int FileSelect(const struct dirent *ent) +{ + const char *mask=selectBuf; + const char *name=ent->d_name; + + //printf("Test:%s %s\n",mask,name); + + if(!strcmp(name,".") || !strcmp(name,"..") ) return 0; + + if(!strcmp(selectBuf,"*.*")) return 1; + + while( *mask && *name ) + { + if(*mask=='*') + { + mask++; // move to the next char in the mask + if(!*mask) // if this is the end of the mask its a match + { + return 1; + } + while(*name && toupper(*name)!=toupper(*mask)) + { // while the two don't meet up again + name++; + } + if(!*name) + { // end of the name + break; + } + } + else if (*mask!='?') + { + if( toupper(*mask) != toupper(*name) ) + { // mismatched! + return 0; + } + else + { + mask++; + name++; + if( !*mask && !*name) + { // if its at the end of the buffer + return 1; + } + + } + + } + else /* mask is "?", we don't care*/ + { + mask++; + name++; + } + } + + return( !*mask && !*name ); // both of the strings are at the end +} + +int FillDataStruct(FIND_DATA *dat) +{ + struct stat fileStat; + + if(dat->curMatch >= dat->numMatches) + return -1; + + char szFullPath[MAX_PATH]; + Q_snprintf( szFullPath, sizeof(szFullPath), "%s/%s", dat->cBaseDir, dat->namelist[dat->curMatch]->d_name ); + + if(!stat(szFullPath,&fileStat)) + { + dat->dwFileAttributes=fileStat.st_mode; + } + else + { + dat->dwFileAttributes=0; + } + + // now just put the filename in the output data + Q_snprintf( dat->cFileName, sizeof(dat->cFileName), "%s", dat->namelist[dat->curMatch]->d_name ); + + //printf("%s\n", dat->namelist[dat->curMatch]->d_name); + free(dat->namelist[dat->curMatch]); + + dat->curMatch++; + return 1; +} + + +HANDLE FindFirstFile( const char *fileName, FIND_DATA *dat) +{ + char nameStore[PATH_MAX]; + char *dir=NULL; + int n,iret=-1; + + Q_strncpy(nameStore,fileName, sizeof( nameStore ) ); + + if(strrchr(nameStore,'/') ) + { + dir=nameStore; + while(strrchr(dir,'/') ) + { + struct stat dirChk; + + // zero this with the dir name + dir=strrchr(nameStore,'/'); + if ( dir == nameStore ) // special case for root dir, '/' + { + dir[1] = '\0'; + } + else + { + *dir='\0'; + dir=nameStore; + } + + + if (stat(dir,&dirChk) < 0) + { + continue; + } + + if( S_ISDIR( dirChk.st_mode ) ) + { + break; + } + } + } + else + { + // couldn't find a dir seperator... + return (HANDLE)-1; + } + + if( strlen(dir)>0 ) + { + if ( strlen(dir) == 1 ) // if it was the root dir + Q_strncpy(selectBuf,fileName+1, sizeof( selectBuf ) ); + else + Q_strncpy(selectBuf,fileName+strlen(dir)+1, sizeof( selectBuf ) ); + + Q_strncpy(dat->cBaseDir,dir, sizeof( dat->cBaseDir ) ); + dat->namelist = NULL; + n = scandir(dir, &dat->namelist, FileSelect, alphasort); + if (n < 0) + { + if ( dat->namelist ) + free(dat->namelist); + // silently return, nothing interesting + } + else + { + dat->numMatches = n; + dat->curMatch = 0; + iret=FillDataStruct(dat); + if(iret<0) + { + if ( dat->namelist ) + free(dat->namelist); + dat->namelist = NULL; + } + + } + } + +// printf("Returning: %i \n",iret); + return (HANDLE)iret; +} + +bool FindNextFile(HANDLE handle, FIND_DATA *dat) +{ + if(dat->curMatch >= dat->numMatches) + { + if ( dat->namelist != NULL ) + free(dat->namelist); + dat->namelist = NULL; + return false; // no matches left + } + + FillDataStruct(dat); + return true; +} + +bool FindClose(HANDLE handle) +{ + return true; +} + + + +// Pass this function a full path and it will look for files in the specified +// directory that match the file name but potentially with different case. +// The directory name itself is not treated specially. +// If multiple names that match are found then lowercase letters take precedence. +bool findFileInDirCaseInsensitive( const char *file, char* output, size_t bufSize) +{ + // Make sure the output buffer is always null-terminated. + output[0] = 0; + + // Find where the file part starts. + const char *dirSep = strrchr(file,'/'); + if( !dirSep ) + { + dirSep=strrchr(file,'\\'); + if( !dirSep ) + { + return false; + } + } + + // Allocate space for the directory portion. + size_t dirSize = ( dirSep - file ) + 1; + char *dirName = static_cast<char *>( alloca( dirSize ) ); + + V_strncpy( dirName , file, dirSize ); + + DIR* pDir = opendir( dirName ); + if ( !pDir ) + return false; + + const char* filePart = dirSep + 1; + // The best matching file name will be placed in this array. + char outputFileName[ MAX_PATH ]; + bool foundMatch = false; + + // Scan through the directory. + for ( dirent* pEntry = NULL; ( pEntry = readdir( pDir ) ); /**/ ) + { + if ( strcasecmp( pEntry->d_name, filePart ) == 0 ) + { + // If we don't have an existing candidate or if this name is + // a better candidate then copy it in. A 'better' candidate + // means that test beats tesT which beats tEst -- more lowercase + // letters earlier equals victory. + if ( !foundMatch || strcmp( outputFileName, pEntry->d_name ) < 0 ) + { + foundMatch = true; + V_strcpy_safe( outputFileName, pEntry->d_name ); + } + } + } + + closedir( pDir ); + + // If we didn't find any matching names then lowercase the passed in + // file name and use that. + if ( !foundMatch ) + { + V_strcpy_safe( outputFileName, filePart ); + V_strlower( outputFileName ); + } + + Q_snprintf( output, bufSize, "%s/%s", dirName, outputFileName ); + return foundMatch; +} diff --git a/filesystem/linux_support.h b/filesystem/linux_support.h new file mode 100644 index 0000000..e090e08 --- /dev/null +++ b/filesystem/linux_support.h @@ -0,0 +1,58 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//=============================================================================// +#ifndef LINUX_SUPPORT_H +#define LINUX_SUPPORT_H + +#include <ctype.h> // tolower() +#include <limits.h> // PATH_MAX define +#include <string.h> //strcmp, strcpy +#include <sys/stat.h> // stat() +#include <unistd.h> +#include <dirent.h> // scandir() +#include <stdlib.h> +#include <stdio.h> +#include "tier0/platform.h" + +#define FILE_ATTRIBUTE_DIRECTORY S_IFDIR + +typedef struct +{ + // public data + int dwFileAttributes; + char cFileName[PATH_MAX]; // the file name returned from the call + char cBaseDir[PATH_MAX]; // the root dir for this find + + int numMatches; + int curMatch; + struct dirent **namelist; +} FIND_DATA; + +#define WIN32_FIND_DATA FIND_DATA + +HANDLE FindFirstFile( const char *findName, FIND_DATA *dat); +bool FindNextFile(HANDLE handle, FIND_DATA *dat); +bool FindClose(HANDLE handle); + +// findFileInDirCaseInsensitive looks for the specified file. It returns +// false if no directory separator is found. Otherwise it looks for the +// requested file by scanning the specified directory and doing a case +// insensitive match. If the file exists (in any case) then the correctly cased +// filename will be returned in the user's buffer and 'true' will be returned. +// If the file does not exist then the filename will be lowercased and 'false' +// will be returned. +// This function uses a static buffer for internal purposes and is therefore +// not thread safe, so it must only be called from the main thread. +bool findFileInDirCaseInsensitive( const char *file, OUT_Z_BYTECAP(bufSize) char* output, size_t bufSize ); +// The _safe version of this function should be preferred since it always infers +// the directory size correctly. +template<size_t bufSize> +bool findFileInDirCaseInsensitive_safe( const char *file, OUT_Z_ARRAY char (&output)[bufSize] ) +{ + return findFileInDirCaseInsensitive( file, output, bufSize ); +} + +#endif // LINUX_SUPPORT_H diff --git a/filesystem/packfile.cpp b/filesystem/packfile.cpp new file mode 100644 index 0000000..5190959 --- /dev/null +++ b/filesystem/packfile.cpp @@ -0,0 +1,1127 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//===========================================================================// + +#include "packfile.h" +#include "zip_utils.h" +#include "tier0/basetypes.h" +#include "tier1/convar.h" +#include "tier1/lzmaDecoder.h" +#include "tier1/utlbuffer.h" +#include "tier1/generichash.h" + +ConVar fs_monitor_read_from_pack( "fs_monitor_read_from_pack", "0", 0, "0:Off, 1:Any, 2:Sync only" ); + +// How many bytes we should decode at a time when doing pseudo-reads to seek forward in a compressed file handle, +// (affects maximum stack allocation by a forward seek) +#define COMPRESSED_SEEK_READ_CHUNK 1024 + +CPackFile::CPackFile() +{ + m_FileLength = 0; + m_hPackFileHandleFS = NULL; + m_fs = NULL; + m_nBaseOffset = 0; + m_bIsMapPath = false; + m_lPackFileTime = 0L; + m_refCount = 0; + m_nOpenFiles = 0; + m_PackFileID = 0; +} + +CPackFile::~CPackFile() +{ + if ( m_nOpenFiles ) + { + Error( "Closing pack file with %d open files!\n", m_nOpenFiles ); + } + + if ( m_hPackFileHandleFS ) + { + m_fs->FS_fclose( m_hPackFileHandleFS ); + m_hPackFileHandleFS = NULL; + } + + m_fs->m_ZipFiles.FindAndRemove( this ); +} + +int CPackFile::GetSectorSize() +{ + if ( m_hPackFileHandleFS ) + { + return m_fs->FS_GetSectorSize( m_hPackFileHandleFS ); + } +#if defined( SUPPORT_PACKED_STORE ) + else if ( m_hPackFileHandleVPK ) + { + return 2048; + } +#endif + else + { + return -1; + } +} + +// Read a bit of the file from the pack file: +int CZipPackFileHandle::Read( void* pBuffer, int nDestSize, int nBytes ) +{ + // Clamp nBytes to not go past the end of the file (async is still possible due to nDestSize) + if ( nBytes + m_nFilePointer > m_nLength ) + { + nBytes = m_nLength - m_nFilePointer; + } + + // Seek to the given file pointer and read + int nBytesRead = m_pOwner->ReadFromPack( m_nIndex, pBuffer, nDestSize, nBytes, m_nBase + m_nFilePointer ); + + m_nFilePointer += nBytesRead; + + return nBytesRead; +} + +// Seek around inside the pack: +int CZipPackFileHandle::Seek( int nOffset, int nWhence ) +{ + if ( nWhence == SEEK_SET ) + { + m_nFilePointer = nOffset; + } + else if ( nWhence == SEEK_CUR ) + { + m_nFilePointer += nOffset; + } + else if ( nWhence == SEEK_END ) + { + m_nFilePointer = m_nLength + nOffset; + } + + // Clamp the file pointer to the actual bounds of the file: + if ( m_nFilePointer > m_nLength ) + { + m_nFilePointer = m_nLength; + } + + return m_nFilePointer; +} + +//----------------------------------------------------------------------------- +// Open a file inside of a pack file. +//----------------------------------------------------------------------------- +CFileHandle *CZipPackFile::OpenFile( const char *pFileName, const char *pOptions ) +{ + int nIndex, nOriginalSize, nCompressedSize; + int64 nPosition; + unsigned short nCompressionMethod; + + // find the file's location in the pack + if ( GetFileInfo( pFileName, nIndex, nPosition, nOriginalSize, nCompressedSize, nCompressionMethod ) ) + { + m_mutex.Lock(); +#if defined( SUPPORT_PACKED_STORE ) + if ( m_nOpenFiles == 0 && m_hPackFileHandleFS == NULL && !m_hPackFileHandleVPK ) +#else + if ( m_nOpenFiles == 0 && m_hPackFileHandleFS == NULL ) +#endif + { + // Try to open it as a regular file first + m_hPackFileHandleFS = m_fs->Trace_FOpen( m_ZipName, "rb", 0, NULL ); + + // !NOTE! Pack files inside of VPK not supported + } + m_nOpenFiles++; + m_mutex.Unlock(); + CPackFileHandle* ph = NULL; + if ( nCompressionMethod == ZIP_COMPRESSION_LZMA ) + { + ph = new CLZMAZipPackFileHandle( this, nPosition, nOriginalSize, nCompressedSize, nIndex ); + } + else + { + AssertMsg( nCompressionMethod == ZIP_COMPRESSION_NONE, "Unsupported compression type in zip pack file" ); + ph = new CZipPackFileHandle( this, nPosition, nOriginalSize, nIndex ); + } + CFileHandle *fh = new CFileHandle( m_fs ); + fh->m_pPackFileHandle = ph; + fh->m_nLength = nOriginalSize; + + // The default mode for fopen is text, so require 'b' for binary + if ( strstr( pOptions, "b" ) == NULL ) + { + fh->m_type = FT_PACK_TEXT; + } + else + { + fh->m_type = FT_PACK_BINARY; + } + +#if !defined( _RETAIL ) + fh->SetName( pFileName ); +#endif + return fh; + } + + return NULL; +} + +//----------------------------------------------------------------------------- +// Get a directory entry from a pack's preload section +//----------------------------------------------------------------------------- +ZIP_PreloadDirectoryEntry* CZipPackFile::GetPreloadEntry( int nEntryIndex ) +{ + if ( !m_pPreloadHeader ) + { + return NULL; + } + + // If this entry doesn't have a corresponding preload entry, fail. + if ( m_PackFiles[nEntryIndex].m_nPreloadIdx == INVALID_PRELOAD_ENTRY ) + { + return NULL; + } + + return &m_pPreloadDirectory[m_PackFiles[nEntryIndex].m_nPreloadIdx]; +} + +//----------------------------------------------------------------------------- +// Read a file from the pack +//----------------------------------------------------------------------------- +int CZipPackFile::ReadFromPack( int nEntryIndex, void* pBuffer, int nDestBytes, int nBytes, int64 nOffset ) +{ + if ( nEntryIndex >= 0 ) + { + if ( nBytes <= 0 ) + { + return 0; + } + + // X360TBD: This is screwy, it works because m_nBaseOffset is 0 for preload capable zips + // It comes into play for files out of the embedded bsp zip, + // this hackery is a pre-bias expecting ReadFromPack() do a symmetric post bias, yuck. + + // Attempt to satisfy request from possible preload section, otherwise fall through + // A preload entry may be compressed + ZIP_PreloadDirectoryEntry *pPreloadEntry = GetPreloadEntry( nEntryIndex ); + if ( pPreloadEntry ) + { + // convert the absolute pack file position to a local file position + int nLocalOffset = nOffset - m_PackFiles[nEntryIndex].m_nPosition; + byte *pPreloadData = (byte*)m_pPreloadData + pPreloadEntry->DataOffset; + + if ( CLZMA::IsCompressed( pPreloadData ) ) + { + unsigned int actualSize = CLZMA::GetActualSize( pPreloadData ); + if ( nLocalOffset + nBytes <= (int)actualSize ) + { + // satisfy from compressed preload + if ( fs_monitor_read_from_pack.GetInt() == 1 ) + { + char szName[MAX_PATH]; + IndexToFilename( nEntryIndex, szName, sizeof( szName ) ); + Msg( "Read From Pack: [Preload] Requested:%d, Compressed:%d, %s\n", nBytes, pPreloadEntry->Length, szName ); + } + + if ( nLocalOffset == 0 && nDestBytes >= (int)actualSize && nBytes == (int)actualSize ) + { + // uncompress directly into caller's buffer + CLZMA::Uncompress( (unsigned char *)pPreloadData, (unsigned char *)pBuffer ); + return nBytes; + } + + // uncompress into temporary memory + CUtlMemory< byte > tempMemory; + tempMemory.EnsureCapacity( actualSize ); + CLZMA::Uncompress( pPreloadData, tempMemory.Base() ); + // copy only what caller expects + V_memcpy( pBuffer, (byte*)tempMemory.Base() + nLocalOffset, nBytes ); + return nBytes; + } + } + else if ( nLocalOffset + nBytes <= (int)pPreloadEntry->Length ) + { + // satisfy from uncompressed preload + if ( fs_monitor_read_from_pack.GetInt() == 1 ) + { + char szName[MAX_PATH]; + IndexToFilename( nEntryIndex, szName, sizeof( szName ) ); + Msg( "Read From Pack: [Preload] Requested:%d, Total:%d, %s\n", nBytes, pPreloadEntry->Length, szName ); + } + + V_memcpy( pBuffer, pPreloadData + nLocalOffset, nBytes ); + return nBytes; + } + } + } + +#if defined ( _X360 ) + // fell through as a direct request from within the pack + // intercept to possible embedded section + if ( m_pSection ) + { + // a section is a special update zip that has no files, only preload + // it has to be in the section + V_memcpy( pBuffer, (byte*)m_pSection + nOffset, nBytes ); + return nBytes; + } +#endif + + // Otherwise, do the read from the pack + m_mutex.Lock(); + + if ( fs_monitor_read_from_pack.GetInt() == 1 || ( fs_monitor_read_from_pack.GetInt() == 2 && ThreadInMainThread() ) ) + { + // spew info about real i/o request + char szName[MAX_PATH]; + IndexToFilename( nEntryIndex, szName, sizeof( szName ) ); + Msg( "Read From Pack: Sync I/O: Requested:%7d, Offset:0x%16.16llx, %s\n", nBytes, m_nBaseOffset + nOffset, szName ); + } + + int nBytesRead = 0; + // Seek to the start of the read area and perform the read: TODO: CHANGE THIS INTO A CFileHandle + if ( m_hPackFileHandleFS ) + { + m_fs->FS_fseek( m_hPackFileHandleFS, m_nBaseOffset + nOffset, SEEK_SET ); + nBytesRead = m_fs->FS_fread( pBuffer, nDestBytes, nBytes, m_hPackFileHandleFS ); + } +#if defined( SUPPORT_PACKED_STORE ) + else + { + // We're a packfile embedded in a VPK + m_hPackFileHandleVPK.Seek( m_nBaseOffset + nOffset, FILESYSTEM_SEEK_HEAD ); + nBytesRead = m_hPackFileHandleVPK.Read( pBuffer, nBytes ); + } +#endif + m_mutex.Unlock(); + + return nBytesRead; +} + +//----------------------------------------------------------------------------- +// Gets size, position, and index for a file in the pack. +//----------------------------------------------------------------------------- +bool CZipPackFile::GetFileInfo( const char *pFileName, int &nBaseIndex, int64 &nFileOffset, int &nOriginalSize, int &nCompressedSize, unsigned short &nCompressionMethod ) +{ + char szCleanName[MAX_FILEPATH]; + Q_strncpy( szCleanName, pFileName, sizeof( szCleanName ) ); +#ifdef _WIN32 + Q_strlower( szCleanName ); +#endif + Q_FixSlashes( szCleanName ); + + if ( !Q_RemoveDotSlashes( szCleanName, CORRECT_PATH_SEPARATOR, false ) ) + { + return false; + } + + CZipPackFile::CPackFileEntry lookup; + + // We may get passed non-canonicalized filenames, so we need to remove the ../ from the path + char szFixedName[MAX_PATH] = {0}; + V_strcpy_safe( szFixedName, pFileName ); + V_RemoveDotSlashes( szFixedName ); + + lookup.m_HashName = HashStringCaselessConventional( szFixedName ); + + int idx = m_PackFiles.Find( lookup ); + if ( -1 != idx ) + { + nFileOffset = m_PackFiles[idx].m_nPosition; + nOriginalSize = m_PackFiles[idx].m_nOriginalSize; + nCompressedSize = m_PackFiles[idx].m_nCompressedSize; + nBaseIndex = idx; + nCompressionMethod = m_PackFiles[idx].m_nCompressionMethod; + return true; + } + + return false; +} + +bool CZipPackFile::IndexToFilename( int nIndex, char *pBuffer, int nBufferSize ) +{ + AssertMsg( nIndex >= 0 && nIndex < m_PackFiles.Count(), "Out of bounds vector access in IndexToFilename" ); + if ( nIndex >= 0 ) + { + m_fs->String( m_PackFiles[nIndex].m_hFileName, pBuffer, nBufferSize ); + return true; + } + + Q_strncpy( pBuffer, "unknown", nBufferSize ); + + return false; +} + +//----------------------------------------------------------------------------- +// Find a file in the pack. +//----------------------------------------------------------------------------- +bool CZipPackFile::ContainsFile( const char *pFileName ) +{ + int nIndex, nOriginalSize, nCompressedSize; + int64 nOffset; + unsigned short nCompressionMethod; + bool bFound = GetFileInfo( pFileName, nIndex, nOffset, nOriginalSize, nCompressedSize, nCompressionMethod ); + return bFound; +} + +//----------------------------------------------------------------------------- +// Build a list of matching files and directories given a FindFirst() style wildcard +//----------------------------------------------------------------------------- +void CZipPackFile::GetFileAndDirLists( const char *pRawWildCard, CUtlStringList &outDirnames, CUtlStringList &outFilenames, bool bSortedOutput ) +{ + // See also: VPKlib function with same name. + + CUtlDict<int,int> AddedDirectories; // Used to remove duplicate paths + + char szWildCard[MAX_PATH] = { 0 }; + char szWildCardPath[MAX_PATH] = { 0 }; + char szWildCardBase[MAX_PATH] = { 0 }; + char szWildCardExt[MAX_PATH] = { 0 }; + + size_t nLenWildcardPath = 0; + size_t nLenWildcardBase = 0; + + bool bBaseWildcard = true; + bool bExtWildcard = true; + + // + // Parse the wildcard string into a base and extension used for string comparisons + // + V_strncpy( szWildCard, pRawWildCard, sizeof( szWildCard ) ); + V_FixSlashes( szWildCard, '/' ); + V_RemoveDotSlashes( szWildCard, '/', /* bRemoveDoubleSlashes */ true ); + + // Workaround edge case in crappy path code. ExtractFilePath extracts a/b/ from a/b/c/ but FileBase would return the empty string. + size_t nLenWildCard = V_strlen( szWildCard ); + if ( nLenWildCard && szWildCard[ nLenWildCard - 1 ] == '/' ) + { + V_strncpy( szWildCardPath, szWildCard, sizeof( szWildCardPath ) ); + } + else + { + V_ExtractFilePath( szWildCard, szWildCardPath, sizeof( szWildCardPath ) ); + } + + V_FileBase( szWildCard, szWildCardBase, sizeof( szWildCardBase ) ); + bool bWildcardHasExt = !!V_strrchr( szWildCard, '.' ); + V_ExtractFileExtension( szWildCard, szWildCardExt, sizeof( szWildCardExt ) ); + + // From the pattern, we now have the directory path up to the file pattern, the filename base, and the filename + // extension. + + // We don't support partial wildcards here (foo*bar.*). This support is massively inconsistent in our codebase and + // there's no one point where we implement it, so rather than trying to match one of our broken implementations + // (windows stdio is the only one I could find that was actually right), I'm going with "you shouldn't use this API + // for that". + bBaseWildcard = ( V_strcmp( szWildCardBase, "*" ) == 0 ); + bExtWildcard = ( V_strcmp( szWildCardExt, "*" ) == 0 ); + + if ( !bWildcardHasExt && bBaseWildcard ) + { + // For the special case of just '*' (and not, e.g., '*.') match '*.*' + bExtWildcard = true; + } + + nLenWildcardPath = V_strlen( szWildCardPath ); + nLenWildcardBase = V_strlen( szWildCardBase ); + + // Generate the list of directories and files that match the wildcard + // + + // For each candidate we attempt to walk up its path and consider the directories it represents as well (the + // directories in a zip only exist in that files contain them, there are no empty directories) + FOR_EACH_VEC( m_PackFiles, filesIdx ) + { + char szCandidateName[MAX_PATH] = { 0 }; + IndexToFilename( filesIdx, szCandidateName, sizeof( szCandidateName )); + + if ( !szCandidateName[0] ) + { + continue; + } + + // Check if this file starts with the wildcard selector's path. + // Note that we only ensure the prefix is the same. There are no specific entries for directories in a zip, they + // only exist in that files in the zip reference them, so handle subdirectory matches from filenames as well. + CUtlDict<int,int> ConsideredDirectories; // Will have duplicate directory matches when multiple files reside in them + if ( ( nLenWildcardPath && ( 0 == V_strnicmp( szCandidateName, szWildCardPath, nLenWildcardPath ) ) ) + || ( !nLenWildcardPath && strchr( szCandidateName, '/' ) ) ) + { + // Check if we matched because of a sub-directory, e.g. a/b/*.* would match /a/b/c/d/foo (in which case we + // want to add /a/b/c to the matched directories list, ignoring the actual specific file) + char szCandidateBaseName[MAX_PATH] = { 0 }; + bool bIsDir = false; + size_t nSubDirLen = 0; + char *pSubDirSlash = strchr( szCandidateName + nLenWildcardPath, '/' ); + if ( pSubDirSlash ) + { + // This is a subdirectory match, drop everything after it and continue with it as the filename + nSubDirLen = (size_t)( (ptrdiff_t)pSubDirSlash - (ptrdiff_t)( szCandidateName + nLenWildcardPath ) ); + V_strncpy( szCandidateBaseName, szCandidateName + nLenWildcardPath, nSubDirLen + 1 ); + bIsDir = true; + + // Early out if we already considered this exact directory from another file + if ( ConsideredDirectories.Find( szCandidateBaseName ) != ConsideredDirectories.InvalidIndex() ) + { + continue; + } + + ConsideredDirectories.Insert( szCandidateBaseName, 0 ); + } + else + { + V_strncpy( szCandidateBaseName, szCandidateName + nLenWildcardPath, sizeof( szCandidateBaseName ) ); + } + + char *pExt = strchr( szCandidateBaseName, '.' ); + if ( pExt ) + { + // Null out the . and move to point to the extension + *pExt = '\0'; + pExt++; + } + + // Determine if this file matches the wildcart (*.*, *.ext, ext.*) + bool bBaseMatch = false; + bool bExtMatch = false; + + // If we have a base dir name, and we have a szWildCardBase to match against + if ( bBaseWildcard ) + bBaseMatch = true; // The base is the wildCard ("*"), so whatever we have as the base matches + else + bBaseMatch = ( 0 == V_stricmp( szCandidateBaseName, szWildCardBase ) ); + + // If we have an extension and we have a szWildCardExtension to mach against + if ( ( bExtWildcard && pExt ) || ( !pExt && !bWildcardHasExt ) ) + bExtMatch = true; + else + bExtMatch = bWildcardHasExt && pExt && ( 0 == V_stricmp( pExt, szWildCardExt ) ); + + // If both parts match, then add it to the list + if ( bBaseMatch && bExtMatch ) + { + if ( bIsDir ) + { + // Pull up to the subdir we considered out of szCandidateName + size_t nMatchSize = nLenWildcardPath + nSubDirLen + 1; + char *pszFullMatch = new char[ nMatchSize ]; + V_strncpy( pszFullMatch, szCandidateName, nMatchSize ); + outDirnames.AddToTail( pszFullMatch ); + } + else + { + size_t nMatchSize = V_strlen( szCandidateName ) + 1; + char *pszFullMatch = new char[ nMatchSize ]; + V_strncpy( pszFullMatch, szCandidateName, nMatchSize ); + outFilenames.AddToTail( pszFullMatch ); + } + } + } + } + + // Sort the output if requested + if ( bSortedOutput ) + { + outDirnames.Sort( &CUtlStringList::SortFunc ); + outFilenames.Sort( &CUtlStringList::SortFunc ); + } +} + + +//----------------------------------------------------------------------------- +// Set up the preload section +//----------------------------------------------------------------------------- +void CZipPackFile::SetupPreloadData() +{ + if ( m_pPreloadHeader || !m_nPreloadSectionSize ) + { + // already loaded or not available + return; + } + + MEM_ALLOC_CREDIT_( "xZip" ); + + void *pPreload; +#if defined ( _X360 ) + if ( m_pSection ) + { + pPreload = (byte*)m_pSection + m_nPreloadSectionOffset; + } + else +#endif + { + pPreload = malloc( m_nPreloadSectionSize ); + if ( !pPreload ) + { + return; + } + + if ( IsX360() ) + { + // 360 XZips are always dvd aligned + Assert( ( m_nPreloadSectionSize % XBOX_DVD_SECTORSIZE ) == 0 ); + Assert( ( m_nPreloadSectionOffset % XBOX_DVD_SECTORSIZE ) == 0 ); + } + + // preload data is loaded as a single unbuffered i/o operation + ReadFromPack( -1, pPreload, -1, m_nPreloadSectionSize, m_nPreloadSectionOffset ); + } + + // setup the header + m_pPreloadHeader = (ZIP_PreloadHeader *)pPreload; + + // setup the preload directory + m_pPreloadDirectory = (ZIP_PreloadDirectoryEntry *)((byte *)m_pPreloadHeader + sizeof( ZIP_PreloadHeader ) ); + + // setup the remap table + m_pPreloadRemapTable = (unsigned short *)((byte *)m_pPreloadDirectory + m_pPreloadHeader->PreloadDirectoryEntries * sizeof( ZIP_PreloadDirectoryEntry ) ); + + // set the preload data base + m_pPreloadData = (byte *)m_pPreloadRemapTable + m_pPreloadHeader->DirectoryEntries * sizeof( unsigned short ); +} + +void CZipPackFile::DiscardPreloadData() +{ + if ( !m_pPreloadHeader ) + { + // already discarded + return; + } + +#if defined ( _X360 ) + // a section is an alias, the header becomes an alias, not owned memory + if ( !m_pSection ) + { + free( m_pPreloadHeader ); + } +#else + free( m_pPreloadHeader ); +#endif + m_pPreloadHeader = NULL; +} + +//----------------------------------------------------------------------------- +// Parse the zip file to build the file directory and preload section +//----------------------------------------------------------------------------- +bool CZipPackFile::Prepare( int64 fileLen, int64 nFileOfs ) +{ + if ( !fileLen || fileLen < sizeof( ZIP_EndOfCentralDirRecord ) ) + { + // nonsense zip + return false; + } + + // Pack files are always little-endian + m_swap.ActivateByteSwapping( IsX360() ); + + m_FileLength = fileLen; + m_nBaseOffset = nFileOfs; + + ZIP_EndOfCentralDirRecord rec = { 0 }; + + // Find and read the central header directory from its expected position at end of the file + bool bCentralDirRecord = false; + int64 offset = fileLen - sizeof( ZIP_EndOfCentralDirRecord ); + + // 360 can have an incompatible format + bool bCompatibleFormat = true; + if ( IsX360() ) + { + // 360 has dependable exact zips, backup to handle possible xzip format + if ( offset - XZIP_COMMENT_LENGTH >= 0 ) + { + offset -= XZIP_COMMENT_LENGTH; + } + + // single i/o operation, scanning forward + char *pTemp = (char *)_alloca( fileLen - offset ); + ReadFromPack( -1, pTemp, -1, fileLen - offset, offset ); + while ( offset <= (int64)(fileLen - sizeof( ZIP_EndOfCentralDirRecord )) ) + { + memcpy( &rec, pTemp, sizeof( ZIP_EndOfCentralDirRecord ) ); + m_swap.SwapFieldsToTargetEndian( &rec ); + if ( rec.signature == PKID( 5, 6 ) ) + { + bCentralDirRecord = true; + if ( rec.commentLength >= 4 ) + { + char *pComment = pTemp + sizeof( ZIP_EndOfCentralDirRecord ); + if ( !V_strnicmp( pComment, "XZP2", 4 ) ) + { + bCompatibleFormat = false; + } + } + break; + } + offset++; + pTemp++; + } + } + else + { + // scan entire file from expected location for central dir + for ( ; offset >= 0; offset-- ) + { + ReadFromPack( -1, (void*)&rec, -1, sizeof( rec ), offset ); + m_swap.SwapFieldsToTargetEndian( &rec ); + if ( rec.signature == PKID( 5, 6 ) ) + { + bCentralDirRecord = true; + break; + } + } + } + Assert( bCentralDirRecord ); + if ( !bCentralDirRecord ) + { + // no zip directory, bad zip + return false; + } + + int numFilesInZip = rec.nCentralDirectoryEntries_Total; + if ( numFilesInZip <= 0 ) + { + // empty valid zip + return true; + } + + int firstFileIdx = 0; + + MEM_ALLOC_CREDIT(); + + // read central directory into memory and parse + CUtlBuffer zipDirBuff( 0, rec.centralDirectorySize, 0 ); + zipDirBuff.EnsureCapacity( rec.centralDirectorySize ); + zipDirBuff.ActivateByteSwapping( IsX360() ); + ReadFromPack( -1, zipDirBuff.Base(), -1, rec.centralDirectorySize, rec.startOfCentralDirOffset ); + zipDirBuff.SeekPut( CUtlBuffer::SEEK_HEAD, rec.centralDirectorySize ); + + ZIP_FileHeader zipFileHeader; + char filename[MAX_PATH] = { 0 }; + + // Check for a preload section, expected to be the first file in the zip + zipDirBuff.GetObjects( &zipFileHeader ); + zipDirBuff.Get( filename, Min( (size_t)zipFileHeader.fileNameLength, sizeof(filename) - 1 ) ); + if ( !V_stricmp( filename, PRELOAD_SECTION_NAME ) ) + { + m_nPreloadSectionSize = zipFileHeader.uncompressedSize; + m_nPreloadSectionOffset = zipFileHeader.relativeOffsetOfLocalHeader + + sizeof( ZIP_LocalFileHeader ) + + zipFileHeader.fileNameLength + + zipFileHeader.extraFieldLength; + SetupPreloadData(); + + // Set up to extract the remaining files + int nextOffset = bCompatibleFormat ? zipFileHeader.extraFieldLength + zipFileHeader.fileCommentLength : 0; + zipDirBuff.SeekGet( CUtlBuffer::SEEK_CURRENT, nextOffset ); + firstFileIdx = 1; + } + else + { + if ( IsX360() ) + { + // all 360 zip files are expected to have preload sections + // only during development, maps are allowed to lack them, due to auto-conversion + if ( !m_bIsMapPath || g_pFullFileSystem->GetDVDMode() == DVDMODE_STRICT ) + { + Warning( "ZipFile '%s' missing preload section\n", m_ZipName.String() ); + } + } + + // No preload section, reset buffer pointer + zipDirBuff.SeekGet( CUtlBuffer::SEEK_HEAD, 0 ); + } + + // Parse out central directory and determine absolute file positions of data. + // Supports uncompressed zip files, with or without preload sections + bool bSuccess = true; + char tmpString[MAX_PATH] = { 0 }; + + m_PackFiles.EnsureCapacity( numFilesInZip ); + + for ( int i = firstFileIdx; i < numFilesInZip; ++i ) + { + CZipPackFile::CPackFileEntry lookup; + zipDirBuff.GetObjects( &zipFileHeader ); + + if ( zipFileHeader.signature != PKID( 1, 2 ) ) + { + Warning( "Invalid pack file signature\n" ); + bSuccess = false; + break; + } + + if ( zipFileHeader.compressionMethod != ZIP_COMPRESSION_NONE && zipFileHeader.compressionMethod != ZIP_COMPRESSION_LZMA ) + { + Warning( "Pack file uses unsupported compression method: %hi\n", zipFileHeader.compressionMethod ); + bSuccess = false; + break; + } + + Assert( zipFileHeader.fileNameLength < sizeof( tmpString ) ); + unsigned int fileNameLen = Min( (size_t)zipFileHeader.fileNameLength, sizeof( tmpString ) - 1 ); + zipDirBuff.Get( (void *)tmpString, fileNameLen ); + tmpString[fileNameLen] = '\0'; + Q_FixSlashes( tmpString ); + + lookup.m_hFileName = m_fs->FindOrAddFileName( tmpString ); + lookup.m_HashName = HashStringCaselessConventional( tmpString ); + lookup.m_nOriginalSize = zipFileHeader.uncompressedSize; + lookup.m_nCompressedSize = zipFileHeader.compressedSize; + lookup.m_nPosition = zipFileHeader.relativeOffsetOfLocalHeader + + sizeof( ZIP_LocalFileHeader ) + + zipFileHeader.fileNameLength + + zipFileHeader.extraFieldLength; + lookup.m_nCompressionMethod = zipFileHeader.compressionMethod; + + // track the index to this file's possible preload directory entry + if ( m_pPreloadRemapTable ) + { + lookup.m_nPreloadIdx = m_pPreloadRemapTable[i]; + } + else + { + lookup.m_nPreloadIdx = INVALID_PRELOAD_ENTRY; + } + m_PackFiles.InsertNoSort( lookup ); + + int nextOffset = bCompatibleFormat ? zipFileHeader.extraFieldLength + zipFileHeader.fileCommentLength : 0; + zipDirBuff.SeekGet( CUtlBuffer::SEEK_CURRENT, nextOffset ); + } + + m_PackFiles.RedoSort(); + + return bSuccess; +} + + +//----------------------------------------------------------------------------- +// +//----------------------------------------------------------------------------- +CZipPackFile::CZipPackFile( CBaseFileSystem* fs, void *pSection ) + : m_PackFiles() +{ + m_fs = fs; + m_pPreloadDirectory = NULL; + m_pPreloadData = NULL; + m_pPreloadHeader = NULL; + m_pPreloadRemapTable = NULL; + m_nPreloadSectionOffset = 0; + m_nPreloadSectionSize = 0; + +#if defined( _X360 ) + m_pSection = pSection; +#endif +} + +CZipPackFile::~CZipPackFile() +{ + DiscardPreloadData(); +} + +//----------------------------------------------------------------------------- +// Purpose: +// Input : src1 - +// src2 - +// Output : Returns true on success, false on failure. +//----------------------------------------------------------------------------- +bool CZipPackFile::CPackFileLessFunc::Less( CZipPackFile::CPackFileEntry const& src1, CZipPackFile::CPackFileEntry const& src2, void *pCtx ) +{ + return ( src1.m_HashName < src2.m_HashName ); +} + +//----------------------------------------------------------------------------- +// Purpose: Zip Pack file handle implementation +//----------------------------------------------------------------------------- +CZipPackFileHandle::CZipPackFileHandle( CZipPackFile* pOwner, int64 nBase, unsigned int nLength, unsigned int nIndex, unsigned int nFilePointer ) +{ + m_pOwner = pOwner; + m_nBase = nBase; + m_nLength = nLength; + m_nIndex = nIndex; + m_nFilePointer = nFilePointer; + pOwner->AddRef(); +} + +CZipPackFileHandle::~CZipPackFileHandle() +{ + m_pOwner->m_mutex.Lock(); + --m_pOwner->m_nOpenFiles; + // XXX(johns) this doesn't go here, the hell + if ( m_pOwner->m_nOpenFiles == 0 && m_pOwner->m_bIsMapPath ) + { + if ( m_pOwner->m_hPackFileHandleFS ) + { + m_pOwner->FileSystem()->Trace_FClose( m_pOwner->m_hPackFileHandleFS ); + m_pOwner->m_hPackFileHandleFS = NULL; + } + } + m_pOwner->Release(); + m_pOwner->m_mutex.Unlock(); +} + +void CZipPackFileHandle::SetBufferSize( int nBytes ) +{ + if ( m_pOwner->m_hPackFileHandleFS ) + { + m_pOwner->FileSystem()->FS_setbufsize( m_pOwner->m_hPackFileHandleFS, nBytes ); + } +} + +int CZipPackFileHandle::GetSectorSize() +{ + return m_pOwner->GetSectorSize(); +} + +int64 CZipPackFileHandle::AbsoluteBaseOffset() +{ + return m_pOwner->GetPackFileBaseOffset() + m_nBase; +} + +#if defined( _DEBUG ) && !defined( OSX ) +#include <atomic> +static std::atomic<int> sLZMAPackFileHandles( 0 ); +#endif // defined( _DEBUG ) && !defined( OSX ) + +CLZMAZipPackFileHandle::CLZMAZipPackFileHandle( CZipPackFile* pOwner, int64 nBase, unsigned int nOriginalSize, unsigned int nCompressedSize, + unsigned int nIndex, unsigned int nFilePointer ) + : CZipPackFileHandle( pOwner, nBase, nCompressedSize, nIndex, nFilePointer ), + m_BackSeekBuffer( 0, PACKFILE_COMPRESSED_FILEHANDLE_SEEK_BUFFER ), + m_ReadBuffer( 0, PACKFILE_COMPRESSED_FILEHANDLE_READ_BUFFER ), + m_pLZMAStream( NULL ), m_nSeekPosition( 0 ), m_nOriginalSize( nOriginalSize ) +{ + Reset(); +#if defined( _DEBUG ) && !defined( OSX ) + if ( ++sLZMAPackFileHandles == PACKFILE_COMPRESSED_FILE_HANDLES_WARNING ) + { + // By my count a live filehandle is currently around 270k, mostly due to the LZMA dictionary (256k) with the + // rest being the read/seek buffers. + Warning( "More than %u compressed file handles in use. " + "These carry large buffers around, and can cause high memory usage\n", + PACKFILE_COMPRESSED_FILE_HANDLES_WARNING ); + } +#endif // defined( _DEBUG ) && !defined( OSX ) +} + +CLZMAZipPackFileHandle::~CLZMAZipPackFileHandle() +{ + delete m_pLZMAStream; + m_pLZMAStream = NULL; +#if defined( _DEBUG ) && !defined( OSX ) + sLZMAPackFileHandles--; + Assert( sLZMAPackFileHandles >= 0 ); +#endif // defined( _DEBUG ) && !defined( OSX ) +} + +int CLZMAZipPackFileHandle::Read( void* pBuffer, int nDestSize, int nBytes ) +{ + int nMaxRead = Min( Min( nDestSize, nBytes ), Size() - Tell() ); + int nBytesRead = 0; + + // If we have seeked backwards into our buffer, read from there first + int nBackSeek = m_BackSeekBuffer.TellPut() - m_BackSeekBuffer.TellGet(); + Assert( nBackSeek >= 0 ); + if ( nBackSeek > 0 ) + { + int nBackSeekRead = Min( nBackSeek, nMaxRead ); + m_BackSeekBuffer.Get( pBuffer, nBackSeekRead ); + nBytesRead += nBackSeekRead; + } + + // Done if nothing to read + if ( nMaxRead - nBytesRead <= 0 ) + { + m_nSeekPosition += nBytesRead; + return nBytesRead; + } + + // Read bytes not fulfilled by backbuffer + Assert( m_BackSeekBuffer.TellPut() == m_BackSeekBuffer.TellGet() ); + while ( nBytesRead < nMaxRead ) + { + // refill read buffer if empty + int nRemainingReadBuffer = FillReadBuffer(); + + // Consume from read buffer + unsigned int nCompressedBytesRead = 0; + unsigned int nOutputBytesWritten = 0; + bool bSuccess = m_pLZMAStream->Read( (unsigned char *)m_ReadBuffer.PeekGet(), nRemainingReadBuffer, + (unsigned char *)pBuffer + nBytesRead, nMaxRead - nBytesRead, + nCompressedBytesRead, nOutputBytesWritten ); + + if ( bSuccess ) + { + // fixup get position + m_ReadBuffer.SeekGet( CUtlBuffer::SEEK_CURRENT, nCompressedBytesRead ); + + nBytesRead += nOutputBytesWritten; + + AssertMsg( nCompressedBytesRead == (unsigned int)nRemainingReadBuffer || nBytesRead == nMaxRead, + "Should have consumed the readbuffer or reached nMaxRead" ); + + if ( nCompressedBytesRead == 0 && nOutputBytesWritten == 0 ) + { + AssertMsg( nCompressedBytesRead > 0 || nOutputBytesWritten > 0, + "Stuck progress in read loop, aborting. Stream may be defunct." ); + break; + } + } + else + { + Warning( "Pack file: reading from LZMA stream failed\n" ); + break; + } + } + + // Finally, store last bytes output to the backseek buffer + + // If we read less than BackSeekBuffer.Size() bytes, shift the end of the old backseek buffer up + int nOldBackSeek = m_BackSeekBuffer.TellPut(); + int nReuseBackSeek = Max( Min( m_BackSeekBuffer.Size() - nBytesRead, nOldBackSeek ), 0 ); + if ( nReuseBackSeek ) + { + // Shift the reused chunk to the front + V_memmove( m_BackSeekBuffer.Base(), + (unsigned char *)m_BackSeekBuffer.Base() + m_BackSeekBuffer.TellPut() - nReuseBackSeek, + nReuseBackSeek ); + } + + // Update get/put position + m_BackSeekBuffer.SeekPut( CUtlBuffer::SEEK_HEAD, nReuseBackSeek ); + m_BackSeekBuffer.SeekGet( CUtlBuffer::SEEK_HEAD, nReuseBackSeek ); + + // Fill in remainder from what we just read + int nReadIntoBackSeek = Min( m_BackSeekBuffer.Size() - nReuseBackSeek, nBytesRead ); + m_BackSeekBuffer.Put( (unsigned char *)pBuffer + nBytesRead - nReadIntoBackSeek, nReadIntoBackSeek ); + m_BackSeekBuffer.SeekGet( CUtlBuffer::SEEK_CURRENT, nReadIntoBackSeek ); + + m_nSeekPosition += nBytesRead; + return nBytesRead; +} + +int CLZMAZipPackFileHandle::Seek( int nOffset, int nWhence ) +{ + int nNewPosition = m_nSeekPosition; + + if ( nWhence == SEEK_CUR ) + { + nNewPosition = m_nSeekPosition + nOffset; + } + else if ( nWhence == SEEK_END ) + { + nNewPosition = Size() + nOffset; + } + else if ( nWhence == SEEK_SET ) + { + nNewPosition = nOffset; + } + else + { + AssertMsg( false, "Unknown seek type" ); + } + + nNewPosition = Min( Size(), nNewPosition ); + nNewPosition = Max( 0, nNewPosition ); + + if ( nNewPosition == m_nSeekPosition ) + { + return nNewPosition; + } + + // Backwards seek + if ( nNewPosition < m_nSeekPosition ) + { + int nBackSeekAvailable = m_BackSeekBuffer.TellGet(); + int nDesiredBackSeek = m_nSeekPosition - nNewPosition; + + if ( nBackSeekAvailable >= nDesiredBackSeek ) + { + // Move get backwards into backseek buffer to account for seek + m_BackSeekBuffer.SeekGet( CUtlBuffer::SEEK_CURRENT, -nDesiredBackSeek ); + m_nSeekPosition = nNewPosition; + } + else + { + // Seeking backwards beyond our backseek buffer. Have to restart stream. This kills the performance. + Warning( "LZMA file handle: seeking backwards beyond backseek buffer size ( %u ), " + "replaying read & decompression of %u bytes. Should avoid large back seeks in compressed files or " + "increase backseek buffer sizing.", + m_BackSeekBuffer.Size(), nNewPosition ); + + // Reset to beginning of underlying stream + Reset(); + + // Fall through to performing a forward seek + } + } + + // Forward seek + if ( nNewPosition > m_nSeekPosition ) + { + // Can't actually seek forward without making decode progress. Issue fake reads until we've reached our target. + unsigned char dummyBuffer[COMPRESSED_SEEK_READ_CHUNK]; + while ( nNewPosition > m_nSeekPosition ) + { + int nReadSize = Min( nNewPosition - m_nSeekPosition, COMPRESSED_SEEK_READ_CHUNK ); + unsigned int nBytesRead = Read( &dummyBuffer, sizeof(dummyBuffer), nReadSize ); + m_nSeekPosition += nBytesRead; + if ( !nBytesRead ) + { + Warning( "LZMA file handle: failed reading forward to desired seek position\n" ); + break; + } + } + } + + return m_nSeekPosition; +} + +int CLZMAZipPackFileHandle::Tell() +{ + return m_nSeekPosition; +} + +int CLZMAZipPackFileHandle::Size() +{ + return m_nOriginalSize; +} + +int CLZMAZipPackFileHandle::FillReadBuffer() +{ + int nRemainingReadBuffer = m_ReadBuffer.TellPut() - m_ReadBuffer.TellGet(); + int nRemainingCompressedBytes = CZipPackFileHandle::Size() - CZipPackFileHandle::Tell(); + + if ( nRemainingReadBuffer > 0 || nRemainingCompressedBytes <= 0 ) + { + // No action if read buffer isn't empty + return nRemainingReadBuffer; + } + + // Reset empty read buffer + m_ReadBuffer.SeekPut( CUtlBuffer::SEEK_HEAD, 0 ); + m_ReadBuffer.SeekGet( CUtlBuffer::SEEK_HEAD, 0 ); + int nRefillSize = Min( nRemainingCompressedBytes, m_ReadBuffer.Size() ); + int nRefillResult = CZipPackFileHandle::Read( m_ReadBuffer.PeekPut(), m_ReadBuffer.Size(), nRefillSize ); + AssertMsg( nRefillSize == nRefillResult, "Don't expect to fail to read here" ); + + // Fixup put pointer after writing into buffer's memory + m_ReadBuffer.SeekPut( CUtlBuffer::SEEK_CURRENT, nRefillResult ); + + return nRefillResult; +} + +void CLZMAZipPackFileHandle::Reset() +{ + // Seek underlying stream back to start + CZipPackFileHandle::Seek( SEEK_SET, 0 ); + + delete m_pLZMAStream; + m_pLZMAStream = new CLZMAStream(); + m_pLZMAStream->InitZIPHeader( CZipPackFileHandle::Size(), m_nOriginalSize ); + m_nSeekPosition = 0; + m_BackSeekBuffer.SeekGet( CUtlBuffer::SEEK_HEAD, 0 ); + m_BackSeekBuffer.SeekPut( CUtlBuffer::SEEK_HEAD, 0 ); + m_ReadBuffer.SeekGet( CUtlBuffer::SEEK_HEAD, 0 ); + m_ReadBuffer.SeekPut( CUtlBuffer::SEEK_HEAD, 0 ); +} diff --git a/filesystem/packfile.h b/filesystem/packfile.h new file mode 100644 index 0000000..7538951 --- /dev/null +++ b/filesystem/packfile.h @@ -0,0 +1,277 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// $NoKeywords: $ +//===========================================================================// + +#ifndef PACKFILE_H +#define PACKFILE_H + +#ifdef _WIN32 +#pragma once +#endif + +// How many bytes compressed filehandles should hold for seeking backwards. Seeks beyond this limit will require +// rewinding and restarting the compression, at significant performance penalty. (Warnings emitted when this occurs) +#define PACKFILE_COMPRESSED_FILEHANDLE_SEEK_BUFFER 4096 + +// How many bytes compressed filehandles should attempt to read (and cache) at a time from the underlying compressed data. +#define PACKFILE_COMPRESSED_FILEHANDLE_READ_BUFFER 4096 + +// Emit warnings in debug builds if we hold more than this many compressed file handles alive to alert to the poor +// memory characteristics. +#define PACKFILE_COMPRESSED_FILE_HANDLES_WARNING 20 + +#include "basefilesystem.h" +#include "tier1/refcount.h" +#include "tier1/utlbuffer.h" +#include "tier1/lzmaDecoder.h" + +class CPackFile; +class CZipPackFile; + +// A pack file handle - essentially represents a file inside the pack file. +class CPackFileHandle +{ +public: + virtual ~CPackFileHandle() {}; + + virtual int Read( void* pBuffer, int nDestSize, int nBytes ) = 0; + virtual int Seek( int nOffset, int nWhence ) = 0; + virtual int Tell() = 0; + virtual int Size() = 0; + + virtual void SetBufferSize( int nBytes ) = 0; + virtual int GetSectorSize() = 0; + virtual int64 AbsoluteBaseOffset() = 0; +}; + +class CZipPackFileHandle : public CPackFileHandle +{ +public: + CZipPackFileHandle( CZipPackFile* pOwner, int64 nBase, unsigned int nLength, unsigned int nIndex = -1, unsigned int nFilePointer = 0 ); + virtual ~CZipPackFileHandle(); + + virtual int Read( void* pBuffer, int nDestSize, int nBytes ) OVERRIDE; + virtual int Seek( int nOffset, int nWhence ) OVERRIDE; + + virtual int Tell() OVERRIDE { return m_nFilePointer; }; + virtual int Size() OVERRIDE { return m_nLength; }; + + virtual void SetBufferSize( int nBytes ) OVERRIDE; + virtual int GetSectorSize() OVERRIDE; + virtual int64 AbsoluteBaseOffset() OVERRIDE; + +protected: + int64 m_nBase; // Base offset of the file inside the pack file. + unsigned int m_nFilePointer; // Current seek pointer (0 based from the beginning of the file). + CZipPackFile* m_pOwner; // Pack file that owns this handle + unsigned int m_nLength; // Length of this file. + unsigned int m_nIndex; // Index into the pack's directory table +}; + +class CLZMAZipPackFileHandle : public CZipPackFileHandle +{ +public: + CLZMAZipPackFileHandle( CZipPackFile* pOwner, int64 nBase, unsigned int nOriginalSize, unsigned int nCompressedSize, + unsigned int nIndex = -1, unsigned int nFilePointer = 0 ); + ~CLZMAZipPackFileHandle(); + + virtual int Read( void* pBuffer, int nDestSize, int nBytes ) OVERRIDE; + virtual int Seek( int nOffset, int nWhence ) OVERRIDE; + + virtual int Tell() OVERRIDE; + virtual int Size() OVERRIDE; + +private: + // Ensure there are bytes in the read buffer, assuming we're not at the end of the underlying data + int FillReadBuffer(); + + // Reset buffers and underlying seek position to 0 + void Reset(); + + // Contains the last PACKFILE_COMPRESSED_FILEHANDLE_SEEK_BUFFER decompressed bytes. The Put and Get locations mimic our + // filehandle -- TellPut() == TelGet() when we are not back seeking. + CUtlBuffer m_BackSeekBuffer; + + // The read buffer from the underlying compressed stream. We read PACKFILE_COMPRESSED_FILEHANDLE_READ_BUFFER bytes + // into this buffer, then consume it via the buffer get position. + CUtlBuffer m_ReadBuffer; + + // The decompress stream we feed our base filehandle into + CLZMAStream *m_pLZMAStream; + + // Current seek position in uncompressed data + int m_nSeekPosition; + + // Size of the decompressed data + unsigned int m_nOriginalSize; +}; + +//----------------------------------------------------------------------------- + +// An abstract pack file +class CPackFile : public CRefCounted<CRefCountServiceMT> +{ +public: + CPackFile(); + virtual ~CPackFile(); + + // The means by which you open files: + virtual CFileHandle *OpenFile( const char *pFileName, const char *pOptions = "rb" ) = 0; + + // Check for existance in pack + virtual bool ContainsFile( const char *pFileName ) = 0; + + // The two functions a pack file must provide + virtual bool Prepare( int64 fileLen = -1, int64 nFileOfs = 0 ) = 0; + + // Returns the filename for a given file in the pack. Returns true if a filename is found, otherwise buffer is filled with "unknown" + virtual bool IndexToFilename( int nIndex, char* buffer, int nBufferSize ) = 0; + + inline int GetSectorSize(); + + virtual void SetupPreloadData() {} + virtual void DiscardPreloadData() {} + virtual int64 GetPackFileBaseOffset() = 0; + + CBaseFileSystem *FileSystem() { return m_fs; } + + // Helper for the filesystem's FindFirst/FindNext() API which mimics the old windows equivalent. pWildcard is the + // same pattern that you would pass to FindFirst, not a true wildcard. + // Mirrors the VPK code's similar call. + virtual void GetFileAndDirLists( const char *pFindWildCard, CUtlStringList &outDirnames, CUtlStringList &outFilenames, bool bSortedOutput ) = 0; + + // Note: threading model for pack files assumes that data + // is segmented into pack files that aggregate files + // meant to be read in one thread. Performance characteristics + // tuned for that case + CThreadFastMutex m_mutex; + + // Path management: + void SetPath( const CUtlSymbol &path ) { m_Path = path; } + const CUtlSymbol& GetPath() const { Assert( m_Path != UTL_INVAL_SYMBOL ); return m_Path; } + CUtlSymbol m_Path; + + // possibly embedded pack + int64 m_nBaseOffset; + + CUtlString m_ZipName; + + bool m_bIsMapPath; + long m_lPackFileTime; + + int m_refCount; + int m_nOpenFiles; + + FILE *m_hPackFileHandleFS; +#if defined( SUPPORT_PACKED_STORE ) + CPackedStoreFileHandle m_hPackFileHandleVPK; +#endif + bool m_bIsExcluded; + + int m_PackFileID; +protected: + // This is the core IO routine for reading anything from a pack file, everything should go through here at some point + virtual int ReadFromPack( int nIndex, void* buffer, int nDestBytes, int nBytes, int64 nOffset ) = 0; + + int64 m_FileLength; + CBaseFileSystem *m_fs; + + friend class CPackFileHandle; +}; + +class CZipPackFile : public CPackFile +{ + friend class CZipPackFileHandle; +public: + CZipPackFile( CBaseFileSystem* fs, void *pSection = NULL ); + virtual ~CZipPackFile(); + + // Loads the pack file + virtual bool Prepare( int64 fileLen = -1, int64 nFileOfs = 0 ) OVERRIDE; + virtual bool ContainsFile( const char *pFileName ) OVERRIDE; + virtual CFileHandle *OpenFile( const char *pFileName, const char *pOptions = "rb" ) OVERRIDE; + + virtual void GetFileAndDirLists( const char *pFindWildCard, CUtlStringList &outDirnames, CUtlStringList &outFilenames, bool bSortedOutput ) OVERRIDE; + + virtual int64 GetPackFileBaseOffset() OVERRIDE { return m_nBaseOffset; } + + virtual bool IndexToFilename( int nIndex, char *pBuffer, int nBufferSize ) OVERRIDE; + +protected: + virtual int ReadFromPack( int nIndex, void* buffer, int nDestBytes, int nBytes, int64 nOffset ) OVERRIDE; + + #pragma pack(1) + + typedef struct + { + char name[ 112 ]; + int64 filepos; + int64 filelen; + } packfile64_t; + + typedef struct + { + char id[ 4 ]; + int64 dirofs; + int64 dirlen; + } packheader64_t; + + typedef struct + { + char id[ 8 ]; + int64 packheaderpos; + int64 originalfilesize; + } packappenededheader_t; + + #pragma pack() + + // A Pack file directory entry: + class CPackFileEntry + { + public: + unsigned int m_nPosition; + unsigned int m_nOriginalSize; + unsigned int m_nCompressedSize; + unsigned int m_HashName; + unsigned short m_nPreloadIdx; + unsigned short pad; + unsigned short m_nCompressionMethod; + FileNameHandle_t m_hFileName; + }; + + class CPackFileLessFunc + { + public: + bool Less( CPackFileEntry const& src1, CPackFileEntry const& src2, void *pCtx ); + }; + + // Find a file inside a pack file: + const CPackFileEntry* FindFile( const char* pFileName ); + + // Entries to the individual files stored inside the pack file. + CUtlSortVector< CPackFileEntry, CPackFileLessFunc > m_PackFiles; + + bool GetFileInfo( const char *pFileName, int &nBaseIndex, int64 &nFileOffset, int &nOriginalSize, int &nCompressedSize, unsigned short &nCompressionMethod ); + + // Preload Support + void SetupPreloadData() OVERRIDE; + void DiscardPreloadData() OVERRIDE; + ZIP_PreloadDirectoryEntry* GetPreloadEntry( int nEntryIndex ); + + int64 m_nPreloadSectionOffset; + unsigned int m_nPreloadSectionSize; + ZIP_PreloadHeader *m_pPreloadHeader; + unsigned short* m_pPreloadRemapTable; + ZIP_PreloadDirectoryEntry *m_pPreloadDirectory; + void* m_pPreloadData; + CByteswap m_swap; + +#if defined ( _X360 ) + void *m_pSection; +#endif +}; + +#endif // PACKFILE_H diff --git a/filesystem/threadsaferefcountedobject.h b/filesystem/threadsaferefcountedobject.h new file mode 100644 index 0000000..89164db --- /dev/null +++ b/filesystem/threadsaferefcountedobject.h @@ -0,0 +1,85 @@ +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +//============================================================================= + +#ifndef THREADSAFEREFCOUNTEDOBJECT_H +#define THREADSAFEREFCOUNTEDOBJECT_H +#ifdef _WIN32 +#pragma once +#endif + + +// This class can be used for fast access to an object from multiple threads, +// and the main thread can wait until the threads are done using the object before it frees the object. +template< class T > +class CThreadSafeRefCountedObject +{ +public: + CThreadSafeRefCountedObject( T initVal ) + { + m_RefCount = 0; + m_pObject = initVal; + m_RefCount = 0; + } + + void Init( T pObj ) + { + Assert( ThreadInMainThread() ); + Assert( !m_pObject ); + m_RefCount = 0; + m_pObject = pObj; + m_RefCount = 1; + } + + // Threads that access the object need to use AddRef/Release to access it. + T AddRef() + { + if ( ++m_RefCount > 1 ) + { + return m_pObject; + } + else + { + // If the refcount was 0 when we called this, then the whitelist is about to be freed. + --m_RefCount; + return NULL; + } + } + void ReleaseRef( T pObj ) + { + if ( --m_RefCount >= 1 ) + { + Assert( m_pObject == pObj ); + } + } + + // The main thread can use this to access the object, since only it can Init() and Free() the object. + T GetInMainThread() + { + Assert( ThreadInMainThread() ); + return m_pObject; + } + + // The main thread calls this after it has released its last reference to the object. + void ResetWhenNoRemainingReferences( T newValue ) + { + Assert( ThreadInMainThread() ); + + // Wait until we can free it. + while ( m_RefCount > 0 ) + { + CThread::Sleep( 20 ); + } + + m_pObject = newValue; + } + +private: + CInterlockedIntT<long> m_RefCount; + T m_pObject; +}; + + +#endif // THREADSAFEREFCOUNTEDOBJECT_H diff --git a/filesystem/xbox/xbox.def b/filesystem/xbox/xbox.def new file mode 100644 index 0000000..a97c613 --- /dev/null +++ b/filesystem/xbox/xbox.def @@ -0,0 +1,3 @@ +LIBRARY filesystem_stdio_360.dll +EXPORTS + CreateInterface @1 |