diff --git a/public/inetchannel.h b/public/inetchannel.h index 2ea380a5..db816651 100644 --- a/public/inetchannel.h +++ b/public/inetchannel.h @@ -127,7 +127,7 @@ public: virtual void PostReceivedNetMessage( INetworkSerializable *pNetMessage, const void *pData, const NetChannelBufType_t *pBufType, int nBits, int nInSequenceNr ) = 0; virtual void InsertReplayMessage( InstantReplayMessage_t &msg ) = 0; - virtual bool HasQueuedPackets( int nMessageId ) const = 0; + virtual bool HasQueuedNetMessages( int nMessageId ) const = 0; virtual void SetPendingDisconnect( ENetworkDisconnectionReason reason ) = 0; virtual ENetworkDisconnectionReason GetPendingDisconnect( void ) const = 0; @@ -135,10 +135,10 @@ public: virtual void SuppressTransmit( bool suppress ) = 0; virtual bool IsSuppressingTransmit( void ) const = 0; - virtual EResult SendMessage( const void *pData, uint32 cbData, int nSendFlags ) = 0; + virtual EResult SendRawMessage( const void *pData, uint32 cbData, int nSendFlags ) = 0; - virtual int GetCurrentMessageBits( void ) const = 0; - virtual int GetCurrentMessageInSequenceNr( void ) const = 0; + virtual int GetCurrentNetMessageBits( void ) const = 0; + virtual int GetCurrentNetMessageInSequenceNr( void ) const = 0; virtual void unk101( void ) = 0; virtual void unk102( void ) = 0; diff --git a/public/schemasystem/schemasystem.h b/public/schemasystem/schemasystem.h index 1ee44cf0..1292e517 100644 --- a/public/schemasystem/schemasystem.h +++ b/public/schemasystem/schemasystem.h @@ -76,8 +76,8 @@ public: virtual void MarkEnumAsRequiringGlobalPromotion( const CSchemaEnumInfo* pEnumInfo ) = 0; virtual void ResolveAtomicInfoThreadsafe( const SchemaAtomicTypeInfo_t** ppAtomicInfo, const char* pszAtomicName, int nAtomicID ) = 0; - virtual void ResolveEnumInfoThreadsafe( const CSchemaEnumInfo** pEnumInfo, const char* pszEnumName ) = 0; - virtual void ResolveClassInfoThreadsafe( const CSchemaClassInfo** pClassInfo, const char* pszClassName ) = 0; + virtual void ResolveEnumInfoThreadsafe( const CSchemaEnumInfo** ppEnumInfo, const char* pszEnumName ) = 0; + virtual void ResolveClassInfoThreadsafe( const CSchemaClassInfo** ppClassInfo, const char* pszClassName ) = 0; }; class CSchemaSystemTypeScope : public ISchemaSystemTypeScope diff --git a/public/tier0/threadtools.h b/public/tier0/threadtools.h index 1536371a..efde3601 100644 --- a/public/tier0/threadtools.h +++ b/public/tier0/threadtools.h @@ -288,8 +288,10 @@ inline bool ThreadInterlockedAssignIf128( volatile int128 *pDest, const int128 & volatile int64 *pDest64 = ( volatile int64 * )pDest; int64 *pValue64 = ( int64 * )&value; int64 *pComperand64 = ( int64 * )&comperand; + + int64 local_comperand[2] = { pComperand64[0], pComperand64[1] }; - return _InterlockedCompareExchange128( pDest64, pValue64[1], pValue64[0], pComperand64 ) == 1; + return _InterlockedCompareExchange128( pDest64, pValue64[1], pValue64[0], local_comperand ) == 1; } #else typedef __int128_t int128; @@ -298,7 +300,10 @@ typedef __int128_t int128; inline bool ThreadInterlockedAssignIf128( volatile int128 *pDest, const int128 &value, const int128 &comperand ) { Assert( (size_t)pDest % 16 == 0 ); - return __sync_bool_compare_and_swap( pDest, comperand, value ); + + int128 local_comperand = comperand; + + return __sync_bool_compare_and_swap( pDest, local_comperand, value ); } #endif #endif diff --git a/public/tier0/tslist.h b/public/tier0/tslist.h index 68795f26..f4bcfba9 100644 --- a/public/tier0/tslist.h +++ b/public/tier0/tslist.h @@ -1,808 +1,1004 @@ -//========== Copyright © 2005, Valve Corporation, All rights reserved. ======== -// -// Purpose: -// -// LIFO from disassembly of Windows API and http://perso.wanadoo.fr/gmem/evenements/jim2002/articles/L17_Fober.pdf -// FIFO from http://perso.wanadoo.fr/gmem/evenements/jim2002/articles/L17_Fober.pdf -// -//============================================================================= - -#ifndef TSLIST_H -#define TSLIST_H - -#if defined( _WIN32 ) -#pragma once -#endif - -#if ( defined( PLATFORM_X360 ) || defined( PLATFORM_WINDOWS_PC64 ) ) -#define USE_NATIVE_SLIST -#endif - -#if defined( USE_NATIVE_SLIST ) && !defined( _X360 ) -#define WIN32_LEAN_AND_MEAN -#include -#endif - -#include "tier0/dbg.h" -#include "tier0/threadtools.h" - -#include "tier0/memdbgon.h" - -//----------------------------------------------------------------------------- - -#if defined(PLATFORM_WINDOWS_PC64) -#define TSLIST_HEAD_ALIGNMENT 16 //MEMORY_ALLOCATION_ALIGNMENT -#define TSLIST_NODE_ALIGNMENT 16 //MEMORY_ALLOCATION_ALIGNMENT -#else -#define TSLIST_HEAD_ALIGNMENT 8 -#define TSLIST_NODE_ALIGNMENT 8 -#endif - -#define TSLIST_HEAD_ALIGN DECL_ALIGN(TSLIST_HEAD_ALIGNMENT) -#define TSLIST_NODE_ALIGN DECL_ALIGN(TSLIST_NODE_ALIGNMENT) - -//----------------------------------------------------------------------------- - -PLATFORM_INTERFACE bool RunTSQueueTests( int nListSize = 10000, int nTests = 1 ); -PLATFORM_INTERFACE bool RunTSListTests( int nListSize = 10000, int nTests = 1 ); - -//----------------------------------------------------------------------------- -// Lock free list. -//----------------------------------------------------------------------------- -//#define USE_NATIVE_SLIST - -#ifdef USE_NATIVE_SLIST -typedef SLIST_ENTRY TSLNodeBase_t; -typedef SLIST_HEADER TSLHead_t; -#else -struct TSLIST_NODE_ALIGN TSLNodeBase_t -{ - TSLNodeBase_t *Next; // name to match Windows -}; - -union TSLHead_t -{ - struct Value_t - { - TSLNodeBase_t *Next; - int16 Depth; - int16 Sequence; - } value; - - int64 value64; -}; -#endif - -//------------------------------------- - -class TSLIST_HEAD_ALIGN CTSListBase -{ -public: - CTSListBase() - { - if ( ((size_t)&m_Head) % TSLIST_HEAD_ALIGNMENT != 0 ) - { - Error( _T( "CTSListBase: Misaligned list\n" ) ); - DebuggerBreak(); - } - -#ifdef USE_NATIVE_SLIST - InitializeSListHead( &m_Head ); -#else - m_Head.value64 = (int64)0; -#endif - } - - ~CTSListBase() - { - Detach(); - } - - TSLNodeBase_t *Push( TSLNodeBase_t *pNode ) - { -#ifdef _DEBUG - if ( (size_t)pNode % TSLIST_NODE_ALIGNMENT != 0 ) - { - Error( _T( "CTSListBase: Misaligned node\n" ) ); - DebuggerBreak(); - } -#endif - -#ifdef USE_NATIVE_SLIST -#ifdef _X360 - // integrated write-release barrier - return (TSLNodeBase_t *)InterlockedPushEntrySListRelease( &m_Head, pNode ); -#else - return (TSLNodeBase_t *)InterlockedPushEntrySList( &m_Head, pNode ); -#endif -#else - TSLHead_t oldHead; - TSLHead_t newHead; - - for (;;) - { - oldHead.value64 = m_Head.value64; - pNode->Next = oldHead.value.Next; - newHead.value.Next = pNode; - *((uint32 *)&newHead.value.Depth) = *((uint32 *)&oldHead.value.Depth) + 0x10001; - - if ( ThreadInterlockedAssignIf64( &m_Head.value64, newHead.value64, oldHead.value64 ) ) - { - break; - } - ThreadPause(); - }; - - return (TSLNodeBase_t *)oldHead.value.Next; -#endif - } - - TSLNodeBase_t *Pop() - { -#ifdef USE_NATIVE_SLIST -#ifdef _X360 - // integrated read-acquire barrier - TSLNodeBase_t *pNode = (TSLNodeBase_t *)InterlockedPopEntrySListAcquire( &m_Head ); -#else - TSLNodeBase_t *pNode = (TSLNodeBase_t *)InterlockedPopEntrySList( &m_Head ); -#endif - return pNode; -#else - TSLHead_t oldHead; - TSLHead_t newHead; - - for (;;) - { - oldHead.value64 = m_Head.value64; - if ( !oldHead.value.Next ) - return NULL; - - newHead.value.Next = oldHead.value.Next->Next; - *((uint32 *)&newHead.value.Depth) = *((uint32 *)&oldHead.value.Depth) - 1; - - if ( ThreadInterlockedAssignIf64( &m_Head.value64, newHead.value64, oldHead.value64 ) ) - { - break; - } - ThreadPause(); - }; - - return (TSLNodeBase_t *)oldHead.value.Next; -#endif - } - - TSLNodeBase_t *Detach() - { -#ifdef USE_NATIVE_SLIST - TSLNodeBase_t *pBase = (TSLNodeBase_t *)InterlockedFlushSList( &m_Head ); -#ifdef _X360 - __lwsync(); // read-acquire barrier -#endif - return pBase; -#else - TSLHead_t oldHead; - TSLHead_t newHead; - - do - { - ThreadPause(); - - oldHead.value64 = m_Head.value64; - if ( !oldHead.value.Next ) - return NULL; - - newHead.value.Next = NULL; - *((uint32 *)&newHead.value.Depth) = *((uint32 *)&oldHead.value.Depth) & 0xffff0000; - - } while( !ThreadInterlockedAssignIf64( &m_Head.value64, newHead.value64, oldHead.value64 ) ); - - return (TSLNodeBase_t *)oldHead.value.Next; -#endif - } - - TSLHead_t *AccessUnprotected() - { - return &m_Head; - } - - int Count() const - { -#ifdef USE_NATIVE_SLIST - return QueryDepthSList( const_cast( &m_Head ) ); -#else - return m_Head.value.Depth; -#endif - } - -private: - TSLHead_t m_Head; -}; - -//------------------------------------- - -template -class TSLIST_HEAD_ALIGN CTSSimpleList : public CTSListBase -{ -public: - void Push( T *pNode ) - { - Assert( sizeof(T) >= sizeof(TSLNodeBase_t) ); - CTSListBase::Push( (TSLNodeBase_t *)pNode ); - } - - T *Pop() - { - return (T *)CTSListBase::Pop(); - } -}; - -//------------------------------------- -// this is a replacement for CTSList<> and CObjectPool<> that does not -// have a per-item, per-alloc new/delete overhead -// similar to CTSSimpleList except that it allocates it's own pool objects -// and frees them on destruct. Also it does not overlay the TSNodeBase_t memory -// on T's memory -template< class T > -class TSLIST_HEAD_ALIGN CTSPool : public CTSListBase -{ - // packs the node and the item (T) into a single struct and pools those - struct TSLIST_NODE_ALIGN simpleTSPoolStruct_t : public TSLNodeBase_t - { - T elem; - }; - -public: - - ~CTSPool() - { - simpleTSPoolStruct_t *pNode = NULL; - while ( 1 ) - { - pNode = (simpleTSPoolStruct_t *)CTSListBase::Pop(); - if ( !pNode ) - break; - delete pNode; - } - } - - void PutObject( T *pInfo ) - { - char *pElem = (char *)pInfo; - pElem -= offsetof(simpleTSPoolStruct_t,elem); - simpleTSPoolStruct_t *pNode = (simpleTSPoolStruct_t *)pElem; - - CTSListBase::Push( pNode ); - } - - T *GetObject() - { - simpleTSPoolStruct_t *pNode = (simpleTSPoolStruct_t *)CTSListBase::Pop(); - if ( !pNode ) - { - pNode = new simpleTSPoolStruct_t; - } - return &pNode->elem; - } - - // omg windows sdk - why do you #define GetObject()? - FORCEINLINE T *Get() - { - return GetObject(); - } - -}; -//------------------------------------- - -template -class TSLIST_HEAD_ALIGN CTSList : public CTSListBase -{ -public: - struct TSLIST_NODE_ALIGN Node_t : public TSLNodeBase_t - { - Node_t() {} - Node_t( const T &init ) : elem( init ) {} - - T elem; - - }; - - ~CTSList() - { - Purge(); - } - - void Purge() - { - Node_t *pCurrent = Detach(); - Node_t *pNext; - while ( pCurrent ) - { - pNext = (Node_t *)pCurrent->Next; - delete pCurrent; - pCurrent = pNext; - } - } - - void RemoveAll() - { - Purge(); - } - - Node_t *Push( Node_t *pNode ) - { - return (Node_t *)CTSListBase::Push( pNode ); - } - - Node_t *Pop() - { - return (Node_t *)CTSListBase::Pop(); - } - - void PushItem( const T &init ) - { - Push( new Node_t( init ) ); - } - - bool PopItem( T *pResult) - { - Node_t *pNode = Pop(); - if ( !pNode ) - return false; - *pResult = pNode->elem; - delete pNode; - return true; - } - - Node_t *Detach() - { - return (Node_t *)CTSListBase::Detach(); - } - -}; - -//------------------------------------- - -template -class TSLIST_HEAD_ALIGN CTSListWithFreeList : public CTSListBase -{ -public: - struct TSLIST_NODE_ALIGN Node_t : public TSLNodeBase_t - { - Node_t() {} - Node_t( const T &init ) : elem( init ) {} - - T elem; - }; - - ~CTSListWithFreeList() - { - Purge(); - } - - void Purge() - { - Node_t *pCurrent = Detach(); - Node_t *pNext; - while ( pCurrent ) - { - pNext = (Node_t *)pCurrent->Next; - delete pCurrent; - pCurrent = pNext; - } - pCurrent = (Node_t *)m_FreeList.Detach(); - while ( pCurrent ) - { - pNext = (Node_t *)pCurrent->Next; - delete pCurrent; - pCurrent = pNext; - } - } - - void RemoveAll() - { - Node_t *pCurrent = Detach(); - Node_t *pNext; - while ( pCurrent ) - { - pNext = (Node_t *)pCurrent->Next; - m_FreeList.Push( pCurrent ); - pCurrent = pNext; - } - } - - Node_t *Push( Node_t *pNode ) - { - return (Node_t *)CTSListBase::Push( pNode ); - } - - Node_t *Pop() - { - return (Node_t *)CTSListBase::Pop(); - } - - void PushItem( const T &init ) - { - Node_t *pNode = (Node_t *)m_FreeList.Pop(); - if ( !pNode ) - { - pNode = new Node_t; - } - pNode->elem = init; - Push( pNode ); - } - - bool PopItem( T *pResult) - { - Node_t *pNode = Pop(); - if ( !pNode ) - return false; - *pResult = pNode->elem; - m_FreeList.Push( pNode ); - return true; - } - - Node_t *Detach() - { - return (Node_t *)CTSListBase::Detach(); - } - - void FreeNode( Node_t *pNode ) - { - m_FreeList.Push( pNode ); - } - -private: - CTSListBase m_FreeList; -}; - -//----------------------------------------------------------------------------- -// Lock free queue -// -// A special consideration: the element type should be simple. This code -// actually dereferences freed nodes as part of pop, but later detects -// that. If the item in the queue is a complex type, only bad things can -// come of that. Also, therefore, if you're using Push/Pop instead of -// push item, be aware that the node memory cannot be freed until -// all threads that might have been popping have completed the pop. -// The PushItem()/PopItem() for handles this by keeping a persistent -// free list. Dont mix Push/PushItem. Note also nodes will be freed at the end, -// and are expected to have been allocated with operator new. -//----------------------------------------------------------------------------- - -template -class TSLIST_HEAD_ALIGN CTSQueue -{ -public: - struct TSLIST_NODE_ALIGN Node_t - { - Node_t() {} - Node_t( const T &init ) : elem( init ) {} - - Node_t *pNext; - T elem; - }; - - union TSLIST_HEAD_ALIGN NodeLink_t - { - struct Value_t - { - Node_t *pNode; - int32 sequence; - } value; - - int64 value64; - }; - - CTSQueue() - { - COMPILE_TIME_ASSERT( sizeof(Node_t) >= sizeof(TSLNodeBase_t) ); - if ( ((size_t)&m_Head) % TSLIST_HEAD_ALIGNMENT != 0 ) - { - Error( "CTSQueue: Misaligned queue\n" ); - DebuggerBreak(); - } - if ( ((size_t)&m_Tail) % TSLIST_HEAD_ALIGNMENT != 0 ) - { - Error( "CTSQueue: Misaligned queue\n" ); - DebuggerBreak(); - } - m_Count = 0; - m_Head.value.sequence = m_Tail.value.sequence = 0; - m_Head.value.pNode = m_Tail.value.pNode = new Node_t; // list always contains a dummy node - m_Head.value.pNode->pNext = End(); - } - - ~CTSQueue() - { - Purge(); - Assert( m_Count == 0 ); - Assert( m_Head.value.pNode == m_Tail.value.pNode ); - Assert( m_Head.value.pNode->pNext == End() ); - delete m_Head.value.pNode; - } - - // Note: Purge, RemoveAll, and Validate are *not* threadsafe - void Purge() - { - if ( IsDebug() ) - { - Validate(); - } - - Node_t *pNode; - while ( ( pNode = Pop() ) != NULL ) - { - delete pNode; - } - - while ( ( pNode = (Node_t *)m_FreeNodes.Pop() ) != NULL ) - { - delete pNode; - } - - Assert( m_Count == 0 ); - Assert( m_Head.value.pNode == m_Tail.value.pNode ); - Assert( m_Head.value.pNode->pNext == End() ); - - m_Head.value.sequence = m_Tail.value.sequence = 0; - } - - void RemoveAll() - { - if ( IsDebug() ) - { - Validate(); - } - - Node_t *pNode; - while ( ( pNode = Pop() ) != NULL ) - { - m_FreeNodes.Push( (TSLNodeBase_t *)pNode ); - } - } - - bool Validate() - { - bool bResult = true; - int nNodes = 0; - if ( m_Tail.value.pNode->pNext != End() ) - { - DebuggerBreakIfDebugging(); - bResult = false; - } - - if ( m_Count == 0 ) - { - if ( m_Head.value.pNode != m_Tail.value.pNode ) - { - DebuggerBreakIfDebugging(); - bResult = false; - } - } - - Node_t *pNode = m_Head.value.pNode; - while ( pNode != End() ) - { - nNodes++; - pNode = pNode->pNext; - } - - nNodes--;// skip dummy node - - if ( nNodes != m_Count ) - { - DebuggerBreakIfDebugging(); - bResult = false; - } - - if ( !bResult ) - { - Msg( "Corrupt CTSQueueDetected" ); - } - - return bResult; - } - - void FinishPush( Node_t *pNode, const NodeLink_t &oldTail ) - { - NodeLink_t newTail; - - newTail.value.pNode = pNode; - newTail.value.sequence = oldTail.value.sequence + 1; - -#ifdef _X360 - __lwsync(); // write-release barrier -#endif - InterlockedCompareExchangeNodeLink( &m_Tail, newTail, oldTail ); - } - - Node_t *Push( Node_t *pNode ) - { -#ifdef _DEBUG - if ( (size_t)pNode % TSLIST_NODE_ALIGNMENT != 0 ) - { - Error( "CTSListBase: Misaligned node\n" ); - DebuggerBreak(); - } -#endif - - NodeLink_t oldTail; - - pNode->pNext = End(); - - for (;;) - { - oldTail = m_Tail; - if ( InterlockedCompareExchangeNode( &(oldTail.value.pNode->pNext), pNode, End() ) == End() ) - { - break; - } - else - { - // Another thread is trying to push, help it along - FinishPush( oldTail.value.pNode->pNext, oldTail ); - } - } - - FinishPush( pNode, oldTail ); - - m_Count++; - - return oldTail.value.pNode; - } - - Node_t *Pop() - { - #define TSQUEUE_BAD_NODE_LINK ((Node_t *)0xdeadbeefULL) - NodeLink_t * volatile pHead = &m_Head; - NodeLink_t * volatile pTail = &m_Tail; - Node_t * volatile * pHeadNode = &m_Head.value.pNode; - volatile int * volatile pHeadSequence = &m_Head.value.sequence; - Node_t * volatile * pTailNode = &pTail->value.pNode; - - NodeLink_t head; - NodeLink_t newHead; - Node_t *pNext; - int tailSequence; - T elem; - - for (;;) - { - head.value.sequence = *pHeadSequence; // must grab sequence first, which allows condition below to ensure pNext is valid -#ifdef _X360 - __lwsync(); // 360 needs a barrier to prevent reordering of these assignments -#endif - head.value.pNode = *pHeadNode; - tailSequence = pTail->value.sequence; - pNext = head.value.pNode->pNext; - - if ( pNext && head.value.sequence == *pHeadSequence ) // Checking pNext only to force optimizer to not reorder the assignment to pNext and the compare of the sequence - { - if ( bTestOptimizer ) - { - if ( pNext == TSQUEUE_BAD_NODE_LINK ) - { - Msg( "Bad node link detected\n" ); - continue; - } - } - if ( head.value.pNode == *pTailNode ) - { - if ( pNext == End() ) - { - return NULL; - } - - // Another thread is trying to push, help it along - NodeLink_t &oldTail = head; // just reuse local memory for head to build old tail - oldTail.value.sequence = tailSequence; // reuse head pNode - FinishPush( pNext, oldTail ); - } - else if ( pNext != End() ) - { - elem = pNext->elem; // NOTE: next could be a freed node here, by design - newHead.value.pNode = pNext; - newHead.value.sequence = head.value.sequence + 1; - if ( InterlockedCompareExchangeNodeLink( pHead, newHead, head ) ) - { -#ifdef _X360 - __lwsync(); // read-acquire barrier -#endif - if ( bTestOptimizer ) - { - head.value.pNode->pNext = TSQUEUE_BAD_NODE_LINK; - } - break; - } - } - } - } - - m_Count--; - head.value.pNode->elem = elem; - return head.value.pNode; - } - - void FreeNode( Node_t *pNode ) - { - m_FreeNodes.Push( (TSLNodeBase_t *)pNode ); - } - - void PushItem( const T &init ) - { - Node_t *pNode = (Node_t *)m_FreeNodes.Pop(); - if ( pNode ) - { - pNode->elem = init; - } - else - { - pNode = new Node_t( init ); - } - Push( pNode ); - } - - bool PopItem( T *pResult) - { - Node_t *pNode = Pop(); - if ( !pNode ) - return false; - *pResult = pNode->elem; - m_FreeNodes.Push( (TSLNodeBase_t *)pNode ); - return true; - } - - int Count() - { - return m_Count; - } - -private: - Node_t *End() { return (Node_t *)this; } // just need a unique signifier - -#ifndef _WIN64 - Node_t *InterlockedCompareExchangeNode( Node_t * volatile *ppNode, Node_t *value, Node_t *comperand ) - { - return (Node_t *)::ThreadInterlockedCompareExchangePointer( (void **)ppNode, value, comperand ); - } - - bool InterlockedCompareExchangeNodeLink( NodeLink_t volatile *pLink, const NodeLink_t &value, const NodeLink_t &comperand ) - { - return ThreadInterlockedAssignIf64( (int64 *)pLink, value.value64, comperand.value64 ); - } - -#else - Node_t *InterlockedCompareExchangeNode( Node_t * volatile *ppNode, Node_t *value, Node_t *comperand ) - { - AUTO_LOCK( m_ExchangeMutex ); - Node_t *retVal = *ppNode; - if ( *ppNode == comperand ) - *ppNode = value; - return retVal; - } - - bool InterlockedCompareExchangeNodeLink( NodeLink_t volatile *pLink, const NodeLink_t &value, const NodeLink_t &comperand ) - { - AUTO_LOCK( m_ExchangeMutex ); - if ( pLink->value64 == comperand.value64 ) - { - pLink->value64 = value.value64; - return true; - } - return false; - } - - CThreadFastMutex m_ExchangeMutex; -#endif - - NodeLink_t m_Head; - NodeLink_t m_Tail; - - CInterlockedInt m_Count; - - CTSListBase m_FreeNodes; -}; - -#include "tier0/memdbgoff.h" - -#endif // TSLIST_H +//========= Copyright Valve Corporation, All rights reserved. ============// +// +// Purpose: +// +// LIFO from disassembly of Windows API and http://perso.wanadoo.fr/gmem/evenements/jim2002/articles/L17_Fober.pdf +// FIFO from http://perso.wanadoo.fr/gmem/evenements/jim2002/articles/L17_Fober.pdf +// +//============================================================================= + +#ifndef TSLIST_H +#define TSLIST_H + +#if defined( _WIN32 ) +#pragma once +// Suppress this spurious warning: +// warning C4700: uninitialized local variable 'oldHead' used +#pragma warning( push ) +#pragma warning( disable : 4700 ) +#endif + +#if defined( USE_NATIVE_SLIST ) && !defined( _X360 ) +#define WIN32_LEAN_AND_MEAN +#include +#endif + +#include "tier0/dbg.h" +#include "tier0/threadtools.h" +#include "tier0/memalloc.h" +#include "tier0/memdbgoff.h" + +#if defined( _X360 ) +#define USE_NATIVE_SLIST +#endif + +//----------------------------------------------------------------------------- + +#if defined( PLATFORM_64BITS ) + +#define TSLIST_HEAD_ALIGNMENT 16 +#define TSLIST_NODE_ALIGNMENT 16 +inline bool ThreadInterlockedAssignIf64x128( volatile int128 *pDest, const int128 &value, const int128 &comperand ) + { return ThreadInterlockedAssignIf128( pDest, value, comperand ); } +#else +#define TSLIST_HEAD_ALIGNMENT 8 +#define TSLIST_NODE_ALIGNMENT 8 +inline bool ThreadInterlockedAssignIf64x128( volatile int64 *pDest, const int64 value, const int64 comperand ) + { return ThreadInterlockedAssignIf64( pDest, value, comperand ); } +#endif + +#ifdef _MSC_VER +#define TSLIST_HEAD_ALIGN DECL_ALIGN(TSLIST_HEAD_ALIGNMENT) +#define TSLIST_NODE_ALIGN DECL_ALIGN(TSLIST_NODE_ALIGNMENT) +#define TSLIST_HEAD_ALIGN_POST +#define TSLIST_NODE_ALIGN_POST +#elif defined( GNUC ) +#define TSLIST_HEAD_ALIGN +#define TSLIST_NODE_ALIGN +#define TSLIST_HEAD_ALIGN_POST DECL_ALIGN(TSLIST_HEAD_ALIGNMENT) +#define TSLIST_NODE_ALIGN_POST DECL_ALIGN(TSLIST_NODE_ALIGNMENT) +#elif defined( _PS3 ) +#define TSLIST_HEAD_ALIGNMENT 8 +#define TSLIST_NODE_ALIGNMENT 8 + +#define TSLIST_HEAD_ALIGN ALIGN8 +#define TSLIST_NODE_ALIGN ALIGN8 +#define TSLIST_HEAD_ALIGN_POST ALIGN8_POST +#define TSLIST_NODE_ALIGN_POST ALIGN8_POST + +#else +#error +#endif + +//----------------------------------------------------------------------------- + +PLATFORM_INTERFACE bool RunTSQueueTests( int nListSize = 10000, int nTests = 1 ); +PLATFORM_INTERFACE bool RunTSListTests( int nListSize = 10000, int nTests = 1 ); + +//----------------------------------------------------------------------------- +// Lock free list. +//----------------------------------------------------------------------------- +//#define USE_NATIVE_SLIST + +#ifdef USE_NATIVE_SLIST +typedef SLIST_ENTRY TSLNodeBase_t; +typedef SLIST_HEADER TSLHead_t; +#else +struct TSLIST_NODE_ALIGN TSLNodeBase_t +{ + TSLNodeBase_t *Next; // name to match Windows +} TSLIST_NODE_ALIGN_POST; + +union TSLIST_HEAD_ALIGN TSLHead_t +{ + struct Value_t + { + TSLNodeBase_t *Next; + // Depth must be in the least significant halfword when atomically loading into register, + // to avoid carrying digits from Sequence. Carrying digits from Depth to Sequence is ok, + // because Sequence can be pretty much random. We could operate on both of them separately, + // but it could perhaps (?) lead to problems with store forwarding. I don't know 'cause I didn't + // performance-test or design original code, I'm just making it work on PowerPC. + #ifdef VALVE_BIG_ENDIAN + int16 Sequence; + int16 Depth; + #else + int16 Depth; + int16 Sequence; + #endif +#ifdef PLATFORM_64BITS + int32 Padding; +#endif + } value; + + struct Value32_t + { + TSLNodeBase_t *Next_do_not_use_me; + int32 DepthAndSequence; + } value32; + +#ifdef PLATFORM_64BITS + int128 value64x128; +#else + int64 value64x128; +#endif +} TSLIST_HEAD_ALIGN_POST; + +#endif + +//------------------------------------- +class TSLIST_HEAD_ALIGN CTSListBase +{ +public: + + // override new/delete so we can guarantee 8-byte aligned allocs + static void * operator new( size_t size ) + { + CTSListBase *pNode = (CTSListBase *)MemAlloc_AllocAlignedFileLine( size, TSLIST_HEAD_ALIGNMENT, __FILE__, __LINE__ ); + return pNode; + } + + static void * operator new( size_t size, int nBlockUse, const char *pFileName, int nLine ) + { + CTSListBase *pNode = (CTSListBase *)MemAlloc_AllocAlignedFileLine( size, TSLIST_HEAD_ALIGNMENT, pFileName, nLine ); + return pNode; + } + + static void operator delete( void *p) + { + MemAlloc_FreeAligned( p ); + } + + static void operator delete( void *p, int nBlockUse, const char *pFileName, int nLine ) + { + MemAlloc_FreeAligned( p ); + } + +private: + // These ain't gonna work + static void * operator new[] ( size_t size ); + static void operator delete [] ( void *p); + +public: + + CTSListBase() + { + if ( ((size_t)&m_Head) % TSLIST_HEAD_ALIGNMENT != 0 ) + { + Plat_FatalErrorFunc( "CTSListBase: Misaligned list\n" ); + DebuggerBreak(); + } + +#ifdef USE_NATIVE_SLIST + InitializeSListHead( &m_Head ); +#elif defined(PLATFORM_64BITS) + m_Head.value64x128 = int128_zero(); +#else + m_Head.value64x128 = (int64)0; +#endif + } + + ~CTSListBase() + { + Detach(); + } + + TSLNodeBase_t *Push( TSLNodeBase_t *pNode ) + { +#ifdef _DEBUG + if ( (size_t)pNode % TSLIST_NODE_ALIGNMENT != 0 ) + { + Plat_FatalErrorFunc( "CTSListBase: Misaligned node\n" ); + DebuggerBreak(); + } +#endif + +#ifdef USE_NATIVE_SLIST +#ifdef _X360 + // integrated write-release barrier + return (TSLNodeBase_t *)InterlockedPushEntrySListRelease( &m_Head, pNode ); +#else + return (TSLNodeBase_t *)InterlockedPushEntrySList( &m_Head, pNode ); +#endif +#else + TSLHead_t oldHead; + TSLHead_t newHead; + + #if defined( PLATFORM_PS3 ) || defined( PLATFORM_X360 ) + __lwsync(); // write-release barrier + #endif + +#ifdef PLATFORM_64BITS + newHead.value.Padding = 0; +#endif + for (;;) + { + oldHead.value64x128 = m_Head.value64x128; + pNode->Next = oldHead.value.Next; + newHead.value.Next = pNode; + + newHead.value32.DepthAndSequence = oldHead.value32.DepthAndSequence + 0x10001; + + + if ( ThreadInterlockedAssignIf64x128( &m_Head.value64x128, newHead.value64x128, oldHead.value64x128 ) ) + { + break; + } + ThreadPause(); + }; + + return (TSLNodeBase_t *)oldHead.value.Next; +#endif + } + + TSLNodeBase_t *Pop() + { +#ifdef USE_NATIVE_SLIST +#ifdef _X360 + // integrated read-acquire barrier + TSLNodeBase_t *pNode = (TSLNodeBase_t *)InterlockedPopEntrySListAcquire( &m_Head ); +#else + TSLNodeBase_t *pNode = (TSLNodeBase_t *)InterlockedPopEntrySList( &m_Head ); +#endif + return pNode; +#else + TSLHead_t oldHead; + TSLHead_t newHead; + +#ifdef PLATFORM_64BITS + newHead.value.Padding = 0; +#endif + for (;;) + { + oldHead.value64x128 = m_Head.value64x128; + if ( !oldHead.value.Next ) + return NULL; + + newHead.value.Next = oldHead.value.Next->Next; + newHead.value32.DepthAndSequence = oldHead.value32.DepthAndSequence - 1; + + + if ( ThreadInterlockedAssignIf64x128( &m_Head.value64x128, newHead.value64x128, oldHead.value64x128 ) ) + { + #if defined( PLATFORM_PS3 ) || defined( PLATFORM_X360 ) + __lwsync(); // read-acquire barrier + #endif + break; + } + ThreadPause(); + }; + + return (TSLNodeBase_t *)oldHead.value.Next; +#endif + } + + TSLNodeBase_t *Detach() + { +#ifdef USE_NATIVE_SLIST + TSLNodeBase_t *pBase = (TSLNodeBase_t *)InterlockedFlushSList( &m_Head ); +#if defined( _X360 ) || defined( _PS3 ) + __lwsync(); // read-acquire barrier +#endif + return pBase; +#else + TSLHead_t oldHead; + TSLHead_t newHead; + +#ifdef PLATFORM_64BITS + newHead.value.Padding = 0; +#endif + do + { + ThreadPause(); + + oldHead.value64x128 = m_Head.value64x128; + if ( !oldHead.value.Next ) + return NULL; + + newHead.value.Next = NULL; + // the reason for AND'ing it instead of poking a short into memory + // is probably to avoid store forward issues, but I'm not sure because + // I didn't construct this code. In any case, leaving it as is on big-endian + newHead.value32.DepthAndSequence = oldHead.value32.DepthAndSequence & 0xffff0000; + + } while( !ThreadInterlockedAssignIf64x128( &m_Head.value64x128, newHead.value64x128, oldHead.value64x128 ) ); + + return (TSLNodeBase_t *)oldHead.value.Next; +#endif + } + + TSLHead_t *AccessUnprotected() + { + return &m_Head; + } + + int Count() const + { +#ifdef USE_NATIVE_SLIST + return QueryDepthSList( const_cast( &m_Head ) ); +#else + return m_Head.value.Depth; +#endif + } + +private: + TSLHead_t m_Head; +} TSLIST_HEAD_ALIGN_POST; + +//------------------------------------- + +template +class TSLIST_HEAD_ALIGN CTSSimpleList : public CTSListBase +{ +public: + void Push( T *pNode ) + { + Assert( sizeof(T) >= sizeof(TSLNodeBase_t) ); + CTSListBase::Push( (TSLNodeBase_t *)pNode ); + } + + T *Pop() + { + return (T *)CTSListBase::Pop(); + } +} TSLIST_HEAD_ALIGN_POST; + +//------------------------------------- +// this is a replacement for CTSList<> and CObjectPool<> that does not +// have a per-item, per-alloc new/delete overhead +// similar to CTSSimpleList except that it allocates it's own pool objects +// and frees them on destruct. Also it does not overlay the TSNodeBase_t memory +// on T's memory +template< class T > +class TSLIST_HEAD_ALIGN CTSPool : public CTSListBase +{ + // packs the node and the item (T) into a single struct and pools those + struct TSLIST_NODE_ALIGN simpleTSPoolStruct_t : public TSLNodeBase_t + { + T elem; + } TSLIST_NODE_ALIGN_POST; + +public: + + ~CTSPool() + { + Purge(); + } + + void Purge() + { + simpleTSPoolStruct_t *pNode = NULL; + while ( 1 ) + { + pNode = (simpleTSPoolStruct_t *)CTSListBase::Pop(); + if ( !pNode ) + break; + delete pNode; + } + } + + void PutObject( T *pInfo ) + { + char *pElem = (char *)pInfo; + pElem -= offsetof(simpleTSPoolStruct_t,elem); + simpleTSPoolStruct_t *pNode = (simpleTSPoolStruct_t *)pElem; + + CTSListBase::Push( pNode ); + } + + T *GetObject() + { + simpleTSPoolStruct_t *pNode = (simpleTSPoolStruct_t *)CTSListBase::Pop(); + if ( !pNode ) + { + pNode = new simpleTSPoolStruct_t; + } + return &pNode->elem; + } + + // omg windows sdk - why do you #define GetObject()? + FORCEINLINE T *Get() + { + return GetObject(); + } +} TSLIST_HEAD_ALIGN_POST; +//------------------------------------- + +template +class TSLIST_HEAD_ALIGN CTSList : public CTSListBase +{ +public: + struct TSLIST_NODE_ALIGN Node_t : public TSLNodeBase_t + { + Node_t() {} + Node_t( const T &init ) : elem( init ) {} + T elem; + + // override new/delete so we can guarantee 8-byte aligned allocs + static void * operator new( size_t size ) + { + Node_t *pNode = (Node_t *)MemAlloc_AllocAlignedFileLine( size, TSLIST_NODE_ALIGNMENT, __FILE__, __LINE__ ); + return pNode; + } + + // override new/delete so we can guarantee 8-byte aligned allocs + static void * operator new( size_t size, int nBlockUse, const char *pFileName, int nLine ) + { + Node_t *pNode = (Node_t *)MemAlloc_AllocAlignedFileLine( size, TSLIST_NODE_ALIGNMENT, pFileName, nLine ); + return pNode; + } + + static void operator delete( void *p) + { + MemAlloc_FreeAligned( p ); + } + static void operator delete( void *p, int nBlockUse, const char *pFileName, int nLine ) + { + MemAlloc_FreeAligned( p ); + } + + } TSLIST_NODE_ALIGN_POST; + + ~CTSList() + { + Purge(); + } + + void Purge() + { + Node_t *pCurrent = Detach(); + Node_t *pNext; + while ( pCurrent ) + { + pNext = (Node_t *)pCurrent->Next; + delete pCurrent; + pCurrent = pNext; + } + } + + void RemoveAll() + { + Purge(); + } + + Node_t *Push( Node_t *pNode ) + { + return (Node_t *)CTSListBase::Push( pNode ); + } + + Node_t *Pop() + { + return (Node_t *)CTSListBase::Pop(); + } + + void PushItem( const T &init ) + { + Push( new Node_t( init ) ); + } + + bool PopItem( T *pResult) + { + Node_t *pNode = Pop(); + if ( !pNode ) + return false; + *pResult = pNode->elem; + delete pNode; + return true; + } + + Node_t *Detach() + { + return (Node_t *)CTSListBase::Detach(); + } + +} TSLIST_HEAD_ALIGN_POST; + +//------------------------------------- + +template +class TSLIST_HEAD_ALIGN CTSListWithFreeList : public CTSListBase +{ +public: + struct TSLIST_NODE_ALIGN Node_t : public TSLNodeBase_t + { + Node_t() {} + Node_t( const T &init ) : elem( init ) {} + + T elem; + } TSLIST_NODE_ALIGN_POST; + + ~CTSListWithFreeList() + { + Purge(); + } + + void Purge() + { + Node_t *pCurrent = Detach(); + Node_t *pNext; + while ( pCurrent ) + { + pNext = (Node_t *)pCurrent->Next; + delete pCurrent; + pCurrent = pNext; + } + pCurrent = (Node_t *)m_FreeList.Detach(); + while ( pCurrent ) + { + pNext = (Node_t *)pCurrent->Next; + delete pCurrent; + pCurrent = pNext; + } + } + + void RemoveAll() + { + Node_t *pCurrent = Detach(); + Node_t *pNext; + while ( pCurrent ) + { + pNext = (Node_t *)pCurrent->Next; + m_FreeList.Push( pCurrent ); + pCurrent = pNext; + } + } + + Node_t *Push( Node_t *pNode ) + { + return (Node_t *)CTSListBase::Push( pNode ); + } + + Node_t *Pop() + { + return (Node_t *)CTSListBase::Pop(); + } + + void PushItem( const T &init ) + { + Node_t *pNode = (Node_t *)m_FreeList.Pop(); + if ( !pNode ) + { + pNode = new Node_t; + } + pNode->elem = init; + Push( pNode ); + } + + bool PopItem( T *pResult) + { + Node_t *pNode = Pop(); + if ( !pNode ) + return false; + *pResult = pNode->elem; + m_FreeList.Push( pNode ); + return true; + } + + Node_t *Detach() + { + return (Node_t *)CTSListBase::Detach(); + } + + void FreeNode( Node_t *pNode ) + { + m_FreeList.Push( pNode ); + } + +private: + CTSListBase m_FreeList; +} TSLIST_HEAD_ALIGN_POST; + +//----------------------------------------------------------------------------- +// Lock free queue +// +// A special consideration: the element type should be simple. This code +// actually dereferences freed nodes as part of pop, but later detects +// that. If the item in the queue is a complex type, only bad things can +// come of that. Also, therefore, if you're using Push/Pop instead of +// push item, be aware that the node memory cannot be freed until +// all threads that might have been popping have completed the pop. +// The PushItem()/PopItem() for handles this by keeping a persistent +// free list. Dont mix Push/PushItem. Note also nodes will be freed at the end, +// and are expected to have been allocated with operator new. +//----------------------------------------------------------------------------- + +template +class TSLIST_HEAD_ALIGN CTSQueue +{ +public: + + // override new/delete so we can guarantee 8-byte aligned allocs + static void * operator new( size_t size ) + { + CTSQueue *pNode = (CTSQueue *)MemAlloc_AllocAlignedFileLine( size, TSLIST_HEAD_ALIGNMENT, __FILE__, __LINE__ ); + return pNode; + } + + // override new/delete so we can guarantee 8-byte aligned allocs + static void * operator new( size_t size, int nBlockUse, const char *pFileName, int nLine ) + { + CTSQueue *pNode = (CTSQueue *)MemAlloc_AllocAlignedFileLine( size, TSLIST_HEAD_ALIGNMENT, pFileName, nLine ); + return pNode; + } + + static void operator delete( void *p) + { + MemAlloc_FreeAligned( p ); + } + + static void operator delete( void *p, int nBlockUse, const char *pFileName, int nLine ) + { + MemAlloc_FreeAligned( p ); + } + +private: + // These ain't gonna work + static void * operator new[] ( size_t size ) throw() + { + return NULL; + } + + static void operator delete [] ( void *p) + { + } + +public: + + struct TSLIST_NODE_ALIGN Node_t + { + // override new/delete so we can guarantee 8-byte aligned allocs + static void * operator new( size_t size ) + { + Node_t *pNode = (Node_t *)MemAlloc_AllocAlignedFileLine( size, TSLIST_NODE_ALIGNMENT, __FILE__, __LINE__ ); + return pNode; + } + + static void * operator new( size_t size, int nBlockUse, const char *pFileName, int nLine ) + { + Node_t *pNode = (Node_t *)MemAlloc_AllocAlignedFileLine( size, TSLIST_NODE_ALIGNMENT, pFileName, nLine ); + return pNode; + } + + static void operator delete( void *p) + { + MemAlloc_FreeAligned( p ); + } + + static void operator delete( void *p, int nBlockUse, const char *pFileName, int nLine ) + { + MemAlloc_FreeAligned( p ); + } + + Node_t() {} + Node_t( const T &init ) : elem( init ) {} + + Node_t *pNext; + T elem; + } TSLIST_NODE_ALIGN_POST; + + union TSLIST_HEAD_ALIGN NodeLink_t + { + // override new/delete so we can guarantee 8-byte aligned allocs + static void * operator new( size_t size ) + { + NodeLink_t *pNode = (NodeLink_t *)MemAlloc_AllocAlignedFileLine( size, TSLIST_HEAD_ALIGNMENT, __FILE__, __LINE__ ); + return pNode; + } + + static void operator delete( void *p) + { + MemAlloc_FreeAligned( p ); + } + + struct Value_t + { + Node_t *pNode; + intp sequence; + } value; + +#ifdef PLATFORM_64BITS + int128 value64x128; +#else + int64 value64x128; +#endif + } TSLIST_HEAD_ALIGN_POST; + + CTSQueue() + { + COMPILE_TIME_ASSERT( sizeof(Node_t) >= sizeof(TSLNodeBase_t) ); + if ( ((size_t)&m_Head) % TSLIST_HEAD_ALIGNMENT != 0 ) + { + Plat_FatalErrorFunc( "CTSQueue: Misaligned queue\n" ); + DebuggerBreak(); + } + if ( ((size_t)&m_Tail) % TSLIST_HEAD_ALIGNMENT != 0 ) + { + Plat_FatalErrorFunc( "CTSQueue: Misaligned queue\n" ); + DebuggerBreak(); + } + m_Count = 0; + m_Head.value.sequence = m_Tail.value.sequence = 0; + m_Head.value.pNode = m_Tail.value.pNode = new Node_t; // list always contains a dummy node + m_Head.value.pNode->pNext = End(); + } + + ~CTSQueue() + { + Purge(); + Assert( m_Count == 0 ); + Assert( m_Head.value.pNode == m_Tail.value.pNode ); + Assert( m_Head.value.pNode->pNext == End() ); + delete m_Head.value.pNode; + } + + // Note: Purge, RemoveAll, and Validate are *not* threadsafe + void Purge() + { + if ( IsDebug() ) + { + ValidateQueue(); + } + + Node_t *pNode; + while ( ( pNode = Pop() ) != NULL ) + { + delete pNode; + } + + while ( ( pNode = (Node_t *)m_FreeNodes.Pop() ) != NULL ) + { + delete pNode; + } + + Assert( m_Count == 0 ); + Assert( m_Head.value.pNode == m_Tail.value.pNode ); + Assert( m_Head.value.pNode->pNext == End() ); + + m_Head.value.sequence = m_Tail.value.sequence = 0; + } + + void RemoveAll() + { + if ( IsDebug() ) + { + ValidateQueue(); + } + + Node_t *pNode; + while ( ( pNode = Pop() ) != NULL ) + { + m_FreeNodes.Push( (TSLNodeBase_t *)pNode ); + } + } + + bool ValidateQueue() + { + if ( IsDebug() ) + { + bool bResult = true; + int nNodes = 0; + if ( m_Tail.value.pNode->pNext != End() ) + { + DebuggerBreakIfDebugging(); + bResult = false; + } + + if ( m_Count == 0 ) + { + if ( m_Head.value.pNode != m_Tail.value.pNode ) + { + DebuggerBreakIfDebugging(); + bResult = false; + } + } + + Node_t *pNode = m_Head.value.pNode; + while ( pNode != End() ) + { + nNodes++; + pNode = pNode->pNext; + } + + nNodes--;// skip dummy node + + if ( nNodes != m_Count ) + { + DebuggerBreakIfDebugging(); + bResult = false; + } + + if ( !bResult ) + { + Msg( "Corrupt CTSQueueDetected" ); + } + + return bResult; + } + else + { + return true; + } + } + + void FinishPush( Node_t *pNode, const NodeLink_t &oldTail ) + { + NodeLink_t newTail; + + newTail.value.pNode = pNode; + newTail.value.sequence = oldTail.value.sequence + 1; + + ThreadMemoryBarrier(); + + InterlockedCompareExchangeNodeLink( &m_Tail, newTail, oldTail ); + } + + Node_t *Push( Node_t *pNode ) + { +#ifdef _DEBUG + if ( (size_t)pNode % TSLIST_NODE_ALIGNMENT != 0 ) + { + Plat_FatalErrorFunc( "CTSQueue: Misaligned node\n" ); + DebuggerBreak(); + } +#endif + + NodeLink_t oldTail; + + pNode->pNext = End(); + + for (;;) + { + oldTail.value.sequence = m_Tail.value.sequence; + oldTail.value.pNode = m_Tail.value.pNode; + if ( InterlockedCompareExchangeNode( &(oldTail.value.pNode->pNext), pNode, End() ) == End() ) + { + break; + } + else + { + // Another thread is trying to push, help it along + FinishPush( oldTail.value.pNode->pNext, oldTail ); + } + } + + FinishPush( pNode, oldTail ); // This can fail if another thread pushed between the sequence and node grabs above. Later pushes or pops corrects + + m_Count++; + + return oldTail.value.pNode; + } + + Node_t *Pop() + { + #define TSQUEUE_BAD_NODE_LINK ( (Node_t *)INT_TO_POINTER( 0xdeadbeef ) ) + NodeLink_t * volatile pHead = &m_Head; + NodeLink_t * volatile pTail = &m_Tail; + Node_t * volatile * pHeadNode = &m_Head.value.pNode; + volatile intp * volatile pHeadSequence = &m_Head.value.sequence; + Node_t * volatile * pTailNode = &pTail->value.pNode; + + NodeLink_t head; + NodeLink_t newHead; + Node_t *pNext; + intp tailSequence; + T elem; + + for (;;) + { + head.value.sequence = *pHeadSequence; // must grab sequence first, which allows condition below to ensure pNext is valid + ThreadMemoryBarrier(); // need a barrier to prevent reordering of these assignments + head.value.pNode = *pHeadNode; + tailSequence = pTail->value.sequence; + pNext = head.value.pNode->pNext; + + // Checking pNext only to force optimizer to not reorder the assignment + // to pNext and the compare of the sequence + if ( !pNext || head.value.sequence != *pHeadSequence ) + continue; + + if ( bTestOptimizer ) + { + if ( pNext == TSQUEUE_BAD_NODE_LINK ) + { + Msg( "Bad node link detected\n" ); + continue; + } + } + + if ( head.value.pNode == *pTailNode ) + { + if ( pNext == End() ) + return NULL; + + // Another thread is trying to push, help it along + NodeLink_t &oldTail = head; // just reuse local memory for head to build old tail + oldTail.value.sequence = tailSequence; // reuse head pNode + FinishPush( pNext, oldTail ); + continue; + } + + if ( pNext != End() ) + { + elem = pNext->elem; // NOTE: next could be a freed node here, by design + newHead.value.pNode = pNext; + newHead.value.sequence = head.value.sequence + 1; + if ( InterlockedCompareExchangeNodeLink( pHead, newHead, head ) ) + { + ThreadMemoryBarrier(); + if ( bTestOptimizer ) + { + head.value.pNode->pNext = TSQUEUE_BAD_NODE_LINK; + } + break; + } + } + } + + m_Count--; + head.value.pNode->elem = elem; + return head.value.pNode; + } + + void FreeNode( Node_t *pNode ) + { + m_FreeNodes.Push( (TSLNodeBase_t *)pNode ); + } + + void PushItem( const T &init ) + { + Node_t *pNode = (Node_t *)m_FreeNodes.Pop(); + if ( pNode ) + { + pNode->elem = init; + } + else + { + pNode = new Node_t( init ); + } + Push( pNode ); + } + + bool PopItem( T *pResult ) + { + Node_t *pNode = Pop(); + if ( !pNode ) + return false; + + *pResult = pNode->elem; + m_FreeNodes.Push( (TSLNodeBase_t *)pNode ); + return true; + } + + int Count() const + { + return m_Count; + } + +private: + Node_t *End() { return (Node_t *)this; } // just need a unique signifier + + Node_t *InterlockedCompareExchangeNode( Node_t * volatile *ppNode, Node_t *value, Node_t *comperand ) + { + return (Node_t *)::ThreadInterlockedCompareExchangePointer( (void **)ppNode, value, comperand ); + } + + bool InterlockedCompareExchangeNodeLink( NodeLink_t volatile *pLink, const NodeLink_t &value, const NodeLink_t &comperand ) + { + return ThreadInterlockedAssignIf64x128( &pLink->value64x128, value.value64x128, comperand.value64x128 ); + } + + NodeLink_t m_Head; + NodeLink_t m_Tail; + + CInterlockedInt m_Count; + + CTSListBase m_FreeNodes; +} TSLIST_HEAD_ALIGN_POST; + +#if defined( _WIN32 ) +// Suppress this spurious warning: +// warning C4700: uninitialized local variable 'oldHead' used +#pragma warning( pop ) +#endif + +#endif // TSLIST_H diff --git a/public/tier1/bufferstring.h b/public/tier1/bufferstring.h index 0d0458a1..135d10ec 100644 --- a/public/tier1/bufferstring.h +++ b/public/tier1/bufferstring.h @@ -153,6 +153,7 @@ public: // Returns NULL if they can't be made relative (on separate drives, for example) DLL_CLASS_IMPORT const char *MakeRelativePath(const char *pFullPath, const char *pDirectory); + // Copies data from pOther and then purges it DLL_CLASS_IMPORT void MoveFrom(CBufferString &pOther); DLL_CLASS_IMPORT void Purge(int nLength); @@ -242,7 +243,7 @@ public: { m_nAllocated |= ALLOW_HEAP_ALLOCATION; } - MoveFrom(const_cast(other)); + Insert(0, other.Get()); } ~CBufferStringGrowable() @@ -259,7 +260,8 @@ public: inline CBufferStringGrowable& operator=(const CBufferStringGrowable& src) { - MoveFrom(const_cast(src)); + Clear(); + Insert(0, src.Get()); return *this; } diff --git a/public/tier1/mempool.h b/public/tier1/mempool.h index f52f52e3..3c9265cd 100644 --- a/public/tier1/mempool.h +++ b/public/tier1/mempool.h @@ -34,7 +34,6 @@ enum MemoryPoolGrowType_t UTLMEMORYPOOL_GROW_NONE=0, // Don't allow new blobs. UTLMEMORYPOOL_GROW_FAST=1, // New blob size is numElements * (i+1) (ie: the blocks it allocates get larger and larger each time it allocates one). UTLMEMORYPOOL_GROW_SLOW=2, // New blob size is numElements. - UTLMEMORYPOOL_GROW_RBTREE=3 // No blobs. All blocks are stored in RBTree. }; class CUtlMemoryPoolBase @@ -51,22 +50,20 @@ public: DLL_CLASS_IMPORT void Free( void *pMem ); // Frees everything - DLL_CLASS_IMPORT void Clear(); - + void Clear() { ClearDestruct( NULL ); } + // returns number of allocated blocks int Count() const { return m_BlocksAllocated; } int PeakCount() const { return m_PeakAlloc; } int BlockSize() const { return m_BlockSize; } - int Size() const { return m_NumBlobs * m_BlocksPerBlob * m_BlockSize; } + int Size() const { return m_TotalSize; } DLL_CLASS_IMPORT bool IsAllocationWithinPool( void *pMem ) const; protected: - struct FreeList_t - { - FreeList_t *m_pNext; - }; + DLL_CLASS_IMPORT void ClearDestruct( void (*)( void* ) ); +private: class CBlob { public: @@ -76,10 +73,8 @@ protected: char m_Padding[3]; // to int align the struct }; - DLL_CLASS_IMPORT FreeList_t* AddNewBlob(); - DLL_CLASS_IMPORT void ResetAllocationCounts(); - DLL_CLASS_IMPORT void InternalClear( CBlob *blob, FreeList_t *free_list ); - DLL_CLASS_IMPORT void ClearDestruct( void (*)( void* ) ); + DLL_CLASS_IMPORT bool AddNewBlob(); + DLL_CLASS_IMPORT void ResetAllocationCounts(); int m_BlockSize; int m_BlocksPerBlob; @@ -90,16 +85,15 @@ protected: CInterlockedInt m_PeakAlloc; unsigned short m_nAlignment; unsigned short m_NumBlobs; - - FreeList_t** m_ppTailOfFreeList; - FreeList_t* m_pHeadOfFreeList; - - CBlob** m_ppBlobTail; - CBlob* m_pBlobHead; - + + CTSListBase m_FreeBlocks; + MemAllocAttribute_t m_AllocAttribute; - - bool m_Unk1; + + bool m_Unk1; + CThreadMutex m_Mutex; + CBlob* m_pBlobHead; + int m_TotalSize; }; diff --git a/public/tier1/utldict.h b/public/tier1/utldict.h index 519ae80f..277b2a2f 100644 --- a/public/tier1/utldict.h +++ b/public/tier1/utldict.h @@ -72,6 +72,9 @@ public: // Number of elements unsigned int Count() const; + // Number of allocated slots + I MaxElement() const; + // Checks if a node is valid and in the tree bool IsValidIndex( I i ) const; @@ -190,7 +193,15 @@ inline unsigned int CUtlDict::Count() const return m_Elements.Count(); } - +//----------------------------------------------------------------------------- +// Number of allocated slots +//----------------------------------------------------------------------------- +template +inline I CUtlDict::MaxElement() const +{ + return m_Elements.MaxElement(); +} + //----------------------------------------------------------------------------- // Checks if a node is valid and in the tree //----------------------------------------------------------------------------- diff --git a/tier1/keyvalues3.cpp b/tier1/keyvalues3.cpp index 543b071d..5a0ca0ae 100644 --- a/tier1/keyvalues3.cpp +++ b/tier1/keyvalues3.cpp @@ -1796,10 +1796,7 @@ void CKeyValues3Context::CopyMetaData( KV3MetaData_t* pDest, const KV3MetaData_t FOR_EACH_MAP_FAST( pSrc->m_Comments, iter ) { - CBufferStringGrowable<8> buff; - buff.Insert( 0, pSrc->m_Comments[ iter ].Get() ); - - pDest->m_Comments.Insert( pSrc->m_Comments.Key( iter ), buff ); + pDest->m_Comments.Insert( pSrc->m_Comments.Key( iter ), pSrc->m_Comments.Element( iter ) ); } }