How do I insert a templated value into a std::map without the templated value destructor being called?

Question

I have a very simple memory manager. The ConstructObject function inserts an ObjectInfo into the m_ObjectInfoMap. But I am doing it in such a way that the ObjectInfo destructor is called. How do I change the function to prevent this? I guess I can use a pointer (ObjectInfo* pObjectInfo = new ObjectInfo()) and insert that into the map, but I am trying to avoid calling the new operator.

#define CONSTRUCT(T) static_cast<T*>(ObjectFactory::ConstructObject<T>( #T ))
#define DESTRUCT(object) ObjectFactory::DestructObject( object )

//dummy class so we can have a templated class as value to a map
class OObjectInfoInterface
{
};

template<class T>
class ObjectInfo : public OObjectInfoInterface
{   
public:
    ObjectInfo(){
        m_Object = NULL; 
        m_Name = NULL;
        m_Size = 0;
    }
    ~ObjectInfo();

    T* m_Object;    

    //used for things like memory profiling
    const char* m_Name;
    size_t m_Size;
};

template<class T>
ObjectInfo<T>::~ObjectInfo()
{
    delete m_Object;
}

class ObjectFactory
{
public:
    template <typename T> 
    static OObjectInterface* ConstructObject( const char* pName )
    {
            T* pNewObject = new T();
            ObjectInfo<T> objectInfo;
            objectInfo.m_Object = pNewObject;
            objectInfo.m_Name = pName;
            m_ObjectInfoMap[pNewObject] = objectInfo;//<--causes destructor of ObjectInfo to be called (which deletes pNewObject)
            return pNewObject;
    }

    void ObjectFactory::DestructObject( void* pObject )
    {
        if ( pObject )
        {
            TObjectInfoMap::iterator it = m_ObjectInfoMap.find( pObject );

            if ( it != m_ObjectInfoMap.end() )
            {
                m_ObjectInfoMap.erase(pObject);
            }
            else
            {
                LOG_ASSERT( 0, "object not found!");
            }
        }
    }
};