I have managed to get MSAA working on a QOpenGLWindow, however when I try to put an OpenGL widget into the main window, MSAA will not work not matter what I try. I have used setFormat and setSamples in the main
#include "test.h"
#include <QtWidgets/QApplication>
#include <QSurfaceFormat>
int main(int argc, char *argv[])
{
    QApplication a(argc, argv);
    QSurfaceFormat format;
    format.setDepthBufferSize(24);
    format.setStencilBufferSize(8);
    format.setVersion(4, 1);
    format.setSamples(16);
    format.setProfile(QSurfaceFormat::CoreProfile);
    QSurfaceFormat::setDefaultFormat(format);
    test w;
    w.show();
    return a.exec();
}
Here is the main window containing gui elements etc:
#include "test.h"
#include "Display.h"
#include <QtWidgets>
#include <QDesktopWidget>
#include <QApplication>
     test::test(QWidget *parent)
        {
            Display *graph = new Display(this);
            qDebug() <<"outide"<< graph->format();
            graph->resize(1000, 300);
            setFixedSize(1000, 500);
            exit = new QPushButton("Exit", this);
            stop_button = new QPushButton("Stop", this);
            stop_button->setGeometry(800, 430, 80, 50);
            connect(exit, SIGNAL(released()), QApplication::instance(), SLOT(quit()));
            //connect(stop_button, SIGNAL(released()), this, SLOT(stop()));
            exit->setGeometry(900, 430, 80, 50);    
            //graph->setFixedSize(1000, 300);
            graph->show();
        }
Display is the opengl widget. If I put a qDebug inside display's constructor the output is identical to the debug in the mainwindow constructor. However in initializeGL() the debug output does not change no matter what I change in the main, the only thing that does change when I change it is the OpenGL version. This exact setup works with a QOpenGLWindow (MSAA works).
Thanks
EDIT: the opengl window, I don't what is relevant so I put all of it in, if I remove the constructor and change the inherited class to QOpenGLWindow in the header file, it works fine
#include "Display.h"
#include <QCoreApplication>
#include <QOpenGLShaderProgram>
#include <QOpenGLPaintDevice>
#include <QOpenGLWidget>
#include <QPainter>
//using namespace Display;
/*static const float vertices[] = {
    -1.0f, -1.0f, 0.0f, 0,1,0,
    1.0f, -1.0f, 0.0f, 0,0,1,
    0.0f,  1.0f, 0.0f, 1,0,0
};*/
Display::Display(QWidget *parent)
    :QOpenGLWidget(parent)
{
    qDebug() << "inside" << this->format().profile();
    qDebug() << "inside" << this->format();
}
Display::~Display()
{
    //qDebug() << "test";
    //teardownGL();
}
const int num_line_segments = 2;
const int num_points = 1 / (0.1 / num_line_segments);
//const int num_points = 200;
void Display::initializeGL()
{   
    const int size = num_points * 6;
    float vertices[size] = { 0 };
    float x_pos = -0.5;
    for (int i = 0; i < size; i += 6) {
        vertices[i] = x_pos;
        vertices[i + 1] = -(x_pos*x_pos) + 0.5;
        vertices[i + 2] = 0.0f;
        vertices[i + 3] = 1;
        x_pos += 0.1 / num_line_segments;
    }
    //qDebug("%d", sizeof(vertices));
    initializeOpenGLFunctions();
    glClearColor(1.0f, 1.0f, 1.0f, 1.0f);
    m_program = new QOpenGLShaderProgram();
    m_program->addShaderFromSourceFile(QOpenGLShader::Vertex, "shader.vert");
    m_program->addShaderFromSourceFile(QOpenGLShader::Fragment, "fragment.frag");
    m_program->link();
    m_program->bind();
    // Create Buffer (Do not release until VAO is created)
    m_vertex.create();
    m_vertex.bind();
    m_vertex.setUsagePattern(QOpenGLBuffer::StaticDraw);
    m_vertex.allocate(vertices, sizeof(vertices));
    // Create Vertex Array Object
    m_object.create();
    m_object.bind();
    m_program->enableAttributeArray(0);
    m_program->enableAttributeArray(1);
    //qDebug("%d", sizeof(vertices));
    //qDebug("%d", sizeof(float));
    m_program->setAttributeBuffer(0, GL_FLOAT, 0, 3, 24);
    m_program->setAttributeBuffer(1, GL_FLOAT, 12,3, 24);
    glEnable(GL_MULTISAMPLE);
    // Release (unbind) all
    m_object.release();
    m_vertex.release();
    m_matrixUniform = m_program->uniformLocation("gWorld");
    m_program->release();
    qDebug() << "initializeGL" << this->format().profile();
    qDebug() << "initializeGL" << this->format();
}
void Display::paintGL()
{
    initializeOpenGLFunctions();
    // Clear
    glClear(GL_COLOR_BUFFER_BIT);
    static float angle = 0.0f;
    static float scale = 0.0f;
    scale += 0.001;
    // Render using our shader
    m_program->bind();
    m_object.bind();
    //qDebug("%d", sizeof(vertices) / sizeof(vertices[0]));
    QMatrix4x4 matrix;
    //matrix.rotate(angle, 0, 1, 0);
    //matrix.scale(1);
    m_program->setUniformValue(m_matrixUniform, matrix);
    glDrawArrays(GL_LINE_STRIP, 0, num_points);
    m_object.release();
    //qDebug() << "inside" << this->format();
    m_program->release();
    if (!is_stopped) {
        angle += 5;
        update();
    }
    update();
    //qDebug("%d", is_stopped);
}
void Display::teardownGL()
{
    // Actually destroy our OpenGL information
    m_object.destroy();
    m_vertex.destroy();
    delete m_program;
}
void Display::resizeGL(int width, int height)
{
    // Currently we are not handling width/height changes.
    (void)width;
    (void)height;
}
   header file
    #include <QOpenGLWidget>
    #include <QOpenGLFunctions>
    #include <QOpenGLBuffer>
    #include <QOpenGLWindow>
    #include <QOpenGLFunctions>
    #include <QOpenGLVertexArrayObject>
    #include <QWidget>
    #include <gl/GLU.h>
    #include <gl/GL.h>
    //#include "logo.h"
    QT_FORWARD_DECLARE_CLASS(QOpenGLShaderProgram)
    class Display : public QOpenGLWidget,protected QOpenGLFunctions
    {
        Q_OBJECT
    public:
        Display(QWidget *parent=0);
        ~Display(); 
        bool is_stopped=false;
        void teardownGL();
    protected:
        void initializeGL();
        void resizeGL(int width, int height);
        void paintGL();
    private:
        QOpenGLBuffer m_vertex;
        QOpenGLContext *m_context;
        QOpenGLVertexArrayObject m_object;
        QOpenGLShaderProgram *m_program;
        GLuint m_matrixUniform;
    };