Loading objects with assimp in OpenGL windows with transparent background

Question

I want to load a 3D object into an OpenGL windows with a transparent background. I'm using C++ on an Ubuntu 16.04 machine with a Nvidia Quadro K3000M (installed driver is the recent 375.66 NVIDIA binary driver). I've got both objectives working independently - I can create an OpenGL window with a transparent background with X11 from this thread and load objects with Assimp and GLFW from this tutorial.

Now I want to combine both and it works at least a bit - I can import an object and load it into the transparent X11 window, but some of the textures are transparent now, which is strange:

GLFW result:

X11 result:

So to make my issue clear: I don't understand why the imported objects has transparencies within the X11 window and not within the GLFW window. I am looking for a fix that the model gets displayed correctly within the X11 window (basically it could be any window-manager, but X11 seemed to be the only one that is capable to display transparent backgrounds).

I use the mesh and model classes from the LearnOpenGL tutorial, and also their shaders. I hope @datenwolf might be able to help. Here's my code:

#include "GL/glew.h"

#include <GL/gl.h>
#include <GL/glx.h>
#include <GL/glxext.h>
#include <X11/Xatom.h>
#include <X11/extensions/Xrender.h>
#include <X11/Xutil.h>

#include <GLFW/glfw3.h>

#include <glm/glm.hpp>
#include <glm/gtc/matrix_transform.hpp>
#include <glm/gtc/type_ptr.hpp>

#include <includes/learnopengl/filesystem.h>
#include <includes/learnopengl/shader_m.h>
#include <includes/learnopengl/camera.h>
#include <includes/learnopengl/model.h>

#include <iostream>
//----------------------------
//----- RGBA X11 -------------
//----------------------------
#define USE_CHOOSE_FBCONFIG

static void fatalError(const char *why)
{
    fprintf(stderr, "%s", why);
    exit(0x666);
}

static int Xscreen;
static Atom del_atom;
static Colormap cmap;
static Display *Xdisplay;
static XVisualInfo *visual;
static XRenderPictFormat *pict_format;
static GLXFBConfig *fbconfigs, fbconfig;
static int numfbconfigs;
static GLXContext render_context;
static Window Xroot, window_handle;
static GLXWindow glX_window_handle;
static int width, height;

std::string window_choice = "GLFW"; //X11 or GLFW

static int VisData[] = {
GLX_RENDER_TYPE, GLX_RGBA_BIT,
GLX_DRAWABLE_TYPE, GLX_WINDOW_BIT,
GLX_DOUBLEBUFFER, True,
GLX_RED_SIZE, 8,
GLX_GREEN_SIZE, 8,
GLX_BLUE_SIZE, 8,
GLX_ALPHA_SIZE, 8,
GLX_DEPTH_SIZE, 16,
None
};

static int isExtensionSupported(const char *extList, const char *extension)
{
  const char *start;
  const char *where, *terminator;

  /* Extension names should not have spaces. */
  where = strchr(extension, ' ');
  if ( where || *extension == '\0' )
    return 0;

  /* It takes a bit of care to be fool-proof about parsing the
     OpenGL extensions string. Don't be fooled by sub-strings,
     etc. */
  for ( start = extList; ; ) {
    where = strstr( start, extension );

    if ( !where )
      break;

    terminator = where + strlen( extension );

    if ( where == start || *(where - 1) == ' ' )
      if ( *terminator == ' ' || *terminator == '\0' )
        return 1;

    start = terminator;
  }
  return 0;
}

static Bool WaitForMapNotify(Display *d, XEvent *e, char *arg)
{
    return d && e && arg && (e->type == MapNotify) && (e->xmap.window == *(Window*)arg);
}

static void describe_fbconfig(GLXFBConfig fbconfig)
{
    int doublebuffer;
    int red_bits, green_bits, blue_bits, alpha_bits, depth_bits;

    glXGetFBConfigAttrib(Xdisplay, fbconfig, GLX_DOUBLEBUFFER, &doublebuffer);
    glXGetFBConfigAttrib(Xdisplay, fbconfig, GLX_RED_SIZE, &red_bits);
    glXGetFBConfigAttrib(Xdisplay, fbconfig, GLX_GREEN_SIZE, &green_bits);
    glXGetFBConfigAttrib(Xdisplay, fbconfig, GLX_BLUE_SIZE, &blue_bits);
    glXGetFBConfigAttrib(Xdisplay, fbconfig, GLX_ALPHA_SIZE, &alpha_bits);
    glXGetFBConfigAttrib(Xdisplay, fbconfig, GLX_DEPTH_SIZE, &depth_bits);

    fprintf(stderr, "FBConfig selected:\n"
        "Doublebuffer: %s\n"
        "Red Bits: %d, Green Bits: %d, Blue Bits: %d, Alpha Bits: %d, Depth Bits: %d\n",
        doublebuffer == True ? "Yes" : "No",
        red_bits, green_bits, blue_bits, alpha_bits, depth_bits);
}

static void createTheWindow()
{
    XEvent event;
    int x,y, attr_mask;
    XSizeHints hints;
    XWMHints *startup_state;
    XTextProperty textprop;
    XSetWindowAttributes attr = {0,};
    static char *title = "ARGB OpenGL Window";

    Xdisplay = XOpenDisplay(NULL);
    if (!Xdisplay) {
        fatalError("Couldn't connect to X server\n");
    }
    Xscreen = DefaultScreen(Xdisplay);
    Xroot = RootWindow(Xdisplay, Xscreen);

    fbconfigs = glXChooseFBConfig(Xdisplay, Xscreen, VisData, &numfbconfigs);
    fbconfig = 0;
    for(int i = 0; i<numfbconfigs; i++) {
        visual = (XVisualInfo*) glXGetVisualFromFBConfig(Xdisplay, fbconfigs[i]);
        if(!visual)
            continue;

        pict_format = XRenderFindVisualFormat(Xdisplay, visual->visual);
        if(!pict_format)
            continue;

        fbconfig = fbconfigs[i];
        if(pict_format->direct.alphaMask > 0) {
            break;
        }
    }

    if(!fbconfig) {
        fatalError("No matching FB config found");
    }

    describe_fbconfig(fbconfig);

    /* Create a colormap - only needed on some X clients, eg. IRIX */
    cmap = XCreateColormap(Xdisplay, Xroot, visual->visual, AllocNone);

    attr.colormap = cmap;
    attr.background_pixmap = None;
    attr.border_pixmap = None;
    attr.border_pixel = 0;
    attr.event_mask =
        StructureNotifyMask |
        EnterWindowMask |
        LeaveWindowMask |
        ExposureMask |
        ButtonPressMask |
        ButtonReleaseMask |
        OwnerGrabButtonMask |
        KeyPressMask |
        KeyReleaseMask;

    attr_mask =
        CWBackPixmap|
        CWColormap|
        CWBorderPixel|
        CWEventMask;

    width = 1920; //DisplayWidth(Xdisplay, DefaultScreen(Xdisplay))/2;
    height = 1080; //DisplayHeight(Xdisplay, DefaultScreen(Xdisplay))/2;
    x=width/2, y=height/2;

    window_handle = XCreateWindow(  Xdisplay,
                    Xroot,
                    x, y, width, height,
                    0,
                    visual->depth,
                    InputOutput,
                    visual->visual,
                    attr_mask, &attr);

    if( !window_handle ) {
        fatalError("Couldn't create the window\n");
    }

#if USE_GLX_CREATE_WINDOW
    int glXattr[] = { None };
    glX_window_handle = glXCreateWindow(Xdisplay, fbconfig, window_handle, glXattr);
    if( !glX_window_handle ) {
        fatalError("Couldn't create the GLX window\n");
    }
#else
    glX_window_handle = window_handle;
#endif

    textprop.value = (unsigned char*)title;
    textprop.encoding = XA_STRING;
    textprop.format = 8;
    textprop.nitems = strlen(title);

    hints.x = x;
    hints.y = y;
    hints.width = width;
    hints.height = height;
    hints.flags = USPosition|USSize;

    startup_state = XAllocWMHints();
    startup_state->initial_state = NormalState;
    startup_state->flags = StateHint;

    XSetWMProperties(Xdisplay, window_handle,&textprop, &textprop,
            NULL, 0,
            &hints,
            startup_state,
            NULL);

    XFree(startup_state);

    XMapWindow(Xdisplay, window_handle);
    XIfEvent(Xdisplay, &event, WaitForMapNotify, (char*)&window_handle);

    if ((del_atom = XInternAtom(Xdisplay, "WM_DELETE_WINDOW", 0)) != None) {
        XSetWMProtocols(Xdisplay, window_handle, &del_atom, 1);
    }
}

static int ctxErrorHandler( Display *dpy, XErrorEvent *ev )
{
    fputs("Error at context creation", stderr);
    return 0;
}

static void createTheRenderContext()
{
    int dummy;
    if (!glXQueryExtension(Xdisplay, &dummy, &dummy)) {
        fatalError("OpenGL not supported by X server\n");
    }

#if USE_GLX_CREATE_CONTEXT_ATTRIB
    #define GLX_CONTEXT_MAJOR_VERSION_ARB       0x2091
    #define GLX_CONTEXT_MINOR_VERSION_ARB       0x2092
    render_context = NULL;
    if( isExtensionSupported( glXQueryExtensionsString(Xdisplay, DefaultScreen(Xdisplay)), "GLX_ARB_create_context" ) ) {
        typedef GLXContext (*glXCreateContextAttribsARBProc)(Display*, GLXFBConfig, GLXContext, Bool, const int*);
        glXCreateContextAttribsARBProc glXCreateContextAttribsARB = (glXCreateContextAttribsARBProc)glXGetProcAddressARB( (const GLubyte *) "glXCreateContextAttribsARB" );
        if( glXCreateContextAttribsARB ) {
            int context_attribs[] =
            {
                GLX_CONTEXT_MAJOR_VERSION_ARB, 3,
                GLX_CONTEXT_MINOR_VERSION_ARB, 0,
                //GLX_CONTEXT_FLAGS_ARB        , GLX_CONTEXT_FORWARD_COMPATIBLE_BIT_ARB,
                None
            };

            int (*oldHandler)(Display*, XErrorEvent*) = XSetErrorHandler(&ctxErrorHandler);

            render_context = glXCreateContextAttribsARB( Xdisplay, fbconfig, 0, True, context_attribs );

            XSync( Xdisplay, False );
            XSetErrorHandler( oldHandler );

            fputs("glXCreateContextAttribsARB failed", stderr);
        } else {
            fputs("glXCreateContextAttribsARB could not be retrieved", stderr);
        }
    } else {
            fputs("glXCreateContextAttribsARB not supported", stderr);
    }

    if(!render_context)
    {
#else
    {
#endif
        render_context = glXCreateNewContext(Xdisplay, fbconfig, GLX_RGBA_TYPE, 0, True);   //GLX_RGBA_TYPE
        if (!render_context) {
            fatalError("Failed to create a GL context\n");
        }
    }

    if (!glXMakeContextCurrent(Xdisplay, glX_window_handle, glX_window_handle, render_context)) {
        fatalError("glXMakeCurrent failed for window\n");
    }
}

static int updateTheMessageQueue()
{
    XEvent event;
    XConfigureEvent *xc;

    while (XPending(Xdisplay))
    {
        XNextEvent(Xdisplay, &event);
        switch (event.type)
        {
        case ClientMessage:
            if (event.xclient.data.l[0] == del_atom)
            {
                return 0;
            }
        break;

        case ConfigureNotify:
            xc = &(event.xconfigure);
            width = xc->width;
            height = xc->height;
            break;
        }
    }
    return 1;
}
//----------------------------
//----- GLFW -----------------
//----------------------------

// settings
const unsigned int SCR_WIDTH = 1920;
const unsigned int SCR_HEIGHT = 1080;

// camera
Camera camera(glm::vec3(0.0f, 0.0f, 3.0f));
float lastX = SCR_WIDTH / 2.0f;
float lastY = SCR_HEIGHT / 2.0f;
bool firstMouse = true;

// timing
float deltaTime = 0.0f;
float lastFrame = 0.0f;

int main(int argc, char *argv[])
{
    GLFWwindow* window;

    if (window_choice == "X11")
    {
        createTheWindow();
        createTheRenderContext();
    }
    else if (window_choice == "GLFW")
    {
        // glfw: initialize and configure
        glfwInit();
        glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3);
        glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 3);
        glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);

        #ifdef __APPLE__
            glfwWindowHint(GLFW_OPENGL_FORWARD_COMPAT, GL_TRUE); // uncomment this statement to fix compilation on OS X
        #endif

        // glfw window creation
        window = glfwCreateWindow(SCR_WIDTH, SCR_HEIGHT, "LearnOpenGL", NULL, NULL);
        if (window == NULL)
        {
            std::cout << "Failed to create GLFW window" << std::endl;
            glfwTerminate();
            return -1;
        }
        glfwMakeContextCurrent(window);
    }

    //Load GLEW
    GLenum err = glewInit();
    if (GLEW_OK != err)
    {
      // Problem: glewInit failed, something is seriously wrong
      fprintf(stderr, "Error: %s\n", glewGetErrorString(err));
    }
    fprintf(stdout, "Status: Using GLEW %s\n", glewGetString(GLEW_VERSION));

    // configure global opengl state
    // -----------------------------
    glEnable(GL_DEPTH_TEST);

    //Build and compile shaders
    Shader ourShader("/home/daniel/aureate/tools/Testing/assimp/1.model_loading.vs", "/home/daniel/aureate/tools/Testing/assimp/1.model_loading.fs");

    //Load models
    Model ourModel(FileSystem::getPath("resources/objects/nanosuit/nanosuit.obj"));

    if (window_choice == "X11")
    {
        while (updateTheMessageQueue())
        {
            //RGBA
            glClearColor(0.0, 0.0, 0.0, 0.0);
            glClear(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT);

            ourShader.use();

            glm::mat4 projection = glm::perspective(glm::radians(camera.Zoom), 1.777f, 0.1f, 100.0f);    //1920/1080 = 1.777
            glm::mat4 view = camera.GetViewMatrix();
            ourShader.setMat4("projection", projection);
            ourShader.setMat4("view", view);

            // render the loaded model
            glm::mat4 model;
            model = glm::translate(model, glm::vec3(0.0f, -1.75f, 0.0f)); // translate it down so it's at the center of the scene
            model = glm::scale(model, glm::vec3(0.2f, 0.2f, 0.2f)); // it's a bit too big for our scene, so scale it down
            ourShader.setMat4("model", model);
            ourModel.Draw(ourShader);

            glXSwapBuffers(Xdisplay, glX_window_handle);
        }
    }
    else if (window_choice == "GLFW")
    {
        while (!glfwWindowShouldClose(window))
        {
           // per-frame time logic
           // --------------------
           float currentFrame = glfwGetTime();
           deltaTime = currentFrame - lastFrame;
           lastFrame = currentFrame;

           // render
           // ------
           glClearColor(0.2f, 0.3f, 0.3f, 0.0f);
           glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

           // don't forget to enable shader before setting uniforms
           ourShader.use();

           // view/projection transformations
           glm::mat4 projection = glm::perspective(glm::radians(camera.Zoom), (float)SCR_WIDTH / (float)SCR_HEIGHT, 0.1f, 100.0f);
           glm::mat4 view = camera.GetViewMatrix();
           ourShader.setMat4("projection", projection);
           ourShader.setMat4("view", view);

           // render the loaded model
           glm::mat4 model;
           model = glm::translate(model, glm::vec3(0.0f, -1.75f, 0.0f)); // translate it down so it's at the center of the scene
           model = glm::scale(model, glm::vec3(0.2f, 0.2f, 0.2f));  // it's a bit too big for our scene, so scale it down
           ourShader.setMat4("model", model);
           ourModel.Draw(ourShader);

           glfwSwapBuffers(window);
        }

        glfwTerminate();
        return 0;
    }
}

Answer 1

So to make my issue clear: I don't understand why the imported objects has transparencies within the X11 window and not within the GLFW window.

When you make a transparent X11 window, the alpha channel of that window will be used as the transparency channel. When you render to a non-transparent window created with GLFW, the alpha channel of the framebuffer will have no effect whatsoever.

Now what you see here is that some of the textures have transparent parts. But as long as you don't enable GL's own blending, those alpha values will never have any observable effect. But your shaders still copy the alpha value from the texture to the output framebuffer. In the GLFW case, it is not even clear if there is an alpha channel in the default framebuffer (you can explicitly request it, but can't be sure it isn't there if you don't), but it is totally irrelevant since that alpha channel isn't used.

In your transparent X11 window code path, you absolutely need to have an alpha channel in the default framebuffer, and the alpha values which end up there will actually have an effect.

If you don't want those transparencies, you could do a couple of things:

• modify the shaders so that they output (r,g,b,1) instead of (r,g,b,a)
• use a GL_RGB based format for the textures, ignoring the alpha values from the file
• fix the textures / material description to not contain any transparencies

X11 window (basically it could be any window-manager, but X11 seemed to be the only one that is capable to display transparent backgrounds).

No, X11 is not a window manager. Not sure what window manager / compositor you are using, but there are several alternatives supporting transparency. On the other hand, you certainly would not be able to get the effect on my system, and this is exactly how I want it to be.