Draw objects on moderngl context in Pygame

Question

I wrote this simple program that displays an image onto a moderngl context, using pygame, and my goal is to draw a rectangle on the screen. (This is obviously just a minimal working example, and the real goal is to animate the texture and make more than a rectangle on the screen).

import moderngl
import numpy as np
import pygame
from PIL import Image


class MyPyGame:
    def __init__(self):
        pygame.init()
        self._surface = pygame.display.set_mode((800, 600), pygame.DOUBLEBUF | pygame.OPENGL)

        self.context = moderngl.create_context()

        image = Image.open("test.jpg").transpose(Image.FLIP_TOP_BOTTOM)
        self.texture = self.context.texture(image.size, 3, image.tobytes())
        self.texture.use(0)

        self.program = self.context.program(
            vertex_shader="""
            #version 330
            in vec2 in_position;
            out vec2 uv0;
            void main() {
                gl_Position = vec4(in_position, 0.0, 1.0);
                uv0 = (0.5 * in_position) + vec2(0.5);
            }
            """,
            fragment_shader="""
            #version 330
            out vec4 fragColor;
            uniform sampler2D texture0;
            in vec2 uv0;
            void main() {
                fragColor = texture(texture0, uv0);
            }
            """)

        vertices_quad_2d = np.array([-1.0, 1.0, -1.0, -1.0, 1.0, -1.0,
                                     -1.0, 1.0, 1.0, -1.0, 1.0, 1.0], dtype=np.float32)
        vertex_buffer_quad_2d = self.context.buffer(vertices_quad_2d.tobytes())
        self.vertex_array = self.context.vertex_array(self.program, [(vertex_buffer_quad_2d, "2f", "in_position")])

    def run(self):
        while True:
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    pygame.quit()
                    quit()
            self.vertex_array.render()
            pygame.display.flip()

if __name__ == '__main__':
    app = MyPyGame()
    app.run()

Some things I tried:

1. Draw a rectangle as is the "regular" way with pygame, that is, add this line in the main run method, either before or after the vertex array rendering (both didn't work):

    pygame.draw.rect(self._surface, (200, 200, 200), [100, 100, 100, 100])
   

2. Instead of flag pygame.OPENGL, use flag pygame.OPENGLBLIT and then blit the rectangle (didn't work, more info here)

3. Use the moderngl context as a window inside the main display, as suggested here, and then blit the rectangle onto the main display. This is not what I am aiming for, I want to able to display texts and shapes onto the context, but still, wanted to at least try this. Didn't work either, the code below (replacing the corresponding code in the __init__) resulted in an exception cannot detect OpenGL context:

    pygame.init()
    self.main_window = pygame.display.set_mode((800, 600))
    self.graphics_window = pygame.Surface((700, 600), pygame.DOUBLEBUF | pygame.OPENGL)
    self.main_window.blit(self.graphics_window, (100, 0))
    self.context = moderngl.create_context()
   

4. Use "out of the box" window from moderngl_window using pygame (code here). Again, didn't succeed to draw a rectangle onto the context itself - tried to add the same line from (1) either inside the window code itself, or when I write my own window that inherits from this pygame window.

[Working on: Windows10, python3.6, pygame 1.9.6, moderngl 5.6.1]

How can I create a window that is both displaying graphics and has a layer of costume objects? (text, buttons, etc.)

EDIT: Perhaps a clarification for my motivation is needed: I would like to have one layer of graphics in the background, that in the example above is some manipulation of an input image. Then, in the foreground, I would like to have some costume PyGame objects like geometric shapes, buttons, etc.

Answer 1

When mixing moderngl and pygame you probably want moderngl to do all the redering to the screen. Things you render with pygame should be rendered to a separate surface. This surface is then converted to a moderngl.Texture so we can draw it to the screen with OpenGL.

There is even an offical example in the moderngl-window project: https://github.com/moderngl/moderngl-window/blob/master/examples/advanced/pygame2.py

If you have questions about the pygame window initialization you can look at the pygame window backend : https://github.com/moderngl/moderngl-window/blob/5cf117ca9c0ef8cbde772d51041a039fc611b6c7/moderngl_window/context/pygame2/window.py#L28-L65

I created a new example that renders a pygame surface to the screen. It's alpha blended on top of the current background. Anything you render in pygame needs to be rendered to this surface and it's uploaded to graphics memory every frame.

The new example is located here: https://github.com/moderngl/moderngl-window/blob/master/examples/advanced/pygame2_simple.py

import math
from pathlib import Path
import pygame
import moderngl
import moderngl_window
from moderngl_window import geometry


class Pygame(moderngl_window.WindowConfig):
    """
    Example using pygame with moderngl.
    Needs to run with ``--window pygame2`` option.
    """
    title = "Pygame"
    window_size = 1280, 720
    resource_dir = (Path(__file__) / '../../resources').absolute()

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        if self.wnd.name != 'pygame2':
            raise RuntimeError('This example only works with --window pygame2 option')

        self.pg_res = (320, 180)
        # Create a 24bit (rgba) offscreen surface pygame can render to
        self.pg_screen = pygame.Surface(self.pg_res, flags=pygame.SRCALPHA)
        # 24 bit (rgba) moderngl texture
        self.pg_texture = self.ctx.texture(self.pg_res, 4)
        self.pg_texture.filter = moderngl.NEAREST, moderngl.NEAREST

        self.texture_program = self.load_program('programs/texture.glsl')
        self.quad_fs = geometry.quad_fs()

    def render(self, time, frametime):
        self.render_pygame(time)

        self.ctx.clear(
            (math.sin(time) + 1.0) / 2,
            (math.sin(time + 2) + 1.0) / 2,
            (math.sin(time + 3) + 1.0) / 2,
        )
    
        self.ctx.enable(moderngl.BLEND)
        self.pg_texture.use()
        self.quad_fs.render(self.texture_program)
        self.ctx.disable(moderngl.BLEND)

    def render_pygame(self, time):
        """Render to offscreen surface and copy result into moderngl texture"""
        self.pg_screen.fill((0, 0, 0, 0))  # Make sure we clear with alpha 0!
        N = 8
        for i in range(N):
            time_offset = 6.28 / N * i
            pygame.draw.circle(
                self.pg_screen,
                ((i * 50) % 255, (i * 100) % 255, (i * 20) % 255),
                (
                    math.sin(time + time_offset) * 55 + self.pg_res[0] // 2,
                    math.cos(time + time_offset) * 55 + self.pg_res[1] // 2),
                math.sin(time) * 4 + 15,
            )

        # Get the buffer view of the Surface's pixels
        # and write this data into the texture
        texture_data = self.pg_screen.get_view('1')
        self.pg_texture.write(texture_data)


if __name__ == '__main__':
    Pygame.run()

Answer 2

Greatly based on @Grimmy's answer above, here is the code that integrates modernl-window and PyGame2, it is based on some examples from the moderngl-window repo:

from pathlib import Path
import pygame
import moderngl
import moderngl_window
from moderngl_window import geometry


class Pygame(moderngl_window.WindowConfig):
    title = "Pygame"
    window_size = 1280, 720
    resource_dir = (Path(__file__) / '../../resources').absolute()

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        # Create a 24bit (rgba) offscreen surface pygame can render to
        self.pg_screen = pygame.Surface(self.window_size, flags=pygame.SRCALPHA)
        # 24 bit (rgba) moderngl texture
        self.pg_texture = self.ctx.texture(self.window_size, 4)
        self.pg_texture.filter = moderngl.NEAREST, moderngl.NEAREST

        self.texture_program = self.load_program('programs/texture.glsl')
        self.quad_texture = self.load_texture_2d('textures/python-bg.png')
        self.quad_fs = geometry.quad_fs()

    def render(self, time, frametime):
        self.ctx.clear()

        self.ctx.enable(moderngl.BLEND)

        # Render background graphics
        self.quad_texture.use()
        self.texture_program['texture0'].value = 0
        self.quad_fs.render(self.texture_program)

        # Render foreground objects
        self.pg_texture.use()
        self.render_pygame(time)
        self.quad_fs.render(self.texture_program)

        self.ctx.disable(moderngl.BLEND)

    def render_pygame(self, time):
        """Render to offscreen surface and copy result into moderngl texture"""
        self.pg_screen.fill((0, 0, 0, 0))  # Make sure we clear with alpha 0!
        pygame.draw.rect(self.pg_screen, (200, 200, 200), [100, 100, 100, 100])

        # Get the buffer view of the Surface's pixels
        # and write this data into the texture
        texture_data = self.pg_screen.get_view('1')
        self.pg_texture.write(texture_data)


if __name__ == '__main__':
    moderngl_window.run_window_config(Pygame, args=('--window', 'pygame2'))

• Has been tested on Ubuntu18.04 with Python3.6