Copy framebuffer of 3D View into custom frame buffer

Question

General question: Is there a reliable way to directly access the OpenGL framebuffer of a View3D for copying?

(Please see the end of the post for more detailed questions)

I need a very fast way to copy the pixel data of a SpaceView3D into an OpenGL texture. The blender internal GPUOffscreen.draw_view3d functionality unfortunately takes 10 to 50 ms per call, which is too slow as can be seen in this short example:

import bpy
import bgl
import gpu
import time
WIDTH = 512
HEIGHT =512
offscreen = gpu.types.GPUOffScreen(WIDTH, HEIGHT)
def draw():
    context = bpy.context
    scene = context.scene
view_matrix = scene.camera.matrix_world.inverted()

projection_matrix = scene.camera.calc_matrix_camera(
    context.evaluated_depsgraph_get(), x=WIDTH, y=HEIGHT)

start_time = time.time()

offscreen.draw_view3d(    # this takes 10 to 50 ms (on a Macbook Pro 2016)
    scene,
    context.view_layer,
    context.space_data,
    context.region,
    view_matrix,
    projection_matrix)

print("draw_view3d call took ", round((time.time() - start_time) * 1000, 2), " ms")



bpy.types.SpaceView3D.draw_handler_add(draw, (), 'WINDOW', 'POST_PIXEL')

What I achieved so far

The following code accesses the currently bound OpenGL draw buffer (Update note: see first comment below) and reads its pixel to a bgl.Buffer object:

import bpy
import bgl
Draw function which copies data from the 3D View
def draw(self, context):
if self.modal_redraw == True:

    # get currently bound framebuffer
    bgl.glGetIntegerv(bgl.GL_DRAW_FRAMEBUFFER_BINDING, self.framebuffer)
    #print("Framebuffer: ", framebuffer)

    # get information on current viewport 
    bgl.glGetIntegerv(bgl.GL_VIEWPORT, self.viewport_info)
    #print("Viewport info: ", viewport_info)

    self.width = self.viewport_info[2]
    self.height = self.viewport_info[3]

    # Write copied data to image
    ######################################################
    # resize image obect to fit the current 3D View size
    self.framebuffer_image.scale(self.width, self.height)
    self.pixelBuffer = bgl.Buffer(bgl.GL_FLOAT, self.width * self.height * 4)

    # obtain pixels from the framebuffer
    bgl.glBindBuffer(bgl.GL_FRAMEBUFFER, self.framebuffer[0]) # POST EDIT 08th Aug 2020: This functions fails for some reason, so it doesn't play a role - but why ... ?
    bgl.glReadPixels(0, 0, self.width, self.height, bgl.GL_RGBA, bgl.GL_FLOAT, self.pixelBuffer)

    # write all pixels into the blender image
    self.framebuffer_image.pixels.foreach_set(self.pixelBuffer)

    # reset draw variable:
    # This is here to prevent excessive redrawing
    self.modal_redraw = False



Modal operator for controlled redrawing of the image object
NOTE: This code is only for a more conveniant testing of the draw function
If you want to stop the test, press 'ESC'
class ModalFramebufferCopy(bpy.types.Operator):
    bl_idname = "view3d.modal_framebuffer_copy"
    bl_label = "Draw 3D View Framebuffer"
def __init__(self):
    print("Start example code")

    # init variables
    self.width = 32
    self.height = 32
    self.modal_redraw = False
    self.image_name = "framebuffer_copy"
    self.framebuffer = bgl.Buffer(bgl.GL_INT, 1)
    self.viewport_info = bgl.Buffer(bgl.GL_INT, 4)
    self.pixelBuffer = bgl.Buffer(bgl.GL_INT, self.width * self.height * 4)

    # create or update image object to which the framebuffer
    # data will be copied
    if not self.image_name in bpy.data.images:
        self.framebuffer_image = bpy.data.images.new(self.image_name , 32, 32)
    else:
        self.framebuffer_image = bpy.data.images[self.image_name ]


# 
def __del__(self):
    print("End example code")


# modal operator for controlled redraw of the image
def modal(self, context, event):
    # stop the execution of this example code if 'ESC' is pressed
    if event.type in {'ESC'}:
        bpy.types.SpaceView3D.draw_handler_remove(self._handle_3d, 'WINDOW')
        print("Removing draw handler")
        return {'CANCELLED'}

    else:

        # set draw variable to update:
        # This is here to prevent excessive redrawing
        self.modal_redraw = True

    return {'PASS_THROUGH'}
    #return {'RUNNING_MODAL'}


def invoke(self, context, event):
    print("Invoking modal operator")

    # Add the region OpenGL drawing callback
    # draw in view space with 'POST_VIEW' and 'PRE_VIEW'
    self._handle_3d = bpy.types.SpaceView3D.draw_handler_add(draw, (self, context), 'WINDOW', 'PRE_VIEW') # this draws the viewport objects alone (without grid)
    #bpy.types.SpaceView3D.draw_handler_add(draw, (self, context), 'WINDOW', 'POST_VIEW') # this draws the grid alone (without objects)

    context.window_manager.modal_handler_add(self)
    return {'RUNNING_MODAL'}



def register():
    bpy.utils.register_class(ModalFramebufferCopy)
def unregister():
    bpy.utils.unregister_class(ModalFramebufferCopy)
if name == "main":
    register()
# Invoke modal operator for the example code
bpy.ops.view3d.modal_framebuffer_copy('INVOKE_DEFAULT')

This basically works, but neither the 'PRE_VIEW' nor the 'POST_VIEW' flag of the draw_handler_add() give the complete result. The following screenshot shows the copied image (upper left area) and the viewport (lower left area) it was copied from via the code above.

For clarification

In the end, I don't want to copy the data into a bpy.types.image object. Instead, I want to copy several SpaceView3D frame buffers into one texture and draw it into another framebuffer for applying another shader. The way my code above approaches this is only, because the blender internal approach GPUOffscreen.draw_view3d is too slow.

My open questions

1. Is there an alternative to GPUOffscreen.draw_view3d that is faster (i.e., < 1 ms)?
2. Is there a more reliable (maybe blender internal) way to access the 3D View framebuffer?
3. How can I access the final framebuffer drawn and used by blender (the one where all controls, etc. are implemented)?
4. The framebuffer I currently access seems to be the object pixel data before any "Color management" settings are applied. Therefore my copied image is much darker than the original 3D View (see attached image). If nobody has an answer to question 3, maybe someone can hint me at how to transform the copied framebuffer data to the correct color space?

Answer 1

There is currently no alternative to GPUOffscreen.draw_view3d. For security reasons the internal frame buffer isn't accessible via python. Scripts/Add-ons could change the internal behavior without the user knowing what happens. When using stereo views there are multiple framebuffers to be aware off. Currently if you want access to the framebuffer you need to make changes directly in the draw manager.

The framebuffer of the 3d viewport has no color management applied. Only during drawing to the screen the color management is applied. External render engine can use the bind_display_space_shader to use the color management shader.

Answer 2

Due to BGL deprecation (https://developer.blender.org/T80730), it's safer to use the new pyGPU API whenever possible. I slightly adapted the code provided by @reg.cs and tested it on blender 3.0. It seems that there is no obvious color space issue, if the image format is set to FLOAT.

import bpy
import gpu
Draw function which copies data from the 3D View
def draw(self, context):
if self.modal_redraw == True:

    # get currently bound framebuffer
    self.framebuffer = gpu.state.active_framebuffer_get()

    # get information on current viewport 
    self.viewport_info = gpu.state.viewport_get()
    self.width = self.viewport_info[2]
    self.height = self.viewport_info[3]

    # Write copied data to image
    ######################################################
    # resize image obect to fit the current 3D View size
    self.framebuffer_image.scale(self.width, self.height)

    # obtain pixels from the framebuffer
    self.pixelBuffer = self.framebuffer.read_color(0, 0, self.width, self.height, 4, 0, 'FLOAT')

    # write all pixels into the blender image
    self.pixelBuffer.dimensions = self.width * self.height * 4
    self.framebuffer_image.pixels.foreach_set(self.pixelBuffer)

    # reset draw variable:
    # This is here to prevent excessive redrawing
    self.modal_redraw = False



Modal operator for controlled redrawing of the image object
NOTE: This code is only for a more conveniant testing of the draw function
If you want to stop the test, press 'ESC'
class ModalFramebufferCopy(bpy.types.Operator):
    bl_idname = "view3d.modal_framebuffer_copy"
    bl_label = "Draw 3D View Framebuffer"
def __init__(self):
    print("Start example code")

    # init variables
    self.width = 32
    self.height = 32
    self.modal_redraw = False
    self.image_name = "color_buffer_copy"
    self.framebuffer = None
    self.viewport_info = None
    self.pixelBuffer = None

    # create or update image object to which the framebuffer
    # data will be copied
    if not self.image_name in bpy.data.images:
        self.framebuffer_image = bpy.data.images.new(self.image_name , 32, 32, float_buffer=True)
    else:
        self.framebuffer_image = bpy.data.images[self.image_name ]


# 
def __del__(self):
    print("End example code")


# modal operator for controlled redraw of the image
def modal(self, context, event):
    # stop the execution of this example code if 'ESC' is pressed
    if event.type in {'ESC'}:
        bpy.types.SpaceView3D.draw_handler_remove(self._handle_3d, 'WINDOW')
        print("Removing draw handler")
        return {'CANCELLED'}

    else:

        # set draw variable to update:
        # This is here to prevent excessive redrawing
        self.modal_redraw = True

    return {'PASS_THROUGH'}
    #return {'RUNNING_MODAL'}


def invoke(self, context, event):
    print("Invoking modal operator")

    # Add the region OpenGL drawing callback
    # draw in view space with 'POST_VIEW' and 'PRE_VIEW'
    self._handle_3d = bpy.types.SpaceView3D.draw_handler_add(draw, (self, context), 'WINDOW', 'PRE_VIEW') # this draws the viewport objects alone (without grid)
    # self._handle_3d = bpy.types.SpaceView3D.draw_handler_add(draw, (self, context), 'WINDOW', 'POST_VIEW') # this draws the grid alone (without objects)

    context.window_manager.modal_handler_add(self)
    return {'RUNNING_MODAL'}



def register():
    bpy.utils.register_class(ModalFramebufferCopy)
def unregister():
    bpy.utils.unregister_class(ModalFramebufferCopy)
if name == "main":
    register()
# Invoke modal operator for the example code
bpy.ops.view3d.modal_framebuffer_copy('INVOKE_DEFAULT')

Here is the result.