Get surface point under mouse using depth buffer


#1

APRIL FOOL!!
BYE.

Maybe not :stuck_out_tongue: .

Hi everyone,

this code snippet will allow you to get the 3D location of the point under the mouse cursor, on any object in your scene, whether it’s a GeoMipTerrain, a loaded or procedurally generated model, or anything that is rendered into the depth buffer (so it should even work on lines and points).

But the best thing of all is: it’s fast, regardless of the complexity of your scene :slight_smile: !
This could make it a very good alternative to traditional ray intersection tests (using CollisionRay with visible geometry), especially with very dense meshes.

from panda3d.core import loadPrcFileData
loadPrcFileData("", "sync-video #f")
loadPrcFileData("", "depth-bits 24")
loadPrcFileData("", "frame-rate-meter-milliseconds true")

from panda3d.core import *
from direct.showbase.ShowBase import ShowBase
from direct.gui.OnscreenText import OnscreenText


def create_cube():

    vertex_format = GeomVertexFormat.get_v3n3()
    vertex_data = GeomVertexData("cube_data", vertex_format, Geom.UH_static)
    tris = GeomTriangles(Geom.UH_static)

    pos_writer = GeomVertexWriter(vertex_data, "vertex")
    normal_writer = GeomVertexWriter(vertex_data, "normal")

    vert_count = 0

    for direction in (-1, 1):

        for i in range(3):

            normal = VBase3()
            normal[i] = direction

            for a, b in ((-1., -1.), (-1., 1.), (1., 1.), (1., -1.)):

              pos = VBase3()
              pos[i] = direction
              pos[(i + direction) % 3] = a
              pos[(i + direction * 2) % 3] = b

              pos_writer.add_data3f(pos)
              normal_writer.add_data3f(normal)

            vert_count += 4

            tris.add_vertices(vert_count - 2, vert_count - 3, vert_count - 4)
            tris.add_vertices(vert_count - 4, vert_count - 1, vert_count - 2)

    geom = Geom(vertex_data)
    geom.add_primitive(tris)
    node = GeomNode("cube_geom_node")
    node.add_geom(geom)

    return node


class DepthCamera(object):

    def __init__(self, showbase):

        self._world = showbase.render

        # Normally, moving the main camera causes the calculation of the
        # surface point distance/depth to lag behind and yield incorrect
        # results.
        # Having the depth buffer rendered by a secondary GraphicsEngine
        # can fix this problem.

        pipe = showbase.win.get_pipe()
        ge = GraphicsEngine(pipe)
        self._gfx_engine = ge
        self._depth_tex = Texture("depth_texture")
        self._depth_tex.set_format(Texture.F_depth_component24)
        fbp = FrameBufferProperties()
        fbp.set_depth_bits(24)
        winprops = WindowProperties.get_default()
        output = ge.make_output(pipe, "depth_buffer", 1, fbp, winprops,
                                GraphicsPipe.BF_refuse_window)
        # create a depth buffer
        buffer = output.make_texture_buffer("buffer", 1, 1,
                                            self._depth_tex,
                                            to_ram=True, fbp=fbp)
        assert buffer

        ge.render_frame()
        self._tex_peeker = self._depth_tex.peek()

        self._origin = showbase.make_camera(buffer)
        node = self._origin.node()
        self._mask = BitMask32.bit(21)
        node.set_camera_mask(self._mask)
        self._lens = lens = node.get_lens()
        # only a very small part of the scene should be rendered to the
        # single-pixel depth texture
        lens.set_fov(.1)

        self._mouse_watcher = showbase.mouseWatcherNode
        self._main_cam_lens = showbase.camLens

        self._pos = Point3()
        # viewing direction in world space
        self._dir_vec = Vec3()

        showbase.taskMgr.add(self.__update, "update_depth_cam", sort=40)

    def __update(self, task):

        if not self._mouse_watcher.has_mouse():
          return task.cont

        screen_pos = self._mouse_watcher.get_mouse()
        far_point = Point3()
        self._main_cam_lens.extrude(screen_pos, Point3(), far_point)
        origin = self._origin
        origin.look_at(far_point)
        self._pos = origin.get_pos(self._world)
        self._dir_vec = self._world.get_relative_vector(origin, Vec3.forward())

        return task.cont

    def get_surface_point(self):
        """ Return nearest surface point in viewing direction of camera """

        self._gfx_engine.render_frame()
        pixel = VBase4()
        self._tex_peeker.lookup(pixel, .5, .5)
        point = Point3()
        self._lens.extrude_depth(Point3(0., 0., pixel[0]), point)
        depth = point[1] * .5

        if depth > 100.:
            # at large distances, the depth precision decreases, so a second
            # depth render is needed;
            # the depth camera is moved forward by the previously retrieved
            # depth (minus a small value, e.g. 1.1, just a little bigger than
            # the near clipping distance of the lens), placing it very close
            # to the surface, which will make the secondary depth value very
            # accurate; the sum of both depth values will therefore yield a
            # very precise distance of the nearest surface point in the viewing
            # direction of the depth camera
            offset = 1.1 # ensures that the camera does not clip the surface
            self._origin.set_y(self._origin, depth - offset)
            self._gfx_engine.render_frame()
            pixel = VBase4()
            self._tex_peeker.lookup(pixel, .5, .5)
            point = Point3()
            self._lens.extrude_depth(Point3(0., 0., pixel[0]), point)
            depth2 = point[1] * .5
            depth += depth2 - offset
            self._origin.set_pos(0., 0., 0.)

        return self._pos + self._dir_vec * depth

    def get_mask(self):

        return self._mask


class MyApp(ShowBase):

    def __init__(self):

        ShowBase.__init__(self)

        self.set_frame_rate_meter(True)
        # the following variables are used to implement a custom frame rate meter
        self._clock1 = ClockObject()
        self._clock2 = ClockObject()
        self._fps_label = OnscreenText("0", pos=(0.01, -0.05), scale=.07,
                                       align=TextNode.A_left, parent=self.a2dTopLeft)

        p_light = PointLight("point_light")
        p_light.set_color((1., 1., 1., 1.))
        self._light = self.camera.attach_new_node(p_light)
        self._light.set_pos(5., -10., 7.)
        self.render.set_light(self._light)

        self._depth_cam = DepthCamera(self)

        self._marker = self.render.attach_new_node(create_cube())
        self._marker.set_color(1., 0., 0.)
        # the marker itself should not be visible to the depth camera
        self._marker.hide(self._depth_cam.get_mask())

        self._test_obj = self.render.attach_new_node(create_cube())
        self._test_obj.set_scale(100.)
        self._test_obj.set_pos(0., 500., 0.)
        self._test_obj.set_hpr(30., 30., 0.)
        self._test_obj.set_color(0., 1., 0.)

        # allow copying the marker at its current location for debugging
        self.accept("enter", self.__copy_marker)

        # place the marker on the surface under the mouse cursor;
        # this task should run after the transformation of the depth camera
        # has been updated, but before the primary GraphicsEngine renders
        # the next frame (in a task with sort=50)
        self.taskMgr.add(self.__set_marker_pos, "set_marker_pos", sort=45)

        # display a custom frame rate meter
        self.taskMgr.add(self.__show_frame_rate, "show_fps")

    def __copy_marker(self):

        self._marker.copy_to(self.render)

    def __set_marker_pos(self, task):

        if not self.mouseWatcherNode.has_mouse():
            return task.cont

        self._marker.set_pos(self._depth_cam.get_surface_point())

        return task.cont

    def __show_frame_rate(self, task):

        t = self._clock1.get_real_time()
        self._clock1.reset()

        # update frame rate every two seconds
        if self._clock2.get_real_time() >= 2.:
            self._fps_label.node().set_text("{:.2f} ms".format(t * 1000.))
            self._clock2.reset()

        return task.cont


app = MyApp()
app.run()

While you move the mouse over the surface of an object, you will see a red “marker” (cube) at the cursor position; press to copy it onto the surface, so you can navigate to it afterwards to see if it is indeed exactly on the surface.

Enjoy :slight_smile: !

Note:
when I limit the framerate to the monitor refresh rate, I see that I actually get a much higher framerate than expected (e.g. 180 instead of 60) - probably because I added a second GraphicsEngine?

EDIT: updated the code to make use of TexturePeeker instead of PNMImage to improve performance. You will need a recent Panda 1.10 dev. version for this to work.


Mouse Position
#2

I know I will sound like “the bad guy” here, but this is really slow. :confused:

Nonetheless did you figure this out, which i think is amazing!

Can you explain how it works? :smiley:


#3

Nah don’t worry about it, feedback is always appreciated :slight_smile: .
But I do find it surprising that it’s so slow for you. On my laptop with integrated graphics, the code sample I posted runs at 60 fps with vsync enabled (i.e. the highest possible frame rate). How much are you getting with that code?

As I noted in the 1st post, the fps given by the built-in frame rate meter doesn’t seem very reliable in this case, so, with vsync disabled, I computed it manually using ClockObject.get_real_time(), which gives me around 1000 fps (1 ms) when the mouse is outside the render window, and around 350 fps (2.85 ms) when the mouse is over it (i.e. when the extra depth rendering and surface point calculations happen). So it adds about 1.85 milliseconds, which doesn’t seem too bad to me.

EDIT:
Updated the code in the first post with a custom frame rate meter at the top left of the window (it’s all in milliseconds now, which is more useful).

Sure. A second camera is created that renders a very small portion of the scene (that the mouse cursor is currently over) to a single-pixel depth buffer, to make it as fast as possible (as you already noticed :stuck_out_tongue: ). The distance to whatever lies directly in front of the camera can then be retrieved from the depth buffer, using Lens.extrude_depth(). One problem was, that the precision of the depth value decreases the further away the rendered object is from the camera. The trick I used to solve this, is to move the second camera to this imprecise position and let it render a second time, yielding a much more accurate result.
The reason I use a second GraphicsEngine is that there was a visible lag between the newly calculated surface point coordinates (visualized with the small red cube) and the mouse cursor, due to the order in which things were rendered and computed. This was the only way I found to fix this.


#4

I missed a lot how to use the, API: set_frame_rate_meter?
It does not work for me.


#5

How do you mean? Doesn’t it show up or does it give unexpected results?

Perhaps the code for the custom frame rate meter I added to my previous post can help you :slight_smile: .


#6

Because of this API, I can not start example

D:\Panda3D\Code\GetSurface>D:\Panda3D-1.9.4-x64\python\python.exe main.py
Known pipe types:
  wglGraphicsPipe
(all display modules loaded.)
Traceback (most recent call last):
  File "main.py", line 200, in <module>
    app = MyApp()
  File "main.py", line 156, in __init__
    self.set_frame_rate_meter(True)
AttributeError: MyApp instance has no attribute 'set_frame_rate_meter'

#7

Ah, I see - you got bitten by a snake_case :stuck_out_tongue: . It would seem that the 1.9 branch of Panda still uses camelCase for some/most/all of the ShowBase method names. Since I use the 1.10 development versions of Panda, I now prefer the snake_case method names, so if you don’t want to upgrade your Panda version, changing set_frame_rate_meter to setFrameRateMeter (there may be others that need changing) should work. Sorry for the trouble :slight_smile: .