FisheyeLens for distortion

I am trying to emulate a real camera in my panda scene, so I need to add some distortion. I think the FisheyeLens is what I want, but I cannot seem to get it to render anything. I figured I would start simple, and just replace the base camera with a fisheye like so:

lens = FisheyeLens()
self.camNode.setLens(lens)

My scene loads, but I just get a gray screen. I have tried positioning the camera to make sure that I have models in view, but no luck. What do I need to do to get some distortion into my camera?

Thanks for the reference. I read most of the issues related to fisheye, but I am not clear on how to use it. I do not want to create a cubemap, I just want to alter the base camera to have some distortion. Is the only way to use the Fisheye lens through a cubemap?

The link was to a post with this code, sorry I wasn’t verbose.

# Make the camera into a fisheye camera using the NonlinearImager.
import direct.directbase.DirectStart
from pandac.PandaModules import *

# scene
smiley = loader.loadModel('smiley')
smiley.reparentTo(render)
room = loader.loadModel('environment')
room.setPos(0, 0, -10)
room.reparentTo(render)

# A node to attach all the screens together.
screens = NodePath('dark_room')

# A node parented to the original camera node to hold all the new cube
# face cameras.
cubeCam = base.cam.attachNewNode('cubeCam')

# Define the forward vector for the cube.  We have this up to the
# upper right, so we can get away with using only the front, right,
# and up faces if we want.
cubeForward = (1, 1, 1)
#cubeForward = (0, 1, 0)


class CubeFace:
    def __init__(self, name, view, up, res):
        self.name = name

        # A camera, for viewing the world under render.
        self.camNode = Camera('cam' + self.name)
        self.camNode.setScene(render)
        self.cam = cubeCam.attachNewNode(self.camNode)

        # A projector, for projecting the generated image of the world
        # onto our screen.
        self.projNode = LensNode('proj' + self.name)
        self.proj = screens.attachNewNode(self.projNode)

        # A perspective lens, for both of the above.  The same lens is
        # used both to film the world and to project it onto the
        # screen.
        self.lens = PerspectiveLens()
        self.lens.setFov(92)
        self.lens.setNear(0.1)
        self.lens.setFar(10000)
        self.lens.setViewVector(view[0], view[1], view[2],
                                up[0], up[1], up[2])

        self.camNode.setLens(self.lens)
        self.projNode.setLens(self.lens)

        # Now the projection screen itself, which is tied to the
        # projector.
        self.psNode = ProjectionScreen('ps' + self.name)
        self.ps = self.proj.attachNewNode(self.psNode)
        self.psNode.setProjector(self.proj)

        # Generate a flat, rectilinear mesh to project the image onto.
        self.psNode.regenerateScreen(self.proj, "screen", res[0], res[1], 10, 0.97)

# Define the six faces.
cubeFaces = [
    CubeFace('Right', (1, 0, 0), (0, 0, 1), (10, 40)),
    CubeFace('Back', (0, -1, 0), (0, 0, 1), (40, 40)),
    CubeFace('Left', (-1, 0, 0), (0, 0, 1), (10, 40)),
    CubeFace('Front', (0, 1, 0), (0, 0, 1), (20, 20)),
    CubeFace('Up', (0, 0, 1), (0, -1, 0), (40, 10)),
    CubeFace('Down', (0, 0, -1), (0, 1, 0), (40, 10)),
    ]

# Indices into the above.
cri = 0
cbi = 1
cli = 2
cfi = 3
cui = 4
cdi = 5


# Rotate the cube to the forward axis.
cubeCam.lookAt(cubeForward[0], cubeForward[1], cubeForward[2])
m = Mat4()
m.invertFrom(cubeCam.getMat())
cubeCam.setMat(m)

# Get the base display region.
dr = base.camNode.getDisplayRegion(0)

# Now make a fisheye lens to view the whole thing.
fcamNode = Camera('fcam')
fcam = screens.attachNewNode(fcamNode)
flens = FisheyeLens()
flens.setViewVector(cubeForward[0], cubeForward[1], cubeForward[2],  0, 0, 1)
flens.setFov(180)
flens.setFilmSize(dr.getPixelWidth() / 2, dr.getPixelHeight())
fcamNode.setLens(flens)

# And a cylindrical lens for fun.
ccamNode = Camera('ccam')
ccam = screens.attachNewNode(ccamNode)
clens = CylindricalLens()
clens.setViewVector(cubeForward[0], cubeForward[1], cubeForward[2],  0, 0, 1)
clens.setFov(120)
clens.setFilmSize(dr.getPixelWidth() / 2, dr.getPixelHeight())
ccamNode.setLens(clens)

# Turn off the base display region and replace it with two
# side-by-side regions.
dr.setActive(0)
window = dr.getWindow()
dr1 = window.makeDisplayRegion(0, 0.5, 0, 1)
dr1.setSort(dr.getSort())
dr2 = window.makeDisplayRegion(0.5, 1, 0, 1)
dr2.setSort(dr.getSort())

# Set the fisheye lens on the left, and the cylindrical lens on the right.
dr1.setCamera(fcam)
dr2.setCamera(ccam)

# And create the NonlinearImager to do all the fancy stuff.
nli = NonlinearImager()
nli.addViewer(dr1)
nli.addViewer(dr2)

for face in cubeFaces:
    i = nli.addScreen(face.ps, face.name)
    nli.setSourceCamera(i, face.cam)
    nli.setTextureSize(i, 256, 256)

def hideAll():
    for i in range(6):
        nli.setScreenActive(i, 0)

def showAll():
    for i in range(6):
        nli.setScreenActive(i, 1)

hideAll() #?
nli.setScreenActive(cfi, 1)
nli.setScreenActive(cri, 1)
nli.setScreenActive(cui, 1)

run()

I cleaned the code.

# Make the camera into a fisheye camera using the NonlinearImager.
from direct.showbase.ShowBase import ShowBase
from panda3d.core import *
from panda3d.fx import FisheyeLens, NonlinearImager, ProjectionScreen

base = ShowBase()

room = loader.loadModel('environment')
room.reparentTo(render)

screens = NodePath('dark_room')
cubeCam = base.cam.attachNewNode('cubeCam')
cubeForward = (1, 1, 1)

class CubeFace:
    def __init__(self, name, view, up, res):

        self.name = name

        # A camera, for viewing the world under render.
        self.camNode = Camera('cam' + self.name)
        self.camNode.setScene(render)
        self.cam = cubeCam.attachNewNode(self.camNode)

        # A projector, for projecting the generated image of the world
        # onto our screen.
        self.projNode = LensNode('proj' + self.name)
        self.proj = screens.attachNewNode(self.projNode)

        self.lens = PerspectiveLens()
        self.lens.setFov(92)
        self.lens.setNear(0.1)
        self.lens.setFar(10000)
        self.lens.setViewVector(view[0], view[1], view[2], up[0], up[1], up[2])

        self.camNode.setLens(self.lens)
        self.projNode.setLens(self.lens)

        # Now the projection screen itself, which is tied to the
        # projector.
        self.psNode = ProjectionScreen('ps' + self.name)
        self.ps = self.proj.attachNewNode(self.psNode)
        self.psNode.setProjector(self.proj)

        # Generate a flat, rectilinear mesh to project the image onto.
        self.psNode.regenerateScreen(self.proj, "screen", res[0], res[1], 10, 0.97)

cubeCam.lookAt(cubeForward)
m = Mat4()
m.invertFrom(cubeCam.getMat())
cubeCam.setMat(m)

# Get the base display region.
dr = base.camNode.getDisplayRegion(0)

# Now make a fisheye lens to view the whole thing.
fcamNode = Camera('fcam')
fcam = screens.attachNewNode(fcamNode)
flens = FisheyeLens()
flens.setViewVector(cubeForward[0], cubeForward[1], cubeForward[2], 0, 0, 1)
flens.setFov(180)
flens.setFilmSize(dr.getPixelWidth(), dr.getPixelHeight())
fcamNode.setLens(flens)

# Set the fisheye lens on the left, and the cylindrical lens on the right.
dr.setCamera(fcam)

# And create the NonlinearImager to do all the fancy stuff.
nli = NonlinearImager()
nli.addViewer(dr)

# Define the six faces.
cubeFaces = [
    CubeFace('Right', (1, 0, 0), (0, 0, 1), (10, 40)),
    CubeFace('Back', (0, -1, 0), (0, 0, 1), (40, 40)),
    CubeFace('Left', (-1, 0, 0), (0, 0, 1), (10, 40)),
    CubeFace('Front', (0, 1, 0), (0, 0, 1), (20, 20)),
    CubeFace('Up', (0, 0, 1), (0, -1, 0), (40, 10)),
    CubeFace('Down', (0, 0, -1), (0, 1, 0), (40, 10)),
    ]

for face in cubeFaces:
    i = nli.addScreen(face.ps, face.name)
    nli.setSourceCamera(i, face.cam)
    nli.setTextureSize(i, 256, 256)

base.run()

1 Like

thanks so much for the responses @serega-kkz. Unfortunately, in my system your code produces very different results than for you. I’m using v 1.10.5. I wonder if there’s something different in my config.prc

I tried it on 1.8.1 and it works. Note that the example just doesn’t have a Panda, but I do.

# Make the camera into a fisheye camera using the NonlinearImager.
from direct.showbase.ShowBase import ShowBase
from panda3d.core import *
from panda3d.fx import FisheyeLens, NonlinearImager, ProjectionScreen

class CubeFace:
    def __init__(self, name, view, up, res):

        self.name = name

        self.camNode = Camera('cam' + self.name)
        self.camNode.setScene(render)
        self.cam = cubeCam.attachNewNode(self.camNode)

        self.projNode = LensNode('proj' + self.name)
        self.proj = screens.attachNewNode(self.projNode)

        self.lens = PerspectiveLens()
        self.lens.setFov(92)
        self.lens.setNear(0.1)
        self.lens.setFar(10000)
        self.lens.setViewVector(view[0], view[1], view[2], up[0], up[1], up[2])

        self.camNode.setLens(self.lens)
        self.projNode.setLens(self.lens)

        self.psNode = ProjectionScreen('ps' + self.name)
        self.ps = self.proj.attachNewNode(self.psNode)
        self.psNode.setProjector(self.proj)
        self.psNode.regenerateScreen(self.proj, "screen", res[0], res[1], 10, 0.97)

base = ShowBase()

room = loader.loadModel('environment')
room.reparentTo(render)

room1 = loader.loadModel('models/panda-model')
room1.setScale(0.005, 0.005, 0.005)
room1.reparentTo(render)

screens = NodePath('dark_room')

cubeCam = base.cam.attachNewNode('cubeCam')
cubeCam.lookAt((1, 1, 1))

m = Mat4()
m.invertFrom(cubeCam.getMat())

cubeCam.setMat(m)

# Get the base display region.
dr = base.camNode.getDisplayRegion(0)

# Now make a fisheye lens to view the whole thing.
fcamNode = Camera('fcam')
fcam = screens.attachNewNode(fcamNode)
flens = FisheyeLens()
flens.setViewVector(1, 1, 1, 0, 0, 1)
flens.setFov(180)
flens.setFilmSize(dr.getPixelWidth(), dr.getPixelHeight())
fcamNode.setLens(flens)

# Set the fisheye lens on the left, and the cylindrical lens on the right.
dr.setCamera(fcam)

# And create the NonlinearImager to do all the fancy stuff.
nli = NonlinearImager()
nli.addViewer(dr)

# Define the six faces.
cubeFaces = [
    CubeFace('Right', (1, 0, 0), (0, 0, 1), (10, 40)),
    CubeFace('Back', (0, -1, 0), (0, 0, 1), (40, 40)),
    CubeFace('Left', (-1, 0, 0), (0, 0, 1), (10, 40)),
    CubeFace('Front', (0, 1, 0), (0, 0, 1), (20, 20)),
    CubeFace('Up', (0, 0, 1), (0, -1, 0), (40, 10)),
    CubeFace('Down', (0, 0, -1), (0, 1, 0), (40, 10)),
    ]

for face in cubeFaces:
    i = nli.addScreen(face.ps, face.name)
    nli.setSourceCamera(i, face.cam)
    nli.setTextureSize(i, 1024, 1024)

base.run()

If you do not touch the mouse at start, it looks like this.

1 Like

@juansg I noticed that you have a large resolution? What are the screen sizes in your configuration? if you scale it after opening it, Yes there will be distortion.

@serega-kkz looks like scaling up/down doesn’t make a noticeable difference. But there’s definitely an issue with the ‘up’ cubeface, even though the code seems correct and it clearly works for you…

This is how it looks for me on start (no scaling or moving)

Regarding screen sizes… I guess you mean this?
win-origin -2 -2
win-size 800 600

I’m pasting my full config.prc in case there’s something I’m missing.

thanks so much for taking the time and helping out :slight_smile:

###########################################################
###                                                     ###
### Panda3D Configuration File -  User-Editable Portion ###
###                                                     ###
###########################################################

load-display pandagl
# Software rendering is the fallback so that when we
#   run in the CI - it will still work when targeting
#   XVFB
aux-display p3tinydisplay

notify-output $THIS_PRC_DIR/../panda3d.log
notify-level warning
default-directnotify-level warning

load-file-type p3ptloader
default-model-extension .egg

model-path    $THIS_PRC_DIR/../../venv/lib/python3.5/site-packages/panda3d/models
model-path    $THIS_PRC_DIR/../../venv/lib/python3.5/site-packages/panda3d/
model-path    $THIS_PRC_DIR/../../src/emulator/sim/assets/models
model-path    $THIS_PRC_DIR/../../

want-directtools  #f
want-tk           #f

# Enable/disable performance profiling tool and frame-rate meter

#want-pstats            #f
show-frame-rate-meter  t

# These control the placement and size of the default rendering window.
# A value of -2 for the origin means to center it on the screen,
# while -1 lets the window manager choose the position.

win-origin -2 -2
win-size 800 600

# Uncomment this line if you want to run Panda fullscreen instead of
# in a window.

fullscreen #f

# The framebuffer-hardware flag forces it to use an accelerated driver.
# The framebuffer-software flag forces it to use a software renderer.
# If you don't set either, it will use whatever's available.

framebuffer-hardware #t
framebuffer-software #f

# These set the minimum requirements for the framebuffer.
# A value of 1 means: get as many bits as possible,
# consistent with the other framebuffer requirements.

depth-bits 1
color-bits 1 1 1
alpha-bits 0
stencil-bits 0
multisamples 0

# Enable audio using the OpenAL audio library by default:

audio-library-name p3openal_audio

# Enable the use of the new movietexture class.

use-movietexture #t

# The new version of panda supports hardware vertex animation, but it's not quite ready

hardware-animated-vertices #f

# Enable the model-cache, but only for models, not textures.

#model-cache-dir $HOME/.cache/panda3d
model-cache-textures #f

# This option specifies the default profiles for Cg shaders.
# Setting it to #t makes them arbvp1 and arbfp1, since these
# seem to be most reliable. Setting it to #f makes Panda use
# the latest profile available.

basic-shaders-only #f

copy-texture-inverted #t

audio-active #f
audio-library-name null

I meant that after opening the window, you change its size. If you change it, there will be distortion.

aux-display p3tinydisplay

Why did you specify a spare display software?

It doesn’t matter, though.

after more tests, it looks like the copy-texture-inverted #t setting in the config is the one that messes things up. Commenting it out, fixes my issue

1 Like