opencv image as a Panda Texture - clarification

General Discussion
1

I’m using recipe here: [Small issue with using numpy arrays as textures)
with 1.8.2 and cv2 (2.4.1)
Alas - not working - no image coming through
Also won’t exit and release control back. so some ptr lossage ?

my code looks like this:

import direct.directbase.DirectStart
from direct.showbase.DirectObject import DirectObject
from direct.gui.DirectGui import *
from direct.gui.OnscreenText import OnscreenText
from panda3d.core import *
import sys

import cv2
import cv2.cv as cv
import numpy as np

def get_cv_img(cap):
    success, img = cap.read() # img is a numpy array
    print "success", success
    if success:
        shape = img.shape # (480,640,3)
        tex = Texture("detect")
        tex.setup2dTexture(shape[0], shape[1], Texture.TUnsignedByte, Texture.FRgb)
        p = PTAUchar.emptyArray(0)
        try:
            p.setData(img)
        except AssertionError:
            pass
        tex.setRamImage(CPTAUchar(p))
        return (tex)

class World(DirectObject):
    def __init__(self):
        ###Standard initialization stuff
        #Standard title that's on screen in every tutorial
        self.title = OnscreenText(text="opencv test", style=1,
                                  fg=(1,1,1,1), pos=(0.9,-0.95),
                                  scale = .04 )
        base.setBackgroundColor(0.5,0.5,0.5)      
        # initiialise cv2 camera
        self.cap = cv2.VideoCapture(0) # first camera
        sm = CardMaker('bg')
        test = render2d.attachNewNode(sm.generate(),2)
        img = get_cv_img(self.cap)
        test.setTexture(img)
        #
        self.accept('escape', sys.exit)  

    def turn(self):
        # continually call cap.read() here to update the texture
        pass


###
w = World()
run()
2

OK. so swapping the X,Y from img.shape fixes the “jamming up on exit” problem.
but still no picture…
Any ideas ?

import direct.directbase.DirectStart
from direct.showbase.DirectObject import DirectObject
from direct.gui.DirectGui import *
from direct.gui.OnscreenText import OnscreenText
from panda3d.core import *
import sys

import cv2
import cv2.cv as cv
import numpy as np

def get_cv_img(cap):
    success, img = cap.read() # img is a numpy array
    print "success", success
    if success:
        shape = img.shape # (480,640,3)
        tex = Texture("detect")
        tex.setup2dTexture(shape[1], shape[0], Texture.TUnsignedByte, Texture.FRgb)
        p = PTAUchar.emptyArray(0)
        p.setData(img)
        tex.setRamImage(CPTAUchar(p))
        return (tex)

class World(DirectObject):
    def __init__(self):
        ###Standard initialization stuff
        #Standard title that's on screen in every tutorial
        self.title = OnscreenText(text="opencv test", style=1,
                                  fg=(1,1,1,1), pos=(0.9,-0.95),
                                  scale = .04 )
        base.setBackgroundColor(0.5,0.5,0.5)      
        # initiialise cv2 camera
        self.cap = cv2.VideoCapture(0) # first camera
        for i in range(10): self.cap.read() # ensure sensor has irised up
        sm = CardMaker('bg')
        test = render2d.attachNewNode(sm.generate(),2)
        img = get_cv_img(self.cap)
        test.setTexture(img)
        #
        self.accept('escape', sys.exit)  

    def turn(self):
        # continually call cap.read() here to update the texture
        pass


###
w = World()
run()
3

Thanks to some fourm help I now have two methods that work - for different reasons.
However both versions do not safely exit on sys.exit and jam up the console.
If you know how to fix that - please leave a note.

The critical problem turned out not to be with the methods but the GC releasing the image arrays because I had not referenced them ‘permanently’ enough. I had wondered about that but spent too long looking for other problems… oh well.
Here is the code with two methods for you to choose from.

import direct.directbase.DirectStart
from direct.showbase.DirectObject import DirectObject
from direct.gui.DirectGui import *
from direct.gui.OnscreenText import OnscreenText
from panda3d.core import *
import sys

import cv2
import cv2.cv as cv
#import numpy as np
import ctypes # just for method 2


    
class World(DirectObject):


    def get_cv_img(self):
        """ This method uses PTAUchar and another bit of pointer trickery.
            The setup2dTexture must be for the right image properties.
        """
        success, img = self.cap.read() # img is a numpy array
        print "success", success
        if success:
            shape = img.shape # (480,640,3)
            self.img = cv2.flip(img, 0) # cv2 image is upside down
            self.tex = Texture("detect")
            self.tex.setCompression(Texture.CMOff) # 1 to 1 copying - default, so is unnecessary
            self.tex.setup2dTexture(shape[1], shape[0],
                                    Texture.TUnsignedByte, Texture.FRgb)#FRgba8) # 3,4 channel
            p = PTAUchar.emptyArray(0)
            p.setData(self.img)
            self.tex.setRamImage(CPTAUchar(p)) 

    def get_cv_img(self):
        """ This method uses setRamMipmapPointerFromInt and
             must have a pointer that points to the other image.
            The setup2dTexture must be for the right image properties.
            This variation also does in-place flipping of the image.
        """
        success, self.img = self.cap.read() # img is a numpy array
        print "success", success
        if success:
            shape = self.img.shape # (480,640,3)
            cv2.flip(self.img, 0, self.img) # cv2 image is upside down
            self.tex = Texture("detect")
            self.tex.setCompression(Texture.CMOff) # 1 to 1 copying - default, so is unnecessary
            self.tex.setup2dTexture(shape[1], shape[0],
                                    Texture.TUnsignedByte, Texture.FRgb)#FRgba8) # 3,4 channel
            self.tex.makeRamImage() # weird flicking results if omit this.
            self.tex.setRamMipmapPointerFromInt(self.img.ctypes.data, 0, 640*480*3)
            
    
    def __init__(self):
        #Standard title that's on screen in every tutorial
        self.title = OnscreenText(text="opencv/numpy image test", style=1,
                                  fg=(1,1,1,1), pos=(0.9,-0.95),
                                  scale = .04 )
        base.setBackgroundColor(0.5,0.5,0.5)      
        # initialise cv2 camera
        self.cap = cv2.VideoCapture(1) # second camera
        # do a few reads so auto-iris has opened up and not black image.
        # which can be very confusing...
        for i in range(10): self.cap.read()
        # bit of ui to show image on.
        sm = CardMaker('bg')
        self.test = render2d.attachNewNode(sm.generate(),2)
        self.get_cv_img()
        self.test.setTexture(self.tex)
        #
        self.accept('escape', sys.exit)
        taskMgr.add(self.turn, "turn")

    def turn(self, task):
        # continually call cap.read() here to update the texture
        self.get_cv_img()
        self.test.setTexture(self.tex)
        print "looping"
        return task.cont

###
w = World()
run()
4

Thanks for that code snippet :slight_smile:
It works well for me when I turn it to use the first camera.

Both methods work very well, btw.

5

Yes - I am using the second camera in script above. I should have changed that for ease of reuse by others.

self.cap = cv2.VideoCapture(1)

should be 0 for the main camera.

I use the ps3eye cameras as they have very low latency. Also cv2 works with more than one camera which many frameworks do not. Just a plug for cv2 here. It is so much easier/clearer to use than previous opencv wrappers. Has many good examples also. Highly recommend it.

Solving the console locking problem:

  • don’t use sys.exit directly and release the cv2 capture device - and the console lock problem goes away.

so replace this line in the init method:

self.accept('escape', self.finish)

and add this as well:

def finish(self):
     self.cap.release()
     sys.exit()
6

[color=black]Panda Jam is a game easy to get started and requires patience to keep going. However the lack of interactions made it less a social game.

7

I know it’s been ages since the last post, but the following worked for me (from function 1)

cvim = cv2.imread("../data/20141020_214655.jpg", cv2.CV_LOAD_IMAGE_GRAYSCALE);
w = cvim.shape[1];
h = cvim.shape[0];

cvim = cv2.flip(cvim, 0);
self.im = Texture("cvIm");
self.im.setCompression(Texture.CMOff);
self.im.setup2dTexture(w, h, Texture.TUnsignedByte, Texture.FLuminance);

[b]self.im.setRamImage(cvim);[/b]

self.screen_im = OnscreenImage(parent=self.render2d, image=self.im, scale=(1, 1, 1), pos=(0, 0, 0));
self.cam2d.node().getDisplayRegion(0).setSort(-20);

The only change is that you can take the opencv ndarray and pass it straight into the Texture.setRamImage()

8

Indeed. It should be noted that the ability to pass such an ndarray straight to setRamImage is a 1.9 feature and only available in development builds as of now, as per this blog post:
panda3d.org/blog/buffer-protocol-support/