Facing issues in implementation of a project which is based around point cloud completion

coder001 · May 17, 2021, 11:03am

I have been facing some issues in running a python script. There are 4 files that are included in this project.
First one that is main file that calls the functions from other scripts and runs it, the other three contains all the functions for the main file.
This is the code of the main file which will call main.py

print "============================================================"
print "Starting Program..."

import cv2
import numpy as np
# import tensorflow as tf
# from tensorflow.python.framework import dtypes

import datetime
import random

import file_util
from pc_completion_util import *
from depth_map_generator import *

image_size = 256 # downscaled to half later
scaling = 0.0001
images_per_obj = 256 # = 256 train, 64 test, 8 val # <---------------- change this for datasets

dataset_folder = "../Data/raw_data/caesar-norm-wsx-fitted-meshes_foot_mesh_BAM/"
all_objects = file_util.read_all_lines("nl_train_legs.txt") # <------- change this for datasets
all_objects.sort()
print "Total Num Obj", len(all_objects)

png_folder = dataset_folder + "../../caesar-norm-wsx_pngs/"
file_util.make_directory(png_folder)
dataset_name = "pc_completion_"
prefix = "train" # <-------------------------------------------------- change this for datasets
train_images_folder = png_folder + dataset_name + prefix + "_pngs/"
file_util.make_directory(train_images_folder)

im_generator = Depth_Generator(image_size, scaling)

start_time = datetime.datetime.now()

for i in range(len(all_objects)):
    name = [all_objects[i] + ".bam"]

    if not file_util.file_exists(dataset_folder + name[0]):
        continue

    print i, name, datetime.datetime.now() - start_time

    obj_folder = train_images_folder + "/" + name[0][:-4] + "/"
    file_util.make_directory(obj_folder)

    im_generator.load_all_objects(name, dataset_folder)

    for j in range(images_per_obj):
        d_images, d_images_jet, extrins = im_generator.generate_images(name, dataset_type = "depth_xray")

        cv2.imwrite(obj_folder + prefix + "_in_im_" + str(j) + ".png", reproject_to128(d_images[0][0]))
        save_array(obj_folder + prefix + "_in_extrin_" + str(j) + ".txt", extrinsics_to_net_rot(extrins[0][0]))
        cv2.imwrite(obj_folder + prefix + "_gt_im_" + str(j) + ".png", reproject_dm_extrinsic_to128(d_images[1][0], extrins[0][0], extrins[1][0]))

    im_generator.unload_all_objects()

print "Total Runtime:", datetime.datetime.now() - start_time

print "============================================================"
print "Ending Program..."

The other 3 files are file_util , pc_completion_file_util and depth map generator
file util :

import sys,os
import shutil

def write_to_file(filename, text):
    with open(filename, 'w') as f:
        f.write(str(text))

def append_to_file(filename, text):
    with open(filename, 'a') as f:
        f.write(str(text) + '\n')

def append_line_to_csv(filename, text):
    with open(filename, 'a') as f:
        f.write(str(text) + ',\n')

def get_foldernames(directory = "."):
    full_dir = os.listdir(directory)
    folders = [name for name in full_dir if os.path.isdir(directory + "/" + name)]
    folders.sort()
    return folders

def get_in_dir(directory = "."):
    full_dir = os.listdir(directory)
    # full_dir.sort()
    return full_dir

def folder_exists(directory = "."):
    return os.path.isdir(directory)

def file_exists(filename):
    return os.path.isfile(filename) 

def make_directory(directory = "new_folder"):
    if not folder_exists(directory):
        os.makedirs(directory)
    return folder_exists(directory)

def get_objs_in_folder(directory = "."):
    folders = get_foldernames(directory)    
    objs = [directory + "/" + name + "/models/model_normalized.obj" for name in folders]

    objs.sort()
    return objs

def read_all_lines(filename):
    with open(filename, 'r') as file:
        lines = file.readlines()

    for l in range(len(lines)):
        lines[l] = lines[l].replace("\n", "")

    return lines

def get_next_availiable_filename(filename):
    name, extension = os.path.splitext(filename)
    filename_num = 1
    while filename_num < 1000:
        try_name = name + str(filename_num) + extension
        if not file_exists(try_name):
            break
        else:
            filename_num += 1

    return try_name

def copy_as_backup(filename):
    name, extension = os.path.splitext(filename)
    backup_name = name + "_backup" + extension
    shutil.copyfile(filename, get_next_availiable_filename(backup_name))

def copy_folder(foldername, destination_folder):
    shutil.copytree(foldername, destination_folder)

def move_folder(foldername, destination_folder):
    shutil.move(foldername, destination_folder)

pc completion util

import tensorflow as tf
import numpy as np
import random
import cv2
from math import pi, sin, cos

import file_util


K1 = np.array([[192.0*2, 0.0,   64.0*2],
               [0.0,   192.0*2, 64.0*2], 
               [0.0 ,  0.0,   1.0]]);
K2 = np.array([[192.0, 0.0,   64.0],
              [0.0,   192.0, 64.0], 
              [0.0 ,  0.0,   1.0]]);
x128, y128 = np.meshgrid(np.arange(128), np.arange(128))
x128 = np.reshape(x128,128*128)
y128 = np.reshape(y128,128*128)
im_p = np.array([x128, y128, np.ones(128*128)])

x256, y256 = np.meshgrid(np.arange(256), np.arange(256))
x256 = np.reshape(x256,256*256)
y256 = np.reshape(y256,256*256)
im_p1 = np.array([x256, y256, np.ones(256*256)])

############################################ Utility Functions
def rollback_losses_csv(file_name, current_itter):
    file_util.copy_as_backup(file_name)
    
    lines = file_util.read_all_lines(file_name)
    new_file = ""
    for i in range(len(lines)):
        line_split = lines[i].split(',')
        if int(line_split[0]) < current_itter:
            new_file += lines[i] + "\n"
    file_util.write_to_file(file_name, new_file)

def read_png16(file_name):
    CV_LOAD_IMAGE_ANYDEPTH = 2 # apparently leftover from cv before cv2
    int16_d_image = cv2.imread(file_name,  CV_LOAD_IMAGE_ANYDEPTH)
    return int16_d_image

def depthmap_16bitTofloat(int16_d_image):
    float_d_image = np.array(int16_d_image, dtype=np.float32)
    float_d_image = (float_d_image  / (2**16 - 1)  - 0.5) * 1.5
    im_shape = float_d_image.shape
    return float_d_image.reshape((im_shape[0], im_shape[1], 1))

def depthmap_floatTo16bit(float_d_image):
    int16_d_image = (float_d_image / 1.5 + 0.5) * (2**16 - 1) 
    int16_d_image[np.where(int16_d_image > 60000)] = 2**16-1
    int16_d_image[np.where(int16_d_image < 0.0)] = 0
    return int16_d_image.astype(np.uint16)

def save_iteration_images(folder, id, in_im, out_gt, out_im, rot, dataset_type = "Train"):
    if dataset_type == "Train" or dataset_type == "Test":
        # only sometimes save the input
        u_in_im = depthmap_floatTo16bit(in_im)
        u_out_gt = depthmap_floatTo16bit(out_gt)
        cv2.imwrite(folder + "in_im_" + str(id) + ".png", u_in_im)
        cv2.imwrite(folder + "gt_im_" + str(id) + ".png", u_out_gt)
        save_array(folder + "out_extrin_" + str(id) + ".txt", rot)
    # always save the output
    u_out_im = depthmap_floatTo16bit(out_im)
    tile3_im = make_3tile_image(in_im, out_gt, out_im)
    cv2.imwrite(folder + "out_im_" + str(id) + ".png", u_out_im)
    cv2.imwrite(folder + "tile3_im_" + str(id) + ".png", tile3_im)

def save_array(file_name, array_list):
    string = ""
    for i in range(len(array_list)):
        string += str(array_list[i])
        if i < len(array_list) - 1:
            string += "\n"
    file_util.write_to_file(file_name, string)

def extrinsics_to_net_rot(extrinsics):
    rot = np.zeros(6) # azimuth, elevation, roll, radius, pitch, heading
    # normalize to 0 - 1
    rot[0] = extrinsics[0] / 360.0
    rot[1] = (extrinsics[1] + 90) / 180.0
    rot[2] = extrinsics[2] / 90.0
    rot[3] = extrinsics[3] / 3.0
    rot[4] = (extrinsics[4] + 5) / 10.0
    rot[5] = (extrinsics[5] + 5) / 10.0
    return rot

def net_rot_to_extrinsics(rot):
    extrinsics = np.zeros(6) # azimuth, elevation, roll, radius, pitch, heading
    # normalize to 0 - 1
    extrinsics[0] = rot[0] * 360.0
    extrinsics[1] = rot[1] * 180.0 -90
    extrinsics[2] = rot[2] * 90.0
    extrinsics[3] = rot[3] * 3.0
    extrinsics[4] = rot[4] * 10.0 - 5
    extrinsics[5] = rot[5] * 10.0 - 5
    return extrinsics

def extrinsics_to_otherside(extrinsics):
    extrin_other = np.zeros(6) # azimuth, elevation, roll, radius, pitch, heading

    extrin_other[0] = (extrinsics[0] + 180) % 360
    extrin_other[1] = extrinsics[1] * -1
    extrin_other[2] = extrinsics[2]
    extrin_other[3] = extrinsics[3]
    extrin_other[4] = extrinsics[4] # not actually doing anything
    extrin_other[5] = extrinsics[5] # not actually doing anything
    return extrin_other

def dm2points(dm):
    dm_flat = np.reshape(dm,128*128)

    p_world = np.dot(np.linalg.inv(K2), im_p) * dm_flat;
    return p_world

def surfnorm_fast(X,Y,Z):
    nz = np.empty((128,128))

    dzdy = np.zeros((128,128))
    dzdy[1:-1,1:-1] = (Z[2:,1:-1] - Z[:-2,1:-1:]) / (Y[2:,1:-1] - Y[:-2,1:-1])
    dzdx = np.zeros((128,128))
    dzdx[1:-1,1:-1] = (Z[1:-1,2:] - Z[1:-1,:-2]) / (X[1:-1,2:] - X[1:-1,:-2])

    nz = 1/np.sqrt((-dzdy)**2 + (-dzdx)**2 + 1)

    angles = np.arccos(nz)
    return nz, angles

def noise_std(z_val, theta_val):
    z_component = 0.0012 + 0.0019 * (z_val - 0.4)**2
    theta_component = (0.0001/np.sqrt(z_val)) * (theta_val**2/(np.pi/2 - theta_val)**2)

    std = z_component + theta_component
    return std

def add_noise(dm_m, Z, angles):
    angles[angles >= np.pi/2] = np.pi/2 - 0.0001 # prevent divide by 0
    std = noise_std(Z, angles)
    dm_m_noise = dm_m + np.random.normal(np.zeros((128,128)), std)

    mask = np.logical_and(Z < 2, angles < 87.5 * np.pi/180)
    dm_m_noise = dm_m_noise * mask + np.logical_not(mask) * np.max(dm_m)

    return dm_m_noise

def add_kinect_v1_noise(dm): # operates on the uint16
    scale = 0.003
    dm_m = dm.astype(float) * 0.0001 / scale / 1000

    p_world = dm2points(dm_m)
    X = np.reshape(p_world[0,:], (128,128))
    Y = np.reshape(p_world[1,:], (128,128))
    Z = np.reshape(p_world[2,:], (128,128))

    if np.min(Z) <= 0.01:
        print "ZERO?"

    nz, angles = surfnorm_fast(X,Y,Z)

    dm_m_noise = add_noise(dm_m, Z, angles)

    return (dm_m_noise * 1000 * scale / 0.0001).astype(np.uint16)

def reproject_to128(dm):
    dm_new = cv2.resize(dm, (128,128), interpolation = cv2.INTER_NEAREST)
    return dm_new.astype(np.uint16)

def reproject_dm_extrinsic_to128(dm, extrin_in, extrin_out):
    z_scale = 0.0001

    dm_new = np.zeros((128,128))

    RT1 = camera_extrinsic_to_RT(extrin_in)
    RT2 = camera_extrinsic_to_RT(extrin_out)

    im1_flat = np.reshape(dm,256*256)

    p_world_1 = np.dot(np.linalg.inv(K1), im_p1) * im1_flat * z_scale;

    p_world_1_5 = np.transpose(np.transpose(np.dot(RT1[:,:3], p_world_1)) + RT1[:,3])

    p_world_2 = np.dot(np.transpose(RT2[:,:3]), np.transpose(np.transpose(p_world_1_5) - RT2[:,3]))

    im_p2 = np.dot(K2, p_world_2)
    im_p2_z = im_p2[2,:]
    im_p2_x = im_p2[0,:] / im_p2_z
    im_p2_y = im_p2[1,:] / im_p2_z

    indx_x = np.logical_and(im_p2_x >= 0, im_p2_x < 128)
    indx_y = np.logical_and(im_p2_y >= 0, im_p2_y < 128)
    indx = np.logical_and(indx_x, indx_y)

    im_p2_z = (im_p2_z[indx] / z_scale).astype(int)
    im_p2_x = (im_p2_x[indx]).astype(int)
    im_p2_y = (im_p2_y[indx]).astype(int)

    for i in range(im_p2_z.shape[0]):
        x_2 = im_p2_x[i]
        y_2 = im_p2_y[i]
        z_2 = im_p2_z[i]
        if y_2 < 128 and y_2 >= 0 and x_2 < 128 and x_2 >= 0:
            if z_2 > dm_new[y_2,x_2]:
                dm_new[y_2,x_2] = z_2

    dm_new[np.where(dm_new <= 0)] = 2**16-1
    return dm_new.astype(np.uint16)

def depthmap_more_visible(depth_image, scaling = 0.0001):
    # make it scaled better, in a uint8 image
    
    # radius was 2 | farthest point seen is at 3, closest at 1
    farthest = (3/scaling/ (2**16 - 1)  - 0.5) * 1.5
    closest = (1/scaling/ (2**16 - 1)  - 0.5) * 1.5
    dist = farthest - closest

    d_im = depth_image - closest
    d_im[np.where(d_im >= dist)] = dist

    d_im = d_im / np.max(d_im) * 255

    return d_im.astype(np.uint8)

def diff_more_visible(diff_image, scaling = 0.0001):
    farthest = np.max(diff_image)
    closest = np.min(diff_image)

    d_im = diff_image - closest
    d_im = d_im / np.max(d_im) * 255

    return d_im.astype(np.uint8)

def depth_2jet(depth_image, scaling = 0.0001):
    # requires input to be a uint8 image
    return cv2.applyColorMap(depthmap_more_visible(depth_image,scaling), cv2.COLORMAP_JET)

def diff_2jet(diff_image, scaling = 0.0001):
    return cv2.applyColorMap(diff_more_visible(diff_image,scaling), cv2.COLORMAP_JET)

def read_dataset_objects(dataset_filename):
    # reads a file containing all the object numbers/keys 
    # returns a list of the object numbers (but as list of strings)
    csv_lines = file_util.read_all_lines(dataset_filename)
    num_files = len(csv_lines)
    print "Number of files:", num_files

    objects = []
    for line in csv_lines:
        line_split = line.split(',')
        object_num = line_split[0]
        # object_name = line_split[1]

        if object_num not in objects:
            objects.append(object_num)

    return objects

def make_3tile_image(in_im, out_gt, out_im):
    show_image = np.concatenate([depth_2jet(in_im), 
                                    depth_2jet(out_gt),
                                    depth_2jet(out_im)], axis=1)
    return show_image

def make_5tile_image(in_im, out_gt, out_im):
    in_diff = (in_im - out_gt)
    out_diff = (out_im - out_gt)
    show_image = np.concatenate([depth_2jet(in_im), diff_2jet(in_diff),
                                    depth_2jet(out_gt),
                                    depth_2jet(out_im), diff_2jet(out_diff)]
                                    , axis=1)
    return show_image

def show_3_images(in_im, out_gt, out_im, wait_time = 0, name = "Depthmaps"):
    show_image = make_3tile_image(in_im, out_gt, out_im)
    cv2.imshow(name, show_image)
    key = cv2.waitKey(wait_time)

def save_3_images(folder, id, in_im, out_gt, out_im, new_scaling):
    u_in_im = undo_preprocess(in_im, new_scaling)
    u_out_gt = undo_preprocess(out_gt, new_scaling)
    u_out_im = undo_preprocess(out_im, new_scaling)
    tile3_im = make_3tile_image(in_im, out_gt, out_im)
    cv2.imwrite(folder + "in_im_" + str(id) + ".png", u_in_im)
    cv2.imwrite(folder + "gt_im_" + str(id) + ".png", u_out_gt)
    cv2.imwrite(folder + "out_im_" + str(id) + ".png", u_out_im)
    cv2.imwrite(folder + "tile3_im_" + str(id) + ".png", tile3_im)

def undo_preprocess(image, scaling):
    original = (image / 1.5 + 0.5) * (2**16-1) / scaling
    original[np.where(original >= 40000.0)] = 2**16-1
    return original.astype(np.uint16)

def camera_extrinsic_to_RT(camera_extrinsics):
    azimuth_rad =           camera_extrinsics[0] * pi / 180
    elevation_rad =         camera_extrinsics[1] * pi / 180
    roll_rad =              camera_extrinsics[2] * pi / 180
    radius =                camera_extrinsics[3]
    pitch_offset_rad =      camera_extrinsics[4] * pi / 180
    heading_offset_rad =    camera_extrinsics[5] * pi / 180
    y = radius * sin(elevation_rad)
    x = radius * sin(azimuth_rad) * cos(elevation_rad)
    z = radius * cos(azimuth_rad) * cos(elevation_rad)

    x_rot = np.zeros((3,3))
    x_rot[0,0] = 1
    x_rot[1,1] = x_rot[2,2] = cos(-(elevation_rad - pitch_offset_rad))
    x_rot[2,1] = sin(-(elevation_rad - pitch_offset_rad))
    x_rot[1,2] = -1 * x_rot[2,1]

    y_rot = np.zeros((3,3))
    y_rot[1,1] = 1
    y_rot[0,0] = y_rot[2,2] = cos(azimuth_rad - heading_offset_rad)
    y_rot[0,2] = sin(azimuth_rad - heading_offset_rad)
    y_rot[2,0] = -1 * y_rot[0,2]

    z_rot = np.zeros((3,3))
    z_rot[2,2] = 1
    z_rot[0,0] = z_rot[1,1] = cos(roll_rad)
    z_rot[1,0] = sin(roll_rad)
    z_rot[0,1] = -1 * z_rot[1,0]

    R = np.dot(y_rot, np.dot(x_rot, z_rot))
    T = -1 * np.array([[x], [y], [z]])

    RT = np.concatenate((R,T), axis=1)

    return RT

def weight_var_nl(shape):
    initial = tf.truncated_normal(shape, stddev=0.1)
    return tf.Variable(initial)

def bias_var_nl(shape):
    initial = tf.constant(0.0, shape=shape)
    return tf.Variable(initial)

def conv2d_nl(input_x, in_depth, out_depth, kernel_size, stride, layer_name):
    with tf.variable_scope(layer_name):
        W = weight_var_nl([kernel_size, kernel_size, in_depth, out_depth])
        b = bias_var_nl([out_depth])
        conv = tf.nn.conv2d(input_x, W, strides=[1, stride, stride, 1], 
                            padding='SAME') + b
        print conv
    return conv

def fc_nl(input_x, in_length, out_length, layer_name):
    with tf.variable_scope(layer_name):
        W = weight_var_nl([in_length, out_length])
        b = bias_var_nl([out_length])
        fc = tf.matmul(input_x, W) + b
        print fc
    return fc

def deconv2d_nl(input_x, in_depth, out_shape, kernel_size, stride, layer_name):
    with tf.variable_scope(layer_name):
        out_depth = out_shape[3]
        W = weight_var_nl([kernel_size, kernel_size, out_depth, in_depth])
        b = bias_var_nl([out_depth])
        dconv = tf.nn.conv2d_transpose(input_x, W, strides=[1, stride, stride, 1], 
                                        padding='SAME', output_shape=out_shape) + b
        print dconv
    return dconv

depth map generator

import cv2
import numpy as np
from math import pi, sin, cos
import random
import sys,os
import time, datetime

import file_util
from pc_completion_util import *

from panda3d.core import *
from direct.showbase.ShowBase import ShowBase
from direct.task import Task

class Depth_Generator(ShowBase):
    def __init__(self, image_size = 128, scaling = 0.0001):

        self.im_size = image_size

        # loadPrcFileData("", "window-type none")
        loadPrcFileData("", "window-type offscreen")
        # loadPrcFileData("", "Generate Depthmaps")
        # loadPrcFileData("", "fullscreen 0") # Set to 1 for fullscreen
        # loadPrcFileData("", "win-origin 100 100")
        loadPrcFileData("", "win-size " + str(image_size) + " " + str(image_size))

        ShowBase.__init__(self)
        # base.setFrameRateMeter(True)

        self.scaling = scaling # ever value 1 (in png) represents scaling distance in game engine
        self.azimuth = 0
        self.elevation = 20
        self.roll = 0
        self.radius = 2
        self.pitch_offset = 0
        self.heading_offset = 0

        self.obj_scale = 0.003
        a_range = 20
        self.az_opt = np.concatenate([np.arange(90-a_range,90+a_range+.1), np.arange(270-a_range,270+a_range+.1)],axis =0)
        self.el_opt = np.arange(-a_range, a_range +0.1)
        self.radius_options = np.arange(1.9, 2.1 +0.05, 0.05)
        self.r_options = np.arange(-5, 5 +.1, 1)

        self.max_16bit = 2**16-1
        self.near_plane = 1.0
        self.far_plane = 10.0
        self.focal_length = 1.5
        self.cam.node().getLens().setNear(self.near_plane)
        self.cam.node().getLens().setFar(self.far_plane)
        self.cam.node().getLens().setFocalLength(self.focal_length)
        self.K = self.get_calib_mat()

        self.object_names_to_index = {}
        self.all_objects = []
        self.object = None
        self.current_object_index = -1

        self.scene = NodePath("Scene")
        self.scene.reparentTo(self.render)
        self.scene.setScale(1, 1, 1)
        self.scene.setTwoSided(True) # important to avoid alot of holes
        self.scene.setPos(0, 0, 0)
        self.scene.setHpr(0, 0, 0)

        self.taskMgr.add(self.everyFrameTask, "EveryFrameTask")
        self.frame = 0
        self.roll_direction = self.elevation_direction = self.radius_direction = 1
        self.rotation_count = 0

    ## source: https://gist.github.com/alexlee-gk/b28fb962c9b2da586d1591bac8888f1f
    ## {
        self.dr = self.camNode.getDisplayRegion(0)

        winprops = WindowProperties.size(self.win.getXSize(), self.win.getYSize())
        fbprops = FrameBufferProperties()
        fbprops.setDepthBits(32)
        self.depthBuffer = self.graphicsEngine.makeOutput(
            self.pipe, "depth buffer", -2,
            fbprops, winprops,
            GraphicsPipe.BFRefuseWindow,
            self.win.getGsg(), self.win)
        self.depthTex = Texture()
        self.depthTex.setFormat(Texture.FDepthComponent)
        self.depthBuffer.addRenderTexture(self.depthTex,
            GraphicsOutput.RTMCopyRam, GraphicsOutput.RTPDepth)
        lens = self.cam.node().getLens()

        self.depthCam = self.makeCamera(self.depthBuffer,
            lens=lens,
            scene=render)
        self.depthCam.reparentTo(self.cam)

    def get_camera_image(self):
        tex = self.dr.getScreenshot()
        data = tex.getRamImage()
        image = np.frombuffer(data.get_data(), np.uint8)
        image.shape = (tex.getYSize(), tex.getXSize(), tex.getNumComponents())
        image = np.flipud(image)
        return image

    def get_camera_depth_image(self):
        # values between 0.0 and 1.0.
        data = self.depthTex.getRamImage()
        depth_image = np.frombuffer(data.get_data(), np.float32)
        # depth_image.shape = (self.depthTex.getYSize(), self.depthTex.getXSize(), self.depthTex.getNumComponents())
        depth_image.shape = (self.depthTex.getYSize(), self.depthTex.getXSize(), 1)
        depth_image = np.flipud(depth_image)
        return depth_image
    ## }

    def close(self):
        self.unload_all_objects()
        base.destroy()

    def linearize_depth(self, d_image):
        # depth function: d = b1/(z + b2) + b3
        b1 = - self.far_plane / (self.far_plane - self.near_plane)
        b2 = 0.0
        b3 = self.far_plane / (self.far_plane - self.near_plane)

        l_d_image = np.round((b1 / (d_image[:,:,0] - b3) - b2) / self.scaling)

        l_d_image[l_d_image >= np.max([self.max_16bit, self.far_plane/self.scaling])] = self.max_16bit
        l_d_image[l_d_image <= 0] = self.max_16bit

        return l_d_image.astype(np.uint16)

    def get_depthmap(self):
        return self.linearize_depth(self.get_camera_depth_image())

    def unload_all_objects(self):
        self.remove_viewable_object()
        for obj in self.all_objects:
            self.loader.unloadModel(obj)
            obj.removeNode()
            obj = None
        self.all_objects = []

        TransformState.garbageCollect()
        RenderState.garbageCollect()

    def load_all_objects(self, object_names, objects_path = ""):
        self.object_names_to_index = {}
        index = 0
        full_object_paths = []
        for name in object_names:
            full_path = objects_path + name 
            full_object_paths.append(full_path)

            self.object_names_to_index[name] = index
            index += 1

        self.all_objects = self.loader.loadModel(full_object_paths)
        # print "Loaded", len(self.all_objects), "objects"

    def pick_viewable_object(self, object_index):
        # print self.all_objects, object_index
        self.object = self.all_objects[object_index]
        self.object.reparentTo(self.scene)
        self.object.setScale(self.obj_scale, self.obj_scale, self.obj_scale)
        self.object.setPos(0,0,0)# - feet
        self.object.setHpr(0,0,0)# - feet
        self.current_object_index = object_index

    def remove_viewable_object(self):
        if self.object is not None:
            self.object.detachNode()
            self.object = None
            self.current_object_index = -1

    def switch_viewable_object(self, object_index):
        self.remove_viewable_object()
        self.pick_viewable_object(object_index)

    def switch_viewable_object_by_name(self, object_name):
        self.switch_viewable_object(self.object_names_to_index[object_name])

    def camera_angles_to_coord(self):
        az_rad = self.azimuth * pi / 180 # input is in degrees
        el_rad = self.elevation * pi / 180

        z = self.radius * sin(el_rad)
        x = self.radius * sin(az_rad) * cos(el_rad)
        y = self.radius * cos(az_rad) * cos(el_rad)

        return x, y, z # z = forward, x = sideways, y = updown

    def camera_angles_to_direction(self):
        h = -(self.azimuth + 180) + self.heading_offset
        p = -(self.elevation) + self.pitch_offset
        r = self.roll
        return h, p, r # yaw/heading, pitch, roll

    def get_calib_mat(self):
        film_size = self.cam.node().getLens().getFilmSize()

        K = np.zeros((3,3))
        K[0,0] = K[1,1] = self.focal_length * self.im_size * film_size[0]
        K[0,2] = self.im_size/2.0
        K[1,2] = self.im_size/2.0
        K[2,2] = 1

        return K

    def update_camera_position(self):
        cam_x, cam_y, cam_z = self.camera_angles_to_coord()
        cam_h, cam_p, cam_r = self.camera_angles_to_direction()
        self.camera.setPos(cam_x, cam_y, cam_z)
        self.camera.setHpr(cam_h, cam_p, cam_r)
        # print self.camera.getPos()

    def depth_2jet(self, depth_image):
        farthest = 4.5/self.scaling
        closest = 1/self.scaling
        dist = farthest - closest

        d_im = depth_image - closest
        d_im[np.where(d_im > dist)] = dist

        d_im = d_im / (np.max(d_im) + 1e-8) * 255
        d_im = d_im.astype(np.uint8)
        return cv2.applyColorMap(d_im, cv2.COLORMAP_JET)

    def generate_image(self, object_name, camera_extrinsics):
        self.switch_viewable_object_by_name(object_name)
        self.azimuth = camera_extrinsics[0]
        self.elevation = camera_extrinsics[1]
        self.roll = camera_extrinsics[2]
        self.radius = camera_extrinsics[3]
        self.pitch_offset = camera_extrinsics[4]
        self.heading_offset = camera_extrinsics[5]

        # print camera_extrinsics

        self.update_camera_position()
        base.graphicsEngine.renderFrame()
        d_image = self.get_depthmap()

        TransformState.garbageCollect()
        RenderState.garbageCollect()

        return d_image.reshape((self.im_size, self.im_size, 1))

    def generate_random_image(self, object_name, image_type = "input", in_extrin = None):
        if image_type == "input":
            az = random.choice(self.az_opt)
            el = random.choice(self.el_opt)
            rad = random.choice(self.radius_options)
            roll = random.choice(self.r_options)
            pit_o = head_o = 0.0
        else:
            az, el, roll, rad, pit_o, head_o = extrinsics_to_otherside(in_extrin)
        # print image_type, [az, el, roll, rad, pit_o, head_o]

        camera_extrinsics = [az, el, roll, rad, pit_o, head_o]
        d_image = self.generate_image(object_name, camera_extrinsics)
        return d_image, camera_extrinsics

    def generate_images(self, object_names):
        d_images = [[], []]
        d_images_jet = [[], []]
        extrins = [[], []]
        for name in object_names:
            d_image, extrin = self.generate_random_image(name, "input")
            d_images[0].append(d_image)
            d_images_jet[0].append(self.depth_2jet(d_image))
            extrins[0].append(extrin)

            d_image, extrin = self.generate_random_image(name, "output", extrin)
            d_images[1].append(d_image)
            d_images_jet[1].append(self.depth_2jet(d_image))
            extrins[1].append(extrin)

        return d_images, d_images_jet, extrins

    def generate_all_outputs_images(self, object_name):
        d_images = []
        extrins = []
        for rad in self.radius_options_out:
            for el in self.elevation_options_out:
                for az in self.azimuth_options_out:
                    extrin = [az, el, 0, rad, 0, 0]
                    d_image = self.generate_image(object_name, extrin)
                    d_images.append(d_image)
                    extrins.append(extrin)

        return d_images, extrins

    def everyFrameTask(self, task):
        if self.frame > 2:
            self.update_camera_position()

            d_image = self.get_depthmap()
            # print np.min(d_image), np.max(d_image)
            cv2.imshow("Images", self.depth_2jet(d_image))
            key = cv2.waitKey(100)

            #### setup for next view
            self.azimuth += 5
            if self.azimuth >= 360:
                self.azimuth = 0
                self.rotation_count += 1
                print "Full Rotation", self.rotation_count

            if self.rotation_count >= 2:
                next_object = self.current_object_index + 1
                if next_object >= len(self.all_objects):
                    next_object = 0
                self.switch_viewable_object(next_object)
                self.rotation_count = 0
            self.elevation += 0.2 * self.elevation_direction * 5
            if self.elevation > 30 or self.elevation < -30:
                self.elevation_direction *= -1
            self.roll += 0.1 * self.roll_direction * 5
            if self.roll > 10 or self.roll < -10:
                self.roll_direction *= -1
            self.radius += 0.01 * self.radius_direction * 5
            if self.radius > 4 or self.radius < 1.5:
                self.radius_direction *= -1            

        self.frame += 1
        # time.sleep(1)
        return Task.cont

The libraries used in this project is panda3d 1.9.3, python 2.7 , opencv 3.4 and tensorflow 0.12.Yes it’s an old project(2017)
So all my libraries all correctly installed in ubuntu and i have all the .bam files
when i try to run the main.py it runs perfectly creating a folder naming pngs but the folder is empty.
Main.py is not calling the functions from the file depth map generator and the code skips it. Everything is correct code runs but still no pngs are created, it doesn’t call im_generator_load_all_objects() from depth map generator.
All the addresses of drives are correct.
Can someone help me out with this issue, it would be a great help. Thank you.

Thaumaturge · May 17, 2021, 11:38am

That’s a fair bit of code that you’ve presented us with. o_o

However, based on what you describe and what I see in main.py, I do have two thoughts:

Have you checked the length of “all_objects”?
- If it’s zero–i.e. if “all_objects” is empty–then the code in the the for-loop presumably won’t run.

and

have you checked that the file exists that’s referenced by “dataset_folder + name[0]”? If it doesn’t, then the “continue” command will run, causing the for-loop to skip the rest of its code and move to the next iteration, I believe.

coder001 · May 17, 2021, 2:48pm

Thanks dude. There was a mistake in the address, now the loop is working.
Now it is showing generate images() got an unexpected keyword argument ‘dataset_type’ but i have installed all the correct libraries of opencv and numpy. Is there some problem in camera extrinsics?

Thaumaturge · May 17, 2021, 2:57pm

Looking at the definition for “generate_images”, you haven’t specified a keyword-argument named “dataset_type”.

To clarify: When you pass an argument into a method using the form “someName = someValue” (e.g. " dataset_type = "depth_xray"", in your case) , you’re specifying that you’re assigning the given value (e.g. “depth_xray”) to the method’s keyword-argument of the given name (e.g. “dataset_type”). If the method doesn’t specify a keyword-argument by the name in question, then you’ll get the error that you’ve been seeing, I believe.

Right now your “generate_images” method only takes two arguments: the automatically-provided “self” argument, and a non-keyword-argument named “object_names”.

coder001 · May 17, 2021, 3:07pm

But what about “camera_extrinsics” ? It is also using this argument and how should i rectify it?
And isn’t self an instance of a class not an argument?

Thaumaturge · May 17, 2021, 3:15pm

It looks like the method “generate_image” (singular) uses “camera_extrinsics”, but “generate_images” (plural) doesn’t. From what you posted above, you were having trouble with the method “generate_images” (plural).

Unless this is a separate issue?

It’s both–in more or less the same way that the word “canary” is both a noun and a type of bird.

That is to say that an “argument” (also called a “parameter”) is a thing passed into a method. What that thing is, specifically–a number, an instance of a class, a string–is another matter entirely.

So, one way to put it is that the “self” argument generally holds a reference to an instance of a class.

coder001 · May 17, 2021, 6:41pm

Yes so the code from the source was wrong only, It should be generate images not generate images which takes only 2 arguments but even after correcting this error it shows
"generate_image got an unexpected keyword argument ‘dataset type’.

Thaumaturge · May 17, 2021, 6:48pm

Well, neither method has a keyword argument (as described above), so indeed it won’t work with either.

Why is it that you expect that the method will take that extra argument?

coder001 · May 17, 2021, 6:51pm

So what do you suggest I do? This project is given by my prof. to implement and I am learning with the project. I really appreciate your help and thanks for taking out time for answering my queries

Thaumaturge · May 17, 2021, 6:57pm

It’s okay.

I’m not familiar with your code, so it’s hard for me to be confident of what’s supposed to be happening at any one point, and thus what would be a good or bad idea at that point.

I suppose that my two questions are these:

Why are you trying to pass that object into the method?

and

Why does the method not take that argument?

(Was one or both of these things specified by your professor?)

coder001 · May 17, 2021, 7:01pm

No none of these things were specified even I am trying to understand why they have passed “depth_xray” through this and why it is not taking the argument. I read on the internet that it can be a issue of some libraries. Do you think so too?

Thaumaturge · May 17, 2021, 7:04pm

Er, no.

It’s literally that you are trying to passing into the method a parameter that it’s never heard of.

If the thing that you’re passing in is to be used by the method, then I suggest changing the method so that it knows about the parameter. If the thing isn’t used, then I suggest just not passing it in.

Let me illustrate by example. Consider the following:

class Cat():
    def mew(self):
        print ("Meow!")

cat = Cat()
cat.mew()

The above should work.

Now:

class Cat():
    def mew(self):
        print ("Meow!")

cat = Cat()
cat.mew(noise = "Purr")

The above, however, shouldn’t work–the “mew” method isn’t expecting a parameter named “noise”!

Similarly:

class Cat():
    def mew(self, noise):
        print (noise)

cat = Cat()
cat.mew()

Here the method is expecting a parameter named “noise”–but we aren’t giving it one, so this, too, shouldn’t work.

However:

class Cat():
    def mew(self, noise):
        print (noise)

cat = Cat()
cat.mew(noise = "Meow")

The above should work:

The method is expecting a parameter
We’re passing in a parameter
And we’re passing in a parameter by the correct name.

coder001 · May 17, 2021, 7:21pm

So if I change camera_extrinsics to dataset_type it should work as then it will know that dataset_type is passed through it. I tried doing it but another error comes
self.switch_viewable_object(self.object_names_to_index[object_name])
unhashable type: ‘list’

Thaumaturge · May 17, 2021, 7:23pm

That indicates that “object_names_to_index” is a dictionary or some such thing that requires a key to access, and that it’s expecting the value of “object_name” to provide such a key. Unfortunately, “object_name” is currently a list, and lists can’t be used as keys.

I’d suggest tracing through the logic of your program and finding out why “object_name” is ending up as a list, and not as, well, a name of some sort.

coder001 · May 17, 2021, 7:30pm

I think there was some error in the code I change the generate image to generate images and passed only the name and it created the images in png format just fine. Thanks for helping me man, really appreciate it

Thaumaturge · May 17, 2021, 7:30pm

Not a problem–I’m glad to have been of service! I’m glad that you got it working, and well done on finding that problem!

coder001 · May 17, 2021, 7:38pm

Couldn’t have done it without you