Pybullet Segmentation Mask not showing anything

[jttaufa]

I'm having a problem where the custom camera I'm capturing frames from is not rendering the segmentation mask. The segmentation mask shows when I use TinyRenderer instead of OpenGL, but the quality of the RGB image is pretty poor (such as the floor plane not displaying).

As seen in the screenshot, there should be two objects in the segmentation mask.
Here is some of my camera code. I'm running in GUI mode with PYBULLET_EGL defined.

class Camera:
  class Params:
    pixelWidth = 240
    pixelHeight = 240
    fov = 70
    aspect = pixelWidth/pixelHeight
    nearplane = 0.01
    farplane = 100

    def get(self):
      return self.fov, self.aspect, self.nearplane, self.farplane

  def __init__(self, pb_client, link):
    self.pb_client = pb_client
    self.link = link
    self.params = Camera.Params()  # Use default params

    self.projection_matrix = self.pb_client.computeProjectionMatrixFOV(*self.params.get())
    # Init view matrix
    self.view_matrix = None
    self._calc_view_matrix()  # = None

  def reset(self):
    self.projection_matrix = self.pb_client.computeProjectionMatrixFOV(*self.params.get())
    self._calc_view_matrix()

  def _calc_view_matrix(self):
    com_p, com_o = self.pb_client.getLinkState(
        self.link.bodies[self.link.bodyIndex],
        self.link.bodyPartIndex,
        computeForwardKinematics=True
        )[:2]
    com_p = np.array(com_p)

    rot_matrix = self.pb_client.getMatrixFromQuaternion(com_o)
    rot_matrix = np.array(rot_matrix).reshape(3, 3)
    # Initial vectors
    init_camera_vector = (1, 0, 0)  # z-axis
    init_up_vector = (0, 0, 1)  # y-axis
    # Rotated vectors
    camera_vector = rot_matrix.dot(init_camera_vector)
    up_vector = rot_matrix.dot(init_up_vector)
    self.view_matrix = self.pb_client.computeViewMatrix(cameraEyePosition=com_p,
                                                        cameraTargetPosition=com_p + 0.01 * camera_vector,
                                                        cameraUpVector=up_vector)

  def get_image(self, n_max_frames=1, accumulate=False):
    assert(n_max_frames >= 1)
    self._calc_view_matrix()
    img = self.pb_client.getCameraImage(height=self.params.pixelHeight,
                                        width=self.params.pixelWidth,
                                        viewMatrix=self.view_matrix,
                                        projectionMatrix=self.projection_matrix,
                                        renderer=pybullet.ER_BULLET_HARDWARE_OPENGL,
                                        flags=pybullet.ER_SEGMENTATION_MASK_OBJECT_AND_LINKINDEX
                                        )
    return (img[0], img[1]), np.array(img[2]), np.array(img[3]), np.array(img[4])

[erwincoumans]

I don't have much time looking into this, but you may want to provide a fully working simplified pybullet example, rather than a small snippet. Also try without EGL and using a default camera view/projection matrix. Thanks!