TypeError when passing HmdMatrix34_t to setOverlayTransformTrackedDeviceRelative

[DASPRiD]

I'm trying to bind an overlay to a user's hand. For this I'm using the following code:

overlay.setOverlayTransformTrackedDeviceRelative(
  handle,
  openvr.c_ulong(leftHandId),
  openvr.HmdMatrix34_t()
)

When trying this though, I'm getting the following error:

TypeError: byref() argument must be a ctypes instance, not 'CArgObject'

I'm not exactly sure what other type might be expected there.

[cmbruns]

I think @matEhickey got overlays working at one point in issue #44. @matEhickey do you have any suggestions?

[sunaurus]

I would love to see a working example of setOverlayTransformTrackedDeviceRelative as well, I get type errors no matter how I try to use it..

[matEhickey]

Hi, sorry for the delay, I just retrieve some code related to this usage, but it's old code, maybe there were breaking changes since..

import os
import time
import math
import openvr

import vrutils

dirname = os.getcwd()
overlaysPath = []
for i in range(1,4):
    path = os.path.join(dirname, "imgTimer/"+str(4-i)+'.png')
    print(path)
    overlaysPath.append(openvr.c_char_p(path.encode('utf-8')))

def switch_generator(ov):
    i = 0
    while(True):
        print("switch : "+str(i))
        overlay.setOverlayFromFile(ov,overlaysPath[i])
        i = i+1 if(i < len(overlaysPath)-1) else 0
        yield

if(__name__=="__main__"):
    try:
        openvr.init(openvr.VRApplication_Overlay)
    except openvr.OpenVRError as e:
        print("Error, HMD probably not connected")
        print(str(e))
        quit()

    overlay=openvr.VROverlay()
    overlayKey = openvr.c_char_p("overlayKey".encode('utf-8'))
    overlayTitle = openvr.c_char_p("overlayTitle".encode('utf-8'))
    res, ov = overlay.createOverlay(overlayKey, overlayTitle)

    switcher = switch_generator(ov)

    overlay.showOverlay(ov)

    poses_t = openvr.TrackedDevicePose_t * openvr.k_unMaxTrackedDeviceCount
    poses = poses_t()
    l, r = None, None
    while(r is None):
        openvr.VRCompositor().waitGetPoses(poses, len(poses), None, 0)
        l,r = vrutils.get_controller_ids()
        print("Right-controller : "+str(r))

    matrixTransformation = openvr.HmdMatrix34_t()
    # identity
    matrixTransformation[0][0] = 0.1
    matrixTransformation[1][1] = 0.1
    matrixTransformation[2][2] = 0.1

    ## Position
    # matrixTransformation[0][3] = 0.1
    matrixTransformation[1][3] = 0.02
    matrixTransformation[2][3] = 0.05


    rotmat = vrutils.rotationXMatrix(math.radians(-90))
    matrixTransformation = vrutils.numpyToMatrix34(vrutils.appliRotationMatrixToAxis(matrixTransformation, rotmat))

    res = openvr.VROverlay().setOverlayTransformTrackedDeviceRelative(ov, r, matrixTransformation)

    # res = openvr.VROverlay().setOverlayTransformAbsolute(ov,openvr.TrackingUniverseStanding, matrixTransformation)
    # res = openvr.VROverlay().setOverlayTransformTrackedDeviceRelative(ov, 3, matrixTransformation)


    while(True):
        next(switcher)
        time.sleep(3)

    # input("Enter a key to return")

The example above was kind of a boilerplate (that only "switch" between differents images) on the right controller.

Integrated code may help you too, but will be harder to decode (It was a app that added a clock on your wrist, so you can know how many time remain in your game session), there were many overlays, and one is rotating according on time:

import sys
import time
import os
import openvr
from math import sin, cos, radians
from colour import Color

class PersonalOverlay:
    """
        An overlay abstraction which aim to be eassily manipulable
        Ex:
            overlay_Clock = PersonalOverlay('Clock', 'OVC', textureClockPath)
            overlay_Clock.setTransform(0, 0.01, 0.1, -85, 0, -10, 0.07, 0.07, 0.07)
            overlay_Clock.changeTexture(pathToFile)
            overlay_Clock.show()
            overlay_Clock.hide()
    """
    def __init__(self, key="0", title="", texturePath=""):
        self.overlayKey = openvr.c_char_p(key.encode('utf8'))
        self.overlayTitle = openvr.c_char_p(title.encode('utf8'))
        self.texturePath = openvr.c_char_p(texturePath.encode('utf8'))

        self.transform = openvr.HmdMatrix34_t()
        self.transform[0][0] = 1
        self.transform[1][1] = 1
        self.transform[2][2] = 1

        vroverlay = openvr.VROverlay()
        res, self.ov = vroverlay.createOverlay(self.overlayKey, self.overlayTitle)
        vroverlay.setOverlayFromFile(self.ov, self.texturePath)

    def setTransform(self, x=0, y=0, z=0, a=0, b=0, c=0, scaleX=1, scaleY=1, scaleZ=1):
        vroverlay = openvr.VROverlay()
        _, orig, _ = vroverlay.getOverlayTransformAbsolute(self.ov)

        self.transform = getTransformMatrix(x, y, z, a, b, c, scaleX, scaleY, scaleZ)
        vroverlay.setOverlayTransformAbsolute(self.ov, orig, self.transform)

    def getTransform(self):
        return self.transform

    def changeTexture(self, texturePath):
        self.texturePath = openvr.c_char_p(texturePath.encode('utf8'))
        openvr.VROverlay().setOverlayFromFile(self.ov, self.texturePath)

    def hide(self):
        openvr.VROverlay().hideOverlay(self.ov)
    def show(self):
        openvr.VROverlay().showOverlay(self.ov)


class PersonalClock :
    debug = False
    def __init__(self, startTime, eventEndTimer=None):
        self.currentTime = startTime
        self.eventEndTimer = eventEndTimer

        #State 0 (> 3min); State 1 (< 1min); State 2 (< 2min); State 3 (< 3min)
        self.currentState = 0

        self.initGradient()
        self.initOverlays()

    def initGradient(self):
        """
        Set the color gradient
        """
        self.gradient = list(Color("#aa0000").range_to(Color("#aa0000"), 30))
        self.gradient = self.gradient + list(Color("#aa0000").range_to(Color("#ff0000"), 30))
        self.gradient = self.gradient + list(Color("#ff0000").range_to(Color("#ffaa00"), 45))
        self.gradient = self.gradient + list(Color("#ffaa00").range_to(Color("#ffff00"), 15))
        self.gradient = self.gradient + list(Color("#ffff00").range_to(Color("#00ff00"), 30))
        self.gradient = self.gradient + list(Color("#00ff00").range_to(Color("#00aa00"), 30))

    def initOverlays(self):
        """
        Create the overlays with corresponding textures
        """
        # Textures loading
        dir_name = os.path.join(os.path.join(os.getcwd(),"timer_overlay"),"imgs")
        textureHandPath = os.path.join(dir_name, 'hand.png')
        textureClockPath = os.path.join(dir_name, 'clock.png')
        self.textureTxt01Path = os.path.join(dir_name, 'txtminInf1.png')
        self.textureTxt02Path = os.path.join(dir_name, 'txtminInf2.png')
        self.textureTxt03Path = os.path.join(dir_name, 'txtminInf3.png')

        #Create overlays and set their transform
        self.overlay_Hand = PersonalOverlay('Hand', 'OVH', textureHandPath)
        self.overlay_Clock = PersonalOverlay('Clock', 'OVC', textureClockPath)
        self.overlay_Clock.setTransform(0, 0.01, 0.1, -85, 0, -10, 0.07, 0.07, 0.07)

        self.overlay_TxtMin = PersonalOverlay('TxtMin', 'OVTM', self.textureTxt03Path)
        self.overlay_TxtMin.setTransform(0, 0.009, 0.145, -85, 0, -10, 0.07, 0.07, 0.07)

    def setRemainingTime (self, remainingTime):
        """
        Set the remaining time in second
        """
        if(PersonalClock.debug): print("Clock timer : update remaining time")
        self.currentTime = remainingTime
        if(self.currentTime > 0):
            self.show()

    def updateTimer(self, controllerIndice):
        """
        Update the display of the timer
        """
        if(PersonalClock.debug): print("updatetimer: ",self.currentTime)

        if (self.currentTime > 0.0) :
            self.show()

            #Change the texture for the text
            if (self.currentState != 1 and 0.0 < self.currentTime and self.currentTime < 60.0) :
                self.currentState = 1
                self.overlay_TxtMin.changeTexture(self.textureTxt01Path)
                if(PersonalClock.debug): print("Clock current state : 1 (< 1min)")
            elif (self.currentState != 2 and 60.0 < self.currentTime and self.currentTime < 120.0):
                self.currentState = 2
                self.overlay_TxtMin.changeTexture(self.textureTxt02Path)
                if(PersonalClock.debug):print("Clock current state : 2 (< 2min)")
            elif (self.currentState != 3 and 120.0 < self.currentTime and self.currentTime < 180.0):
                self.currentState = 3
                self.overlay_TxtMin.changeTexture(self.textureTxt03Path)
                if(PersonalClock.debug): print("Clock current state : 3 (< 3min)")
            elif (self.currentState != 0 and 180.0 < self.currentTime):
                self.currentState = 0
                self.hide()
                if(PersonalClock.debug): print("Clock current state : 0 (> 3min)")

            if (not self.currentState == 0) :
                #Rotate the hand of the clock
                self.overlay_Hand.setTransform(0, 0.011, 0.1, -85, -self.currentTime * 6.0, -10, 0.07, 0.07, 0.07)

                #Change the color of the color
                red = self.gradient[int(self.currentTime)].red
                green = self.gradient[int(self.currentTime)].green
                blue = self.gradient[int(self.currentTime)].blue

                vroverlay = openvr.VROverlay()
                vroverlay.setOverlayColor(self.overlay_Clock.ov, red, green, blue)

                #Update transform of the overlays
                vroverlay.setOverlayTransformTrackedDeviceRelative(self.overlay_Clock.ov, openvr.c_ulong(controllerIndice), self.overlay_Clock.getTransform())
                vroverlay.setOverlayTransformTrackedDeviceRelative(self.overlay_Hand.ov, openvr.c_ulong(controllerIndice), self.overlay_Hand.getTransform())
                vroverlay.setOverlayTransformTrackedDeviceRelative(self.overlay_TxtMin.ov, openvr.c_ulong(controllerIndice), self.overlay_TxtMin.getTransform())

        else :
            if(PersonalClock.debug): print("Clock timer : ENDED")
            self.hide()
            self.isOver = True
            if(self.eventEndTimer):
                self.eventEndTimer()

    def hide(self):
        self.overlay_Hand.hide()
        self.overlay_Clock.hide()
        self.overlay_TxtMin.hide()
    def show(self):
        self.overlay_Hand.show()
        self.overlay_Clock.show()
        self.overlay_TxtMin.show()



def getTransformMatrix(x=0, y=0, z=0, a=0, b=0, c=0, scaleX=1, scaleY=1, scaleZ=1):
    result = openvr.HmdMatrix34_t()
    #Translation
    result[0][3] = x
    result[1][3] = y
    result[2][3] = z

    #Rotations....
    a = radians(a)
    b = radians(b)
    c = radians(c)
    result[0][0] = cos(c) * cos(b)
    result[0][1] = cos(c) * sin(b) * sin(a) - sin(c) * cos(a)
    result[0][2] = cos(c) * sin(b) * cos(a) + sin(c) * sin(a)

    result[1][0] = sin(c) * cos(b)
    result[1][1] = sin(c) * sin(b) * sin(a) + cos(c) * cos(a)
    result[1][2] = sin(c) * sin(b) * cos(a) - cos(c) * sin(a)

    result[2][0] = - sin(b)
    result[2][1] = cos(b) * sin(a)
    result[2][2] = cos(b) * cos(a)
    #Scale
    result[0][0] = result[0][0] * scaleX
    result[0][1] = result[0][1] * scaleY
    result[0][2] = result[0][2] * scaleZ
    result[1][0] = result[1][0] * scaleX
    result[1][1] = result[1][1] * scaleY
    result[1][2] = result[1][2] * scaleZ
    result[2][0] = result[2][0] * scaleX
    result[2][1] = result[2][1] * scaleY
    result[2][2] = result[2][2] * scaleZ
    return result

def getControllerIDs():
    left, right = None, None
    vrsystem = openvr.VRSystem()

    while(right is None or left is None):
        poses_t = openvr.TrackedDevicePose_t * openvr.k_unMaxTrackedDeviceCount
        poses = poses_t()
        openvr.VRCompositor().waitGetPoses(poses, len(poses), None, 0)

        for i in range(openvr.k_unMaxTrackedDeviceCount):
            device_class = vrsystem.getTrackedDeviceClass(i)
            if device_class == openvr.TrackedDeviceClass_Controller:
                role = vrsystem.getControllerRoleForTrackedDeviceIndex(i)
                if role == openvr.TrackedControllerRole_RightHand:
                    right = i
                if role == openvr.TrackedControllerRole_LeftHand:
                    left = i
    return left, right


if(__name__=="__main__"):

    try:
        openvr.init(openvr.VRApplication_Overlay)
    except openvr.OpenVRError as e:
        print("Error, HMD probably not connected")
        print(str(e))
        quit()

    # Get the right controller ID
    left, right = getControllerIDs()
    print("Right-controller ID : "+str(right))

    ####-------------------------------------------------------------
    myClock = PersonalClock(3.25 * 60.0, eventEndTimer=lambda : print("ENDTIMER"))

    current = 200
    while(current > 0):
        time.sleep(0.02)
        current -= 1
        myClock.setRemainingTime(current)
        myClock.updateTimer(right)

Sorry for some of the bad code (I promise it's very old haha), and I can't tweak it now or I could break it since I can't test it anymore.

Hope this help !

[matEhickey]

TLDR:
The part you interessed of is probably just:

matrixTransformation = openvr.HmdMatrix34_t()
rotmat = vrutils.rotationXMatrix(math.radians(-90))
matrixTransformation = vrutils.numpyToMatrix34(vrutils.appliRotationMatrixToAxis(matrixTransformation, rotmat))
openvr.VROverlay().setOverlayTransformTrackedDeviceRelative(ov, r, matrixTransformation)

[matEhickey]

Oups, didn't saw my vrutils dependencies..

here it is:

# vrutils
import openvr
import math
import time
import numpy as np
import transformations

def get_controller_ids(vrsys=None):
    # return the ids of the controllers by checking the type of each tracked devices
    if vrsys is None:
        vrsys = openvr.VRSystem()
    else:
        vrsys = vrsys
    left, right = None, None
    for i in range(openvr.k_unMaxTrackedDeviceCount):
        device_class = vrsys.getTrackedDeviceClass(i)
        if device_class == openvr.TrackedDeviceClass_Controller:
            role = vrsys.getControllerRoleForTrackedDeviceIndex(i)
            if role == openvr.TrackedControllerRole_RightHand:
                right = i
            if role == openvr.TrackedControllerRole_LeftHand:
                left = i
    return left, right

def from_controller_state_to_dict(pControllerState):
    # docs: https://github.com/ValveSoftware/openvr/wiki/IVRSystem::GetControllerState
    d = {}
    # d['ulButtonPressed'] = pControllerState.ulButtonPressed
    # d['ulButtonTouched'] = pControllerState.ulButtonTouched -> to use with bitmask
    d['unPacketNum'] = pControllerState.unPacketNum
    d['trigger'] = pControllerState.rAxis[1].x
    d['trackpad_x'] = pControllerState.rAxis[0].x
    d['trackpad_y'] = pControllerState.rAxis[0].y
    d['menu_button'] = bool(pControllerState.ulButtonPressed >> 1 & 1)
    d['trackpad_pressed'] = bool(pControllerState.ulButtonPressed >> 32 & 1)
    d['trackpad_touched'] = bool(pControllerState.ulButtonTouched >> 32 & 1)
    d['grip_button'] = bool(pControllerState.ulButtonPressed >> 2 & 1)
    # System button can't be read, if you press it
    # the controllers stop reporting
    return d

def showInputs():
    import pprint
    vrsystem = openvr.VRSystem()

    left_id, right_id = get_controller_ids(vrsystem)

    print("Left controller ID: " + str(left_id))
    print("Right controller ID: " + str(right_id))
    print("===========================")
    try:
        while True:
            time.sleep(0.2)

            if(left_id):
                result, pControllerState = vrsystem.getControllerState(left_id)
                pprint.pprint(from_controller_state_to_dict(pControllerState))
            if(right_id):
                result, pControllerState = vrsystem.getControllerState(right_id)
                pprint.pprint(from_controller_state_to_dict(pControllerState))


    except KeyboardInterrupt:
        print("Control+C pressed, shutting down...")

def identityMatrix():
    m = openvr.HmdMatrix34_t()
    m[0][0] = m[1][1] = m[2][2] = 1
    return(m)
def getPosition(matrix):
    x = matrix[0][3]
    y = matrix[1][3]
    z = matrix[2][3]
    return(x,y,z)

def getQuaternion(matrix):
    w = math.sqrt(max(0, 1 + matrix[0][0] + matrix[1][1]+ matrix[2][2])) / 2;
    x = math.sqrt(max(0, 1 + matrix[0][0] - matrix[1][1] - matrix[2][2])) / 2;
    y = math.sqrt(max(0, 1 - matrix[0][0] + matrix[1][1] - matrix[2][2])) / 2;
    z = math.sqrt(max(0, 1 - matrix[0][0] - matrix[1][1] + matrix[2][2])) / 2;

    x = x if(matrix[2][1] - matrix[1][2] > 0) else -x
    y = y if(matrix[0][2] - matrix[2][0] > 0) else -y
    z = z if(matrix[1][0] - matrix[0][1] > 0) else -z

    return(w,x,y,z)

def rotationXMatrix(angle):
    xaxis = [1, 0, 0]
    return(transformations.rotation_matrix(angle, xaxis))

def rotationYMatrix(angle):
    yaxis = [0, 1, 0]
    return(transformations.rotation_matrix(angle, yaxis))

def rotationZMatrix(angle):
    zaxis = [0, 0, 1]
    return(transformations.rotation_matrix(angle, zaxis))

def translationMatrix(x,y,z):
    return(transformations.translation_matrix((x,y,z)))

def appliRotationMatrixToAxis(hmdMatrix, rotationMatrix):
    # rotate an transform on himself (dont change his position)
    h = matrix34ToNumpy(hmdMatrix)
    r = rotationMatrix
    return(h.dot(r))

def npArray(obj):
    return(np.array(obj))

def appliRotationMatrixToAxisOrigin(hmdMatrix, rotationMatrix):
    # rotate the transform origin base on point (0.0.0)
    # consider the transform has a vector, so it modify the position of his origin, but not the orientation
    mat = matrix34ToNumpy(hmdMatrix) # only the position (so we consider a vector)
    h = npArray(list(map(lambda x:x[3], mat)))
    h = np.append(h,[0]) #add last attribute to have a 1*4 multiply by a 4*4

    r = rotationMatrix

    res = h.dot(r)

    # set back the rotation to the matrix
    for i in range(3):
        mat[i][3] = res[i]
    return(mat)

def matrix34ToNumpy(mat):
    n = npArray(mat.m)
    return(n)

def numpyToMatrix34(nparray):
    hmd = openvr.HmdMatrix34_t()
    for i in range(3):
        for j in range(4):
            hmd[i][j] = nparray[i][j]
    return(hmd)

transformations lib is the well known transformations.py