Skip to content

Commit 1aa4b8b

Browse files
EnoxSoftwareEnoxSoftware
committed
Fixed UI objects to display in the optimal zone for hologram placement (between 1.25m and 5m).
Fixed the method of judging whether a Calib3d.solvePnP result is valid or not.
1 parent b727fcc commit 1aa4b8b

File tree

9 files changed

+506
-339
lines changed

9 files changed

+506
-339
lines changed

‎Assets/HoloLensWithDlibFaceLandmarkDetectorExample/HoloLensARHeadExample/HoloLensARHeadExample.cs‎

Lines changed: 35 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,11 @@ public class HoloLensARHeadExample : ExampleSceneBase
211211
/// </summary>
212212
Matrix4x4 invertZM;
213213

214+
/// <summary>
215+
/// The matrix that AR camera P * V.
216+
/// </summary>
217+
Matrix4x4 VP;
218+
214219
/// <summary>
215220
/// The transformation matrix.
216221
/// </summary>
@@ -473,7 +478,7 @@ public void OnWebCamTextureToMatHelperInitialized ()
473478
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
474479

475480
previewQuad.GetComponent<MeshRenderer>().material.mainTexture = texture;
476-
previewQuad.transform.localScale = new Vector3 (1, height/width, 1);
481+
previewQuad.transform.localScale = new Vector3 (0.2f * width / height, 0.2f, 1);
477482
previewQuad.SetActive (displayCameraPreview);
478483

479484

@@ -497,6 +502,12 @@ public void OnWebCamTextureToMatHelperInitialized ()
497502
distCoeffs = new MatOfDouble (distCoeffs1, distCoeffs2, distCoeffs3, distCoeffs4, distCoeffs5);
498503
Debug.Log ("distCoeffs " + distCoeffs.dump ());
499504

505+
506+
// create AR camera P * V Matrix
507+
Matrix4x4 P = ARUtils.CalculateProjectionMatrixFromCameraMatrixValues((float)fx, (float)fy, (float)cx, (float)cy, width, height, 0.3f, 5f);
508+
Matrix4x4 V = Matrix4x4.TRS(Vector3.zero, Quaternion.identity, new Vector3(1, 1, -1));
509+
VP = P * V;
510+
500511
//Calibration camera
501512
Size imageSize = new Size (width, height);
502513
double apertureWidth = 0;
@@ -796,7 +807,7 @@ public void OnFrameMatAcquired (Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4
796807
}
797808

798809
if (points != null) {
799-
UpdateARHeadTransform (points, cameraToWorldMatrix, cameraToWorldMatrix.inverse, projectionMatrix);
810+
UpdateARHeadTransform (points, cameraToWorldMatrix);
800811
}
801812

802813
bgraMat.Dispose ();
@@ -883,7 +894,7 @@ void Update ()
883894
opticalFlowFilter.Process (grayMat, points, points, false);
884895
}
885896

886-
UpdateARHeadTransform (points, arCamera.cameraToWorldMatrix, arCamera.worldToCameraMatrix, arCamera.projectionMatrix);
897+
UpdateARHeadTransform (points, arCamera.cameraToWorldMatrix);
887898

888899
if (displayCameraPreview) {
889900
// draw landmark points
@@ -960,7 +971,7 @@ void Update ()
960971
opticalFlowFilter.Process (grayMat, points, points, false);
961972
}
962973

963-
UpdateARHeadTransform (points, arCamera.cameraToWorldMatrix, arCamera.worldToCameraMatrix, arCamera.projectionMatrix);
974+
UpdateARHeadTransform (points, arCamera.cameraToWorldMatrix);
964975

965976
if (displayCameraPreview) {
966977
// draw landmark points
@@ -979,7 +990,7 @@ void Update ()
979990
}
980991
#endif
981992

982-
private void UpdateARHeadTransform (List<Vector2> points, Matrix4x4 cameraToWorldMatrix, Matrix4x4 worldToCameraMatrix, Matrix4x4 projectionMatrix)
993+
private void UpdateARHeadTransform (List<Vector2> points, Matrix4x4 cameraToWorldMatrix)
983994
{
984995
MatOfPoint3f objectPoints = null;
985996
bool isRightEyeOpen = false;
@@ -1092,13 +1103,27 @@ private void UpdateARHeadTransform (List<Vector2> points, Matrix4x4 cameraToWorl
10921103
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec);
10931104
}
10941105

1095-
10961106

10971107
/*
1108+
double tvec_z = tvec.get(2, 0)[0];
1109+
1110+
if (double.IsNaN(tvec_z) || tvec_z < 0)
1111+
{ // if tvec is wrong data, do not use extrinsic guesses.
1112+
Calib3d.solvePnP(objectPoints68, imagePoints, camMatrix, distCoeffs, rvec, tvec);
1113+
}
1114+
else
1115+
{
1116+
Calib3d.solvePnP(objectPoints68, imagePoints, camMatrix, distCoeffs, rvec, tvec, true, Calib3d.SOLVEPNP_ITERATIVE);
1117+
}
1118+
1119+
if (applyEstimationPose && !double.IsNaN(tvec_z))
1120+
{
1121+
*/
1122+
1123+
10981124
double tvec_x = tvec.get(0, 0)[0], tvec_y = tvec.get(1, 0)[0], tvec_z = tvec.get(2, 0)[0];
1099-
1125+
11001126
bool isNotInViewport = false;
1101-
Matrix4x4 VP = projectionMatrix * worldToCameraMatrix;
11021127
Vector4 pos = VP * new Vector4((float)tvec_x, (float)tvec_y, (float)tvec_z, 1.0f);
11031128
if (pos.w != 0)
11041129
{
@@ -1115,25 +1140,10 @@ private void UpdateARHeadTransform (List<Vector2> points, Matrix4x4 cameraToWorl
11151140
{
11161141
Calib3d.solvePnP(objectPoints, imagePoints, camMatrix, distCoeffs, rvec, tvec, true, Calib3d.SOLVEPNP_ITERATIVE);
11171142
}
1118-
//Debug.Log (tvec.dump() + " " + isNotInViewport);
1119-
*/
1120-
1121-
1122-
1123-
double tvec_z = tvec.get(2, 0)[0];
1124-
1125-
if (double.IsNaN(tvec_z) || tvec_z < 0)
1126-
{ // if tvec is wrong data, do not use extrinsic guesses.
1127-
Calib3d.solvePnP(objectPoints68, imagePoints, camMatrix, distCoeffs, rvec, tvec);
1128-
}
1129-
else
1130-
{
1131-
Calib3d.solvePnP(objectPoints68, imagePoints, camMatrix, distCoeffs, rvec, tvec, true, Calib3d.SOLVEPNP_ITERATIVE);
1132-
}
11331143

1144+
//Debug.Log (tvec.dump());
11341145

1135-
1136-
if (applyEstimationPose && !double.IsNaN(tvec_z))
1146+
if (applyEstimationPose && !isNotInViewport)
11371147
{
11381148

11391149
// Display effects.

‎Assets/HoloLensWithDlibFaceLandmarkDetectorExample/HoloLensARHeadExample/HoloLensARHeadExample.unity‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -669,6 +669,11 @@ PrefabInstance:
669669
propertyPath: m_RootOrder
670670
value: 4
671671
objectReference: {fileID: 0}
672+
- target: {fileID: 20770519707920992, guid: d29bc40b7f3df26479d6a0aac211c355,
673+
type: 3}
674+
propertyPath: far clip plane
675+
value: 5
676+
objectReference: {fileID: 0}
672677
m_RemovedComponents: []
673678
m_SourcePrefab: {fileID: 100100000, guid: d29bc40b7f3df26479d6a0aac211c355, type: 3}
674679
--- !u!4 &105640272 stripped
@@ -3848,8 +3853,8 @@ Transform:
38483853
m_PrefabAsset: {fileID: 0}
38493854
m_GameObject: {fileID: 1273041842}
38503855
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
3851-
m_LocalPosition: {x: -1, y: 1, z: 10}
3852-
m_LocalScale: {x: 1, y: 1, z: 1}
3856+
m_LocalPosition: {x: -0.3, y: 0.2, z: 2.2}
3857+
m_LocalScale: {x: 0.354, y: 0.2, z: 1}
38533858
m_Children: []
38543859
m_Father: {fileID: 105640272}
38553860
m_RootOrder: 2
@@ -4898,8 +4903,8 @@ RectTransform:
48984903
m_PrefabAsset: {fileID: 0}
48994904
m_GameObject: {fileID: 1478280149}
49004905
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
4901-
m_LocalPosition: {x: 0, y: 0, z: 1.1}
4902-
m_LocalScale: {x: 0.0005, y: 0.0005, z: 0}
4906+
m_LocalPosition: {x: 0, y: 0, z: 2}
4907+
m_LocalScale: {x: 0.001, y: 0.001, z: 0}
49034908
m_Children:
49044909
- {fileID: 270024293}
49054910
- {fileID: 62512752}
@@ -4923,7 +4928,7 @@ MonoBehaviour:
49234928
m_Script: {fileID: 11500000, guid: 2be8bd2ebd8277c448d6d81c75517fee, type: 3}
49244929
m_Name:
49254930
m_EditorClassIdentifier:
4926-
TagalongDistance: 1.1
4931+
TagalongDistance: 2
49274932
EnforceDistance: 1
49284933
PositionUpdateSpeed: 9.8
49294934
SmoothMotion: 1
@@ -5190,10 +5195,6 @@ PrefabInstance:
51905195
propertyPath: m_MaxParticleSize
51915196
value: 0.01
51925197
objectReference: {fileID: 0}
5193-
- target: {fileID: 100000, guid: 0ff7fb238b264d249b06d5faab227e44, type: 3}
5194-
propertyPath: m_IsActive
5195-
value: 1
5196-
objectReference: {fileID: 0}
51975198
- target: {fileID: 19900006, guid: 0ff7fb238b264d249b06d5faab227e44, type: 3}
51985199
propertyPath: m_MinParticleSize
51995200
value: 0.1
@@ -5202,6 +5203,10 @@ PrefabInstance:
52025203
propertyPath: m_MaxParticleSize
52035204
value: 0.2
52045205
objectReference: {fileID: 0}
5206+
- target: {fileID: 100000, guid: 0ff7fb238b264d249b06d5faab227e44, type: 3}
5207+
propertyPath: m_IsActive
5208+
value: 1
5209+
objectReference: {fileID: 0}
52055210
m_RemovedComponents: []
52065211
m_SourcePrefab: {fileID: 100100000, guid: 0ff7fb238b264d249b06d5faab227e44, type: 3}
52075212
--- !u!4 &1612021276 stripped

‎Assets/HoloLensWithDlibFaceLandmarkDetectorExample/HoloLensFaceLandmarkDetectionExample/HoloLensFaceLandmarkDetectionExample.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -504,7 +504,7 @@ public void OnFrameMatAcquired (Mat bgraMat, Matrix4x4 projectionMatrix, Matrix4
504504
// Position the canvas object slightly in front
505505
// of the real world web camera.
506506
Vector3 position = cameraToWorldMatrix.GetColumn (3) - cameraToWorldMatrix.GetColumn (2);
507-
position *= 1.2f;
507+
position *= 2.2f;
508508

509509
// Rotate the canvas object so that it faces the user.
510510
Quaternion rotation = Quaternion.LookRotation (-cameraToWorldMatrix.GetColumn (2), cameraToWorldMatrix.GetColumn (1));
@@ -690,7 +690,7 @@ void Update()
690690
// Position the canvas object slightly in front
691691
// of the real world web camera.
692692
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2);
693-
position *= 1.2f;
693+
position *= 2.2f;
694694

695695
// Rotate the canvas object so that it faces the user.
696696
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));

‎Assets/HoloLensWithDlibFaceLandmarkDetectorExample/HoloLensFaceLandmarkDetectionExample/HoloLensFaceLandmarkDetectionExample.unity‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -772,6 +772,11 @@ PrefabInstance:
772772
propertyPath: m_RootOrder
773773
value: 2
774774
objectReference: {fileID: 0}
775+
- target: {fileID: 20770519707920992, guid: d29bc40b7f3df26479d6a0aac211c355,
776+
type: 3}
777+
propertyPath: far clip plane
778+
value: 5
779+
objectReference: {fileID: 0}
775780
m_RemovedComponents: []
776781
m_SourcePrefab: {fileID: 100100000, guid: d29bc40b7f3df26479d6a0aac211c355, type: 3}
777782
--- !u!1 &262962544
@@ -2045,8 +2050,8 @@ Transform:
20452050
m_PrefabAsset: {fileID: 0}
20462051
m_GameObject: {fileID: 1076083694}
20472052
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
2048-
m_LocalPosition: {x: 0, y: 0, z: 1.2}
2049-
m_LocalScale: {x: 1.77, y: 1, z: 1}
2053+
m_LocalPosition: {x: 0, y: 0, z: 2.2}
2054+
m_LocalScale: {x: 2.5, y: 2.5, z: 1}
20502055
m_Children: []
20512056
m_Father: {fileID: 0}
20522057
m_RootOrder: 0
@@ -3044,8 +3049,8 @@ RectTransform:
30443049
m_PrefabAsset: {fileID: 0}
30453050
m_GameObject: {fileID: 1478280149}
30463051
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
3047-
m_LocalPosition: {x: 0, y: 0, z: 1.1}
3048-
m_LocalScale: {x: 0.0005, y: 0.0005, z: 0}
3052+
m_LocalPosition: {x: 0, y: 0, z: 2}
3053+
m_LocalScale: {x: 0.001, y: 0.001, z: 0}
30493054
m_Children:
30503055
- {fileID: 270024293}
30513056
- {fileID: 62512752}
@@ -3069,7 +3074,7 @@ MonoBehaviour:
30693074
m_Script: {fileID: 11500000, guid: 2be8bd2ebd8277c448d6d81c75517fee, type: 3}
30703075
m_Name:
30713076
m_EditorClassIdentifier:
3072-
TagalongDistance: 1.1
3077+
TagalongDistance: 2
30733078
EnforceDistance: 1
30743079
PositionUpdateSpeed: 9.8
30753080
SmoothMotion: 1
@@ -3178,11 +3183,6 @@ PrefabInstance:
31783183
propertyPath: m_Range
31793184
value: 0.01
31803185
objectReference: {fileID: 0}
3181-
- target: {fileID: 114000013851064060, guid: b2db04283121ca74495c2ee000fb4243,
3182-
type: 3}
3183-
propertyPath: DefaultCursorDistance
3184-
value: 1.2
3185-
objectReference: {fileID: 0}
31863186
m_RemovedComponents: []
31873187
m_SourcePrefab: {fileID: 100100000, guid: b2db04283121ca74495c2ee000fb4243, type: 3}
31883188
--- !u!114 &1492569180 stripped

‎Assets/HoloLensWithDlibFaceLandmarkDetectorExample/HoloLensPhotoCaptureExample/HoloLensPhotoCaptureExample.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,7 @@ void OnPhotoCaptured (PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame
213213
// Position the canvas object slightly in front
214214
// of the real world web camera.
215215
Vector3 position = cameraToWorldMatrix.GetColumn (3) - cameraToWorldMatrix.GetColumn (2);
216-
position *= 1.2f;
216+
position *= 2.2f;
217217

218218
// Rotate the canvas object so that it faces the user.
219219
Quaternion rotation = Quaternion.LookRotation (-cameraToWorldMatrix.GetColumn (2), cameraToWorldMatrix.GetColumn (1));

0 commit comments

Comments
 (0)