Skip to content

Commit b64f054

Browse files
lanreolokobalanreolokoba
committed
Migrate to VideoKit
1 parent 93f5fd6 commit b64f054

File tree

8 files changed

+459
-426
lines changed

8 files changed

+459
-426
lines changed

‎Assets/BlazePoseSample.cs‎

Lines changed: 21 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -8,54 +8,47 @@ namespace NatML.Examples {
88
using UnityEngine;
99
using NatML.Devices;
1010
using NatML.Devices.Outputs;
11+
using NatML.VideoKit;
1112
using NatML.Vision;
1213
using Visualizers;
1314

1415
public sealed class BlazePoseSample : MonoBehaviour {
1516

17+
[Header(@"VideoKit")]
18+
public VideoKitCameraManager cameraManager;
19+
1620
[Header(@"Visualizers")]
1721
public LandmarkVisualizer visualizer;
1822
public Landmark3DVisualizer visualizer3D;
1923

20-
CameraDevice cameraDevice;
21-
TextureOutput cameraTextureOutput;
22-
BlazePosePipeline posePipeline;
24+
private BlazePosePipeline pipeline;
2325

24-
async void Start () {
25-
// Request camera permissions
26-
var permissionStatus = await MediaDeviceQuery.RequestPermissions<CameraDevice>();
27-
if (permissionStatus != PermissionStatus.Authorized) {
28-
Debug.Log(@"User did not grant camera permissions");
29-
return;
30-
}
31-
// Get a camera device
32-
var query = new MediaDeviceQuery(MediaDeviceCriteria.CameraDevice);
33-
cameraDevice = query.current as CameraDevice;
34-
// Start the camera preview
35-
cameraTextureOutput = new TextureOutput();
36-
cameraDevice.StartRunning(cameraTextureOutput);
37-
// Display the preview
38-
await cameraTextureOutput.textureCreated;
39-
visualizer.image = cameraTextureOutput.texture;
26+
private async void Start () {
4027
// Create the BlazePose pipeline
4128
var detectorModelData = await MLModelData.FromHub("@natml/blazepose-detector");
4229
var predictorModelData = await MLModelData.FromHub("@natml/blazepose-landmark");
43-
posePipeline = new BlazePosePipeline(detectorModelData, predictorModelData, maxDetections: 1);
30+
pipeline = new BlazePosePipeline(detectorModelData, predictorModelData, maxDetections: 1);
31+
// Listen for camera frames
32+
cameraManager.OnFrame.AddListener(OnCameraFrame);
4433
}
4534

46-
void Update () {
47-
// Check that pipeline has been created
48-
if (posePipeline == null)
49-
return;
35+
private void OnCameraFrame (CameraFrame frame) {
5036
// Predict
51-
var poses = posePipeline.Predict(cameraTextureOutput.texture);
52-
// Visualize
37+
var poses = pipeline.Predict(frame);
38+
// Check
5339
if (poses.Length == 0)
5440
return;
41+
// Visualize
5542
var pose = poses[0];
56-
visualizer.Clear();
57-
visualizer.Render(pose.keypoints, Color.green);
43+
visualizer.Render(pose.keypoints);
5844
visualizer3D.Render(pose.keypoints3D);
5945
}
46+
47+
private void OnDisable () {
48+
// Stop listening for camera frames
49+
cameraManager.OnFrame.RemoveListener(OnCameraFrame);
50+
// Dispose the pipeline
51+
pipeline?.Dispose();
52+
}
6053
}
6154
}

‎Assets/BlazePoseSample.unity‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,7 @@ GameObject:
202202
- component: {fileID: 357584624}
203203
- component: {fileID: 357584623}
204204
- component: {fileID: 357584625}
205+
- component: {fileID: 357584627}
205206
- component: {fileID: 357584626}
206207
m_Layer: 5
207208
m_Name: Preview
@@ -292,6 +293,20 @@ MonoBehaviour:
292293
m_Name:
293294
m_EditorClassIdentifier:
294295
keypoint: {fileID: 878973901}
296+
--- !u!114 &357584627
297+
MonoBehaviour:
298+
m_ObjectHideFlags: 0
299+
m_CorrespondingSourceObject: {fileID: 0}
300+
m_PrefabInstance: {fileID: 0}
301+
m_PrefabAsset: {fileID: 0}
302+
m_GameObject: {fileID: 357584621}
303+
m_Enabled: 1
304+
m_EditorHideFlags: 0
305+
m_Script: {fileID: 11500000, guid: 145a5a333d6e14379ae91956c0b3373e, type: 3}
306+
m_Name:
307+
m_EditorClassIdentifier:
308+
cameraManager: {fileID: 2111620525}
309+
viewMode: 0
295310
--- !u!1 &705507993
296311
GameObject:
297312
m_ObjectHideFlags: 0
@@ -1025,6 +1040,7 @@ GameObject:
10251040
serializedVersion: 6
10261041
m_Component:
10271042
- component: {fileID: 2111620523}
1043+
- component: {fileID: 2111620525}
10281044
- component: {fileID: 2111620524}
10291045
m_Layer: 0
10301046
m_Name: BlazePoseSample
@@ -1060,5 +1076,26 @@ MonoBehaviour:
10601076
m_Script: {fileID: 11500000, guid: 0dc1abe2732cf4c29bd30fa0833535e3, type: 3}
10611077
m_Name:
10621078
m_EditorClassIdentifier:
1079+
cameraManager: {fileID: 2111620525}
10631080
visualizer: {fileID: 357584626}
10641081
visualizer3D: {fileID: 1522798682}
1082+
--- !u!114 &2111620525
1083+
MonoBehaviour:
1084+
m_ObjectHideFlags: 0
1085+
m_CorrespondingSourceObject: {fileID: 0}
1086+
m_PrefabInstance: {fileID: 0}
1087+
m_PrefabAsset: {fileID: 0}
1088+
m_GameObject: {fileID: 2111620522}
1089+
m_Enabled: 1
1090+
m_EditorHideFlags: 0
1091+
m_Script: {fileID: 11500000, guid: 98622613d505143a9a06b4b11b0a82cf, type: 3}
1092+
m_Name:
1093+
m_EditorClassIdentifier:
1094+
resolution: 3
1095+
capabilities: 1
1096+
playOnAwake: 1
1097+
focusMode: 0
1098+
exposureMode: 0
1099+
OnFrame:
1100+
m_PersistentCalls:
1101+
m_Calls: []

‎Assets/LandmarkVisualizer.cs‎

Lines changed: 7 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,12 @@ namespace NatML.Examples.Visualizers {
88
using System.Collections.Generic;
99
using UnityEngine;
1010
using UnityEngine.UI;
11+
using NatML.VideoKit.UI;
1112

1213
/// <summary>
1314
/// Lightweight 2D pose visualizer.
1415
/// </summary>
15-
[RequireComponent(typeof(RawImage), typeof(AspectRatioFitter))]
16+
[RequireComponent(typeof(VideoKitCameraView))]
1617
public sealed class LandmarkVisualizer : MonoBehaviour {
1718

1819
#region --Inspector--
@@ -24,56 +25,30 @@ public sealed class LandmarkVisualizer : MonoBehaviour {
2425

2526

2627
#region --Client API--
27-
/// <summary>
28-
/// Get or set the detection image.
29-
/// </summary>
30-
public Texture2D image {
31-
get => rawImage.texture as Texture2D;
32-
set {
33-
rawImage.texture = value;
34-
aspectFitter.aspectRatio = (float)value.width / value.height;
35-
}
36-
}
37-
3828
/// <summary>
3929
/// Render a 2D pose.
4030
/// </summary>
4131
/// <param name="pose">Pose keypoints in normalized coordinates.</param>
4232
/// <param name="color">Pose keypoint color.</param>
43-
public void Render (IEnumerable<Vector4> pose, Color color) {
44-
foreach (var point in pose)
45-
AddKeypoint(point, color);
46-
}
47-
48-
/// <summary>
49-
/// Clear all rendered poses.
50-
/// </summary>
51-
public void Clear () {
52-
foreach (var obj in current)
53-
GameObject.Destroy(obj);
54-
current.Clear();
33+
public void Render (params IEnumerable<Vector4>[] poses) {
34+
foreach (var pose in poses)
35+
foreach (var point in pose)
36+
AddKeypoint(point, Color.green);
5537
}
5638
#endregion
5739

5840

5941
#region --Operations--
60-
private RawImage rawImage;
61-
private AspectRatioFitter aspectFitter;
6242
private readonly List<GameObject> current = new List<GameObject>();
6343

64-
private void Awake () {
65-
rawImage = GetComponent<RawImage>();
66-
aspectFitter = GetComponent<AspectRatioFitter>();
67-
}
68-
6944
private void AddKeypoint (Vector2 point, Color color) {
7045
// Instantiate
7146
var prefab = Instantiate(keypoint, transform);
7247
prefab.gameObject.SetActive(true);
7348
prefab.color = color;
7449
// Position
7550
var prefabTransform = prefab.transform as RectTransform;
76-
var imageTransform = rawImage.transform as RectTransform;
51+
var imageTransform = transform as RectTransform;
7752
prefabTransform.anchorMin = 0.5f * Vector2.one;
7853
prefabTransform.anchorMax = 0.5f * Vector2.one;
7954
prefabTransform.pivot = 0.5f * Vector2.one;

‎Packages/manifest.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
}
1313
],
1414
"dependencies": {
15-
"ai.natml.natdevice": "1.3.1",
15+
"ai.natml.videokit": "0.0.3",
1616
"com.unity.collab-proxy": "1.15.17",
1717
"com.unity.feature.development": "1.0.1",
1818
"com.unity.ide.rider": "3.0.14",

‎Packages/packages-lock.json‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,23 @@
22
"dependencies": {
33
"ai.natml.hub": {
44
"version": "1.0.15",
5-
"depth": 1,
5+
"depth": 2,
66
"source": "registry",
77
"dependencies": {},
88
"url": "https://registry.npmjs.com"
99
},
10+
"ai.natml.natcorder": {
11+
"version": "1.9.1",
12+
"depth": 1,
13+
"source": "registry",
14+
"dependencies": {
15+
"ai.natml.hub": "1.0.15"
16+
},
17+
"url": "https://registry.npmjs.com"
18+
},
1019
"ai.natml.natdevice": {
1120
"version": "1.3.1",
12-
"depth": 0,
21+
"depth": 1,
1322
"source": "registry",
1423
"dependencies": {
1524
"ai.natml.hub": "1.0.15"
@@ -25,6 +34,27 @@
2534
},
2635
"url": "https://registry.npmjs.com"
2736
},
37+
"ai.natml.natshare": {
38+
"version": "1.3.0",
39+
"depth": 1,
40+
"source": "registry",
41+
"dependencies": {
42+
"ai.natml.hub": "1.0.12"
43+
},
44+
"url": "https://registry.npmjs.com"
45+
},
46+
"ai.natml.videokit": {
47+
"version": "0.0.3",
48+
"depth": 0,
49+
"source": "registry",
50+
"dependencies": {
51+
"ai.natml.natcorder": "1.9.1",
52+
"ai.natml.natdevice": "1.3.1",
53+
"ai.natml.natml": "1.0.19",
54+
"ai.natml.natshare": "1.3.0"
55+
},
56+
"url": "https://registry.npmjs.com"
57+
},
2858
"ai.natml.vision.blazepose": {
2959
"version": "file:ai.natml.vision.blazepose",
3060
"depth": 0,

‎README.md‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ Retrieve your access key from [NatML Hub](https://hub.natml.ai/profile) and add
1717

1818
## Supported Platforms
1919
- Android API level 24+
20-
- iOS 13+
20+
- iOS 14+
2121
- macOS 10.15+ (Apple Silicon and Intel)
2222
- Windows 10+ (64-bit only)
2323
- WebGL:
@@ -27,11 +27,9 @@ Retrieve your access key from [NatML Hub](https://hub.natml.ai/profile) and add
2727
## Resources
2828
- Join the [NatML community on Discord](https://hub.natml.ai/community).
2929
- See the [NatML documentation](https://docs.natml.ai/unity).
30-
- See the [NatDevice documentation](https://docs.natml.ai/natdevice).
30+
- See the [VideoKit documentation](https://docs.natml.ai/videokit).
3131
- Check out [NatML on GitHub](https://github.com/natmlx).
3232
- Read the [NatML blog](https://blog.natml.ai/).
33-
- Discuss [NatML on Unity Forums](https://forum.unity.com/threads/open-beta-natml-machine-learning-runtime.1109339/).
34-
- Discuss [NatDevice on Unity Forums](https://forum.unity.com/threads/natdevice-media-device-api.374690/).
3533
- Contact us at [hi@natml.ai](mailto:hi@natml.ai).
3634

3735
Thank you very much!

0 commit comments

Comments
 (0)