Skip to content

Commit e55f47b

Browse files
paulparkinsonpaulparkinson
committed
init - unity C# client
1 parent 22658af commit e55f47b

File tree

1 file changed

+189
-0
lines changed

1 file changed

+189
-0
lines changed

‎unity-client/TakePicAndProcessAI.cs

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
using UnityEngine;
2+
using System.Collections;
3+
using System.Linq;
4+
using UnityEngine.Windows.WebCam;
5+
6+
using System;
7+
using System.IO;
8+
using XRCloudServices;
9+
10+
using System.Collections.Generic;
11+
using System.IO;
12+
using UnityEngine.UI;
13+
using UnityEngine.Networking;
14+
using TMPro;
15+
using Microsoft.MixedReality.Toolkit.Audio;
16+
17+
18+
public class TakePicAndProcessAI : MonoBehaviour
19+
{
20+
public Material quadMaterial = null;
21+
UnityEngine.Windows.WebCam.PhotoCapture photoCaptureObject = null;
22+
Texture2D targetTexture = null;
23+
int tapsCount = 1;
24+
string filenameFromStorage;
25+
string filePath;
26+
27+
public string backendURL = "https://yourhost:yourport";
28+
public GameObject gameojbectToActivate;
29+
public GameObject textToSpeechObject;
30+
public TextToSpeechLogic textToSpeechLogic;
31+
32+
[SerializeField] private TMP_Text textMesh;
33+
GameObject quad;
34+
35+
private bool isInit = false;
36+
37+
private string endPoint;
38+
39+
private string genopts;
40+
41+
public void Start()
42+
{
43+
TakePicAndProcess();
44+
}
45+
public void ExplainThis()
46+
{
47+
endPoint = "visionai/explain";
48+
TakePicAndProcess();
49+
}
50+
public void SummarizeThis()
51+
{
52+
endPoint = "visionai/summarize";
53+
TakePicAndProcess();
54+
}
55+
public void Transcribe()
56+
{
57+
endPoint = "visionai/transcribe";
58+
TakePicAndProcess();
59+
}
60+
public void TellAScaryStoryAndSentiments()
61+
{
62+
endPoint = "tellastory/tellastory";
63+
genopts = "scary";
64+
TakePicAndProcess();
65+
}
66+
public void TellADystopianStoryAndSentiments()
67+
{
68+
endPoint = "tellastory/tellastory";
69+
genopts = "dystopian";
70+
TakePicAndProcess();
71+
}
72+
public void GenerateAnImageFromDescription()
73+
{
74+
endPoint = "tellastory/tellastory";
75+
genopts = "scary";
76+
TakePicAndProcess();
77+
}
78+
79+
private void TakePicAndProcess()
80+
{
81+
Resolution cameraResolution = UnityEngine.Windows.WebCam.PhotoCapture.SupportedResolutions.OrderByDescending((res) => res.width * res.height).First();
82+
Texture2D targetTexture = new Texture2D(cameraResolution.width, cameraResolution.height);
83+
UnityEngine.Windows.WebCam.PhotoCapture.CreateAsync(false, delegate (UnityEngine.Windows.WebCam.PhotoCapture captureObject)
84+
{
85+
photoCaptureObject = captureObject;
86+
CameraParameters camParameters = new CameraParameters();
87+
camParameters.hologramOpacity = 0.0f;
88+
camParameters.cameraResolutionWidth = targetTexture.width;
89+
camParameters.cameraResolutionHeight = targetTexture.height;
90+
camParameters.pixelFormat = CapturePixelFormat.BGRA32; //other options include JPEG, NV12, etc.;
91+
captureObject.StartPhotoModeAsync(camParameters, delegate (UnityEngine.Windows.WebCam.PhotoCapture.PhotoCaptureResult result)
92+
{
93+
filenameFromStorage = string.Format(@"CapturedImage{0}.jpg", tapsCount++);
94+
filePath = Path.Combine(Application.persistentDataPath, filenameFromStorage);
95+
if (textMesh != null) textMesh.text = "pic file:" + filePath;
96+
photoCaptureObject.TakePhotoAsync(filePath, PhotoCaptureFileOutputFormat.JPG, OnCapturedPhotoToDisk);
97+
});
98+
});
99+
}
100+
void OnCapturedPhotoToDisk(UnityEngine.Windows.WebCam.PhotoCapture.PhotoCaptureResult result)
101+
{
102+
if (!isInit)
103+
{
104+
photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
105+
isInit = true;
106+
return;
107+
}
108+
StartCoroutine(ProcessAI(filePath));
109+
Texture2D tex = new Texture2D(2, 2);
110+
tex.LoadImage(File.ReadAllBytes(filePath));
111+
112+
GameObject quad = GameObject.CreatePrimitive(PrimitiveType.Quad);
113+
quad.transform.SetParent(Camera.main.transform);
114+
quad.transform.SetParent(gameojbectToActivate.transform);
115+
116+
quad.transform.localPosition = new Vector3(0, 0, 2); // Adjust this to position the quad correctly
117+
quad.transform.localScale = new Vector3(0.2f, 0.2f, 0.2f); // Adjust this to scale the quad correctly
118+
119+
quad.transform.localEulerAngles = Vector3.zero; // Rotate it to face the camera
120+
quad.GetComponent<Renderer>().material.mainTexture = tex;
121+
122+
123+
photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
124+
}
125+
126+
private IEnumerator ProcessAI(string filePath)
127+
{
128+
Texture2D texture2D = new Texture2D(2, 2);
129+
byte[] imageData = File.ReadAllBytes(filePath);
130+
texture2D.LoadImage(imageData);
131+
yield return ProcessAI(filePath, texture2D);
132+
}
133+
134+
private IEnumerator ProcessAI(string filePath, Texture2D image)
135+
{
136+
if (genopts != null)
137+
{
138+
if (textMesh != null) textMesh.text = generatedStory;
139+
textToSpeechLogic.talk(textToSpeechObject.GetComponent<TextToSpeech>(), generatedStory);
140+
yield return null;
141+
}
142+
if (textMesh != null) textMesh.text = "in " + endPoint +" making call to Vision AI + GPT for file: " + filePath + "...";
143+
byte[] imageData = image.EncodeToPNG();
144+
WWWForm form = new WWWForm();
145+
form.AddBinaryData("file", imageData, "image.png", "image/png");
146+
if (genopts != null)
147+
{
148+
form.AddField("genopts", genopts);
149+
genopts = null;
150+
}
151+
Debug.Log("Making AI etc. calls, providing file: " + filePath + "...");
152+
if (textMesh != null)
153+
{
154+
textMesh.text = "in " + endPoint +" about to textToSpeechObject: " + textToSpeechObject;
155+
textToSpeechObject.SetActive(true);
156+
if (textMesh != null) textMesh.text = "in " + endPoint +" about to gameojbectToActivate and call server: " + gameojbectToActivate;
157+
}
158+
UnityWebRequest request = UnityWebRequest.Post(backendURL + "/" + endPoint, form);
159+
yield return request.SendWebRequest();
160+
if (request.result == UnityWebRequest.Result.Success)
161+
{
162+
string jsonResponse = request.downloadHandler.text;
163+
if (textMesh != null) textMesh.text = jsonResponse;
164+
textToSpeechLogic.talk(textToSpeechObject.GetComponent<TextToSpeech>(), jsonResponse);
165+
Debug.Log(jsonResponse);
166+
}
167+
else
168+
{
169+
if (textMesh != null) textMesh.text = "request.error=" + request.error;
170+
textToSpeechLogic.talk(textToSpeechObject.GetComponent<TextToSpeech>(), request.error);
171+
Debug.Log(request.error);
172+
}
173+
request.Dispose();
174+
}
175+
176+
void OnStoppedPhotoMode(UnityEngine.Windows.WebCam.PhotoCapture.PhotoCaptureResult result)
177+
{
178+
photoCaptureObject.Dispose();
179+
photoCaptureObject = null;
180+
}
181+
182+
public void Clear()
183+
{
184+
quad.SetActive(false);
185+
quad = null;
186+
gameojbectToActivate.SetActive(false);
187+
gameojbectToActivate = null;
188+
}
189+
}

0 commit comments

Comments
 (0)