Skip to content

Commit eaadfec

Browse files
fquirinfquirin
authored
Merge pull request #20 from SEPIA-Framework/dev
v0.19.1; iOS build fix; audio handling improved; chat fixes; 'env' URL parameter; and more
2 parents 624b8fc + 35894b5 commit eaadfec

30 files changed

+1093
-257
lines changed

‎config.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
<?xml version='1.0' encoding='utf-8'?>
2-
<widget id="org.example.sepia.app.web" version="0.19.0" xmlns="http://www.w3.org/ns/widgets" xmlns:cdv="http://cordova.apache.org/ns/1.0">
2+
<widget id="org.example.sepia.app.web" version="0.19.1" xmlns="http://www.w3.org/ns/widgets" xmlns:cdv="http://cordova.apache.org/ns/1.0">
33
<name>S.E.P.I.A.</name>
44
<description>
55
Prototype of S.E.P.I.A. framework client

‎create-ios-project-wkwebview.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ npm install xcode
5959
# add ios platform
6060
sleep 2
6161
echo "#Adding platform ..."
62-
cordova platform add ios@4.5.3
62+
cordova platform add ios@5.0.1
6363
#
6464
# prepare build
6565
echo "#Preparing build ..."

‎plugin_mods/speechrecognition/org.apache.cordova.speech.speechrecognition/src/ios/CDVSpeechRecognitionViewController.swift‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -126,13 +126,13 @@ public class CDVSpeechRecognitionViewController: UIViewController, SFSpeechRecog
126126
let url = Bundle.main.url(forAuxiliaryExecutable: self.micOpenDest)!
127127
do {
128128
let audioSession = AVAudioSession.sharedInstance()
129-
try audioSession.setCategory(AVAudioSessionCategoryPlayback)
129+
try audioSession.setCategory(AVAudioSession.Category.playback)
130130

131131
if #available(iOS 9, *) {
132-
try audioSession.setMode(AVAudioSessionModeSpokenAudio)
132+
try audioSession.setMode(AVAudioSession.Mode.spokenAudio)
133133
}
134134

135-
try audioSession.setActive(true, with: .notifyOthersOnDeactivation)
135+
try audioSession.setActive(true)
136136

137137
player = try AVAudioPlayer(contentsOf: url)
138138
guard let player = player else { return }
@@ -148,11 +148,11 @@ public class CDVSpeechRecognitionViewController: UIViewController, SFSpeechRecog
148148
private func playMicSoundOff() {
149149
do {
150150
let audioSession = AVAudioSession.sharedInstance()
151-
try audioSession.setCategory(AVAudioSessionCategoryPlayback)
151+
try audioSession.setCategory(AVAudioSession.Category.playback)
152152
if #available(iOS 9, *) {
153-
try audioSession.setMode(AVAudioSessionModeSpokenAudio)
153+
try audioSession.setMode(AVAudioSession.Mode.spokenAudio)
154154
}
155-
try audioSession.setActive(true, with: .notifyOthersOnDeactivation)
155+
try audioSession.setActive(true)
156156

157157
/*
158158
let url = Bundle.main.url(forAuxiliaryExecutable: self.micCloseDest)!
@@ -188,13 +188,13 @@ public class CDVSpeechRecognitionViewController: UIViewController, SFSpeechRecog
188188
startVibration(startV: true)
189189

190190
let audioSession = AVAudioSession.sharedInstance()
191-
try audioSession.setCategory(AVAudioSessionCategoryRecord)
192-
try audioSession.setMode(AVAudioSessionModeMeasurement)
193-
try audioSession.setActive(true, with: .notifyOthersOnDeactivation)
191+
try audioSession.setCategory(AVAudioSession.Category.record)
192+
try audioSession.setMode(AVAudioSession.Mode.measurement)
193+
try audioSession.setActive(true)
194194

195195
recognitionRequest = SFSpeechAudioBufferRecognitionRequest()
196196

197-
guard let inputNode = audioEngine.inputNode else { fatalError("Audio engine has no input node") }
197+
198198

199199
guard let recognitionRequest = recognitionRequest else { fatalError("Unable to created a SFSpeechAudioBufferRecognitionRequest object") }
200200

@@ -221,7 +221,7 @@ public class CDVSpeechRecognitionViewController: UIViewController, SFSpeechRecog
221221
self.delegate?.returnResult(statusIsOK: true, returnString: "", messageType: "end")
222222
self.audioEngine.stop()
223223
self.recognitionRequest?.endAudio()
224-
inputNode.removeTap(onBus: 0)
224+
self.audioEngine.inputNode.removeTap(onBus: 0)
225225
self.recognitionRequest = nil
226226
self.recognitionTask = nil
227227
self.startVibration(startV: false)
@@ -232,8 +232,8 @@ public class CDVSpeechRecognitionViewController: UIViewController, SFSpeechRecog
232232
})
233233
self.startTimer()
234234

235-
let recordingFormat = inputNode.outputFormat(forBus: 0)
236-
inputNode.installTap(onBus: 0, bufferSize: 1024, format: recordingFormat) { (buffer: AVAudioPCMBuffer, when: AVAudioTime) in
235+
let recordingFormat = audioEngine.inputNode.outputFormat(forBus: 0)
236+
audioEngine.inputNode.installTap(onBus: 0, bufferSize: 1024, format: recordingFormat) { (buffer: AVAudioPCMBuffer, when: AVAudioTime) in
237237
self.recognitionRequest?.append(buffer)
238238
}
239239
self.delegate?.returnResult(statusIsOK: true, returnString: "", messageType: "speechstart")
@@ -297,7 +297,7 @@ public class CDVSpeechRecognitionViewController: UIViewController, SFSpeechRecog
297297
}
298298
}
299299

300-
func InterruptEvent() {
300+
@objc func InterruptEvent() {
301301
if audioEngine.isRunning {
302302
audioEngine.stop()
303303
recognitionRequest?.endAudio()

‎www/css/sepiaFW-alwaysOn.css‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,10 @@
6464
}
6565
}
6666

67+
#sepiaFW-alwaysOn-notifications.check-channels .material-icons {
68+
color: #ff2550;
69+
}
70+
6771
.sepiaFW-alwaysOn-alarm-anim {
6872
animation: alarm 1s infinite ease-in;
6973
}

‎www/css/sepiaFW-style.css‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1317,6 +1317,12 @@ body {
13171317
#sepiaFW-nav-label-online-status.connecting {
13181318
opacity: 0.50;
13191319
}
1320+
#sepiaFW-nav-label-online-status.wake-word-active:after,
1321+
#sepiaFW-nav-label-online-status.wake-word-active:before {
1322+
content: ' Ξ ';
1323+
/*color: #ff2550;*/
1324+
font-family: monospace;
1325+
}
13201326
#sepiaFW-nav-label-note {
13211327
display: flex;
13221328
align-items: center;

‎www/index.html‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -304,10 +304,11 @@ <h2><i class="material-icons md-txt">people</i>&nbsp;<script>SepiaFW.local.w('ac
304304
<link rel="stylesheet alternate" href="css/sepiaFW-skin-spot.css" class='sepiaFW-style-skin' title='' data-name='Spots'>
305305
<link rel="stylesheet alternate" href="css/sepiaFW-skin-canary-dark.css" class='sepiaFW-style-skin' title='' data-name='DarkCanary'>
306306
<!-- START JS -->
307-
<script src="scripts/jquery-3.1.1.min.js"></script>
307+
<script src="scripts/jquery-3.4.1.min.js"></script>
308308
<!--<script src="https://code.jquery.com/jquery-3.1.1.min.js" integrity="sha256-hVVnYaiADRTO2PzUGmuLJr8BLUSjGIZsDYGmIJLv2b8=" crossorigin="anonymous"></script>-->
309309
<script src="scripts/sjcl.js"></script>
310310
<script src="scripts/recorder.js"></script>
311+
<script src="scripts/vad.js"></script>
311312
<script src="scripts/hammer.min.js"></script>
312313
<script src="scripts/hammer-time.min.js"></script>
313314
<script src="scripts/clexi-0.8.0.js"></script>

‎www/scripts/jquery-3.1.1.min.js‎

Lines changed: 0 additions & 4 deletions
This file was deleted.

‎www/scripts/jquery-3.4.1.min.js‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎www/scripts/recorder.js‎

Lines changed: 129 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -21,70 +21,88 @@ DEALINGS IN THE SOFTWARE.
2121

2222
(function (window) {
2323

24-
var Recorder = function (source) {
25-
var bufferLen = 4096;
24+
var Recorder = function(source, audioProcessor, startFun, stopFun) {
2625
var websocket;
27-
this.context = source.context;
28-
if (!this.context.createScriptProcessor) {
29-
this.node = this.context.createJavaScriptNode(bufferLen, 2, 2);
30-
} else {
31-
this.node = this.context.createScriptProcessor(bufferLen, 2, 2);
32-
}
26+
var audioContext = source.context;
27+
var inputSampleRate = (audioContext)? audioContext.sampleRate : source.sampleRate;
28+
var outputSampleRate = 16000;
3329

3430
var recording = false;
35-
36-
this.node.onaudioprocess = function (e) {
37-
if (!recording) return;
3831

39-
var inputL = e.inputBuffer.getChannelData(0);
40-
var length = Math.floor(inputL.length / 3);
41-
var result = new Float32Array(length);
42-
43-
var index = 0,
44-
inputIndex = 0;
32+
function processAudio(inputAudioFrame){
33+
if (!recording) return;
4534

46-
while (index < length) {
47-
result[index++] = inputL[inputIndex];
48-
inputIndex += 3;
49-
}
35+
//downsample
36+
var result = downsampleFloat32(inputAudioFrame, inputSampleRate, outputSampleRate);
5037

5138
var offset = 0;
52-
var buffer = new ArrayBuffer(length * 2);
39+
var buffer = new ArrayBuffer(result.length * 2);
5340
var view = new DataView(buffer);
54-
for (var i = 0; i < result.length; i++, offset += 2) {
55-
var s = Math.max(-1, Math.min(1, result[i]));
56-
view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
57-
}
41+
floatTo16BitPCM(view, offset, result);
5842

59-
//console.log('Recorder onaudioprocess - view: ' + view); //DEBUG
60-
websocket.send(view);
43+
//console.log('Recorder onaudioprocess - view: ' + view); //DEBUG
44+
if (websocket){
45+
websocket.send(view);
46+
}
6147
}
6248

63-
//Move this to 'start'?
64-
source.connect(this.node);
49+
//Custom
50+
if (audioProcessor && startFun && stopFun){
51+
audioProcessor.onaudioprocess = function(inputAudioFrame){
52+
processAudio(inputAudioFrame);
53+
}
54+
//Web-Audio
55+
}else if (audioContext){
56+
var bufferLen = 4096;
57+
let processNode;
58+
if ('createScriptProcessor' in audioContext){
59+
processNode = audioContext.createScriptProcessor(bufferLen, 1, 1);
60+
}else if ('createJavaScriptNode' in audioContext){
61+
processNode = audioContext.createJavaScriptNode(bufferLen, 1, 1);
62+
}else{
63+
console.error("Recorder - cannot create an audio processor!");
64+
return;
65+
}
66+
processNode.onaudioprocess = function(e) {
67+
var inputAudioFrame = e.inputBuffer.getChannelData(0);
68+
processAudio(inputAudioFrame);
69+
}
70+
startFun = function(){
71+
source.connect(processNode);
72+
processNode.connect(audioContext.destination); //if the script node is not connected to an output the "onaudioprocess" event is not triggered in chrome.
73+
}
74+
stopFun = function(){
75+
source.disconnect(processNode);
76+
processNode.disconnect(audioContext.destination);
77+
//processNode.disconnect();
78+
}
79+
//Error
80+
}else{
81+
console.error('Recorder - no valid audio processor found!');
82+
return;
83+
}
6584

6685
//Will be called at beginning of SepiaFW.audioRecorder.start();
67-
this.start = function() {
68-
this.node.connect(this.context.destination); // if the script node is not connected to an output the "onaudioprocess" event is not triggered in chrome.
86+
this.start = function(){
87+
recording = true;
88+
startFun();
6989
}
7090

7191
//Will be called at beginning of SepiaFW.audioRecorder.stop();
72-
this.stop = function () {
92+
this.stop = function(){
7393
recording = false;
74-
this.node.disconnect(0);
94+
stopFun();
7595
}
7696

77-
this.record = function (ws) {
97+
this.connect = function(ws){
7898
websocket = ws;
79-
recording = true;
8099
//console.log('Recorder record - ws: ' + ws); //DEBUG
81-
//console.log(this.node); //DEBUG
100+
//console.log(processNode); //DEBUG
82101
}
83102

84-
this.sendHeader = function (ws) {
103+
this.sendHeader = function(ws){
85104
var sampleLength = 1000000;
86105
var mono = true;
87-
var sampleRate = 16000; //TODO: this might not be true! We cannot guarantee that!
88106
var buffer = new ArrayBuffer(44);
89107
var view = new DataView(buffer);
90108

@@ -103,9 +121,9 @@ DEALINGS IN THE SOFTWARE.
103121
//channel count
104122
view.setUint16(22, mono ? 1 : 2, true);
105123
//sample rate
106-
view.setUint32(24, sampleRate, true);
124+
view.setUint32(24, outputSampleRate, true);
107125
//byte rate (sample rate * block align)
108-
view.setUint32(28, sampleRate * 2, true);
126+
view.setUint32(28, outputSampleRate * 2, true);
109127
//block align (channel count * bytes per sample)
110128
view.setUint16(32, 2, true);
111129
//bits per sample
@@ -118,11 +136,78 @@ DEALINGS IN THE SOFTWARE.
118136
//console.log('Recorder sendHeader - view: ' + view); //DEBUG
119137
ws.send(view);
120138
}
121-
function writeString(view, offset, string) {
122-
for (var i = 0; i < string.length; i++) {
139+
140+
//--- conversion methods ---
141+
142+
function writeString(view, offset, string){
143+
for (let i = 0; i < string.length; i++){
123144
view.setUint8(offset + i, string.charCodeAt(i));
124145
}
125-
}
146+
}
147+
148+
function floatTo16BitPCM(output, offset, input){
149+
for (let i = 0; i < input.length; i++, offset += 2){
150+
let s = Math.max(-1, Math.min(1, input[i]));
151+
output.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
152+
}
153+
}
154+
155+
function downsampleAndConvertToInt16(buffer, sampleRate, outSampleRate){
156+
if (outSampleRate == sampleRate){
157+
return buffer;
158+
}
159+
if (outSampleRate > sampleRate){
160+
console.error("Recorder - Downsampling to " + outSampleRate + " failed! Input sampling rate was too low: " + sampleRate);
161+
return buffer;
162+
}
163+
var sampleRateRatio = sampleRate / outSampleRate;
164+
var newLength = Math.round(buffer.length / sampleRateRatio);
165+
var result = new Int16Array(newLength);
166+
var offsetResult = 0;
167+
var offsetBuffer = 0;
168+
while (offsetResult < result.length){
169+
var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
170+
var accum = 0, count = 0;
171+
for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++){
172+
accum += buffer[i];
173+
count++;
174+
}
175+
result[offsetResult] = Math.min(1, accum / count)*0x7FFF;
176+
offsetResult++;
177+
offsetBuffer = nextOffsetBuffer;
178+
}
179+
return result.buffer;
180+
}
181+
function downsampleFloat32(array, sampleRate, outSampleRate){
182+
if (outSampleRate == sampleRate){
183+
var result = new Float32Array(array.length);
184+
for (let i = 0 ; i < array.length ; i++){
185+
result[i] = array[i];
186+
}
187+
return array;
188+
}
189+
if (outSampleRate > sampleRate){
190+
console.error("Recorder - Downsampling to " + outSampleRate + " failed! Input sampling rate was too low: " + sampleRate);
191+
return downsampleFloat32(array, sampleRate, sampleRate);
192+
}
193+
var sampleRateRatio = sampleRate / outSampleRate;
194+
var newLength = Math.round(array.length / sampleRateRatio);
195+
var result = new Float32Array(newLength);
196+
var offsetResult = 0;
197+
var offsetBuffer = 0;
198+
while (offsetResult < result.length){
199+
var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
200+
var accum = 0, count = 0;
201+
for (var i = offsetBuffer; i < nextOffsetBuffer && i < array.length; i++){
202+
accum += array[i];
203+
count++;
204+
}
205+
result[offsetResult] = accum / count;
206+
offsetResult++;
207+
offsetBuffer = nextOffsetBuffer;
208+
}
209+
return result;
210+
}
126211
};
127212

128213
window.RecorderJS = Recorder;

‎www/scripts/sepiaFW.account.js‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,19 @@ function sepiaFW_build_account(){
118118
}
119119

120120
//----------------------
121+
122+
//get general ID prefix
123+
Account.getIdPrefix = function(){
124+
if (userId){
125+
return userId.split(/\d/,2)[0];
126+
}else{
127+
return undefined;
128+
}
129+
}
130+
//does the given string look like an ID
131+
Account.stringLooksLikeAnID = function(str){
132+
return !!str.match(new RegExp("^" + Account.getIdPrefix().toLowerCase() + "\\d+$"));
133+
}
121134

122135
//get user id
123136
Account.getUserId = function(){
@@ -490,13 +503,13 @@ function sepiaFW_build_account(){
490503
});
491504
//id placeholder
492505
var idInput = document.getElementById("sepiaFW-login-id");
493-
idInput.placeholder = SepiaFW.local.username;
506+
idInput.placeholder = SepiaFW.local.g('username');
494507
$(idInput).off().on("keypress", function(e){
495508
if (e.keyCode === 13) { sendLoginFromBox(); }
496509
});
497510
//keypress on pwd
498511
var pwdInput = document.getElementById("sepiaFW-login-pwd");
499-
pwdInput.placeholder = SepiaFW.local.password;
512+
pwdInput.placeholder = SepiaFW.local.g('password');
500513
$(pwdInput).off().on("keypress", function (e) {
501514
if (e.keyCode === 13) { sendLoginFromBox(); }
502515
});
@@ -773,6 +786,8 @@ function sepiaFW_build_account(){
773786
if (status == "fail"){
774787
if (data.code && data.code == 3){
775788
if (errorCallback) errorCallback(SepiaFW.local.g('loginFailedServer'));
789+
}else if (data.code && data.code == 10){
790+
if (errorCallback) errorCallback(SepiaFW.local.g('loginFailedBlocked'));
776791
}else{
777792
if (errorCallback) errorCallback(SepiaFW.local.g('loginFailedUser'));
778793
}

0 commit comments

Comments
 (0)