Skip to content

Commit a51f08b

Browse files
JarbasAlJarbasAl
authored
active skill end2end tests (#717)
* active skill tests * test final session * fix stop tests - async messages support for when order is not known, eg. messages emitted from a thread
1 parent d40b2f7 commit a51f08b

File tree

9 files changed

+286
-34
lines changed

9 files changed

+286
-34
lines changed

‎requirements/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,4 @@ ovos-utils[extras]>=0.6.0,<1.0.0
77
ovos_bus_client>=0.1.4,<2.0.0
88
ovos-plugin-manager>=1.0.3,<2.0.0
99
ovos-config>=0.0.13,<3.0.0
10-
ovos-workshop>=7.0.5,<8.0.0
10+
ovos-workshop>=7.0.6,<8.0.0

‎requirements/tests.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@ pytest-cov>=2.8.1
55
pytest-testmon>=2.1.3
66
pytest-randomly>=3.16.0
77
cov-core>=1.15.0
8-
ovoscope>=0.7.1,<1.0.0
8+
ovoscope>=0.7.2,<1.0.0

‎test/end2end/test_activate.py‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
from unittest import TestCase
2+
3+
from ovos_bus_client.message import Message
4+
from ovos_bus_client.session import Session
5+
from ovos_utils.log import LOG
6+
7+
from ovos_workshop.skills.converse import ConversationalSkill
8+
from ovoscope import End2EndTest, get_minicroft
9+
10+
11+
class TestSkill(ConversationalSkill):
12+
13+
def initialize(self):
14+
self.add_event("test_activate", self.handle_activate_test)
15+
self.add_event("test_deactivate", self.handle_deactivate_test)
16+
17+
def handle_activate_test(self, message: Message):
18+
self.activate()
19+
20+
def handle_deactivate_test(self, message: Message):
21+
self.deactivate()
22+
23+
def can_converse(self, message: Message) -> bool:
24+
return True
25+
26+
def converse(self, message: Message):
27+
self.log.debug("I dont wanna converse anymore")
28+
self.deactivate()
29+
30+
31+
class TestDeactivate(TestCase):
32+
33+
def setUp(self):
34+
LOG.set_level("DEBUG")
35+
self.skill_id = "test_activation.openvoiceos"
36+
self.minicroft = get_minicroft([self.skill_id],
37+
extra_skills={self.skill_id: TestSkill})
38+
39+
def tearDown(self):
40+
if self.minicroft:
41+
self.minicroft.stop()
42+
LOG.set_level("CRITICAL")
43+
44+
def test_activate(self):
45+
session = Session("123")
46+
session.lang = "en-US"
47+
session.deactivate_skill(self.skill_id) # start with skill inactive
48+
49+
message = Message("test_activate",
50+
context={"session": session.serialize(),
51+
"source": "A", "destination": "B"})
52+
53+
final_session = Session("123")
54+
final_session.lang = "en-US"
55+
final_session.active_skills = [(self.skill_id, 0.0)]
56+
57+
test = End2EndTest(
58+
minicroft=self.minicroft,
59+
skill_ids=[self.skill_id],
60+
source_message=message,
61+
deactivation_points=[message.msg_type],
62+
final_session=final_session,
63+
activation_points=["intent.service.skills.activated"],
64+
# messages internal to ovos-core, i.e. would not be sent to clients such as hivemind
65+
keep_original_src=[
66+
#"intent.service.skills.activate", # TODO
67+
#f"{self.skill_id}.activate", # TODO
68+
],
69+
expected_messages=[
70+
message,
71+
# handler code
72+
Message("intent.service.skills.activate",
73+
data={"skill_id": self.skill_id},
74+
context={"skill_id": self.skill_id}),
75+
Message("intent.service.skills.activated",
76+
data={"skill_id": self.skill_id},
77+
context={"skill_id": self.skill_id}),
78+
Message(f"{self.skill_id}.activate",
79+
data={},
80+
context={"skill_id": self.skill_id}),
81+
]
82+
)
83+
84+
test.execute(timeout=10)
85+
86+
def test_deactivate(self):
87+
session = Session("123")
88+
session.lang = "en-US"
89+
session.activate_skill(self.skill_id) # start with skill active
90+
91+
message = Message("test_deactivate",
92+
context={"session": session.serialize(),
93+
"source": "A", "destination": "B"})
94+
95+
final_session = Session("123")
96+
final_session.lang = "en-US"
97+
final_session.active_skills = []
98+
99+
test = End2EndTest(
100+
minicroft=self.minicroft,
101+
skill_ids=[self.skill_id],
102+
source_message=message,
103+
final_session=final_session,
104+
activation_points=[message.msg_type], # starts activated
105+
deactivation_points=["intent.service.skills.deactivated"],
106+
# messages internal to ovos-core, i.e. would not be sent to clients such as hivemind
107+
keep_original_src=[
108+
#"intent.service.skills.deactivate", # TODO
109+
#f"{self.skill_id}.deactivate", # TODO
110+
#f"{self.skill_id}.activate", # TODO
111+
],
112+
expected_messages=[
113+
message,
114+
# handler code
115+
Message("intent.service.skills.deactivate",
116+
data={"skill_id": self.skill_id},
117+
context={"skill_id": self.skill_id}),
118+
Message("intent.service.skills.deactivated",
119+
data={"skill_id": self.skill_id},
120+
context={"skill_id": self.skill_id}),
121+
Message(f"{self.skill_id}.deactivate",
122+
data={},
123+
context={"skill_id": self.skill_id}),
124+
]
125+
)
126+
127+
test.execute(timeout=10)
128+
129+
def test_deactivate_inside_converse(self):
130+
session = Session("123")
131+
session.lang = "en-US"
132+
session.activate_skill(self.skill_id) # start with skill active
133+
134+
message = Message("recognizer_loop:utterance",
135+
{"utterances": ["deactivate skill from within converse"], "lang": session.lang},
136+
{"session": session.serialize(), "source": "A", "destination": "B"})
137+
138+
final_session = Session("123")
139+
final_session.lang = "en-US"
140+
final_session.active_skills = []
141+
142+
test = End2EndTest(
143+
minicroft=self.minicroft,
144+
skill_ids=[self.skill_id],
145+
source_message=message,
146+
final_session=final_session,
147+
activation_points=[message.msg_type], # starts activated
148+
deactivation_points=["intent.service.skills.deactivated"],
149+
# messages internal to ovos-core, i.e. would not be sent to clients such as hivemind
150+
keep_original_src=[
151+
f"{self.skill_id}.converse.ping",
152+
f"{self.skill_id}.converse.request",
153+
#"intent.service.skills.deactivate", # TODO
154+
#f"{self.skill_id}.deactivate", # TODO
155+
#f"{self.skill_id}.activate", # TODO
156+
],
157+
expected_messages=[
158+
message,
159+
Message(f"{self.skill_id}.converse.ping",
160+
data={"utterances": ["deactivate skill from within converse"], "skill_id": self.skill_id},
161+
context={}),
162+
Message("skill.converse.pong",
163+
data={"can_handle": True, "skill_id": self.skill_id},
164+
context={"skill_id": self.skill_id}),
165+
Message(f"{self.skill_id}.activate",
166+
data={},
167+
context={"skill_id": self.skill_id}),
168+
Message("converse:skill",
169+
data={"utterances": ["deactivate skill from within converse"], "lang": session.lang,
170+
"skill_id": self.skill_id},
171+
context={"skill_id": self.skill_id}),
172+
Message(f"{self.skill_id}.converse.request",
173+
data={"utterances": ["deactivate skill from within converse"], "lang": session.lang},
174+
context={"skill_id": self.skill_id}),
175+
# converse handler code
176+
Message("intent.service.skills.deactivate",
177+
data={"skill_id": self.skill_id},
178+
context={"skill_id": self.skill_id}),
179+
Message("intent.service.skills.deactivated",
180+
data={"skill_id": self.skill_id},
181+
context={"skill_id": self.skill_id}),
182+
Message(f"{self.skill_id}.deactivate",
183+
data={},
184+
context={"skill_id": self.skill_id}),
185+
# post converse handler
186+
Message("skill.converse.response",
187+
data={"skill_id": self.skill_id},
188+
context={"skill_id": self.skill_id}),
189+
Message("ovos.utterance.handled",
190+
data={},
191+
context={"skill_id": self.skill_id})
192+
193+
]
194+
)
195+
196+
test.execute(timeout=10)

‎test/end2end/test_adapt.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
from unittest import TestCase
2-
2+
from copy import deepcopy
33
from ovos_bus_client.message import Message
44
from ovos_bus_client.session import Session
55
from ovos_utils.log import LOG
@@ -27,10 +27,16 @@ def test_adapt_match(self):
2727
{"utterances": ["hello world"], "lang": session.lang},
2828
{"session": session.serialize(), "source": "A", "destination": "B"})
2929

30+
final_session = deepcopy(session)
31+
final_session.active_skills = [(self.skill_id, 0.0)]
32+
3033
test = End2EndTest(
3134
minicroft=self.minicroft,
3235
skill_ids=[self.skill_id],
3336
source_message=message,
37+
final_session=final_session,
38+
activation_points=[f"{self.skill_id}:HelloWorldIntent"],
39+
# keep_original_src=[f"{self.skill_id}.activate"], # TODO
3440
expected_messages=[
3541
message,
3642
Message(f"{self.skill_id}.activate",
@@ -76,6 +82,7 @@ def test_skill_blacklist(self):
7682
minicroft=self.minicroft,
7783
skill_ids=[self.skill_id],
7884
source_message=message,
85+
final_session=session,
7986
expected_messages=[
8087
message,
8188
Message("mycroft.audio.play_sound", {"uri": "snd/error.mp3"}),
@@ -99,6 +106,7 @@ def test_intent_blacklist(self):
99106
minicroft=self.minicroft,
100107
skill_ids=[self.skill_id],
101108
source_message=message,
109+
final_session=session,
102110
expected_messages=[
103111
message,
104112
Message("mycroft.audio.play_sound", {"uri": "snd/error.mp3"}),
@@ -120,6 +128,7 @@ def test_padatious_no_match(self):
120128
test = End2EndTest(
121129
minicroft=self.minicroft,
122130
skill_ids=[self.skill_id],
131+
final_session=session,
123132
source_message=message,
124133
expected_messages=[
125134
message,

‎test/end2end/test_cancel_plugin.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ def test_cancel_match(self):
3030
minicroft=self.minicroft,
3131
skill_ids=[self.skill_id],
3232
source_message=message,
33+
final_session=session,
3334
expected_messages=[
3435
message,
3536
Message("mycroft.audio.play_sound", {"uri": "snd/cancel.mp3"}),

‎test/end2end/test_converse.py‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
from copy import deepcopy
12
from unittest import TestCase
23

34
from ovos_bus_client.message import Message
@@ -25,15 +26,16 @@ def test_parrot_mode(self):
2526
session.pipeline = ["ovos-converse-pipeline-plugin", "ovos-padatious-pipeline-plugin-high"]
2627

2728
message1 = Message("recognizer_loop:utterance",
28-
{"utterances": ["start parrot mode"], "lang": session.lang},
29-
{"session": session.serialize(), "source": "A", "destination": "B"})
30-
# NOTE: we dont pass session, End2EndTest will inject/update the session from message1
29+
{"utterances": ["start parrot mode"], "lang": session.lang},
30+
{"session": session.serialize(), "source": "A", "destination": "B"})
31+
# NOTE: we dont pass session after first message
32+
# End2EndTest will inject/update the session from message1
3133
message2 = Message("recognizer_loop:utterance",
32-
{"utterances": ["echo test"], "lang": session.lang},
33-
{"source": "A", "destination": "B"})
34+
{"utterances": ["echo test"], "lang": session.lang},
35+
{"source": "A", "destination": "B"})
3436
message3 = Message("recognizer_loop:utterance",
35-
{"utterances": ["stop parrot"], "lang": session.lang},
36-
{"source": "A", "destination": "B"})
37+
{"utterances": ["stop parrot"], "lang": session.lang},
38+
{"source": "A", "destination": "B"})
3739
message4 = Message("recognizer_loop:utterance",
3840
{"utterances": ["echo test"], "lang": session.lang},
3941
{"source": "A", "destination": "B"})
@@ -139,20 +141,27 @@ def test_parrot_mode(self):
139141
Message("skill.converse.pong",
140142
data={"can_handle": False, "skill_id": self.skill_id},
141143
context={"skill_id": self.skill_id}),
142-
Message("mycroft.audio.play_sound", data={"uri": "snd/error.mp3"}),
144+
Message("mycroft.audio.play_sound", data={"uri": "snd/error.mp3"}),
143145
Message("complete_intent_failure"),
144146
Message("ovos.utterance.handled")
145147
]
148+
149+
final_session = deepcopy(session)
150+
final_session.active_skills = [(self.skill_id, 0.0)]
151+
146152
test = End2EndTest(
147153
minicroft=self.minicroft,
148154
skill_ids=[self.skill_id],
149155
eof_msgs=["ovos.utterance.handled"],
150156
flip_points=["recognizer_loop:utterance"],
157+
final_session=final_session,
151158
source_message=[message1, message2, message3, message4],
152159
expected_messages=expected1 + expected2 + expected3 + expected4,
153-
activation_points={f"{self.skill_id}:start_parrot.intent": self.skill_id},
160+
activation_points=[f"{self.skill_id}:start_parrot.intent"],
154161
# messages internal to ovos-core, i.e. would not be sent to clients such as hivemind
155162
keep_original_src=[f"{self.skill_id}.converse.ping",
156-
f"{self.skill_id}.converse.request"]
163+
f"{self.skill_id}.converse.request"
164+
# f"{self.skill_id}.activate", # TODO
165+
]
157166
)
158167
test.execute(timeout=10)

‎test/end2end/test_fallback.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
from unittest import TestCase
2-
2+
from copy import deepcopy
33
from ovos_bus_client.message import Message
44
from ovos_bus_client.session import Session
55
from ovos_utils.log import LOG
@@ -27,10 +27,19 @@ def test_fallback_match(self):
2727
{"utterances": ["hello world"], "lang": session.lang},
2828
{"session": session.serialize(), "source": "A", "destination": "B"})
2929

30+
final_session = deepcopy(session)
31+
# final_session.active_skills = [(self.skill_id, 0.0)] # TODO - failing
32+
33+
3034
test = End2EndTest(
3135
minicroft=self.minicroft,
3236
skill_ids=[self.skill_id],
33-
keep_original_src=["ovos.skills.fallback.ping"], # for routing tests this is an exception
37+
final_session=final_session,
38+
keep_original_src=[
39+
"ovos.skills.fallback.ping",
40+
# "ovos.skills.fallback.pong", # TODO
41+
],
42+
activation_points=[f"ovos.skills.fallback.{self.skill_id}.request"],
3443
source_message=message,
3544
expected_messages=[
3645
message,

0 commit comments

Comments
 (0)