Skip to content

Commit 59861cf

Browse files
author
zomint
authored
Merge pull request #9 from nulllaborg/develop
串口示例
2 parents b48ba94 + fbf27a7 commit 59861cf

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+144996
-0
lines changed

‎ai_vox_engine_led_iot/.clang-format‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
BasedOnStyle: Google
2+
3+
ColumnLimit: 150
4+
5+
AllowShortFunctionsOnASingleLine: None
6+
7+
BinPackArguments: false
8+
9+
BinPackParameters: false

‎ai_vox_engine_led_iot/ai_vox_engine_led_iot.ino‎

Lines changed: 336 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,336 @@
1+
#include <WiFi.h>
2+
#include <driver/spi_common.h>
3+
#include <esp_heap_caps.h>
4+
#include <esp_lcd_panel_io.h>
5+
#include <esp_lcd_panel_ops.h>
6+
#include <esp_lcd_panel_vendor.h>
7+
8+
#include "ai_vox_engine.h"
9+
#include "ai_vox_observer.h"
10+
#include "audio_input_device_sph0645.h"
11+
#include "display.h"
12+
#include "i2s_std_audio_output_device.h"
13+
14+
#ifndef ARDUINO_ESP32S3_DEV
15+
#error "This example only supports ESP32S3-Dev board."
16+
#endif
17+
18+
#ifndef CONFIG_SPIRAM_MODE_OCT
19+
#error "This example requires PSRAM to OPI PSRAM. Please enable it in Arduino IDE."
20+
#endif
21+
22+
#ifndef WIFI_SSID
23+
#define WIFI_SSID "ssid"
24+
#endif
25+
26+
#ifndef WIFI_PASSWORD
27+
#define WIFI_PASSWORD "password"
28+
#endif
29+
30+
namespace {
31+
constexpr gpio_num_t kLedPin = GPIO_NUM_46; // led Pin
32+
33+
constexpr gpio_num_t kMicPinBclk = GPIO_NUM_5;
34+
constexpr gpio_num_t kMicPinWs = GPIO_NUM_2;
35+
constexpr gpio_num_t kMicPinDin = GPIO_NUM_4;
36+
37+
constexpr gpio_num_t kSpeakerPinBclk = GPIO_NUM_13;
38+
constexpr gpio_num_t kSpeakerPinWs = GPIO_NUM_14;
39+
constexpr gpio_num_t kSpeakerPinDout = GPIO_NUM_1;
40+
41+
constexpr gpio_num_t kTriggerPin = GPIO_NUM_0;
42+
43+
constexpr gpio_num_t kDisplayBacklightPin = GPIO_NUM_11;
44+
constexpr gpio_num_t kDisplayMosiPin = GPIO_NUM_17;
45+
constexpr gpio_num_t kDisplayClkPin = GPIO_NUM_16;
46+
constexpr gpio_num_t kDisplayDcPin = GPIO_NUM_12;
47+
constexpr gpio_num_t kDisplayRstPin = GPIO_NUM_21;
48+
constexpr gpio_num_t kDisplayCsPin = GPIO_NUM_15;
49+
50+
constexpr auto kDisplaySpiMode = 0;
51+
constexpr uint32_t kDisplayWidth = 240;
52+
constexpr uint32_t kDisplayHeight = 240;
53+
constexpr bool kDisplayMirrorX = false;
54+
constexpr bool kDisplayMirrorY = false;
55+
constexpr bool kDisplayInvertColor = true;
56+
constexpr bool kDisplaySwapXY = false;
57+
constexpr auto kDisplayRgbElementOrder = LCD_RGB_ELEMENT_ORDER_RGB;
58+
59+
std::shared_ptr<ai_vox::iot::Entity> g_led_iot_entity;
60+
auto g_audio_output_device = std::make_shared<ai_vox::I2sStdAudioOutputDevice>(kSpeakerPinBclk, kSpeakerPinWs, kSpeakerPinDout);
61+
62+
std::unique_ptr<Display> g_display;
63+
auto g_observer = std::make_shared<ai_vox::Observer>();
64+
65+
void InitDisplay() {
66+
pinMode(kDisplayBacklightPin, OUTPUT);
67+
analogWrite(kDisplayBacklightPin, 255);
68+
69+
spi_bus_config_t buscfg{
70+
.mosi_io_num = kDisplayMosiPin,
71+
.miso_io_num = GPIO_NUM_NC,
72+
.sclk_io_num = kDisplayClkPin,
73+
.quadwp_io_num = GPIO_NUM_NC,
74+
.quadhd_io_num = GPIO_NUM_NC,
75+
.data4_io_num = GPIO_NUM_NC,
76+
.data5_io_num = GPIO_NUM_NC,
77+
.data6_io_num = GPIO_NUM_NC,
78+
.data7_io_num = GPIO_NUM_NC,
79+
.data_io_default_level = false,
80+
.max_transfer_sz = kDisplayWidth * kDisplayHeight * sizeof(uint16_t),
81+
.flags = 0,
82+
.isr_cpu_id = ESP_INTR_CPU_AFFINITY_AUTO,
83+
.intr_flags = 0,
84+
};
85+
ESP_ERROR_CHECK(spi_bus_initialize(SPI3_HOST, &buscfg, SPI_DMA_CH_AUTO));
86+
87+
esp_lcd_panel_io_handle_t panel_io = nullptr;
88+
esp_lcd_panel_handle_t panel = nullptr;
89+
// 液晶屏控制IO初始化
90+
ESP_LOGD(TAG, "Install panel IO");
91+
esp_lcd_panel_io_spi_config_t io_config = {};
92+
io_config.cs_gpio_num = kDisplayCsPin;
93+
io_config.dc_gpio_num = kDisplayDcPin;
94+
io_config.spi_mode = kDisplaySpiMode;
95+
io_config.pclk_hz = 40 * 1000 * 1000;
96+
io_config.trans_queue_depth = 10;
97+
io_config.lcd_cmd_bits = 8;
98+
io_config.lcd_param_bits = 8;
99+
ESP_ERROR_CHECK(esp_lcd_new_panel_io_spi(SPI3_HOST, &io_config, &panel_io));
100+
101+
// 初始化液晶屏驱动芯片
102+
ESP_LOGD(TAG, "Install LCD driver");
103+
esp_lcd_panel_dev_config_t panel_config = {};
104+
panel_config.reset_gpio_num = kDisplayRstPin;
105+
panel_config.rgb_ele_order = kDisplayRgbElementOrder;
106+
panel_config.bits_per_pixel = 16;
107+
ESP_ERROR_CHECK(esp_lcd_new_panel_st7789(panel_io, &panel_config, &panel));
108+
109+
esp_lcd_panel_reset(panel);
110+
111+
esp_lcd_panel_init(panel);
112+
esp_lcd_panel_invert_color(panel, kDisplayInvertColor);
113+
esp_lcd_panel_swap_xy(panel, kDisplaySwapXY);
114+
esp_lcd_panel_mirror(panel, kDisplayMirrorX, kDisplayMirrorY);
115+
116+
g_display = std::make_unique<Display>(panel_io, panel, kDisplayWidth, kDisplayHeight, 0, 0, kDisplayMirrorX, kDisplayMirrorY, kDisplaySwapXY);
117+
g_display->Start();
118+
}
119+
120+
void InitIot() {
121+
auto& ai_vox_engine = ai_vox::Engine::GetInstance();
122+
123+
// LED
124+
// 1.Define the properties for the LED entity
125+
std::vector<ai_vox::iot::Property> led_properties({
126+
{
127+
"state", // property name
128+
"LED灯开关状态", // property description
129+
ai_vox::iot::ValueType::kBool // property type
130+
},
131+
});
132+
133+
// 2.Define the functions for the LED entity
134+
std::vector<ai_vox::iot::Function> led_functions({
135+
{"TurnOn", // function name
136+
"打开LED灯", // function description
137+
{}},
138+
{"TurnOff", // function name
139+
"关闭LED灯", // function description
140+
{}},
141+
});
142+
143+
// 3.Create the LED entity
144+
g_led_iot_entity = std::make_shared<ai_vox::iot::Entity>("Led", // name
145+
"LED灯", // description
146+
std::move(led_properties), // properties
147+
std::move(led_functions) // functions
148+
);
149+
150+
// 4.Initialize the LED entity with default values
151+
g_led_iot_entity->UpdateState("state", false);
152+
153+
// 5.Register the LED entity with the AI Vox engine
154+
ai_vox_engine.RegisterIotEntity(g_led_iot_entity);
155+
}
156+
157+
#ifdef PRINT_HEAP_INFO_INTERVAL
158+
void PrintMemInfo() {
159+
if (heap_caps_get_total_size(MALLOC_CAP_SPIRAM) > 0) {
160+
const auto total_size = heap_caps_get_total_size(MALLOC_CAP_SPIRAM);
161+
const auto free_size = heap_caps_get_free_size(MALLOC_CAP_SPIRAM);
162+
const auto min_free_size = heap_caps_get_minimum_free_size(MALLOC_CAP_SPIRAM);
163+
printf("SPIRAM total size: %zu B (%zu KB), free size: %zu B (%zu KB), minimum free size: %zu B (%zu KB)\n",
164+
total_size,
165+
total_size >> 10,
166+
free_size,
167+
free_size >> 10,
168+
min_free_size,
169+
min_free_size >> 10);
170+
}
171+
172+
if (heap_caps_get_total_size(MALLOC_CAP_INTERNAL) > 0) {
173+
const auto total_size = heap_caps_get_total_size(MALLOC_CAP_INTERNAL);
174+
const auto free_size = heap_caps_get_free_size(MALLOC_CAP_INTERNAL);
175+
const auto min_free_size = heap_caps_get_minimum_free_size(MALLOC_CAP_INTERNAL);
176+
printf("IRAM total size: %zu B (%zu KB), free size: %zu B (%zu KB), minimum free size: %zu B (%zu KB)\n",
177+
total_size,
178+
total_size >> 10,
179+
free_size,
180+
free_size >> 10,
181+
min_free_size,
182+
min_free_size >> 10);
183+
}
184+
185+
if (heap_caps_get_total_size(MALLOC_CAP_DEFAULT) > 0) {
186+
const auto total_size = heap_caps_get_total_size(MALLOC_CAP_DEFAULT);
187+
const auto free_size = heap_caps_get_free_size(MALLOC_CAP_DEFAULT);
188+
const auto min_free_size = heap_caps_get_minimum_free_size(MALLOC_CAP_DEFAULT);
189+
printf("DRAM total size: %zu B (%zu KB), free size: %zu B (%zu KB), minimum free size: %zu B (%zu KB)\n",
190+
total_size,
191+
total_size >> 10,
192+
free_size,
193+
free_size >> 10,
194+
min_free_size,
195+
min_free_size >> 10);
196+
}
197+
}
198+
#endif
199+
} // namespace
200+
201+
void setup() {
202+
Serial.begin(115200);
203+
204+
pinMode(kLedPin, OUTPUT);
205+
206+
InitDisplay();
207+
208+
if (heap_caps_get_total_size(MALLOC_CAP_SPIRAM) == 0) {
209+
g_display->SetChatMessage(Display::Role::kSystem, "No SPIRAM available, please check your board.");
210+
while (true) {
211+
printf("No SPIRAM available, please check your board.\n");
212+
delay(1000);
213+
}
214+
}
215+
216+
g_display->ShowStatus("Wifi connecting...");
217+
WiFi.useStaticBuffers(true);
218+
WiFi.begin(WIFI_SSID, WIFI_PASSWORD);
219+
while (WiFi.status() != WL_CONNECTED) {
220+
delay(1000);
221+
printf("Connecting to WiFi, ssid: %s, password: %s\n", WIFI_SSID, WIFI_PASSWORD);
222+
}
223+
printf("WiFi connected, IP address: %s\n", WiFi.localIP().toString().c_str());
224+
g_display->ShowStatus("Wifi connected");
225+
226+
InitIot();
227+
228+
auto audio_input_device = std::make_shared<AudioInputDeviceSph0645>(kMicPinBclk, kMicPinWs, kMicPinDin);
229+
auto& ai_vox_engine = ai_vox::Engine::GetInstance();
230+
ai_vox_engine.SetObserver(g_observer);
231+
ai_vox_engine.SetTrigger(kTriggerPin);
232+
ai_vox_engine.SetOtaUrl("https://api.tenclass.net/xiaozhi/ota/");
233+
ai_vox_engine.ConfigWebsocket("wss://api.tenclass.net/xiaozhi/v1/",
234+
{
235+
{"Authorization", "Bearer test-token"},
236+
});
237+
ai_vox_engine.Start(audio_input_device, g_audio_output_device);
238+
g_display->ShowStatus("AI Vox Engine starting...");
239+
}
240+
241+
void loop() {
242+
#ifdef PRINT_HEAP_INFO_INTERVAL
243+
static uint32_t s_print_heap_info_time = 0;
244+
if (s_print_heap_info_time == 0 || millis() - s_print_heap_info_time >= PRINT_HEAP_INFO_INTERVAL) {
245+
s_print_heap_info_time = millis();
246+
PrintMemInfo();
247+
}
248+
#endif
249+
250+
const auto events = g_observer->PopEvents();
251+
for (auto& event : events) {
252+
if (auto activation_event = std::get_if<ai_vox::Observer::ActivationEvent>(&event)) {
253+
printf("activation code: %s, message: %s\n", activation_event->code.c_str(), activation_event->message.c_str());
254+
g_display->ShowStatus("激活设备");
255+
g_display->SetChatMessage(Display::Role::kSystem, activation_event->message);
256+
} else if (auto state_changed_event = std::get_if<ai_vox::Observer::StateChangedEvent>(&event)) {
257+
switch (state_changed_event->new_state) {
258+
case ai_vox::ChatState::kIdle: {
259+
printf("Idle\n");
260+
break;
261+
}
262+
case ai_vox::ChatState::kIniting: {
263+
printf("Initing...\n");
264+
g_display->ShowStatus("初始化");
265+
break;
266+
}
267+
case ai_vox::ChatState::kStandby: {
268+
printf("Standby\n");
269+
g_display->ShowStatus("待命");
270+
break;
271+
}
272+
case ai_vox::ChatState::kConnecting: {
273+
printf("Connecting...\n");
274+
g_display->ShowStatus("连接中...");
275+
break;
276+
}
277+
case ai_vox::ChatState::kListening: {
278+
printf("Listening...\n");
279+
g_display->ShowStatus("聆听中");
280+
break;
281+
}
282+
case ai_vox::ChatState::kSpeaking: {
283+
printf("Speaking...\n");
284+
g_display->ShowStatus("说话中");
285+
break;
286+
}
287+
default: {
288+
break;
289+
}
290+
}
291+
} else if (auto emotion_event = std::get_if<ai_vox::Observer::EmotionEvent>(&event)) {
292+
printf("emotion: %s\n", emotion_event->emotion.c_str());
293+
g_display->SetEmotion(emotion_event->emotion);
294+
} else if (auto chat_message_event = std::get_if<ai_vox::Observer::ChatMessageEvent>(&event)) {
295+
switch (chat_message_event->role) {
296+
case ai_vox::ChatRole::kAssistant: {
297+
printf("role: assistant, content: %s\n", chat_message_event->content.c_str());
298+
g_display->SetChatMessage(Display::Role::kAssistant, chat_message_event->content);
299+
break;
300+
}
301+
case ai_vox::ChatRole::kUser: {
302+
printf("role: user, content: %s\n", chat_message_event->content.c_str());
303+
g_display->SetChatMessage(Display::Role::kUser, chat_message_event->content);
304+
break;
305+
}
306+
}
307+
} else if (auto iot_message_event = std::get_if<ai_vox::Observer::IotMessageEvent>(&event)) {
308+
printf("IOT message: %s, function: %s\n", iot_message_event->name.c_str(), iot_message_event->function.c_str());
309+
for (const auto& [key, value] : iot_message_event->parameters) {
310+
if (std::get_if<bool>(&value)) {
311+
printf("key: %s, value: %s\n", key.c_str(), std::get<bool>(value) ? "true" : "false");
312+
} else if (std::get_if<std::string>(&value)) {
313+
printf("key: %s, value: %s\n", key.c_str(), std::get<std::string>(value).c_str());
314+
} else if (std::get_if<int64_t>(&value)) {
315+
printf("key: %s, value: %lld\n", key.c_str(), std::get<int64_t>(value));
316+
}
317+
}
318+
319+
if (iot_message_event->name == "Led") {
320+
if (iot_message_event->function == "TurnOn") {
321+
printf("turn on led\n");
322+
digitalWrite(kLedPin, HIGH);
323+
Serial.println(1);
324+
325+
g_led_iot_entity->UpdateState("state", true);
326+
} else if (iot_message_event->function == "TurnOff") {
327+
printf("turn off led\n");
328+
digitalWrite(kLedPin, LOW);
329+
Serial.println(0);
330+
}
331+
}
332+
}
333+
}
334+
335+
taskYIELD();
336+
}

0 commit comments

Comments
 (0)