Skip to content
This repository was archived by the owner on Dec 14, 2025. It is now read-only.

Commit 48e1e8d

Browse files
lorowlorow
committed
initial experimentation with streaming over UDP/TCP
1 parent 27177e4 commit 48e1e8d

File tree

4 files changed

+279
-118
lines changed

4 files changed

+279
-118
lines changed

‎ESP/lib/src/io/camera/cameraHandler.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ void CameraHandler::setupCameraPinout() {
1616
// 16500000 optimal freq on ESP32-CAM (default)
1717
// 20000000 max freq on ESP32-CAM
1818
// 24000000 optimal freq on ESP32-S3
19-
int xclk_freq_hz = DEFAULT_XCLK_FREQ_HZ;
19+
// int xclk_freq_hz = DEFAULT_XCLK_FREQ_HZ;
20+
int xclk_freq_hz = USB_DEFAULT_XCLK_FREQ_HZ;
2021

2122
#if CONFIG_CAMERA_MODULE_ESP_EYE
2223
/* IO13, IO14 is designed for JTAG by default,

‎ESP/lib/src/network/api/webserverHandler.cpp‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,10 @@ void APIServer::setup() {
2828
"^\\%s\\/([a-zA-Z0-9]+)\\/command\\/([a-zA-Z0-9]+)$",
2929
this->api_url.c_str());
3030
log_d("API URL: %s", buffer);
31-
server.on(buffer, 0b01111111, [&](AsyncWebServerRequest* request) {
32-
handleRequest(request);
33-
});
31+
server.on(buffer, 0b01111111,
32+
[&](AsyncWebServerRequest* request) { handleRequest(request); });
3433
#ifndef SIM_ENABLED
35-
//this->_authRequired = true;
34+
// this->_authRequired = true;
3635
#endif // SIM_ENABLED
3736
beginOTA();
3837
server.begin();

‎ESP/lib/src/network/stream/streamServer.cpp‎

Lines changed: 237 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -1,114 +1,250 @@
11
#include "streamServer.hpp"
22

3-
constexpr static const char *STREAM_CONTENT_TYPE = "multipart/x-mixed-replace;boundary=" PART_BOUNDARY;
4-
constexpr static const char *STREAM_BOUNDARY = "\r\n--" PART_BOUNDARY "\r\n";
5-
constexpr static const char *STREAM_PART = "Content-Type: image/jpeg\r\nContent-Length: %u\r\nX-Timestamp: %d.%06d\r\n\r\n";
3+
constexpr static const char* STREAM_CONTENT_TYPE =
4+
"multipart/x-mixed-replace;boundary=" PART_BOUNDARY;
5+
constexpr static const char* STREAM_BOUNDARY = "\r\n--" PART_BOUNDARY "\r\n";
6+
constexpr static const char* STREAM_PART =
7+
"Content-Type: image/jpeg\r\nContent-Length: %u\r\nX-Timestamp: "
8+
"%d.%06d\r\n\r\n";
69

7-
esp_err_t StreamHelpers::stream(httpd_req_t *req)
8-
{
9-
long last_request_time = 0;
10-
camera_fb_t *fb = NULL;
11-
struct timeval _timestamp;
10+
esp_err_t StreamHelpers::stream(httpd_req_t* req) {
11+
long last_request_time = 0;
12+
camera_fb_t* fb = NULL;
13+
struct timeval _timestamp;
1214

13-
esp_err_t res = ESP_OK;
15+
esp_err_t res = ESP_OK;
1416

15-
size_t _jpg_buf_len = 0;
16-
uint8_t *_jpg_buf = NULL;
17+
size_t _jpg_buf_len = 0;
18+
uint8_t* _jpg_buf = NULL;
1719

18-
char *part_buf[256];
20+
char* part_buf[256];
1921

20-
static int64_t last_frame = 0;
21-
if (!last_frame)
22-
last_frame = esp_timer_get_time();
22+
static int64_t last_frame = 0;
23+
if (!last_frame)
24+
last_frame = esp_timer_get_time();
2325

24-
res = httpd_resp_set_type(req, STREAM_CONTENT_TYPE);
25-
if (res != ESP_OK)
26-
return res;
27-
28-
httpd_resp_set_hdr(req, "Access-Control-Allow-Origin", "*");
29-
httpd_resp_set_hdr(req, "X-Framerate", "60");
30-
31-
while (true)
32-
{
33-
fb = esp_camera_fb_get();
34-
if (!fb)
35-
{
36-
log_e("Camera capture failed with response: %s", esp_err_to_name(res));
37-
res = ESP_FAIL;
38-
}
39-
else
40-
{
41-
_timestamp.tv_sec = fb->timestamp.tv_sec;
42-
_timestamp.tv_usec = fb->timestamp.tv_usec;
43-
_jpg_buf_len = fb->len;
44-
_jpg_buf = fb->buf;
45-
}
46-
if (res == ESP_OK)
47-
res = httpd_resp_send_chunk(req, STREAM_BOUNDARY, strlen(STREAM_BOUNDARY));
48-
if (res == ESP_OK)
49-
{
50-
size_t hlen = snprintf((char *)part_buf, 128, STREAM_PART, _jpg_buf_len, _timestamp.tv_sec, _timestamp.tv_usec);
51-
res = httpd_resp_send_chunk(req, (const char *)part_buf, hlen);
52-
}
53-
if (res == ESP_OK)
54-
res = httpd_resp_send_chunk(req, (const char *)_jpg_buf, _jpg_buf_len);
55-
if (fb)
56-
{
57-
esp_camera_fb_return(fb);
58-
fb = NULL;
59-
_jpg_buf = NULL;
60-
}
61-
else if (_jpg_buf)
62-
{
63-
free(_jpg_buf);
64-
_jpg_buf = NULL;
65-
}
66-
if (res != ESP_OK)
67-
break;
68-
long request_end = millis();
69-
long latency = (request_end - last_request_time);
70-
last_request_time = request_end;
71-
log_d("Size: %uKB, Time: %ums (%ifps)\n", _jpg_buf_len / 1024, latency, 1000 / latency);
72-
}
73-
last_frame = 0;
26+
res = httpd_resp_set_type(req, STREAM_CONTENT_TYPE);
27+
if (res != ESP_OK)
7428
return res;
29+
30+
httpd_resp_set_hdr(req, "Access-Control-Allow-Origin", "*");
31+
httpd_resp_set_hdr(req, "X-Framerate", "60");
32+
33+
while (true) {
34+
fb = esp_camera_fb_get();
35+
if (!fb) {
36+
log_e("Camera capture failed with response: %s", esp_err_to_name(res));
37+
res = ESP_FAIL;
38+
} else {
39+
_timestamp.tv_sec = fb->timestamp.tv_sec;
40+
_timestamp.tv_usec = fb->timestamp.tv_usec;
41+
_jpg_buf_len = fb->len;
42+
_jpg_buf = fb->buf;
43+
}
44+
if (res == ESP_OK)
45+
res =
46+
httpd_resp_send_chunk(req, STREAM_BOUNDARY, strlen(STREAM_BOUNDARY));
47+
if (res == ESP_OK) {
48+
size_t hlen = snprintf((char*)part_buf, 128, STREAM_PART, _jpg_buf_len,
49+
_timestamp.tv_sec, _timestamp.tv_usec);
50+
res = httpd_resp_send_chunk(req, (const char*)part_buf, hlen);
51+
}
52+
if (res == ESP_OK)
53+
res = httpd_resp_send_chunk(req, (const char*)_jpg_buf, _jpg_buf_len);
54+
if (fb) {
55+
esp_camera_fb_return(fb);
56+
fb = NULL;
57+
_jpg_buf = NULL;
58+
} else if (_jpg_buf) {
59+
free(_jpg_buf);
60+
_jpg_buf = NULL;
61+
}
62+
if (res != ESP_OK)
63+
break;
64+
long request_end = millis();
65+
long latency = (request_end - last_request_time);
66+
last_request_time = request_end;
67+
log_d("Size: %uKB, Time: %ums (%ifps)\n", _jpg_buf_len / 1024, latency,
68+
1000 / latency);
69+
}
70+
last_frame = 0;
71+
return res;
7572
}
7673

77-
StreamServer::StreamServer(const int STREAM_PORT) : STREAM_SERVER_PORT(STREAM_PORT) {}
78-
79-
int StreamServer::startStreamServer()
80-
{
81-
// WRITE_PERI_REG(RTC_CNTL_BROWN_OUT_REG, 0); //! Turn-off the 'brownout detector'
82-
httpd_config_t config = HTTPD_DEFAULT_CONFIG();
83-
config.stack_size = 20480;
84-
config.max_uri_handlers = 1;
85-
config.server_port = this->STREAM_SERVER_PORT;
86-
config.ctrl_port = this->STREAM_SERVER_PORT;
87-
config.stack_size = 20480;
88-
89-
httpd_uri_t stream_page = {
90-
.uri = "/",
91-
.method = HTTP_GET,
92-
.handler = &StreamHelpers::stream,
93-
.user_ctx = nullptr};
94-
95-
int status = httpd_start(&camera_stream, &config);
96-
97-
if (status != ESP_OK)
98-
return -1;
99-
else
100-
{
101-
httpd_register_uri_handler(camera_stream, &stream_page);
102-
Serial.println("Stream server initialized");
103-
switch (wifiStateManager.getCurrentState())
104-
{
105-
case WiFiState_e::WiFiState_ADHOC:
106-
Serial.printf("\n\rThe stream is under: http://%s:%i\n\r", WiFi.softAPIP().toString().c_str(), this->STREAM_SERVER_PORT);
107-
break;
108-
default:
109-
Serial.printf("\n\rThe stream is under: http://%s:%i\n\r", WiFi.localIP().toString().c_str(), this->STREAM_SERVER_PORT);
110-
break;
111-
}
112-
return 0;
74+
StreamServer::StreamServer(const int STREAM_PORT)
75+
: STREAM_SERVER_PORT(STREAM_PORT) {
76+
memcpy(initial_packet_buffer, ETVR_HEADER, sizeof(ETVR_HEADER));
77+
}
78+
79+
int StreamServer::startStreamServer() {
80+
httpd_config_t config = HTTPD_DEFAULT_CONFIG();
81+
config.stack_size = 20480;
82+
config.max_uri_handlers = 1;
83+
config.server_port = this->STREAM_SERVER_PORT;
84+
config.ctrl_port = this->STREAM_SERVER_PORT;
85+
config.stack_size = 20480;
86+
87+
httpd_uri_t stream_page = {.uri = "/",
88+
.method = HTTP_GET,
89+
.handler = &StreamHelpers::stream,
90+
.user_ctx = nullptr};
91+
92+
int status = httpd_start(&camera_stream, &config);
93+
94+
if (status != ESP_OK)
95+
return -1;
96+
else {
97+
httpd_register_uri_handler(camera_stream, &stream_page);
98+
Serial.println("Stream server initialized");
99+
switch (wifiStateManager.getCurrentState()) {
100+
case WiFiState_e::WiFiState_ADHOC:
101+
Serial.printf("\n\rThe stream is under: http://%s:%i\n\r",
102+
WiFi.softAPIP().toString().c_str(),
103+
this->STREAM_SERVER_PORT);
104+
break;
105+
default:
106+
Serial.printf("\n\rThe stream is under: http://%s:%i\n\r",
107+
WiFi.localIP().toString().c_str(),
108+
this->STREAM_SERVER_PORT);
109+
break;
113110
}
111+
return 0;
112+
}
113+
}
114+
115+
bool StreamServer::startUDPStreamServer() {
116+
socket = AsyncUDP();
117+
return socket.listen(this->STREAM_SERVER_PORT + 1);
118+
}
119+
120+
void StreamServer::sendUDPFrame() {
121+
///////////////////////////////////////////////////////
122+
///////////////////////////////////////////////////////
123+
// TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO
124+
//
125+
// ADD PROTOCOL VERSION
126+
//
127+
// TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO
128+
///////////////////////////////////////////////////////
129+
///////////////////////////////////////////////////////
130+
131+
if (!last_frame)
132+
last_frame = esp_timer_get_time();
133+
134+
size_t len = 0;
135+
uint8_t* buf = NULL;
136+
137+
auto fb = esp_camera_fb_get();
138+
if (fb) {
139+
len = fb->len;
140+
buf = fb->buf;
141+
} else {
142+
log_e("Camera capture failed");
143+
return;
144+
}
145+
146+
// we're sending the initial header with the total number of chunks first
147+
// we can then later detect new frame with the header packets
148+
uint8_t totalChunks = (len + CHUNK_SIZE - 1) / CHUNK_SIZE;
149+
initial_packet_buffer[sizeof(ETVR_HEADER)] = totalChunks;
150+
socket.broadcastTo(initial_packet_buffer, sizeof(initial_packet_buffer),
151+
this->STREAM_SERVER_PORT);
152+
153+
for (uint8_t i = 0; i < totalChunks; i++) {
154+
auto offset = i * CHUNK_SIZE;
155+
// we need to make sure we don't overread
156+
auto chunkSize = (offset + CHUNK_SIZE <= len) ? CHUNK_SIZE : len - offset;
157+
packet_buffer[0] = static_cast<uint8_t>(i);
158+
// since this is a pointer, we can just add an offset to it, with a
159+
// chunksize to read and we're done
160+
memcpy(packet_buffer + 1, buf + offset, chunkSize);
161+
socket.broadcastTo(packet_buffer, chunkSize + 1, this->STREAM_SERVER_PORT);
162+
}
163+
164+
if (fb) {
165+
esp_camera_fb_return(fb);
166+
fb = NULL;
167+
buf = NULL;
168+
} else if (buf) {
169+
free(buf);
170+
buf = NULL;
171+
}
172+
173+
long request_end = millis();
174+
long latency = request_end - last_request_time;
175+
last_request_time = request_end;
176+
177+
log_d("Size: %uKB, Time: %ums (%ifps) chunks: %u \n", len / 1024, latency,
178+
1000 / latency, totalChunks);
114179
}
180+
181+
bool StreamServer::startTCPStreamServer() {
182+
tcp_server = new AsyncServer(this->STREAM_SERVER_PORT);
183+
tcp_server->onClient(
184+
[this](void* arg, AsyncClient* client) {
185+
this->handleNewTCPClient(arg, client);
186+
},
187+
tcp_server);
188+
189+
tcp_server->begin();
190+
return true;
191+
}
192+
193+
void StreamServer::handleNewTCPClient(void* arg, AsyncClient* client) {
194+
Serial.printf("Client connecting with ip: %s \n\r",
195+
client->remoteIP().toString().c_str());
196+
197+
if (this->tcp_connected_client == nullptr) {
198+
this->tcp_connected_client = client;
199+
200+
this->tcp_connected_client->onError(
201+
[this](void* arg, AsyncClient* client, int8_t error) {
202+
Serial.printf("\n connection error %s from client %s \n",
203+
client->errorToString(error),
204+
client->remoteIP().toString().c_str());
205+
206+
this->tcp_connected_client = nullptr;
207+
});
208+
209+
this->tcp_connected_client->onDisconnect(
210+
[this](void* arg, AsyncClient* client) {
211+
this->tcp_connected_client = nullptr;
212+
});
213+
214+
Serial.println("Client connected!");
215+
} else {
216+
client->close();
217+
Serial.println("Rejected client, only one connection allowed!");
218+
}
219+
}
220+
221+
void StreamServer::sendTCPFrame() {
222+
if (this->tcp_connected_client == nullptr ||
223+
!this->tcp_connected_client->connected()) {
224+
return;
225+
}
226+
227+
if (!last_frame)
228+
last_frame = esp_timer_get_time();
229+
230+
auto fb = esp_camera_fb_get();
231+
if (!fb) {
232+
log_e("Camera capture failed");
233+
return;
234+
}
235+
236+
size_t len = fb->len;
237+
238+
this->tcp_connected_client->write(ETVR_HEADER_BYTES, 4);
239+
this->tcp_connected_client->write((const char*)fb->buf, fb->len);
240+
241+
if (fb) {
242+
esp_camera_fb_return(fb);
243+
}
244+
245+
long request_end = millis();
246+
long latency = request_end - last_request_time;
247+
last_request_time = request_end;
248+
log_d("Size: %uKB, Time: %ums (%ifps)\n", len / 1024, latency,
249+
1000 / latency);
250+
}

0 commit comments

Comments
 (0)