Skip to content

Commit c3d27c1

Browse files
randomsjihoonl
randoms
authored and
jihoonl
committed
Fix vp8 in kinetic add vp9 and h264 support (#52)
* fix vp8 in kinetic * add h264 and vp9 support
1 parent 913ad9e commit c3d27c1

File tree

8 files changed

+210
-34
lines changed

8 files changed

+210
-34
lines changed

‎CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ add_executable(${PROJECT_NAME}
4343
src/image_streamer.cpp
4444
src/libav_streamer.cpp
4545
src/vp8_streamer.cpp
46+
src/h264_streamer.cpp
47+
src/vp9_streamer.cpp
4648
src/multipart_stream.cpp
4749
src/ros_compressed_streamer.cpp
4850
src/jpeg_streamers.cpp)

‎include/web_video_server/h264_streamer.h

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#ifndef H264_STREAMERS_H_
2+
#define H264_STREAMERS_H_
3+
4+
#include <image_transport/image_transport.h>
5+
#include "web_video_server/libav_streamer.h"
6+
#include "async_web_server_cpp/http_request.hpp"
7+
#include "async_web_server_cpp/http_connection.hpp"
8+
9+
namespace web_video_server
10+
{
11+
12+
class H264Streamer : public LibavStreamer
13+
{
14+
public:
15+
H264Streamer(const async_web_server_cpp::HttpRequest& request, async_web_server_cpp::HttpConnectionPtr connection,
16+
ros::NodeHandle& nh);
17+
~H264Streamer();
18+
protected:
19+
virtual void initializeEncoder();
20+
std::string preset_;
21+
};
22+
23+
class H264StreamerType : public LibavStreamerType
24+
{
25+
public:
26+
H264StreamerType();
27+
virtual boost::shared_ptr<ImageStreamer> create_streamer(const async_web_server_cpp::HttpRequest& request,
28+
async_web_server_cpp::HttpConnectionPtr connection,
29+
ros::NodeHandle& nh);
30+
};
31+
32+
}
33+
34+
#endif
35+

‎include/web_video_server/libav_streamer.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ class LibavStreamer : public ImageTransportImageStreamer
4040
AVCodecContext* codec_context_;
4141
AVStream* video_stream_;
4242

43+
AVDictionary* opt_; // container format options
44+
4345
private:
4446
AVFrame* frame_;
4547
struct SwsContext* sws_context_;
@@ -53,6 +55,8 @@ class LibavStreamer : public ImageTransportImageStreamer
5355
int qmin_;
5456
int qmax_;
5557
int gop_;
58+
59+
uint8_t* io_buffer_; // custom IO buffer
5660
};
5761

5862
class LibavStreamerType : public ImageStreamerType

‎include/web_video_server/vp9_streamer.h

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#ifndef VP9_STREAMERS_H_
2+
#define VP9_STREAMERS_H_
3+
4+
#include <image_transport/image_transport.h>
5+
#include "web_video_server/libav_streamer.h"
6+
#include "async_web_server_cpp/http_request.hpp"
7+
#include "async_web_server_cpp/http_connection.hpp"
8+
9+
namespace web_video_server
10+
{
11+
12+
class Vp9Streamer : public LibavStreamer
13+
{
14+
public:
15+
Vp9Streamer(const async_web_server_cpp::HttpRequest& request, async_web_server_cpp::HttpConnectionPtr connection,
16+
ros::NodeHandle& nh);
17+
~Vp9Streamer();
18+
protected:
19+
virtual void initializeEncoder();
20+
};
21+
22+
class Vp9StreamerType : public LibavStreamerType
23+
{
24+
public:
25+
Vp9StreamerType();
26+
virtual boost::shared_ptr<ImageStreamer> create_streamer(const async_web_server_cpp::HttpRequest& request,
27+
async_web_server_cpp::HttpConnectionPtr connection,
28+
ros::NodeHandle& nh);
29+
};
30+
31+
}
32+
33+
#endif

‎src/h264_streamer.cpp

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#include "web_video_server/h264_streamer.h"
2+
3+
namespace web_video_server
4+
{
5+
6+
H264Streamer::H264Streamer(const async_web_server_cpp::HttpRequest& request,
7+
async_web_server_cpp::HttpConnectionPtr connection, ros::NodeHandle& nh) :
8+
LibavStreamer(request, connection, nh, "mp4", "libx264", "video/mp4")
9+
{
10+
/* possible quality presets:
11+
* ultrafast, superfast, veryfast, faster, fast, medium, slow, slower, veryslow, placebo
12+
* no latency improvements observed with ultrafast instead of medium
13+
*/
14+
preset_ = request.get_query_param_value_or_default("preset", "ultrafast");
15+
}
16+
17+
H264Streamer::~H264Streamer()
18+
{
19+
}
20+
21+
void H264Streamer::initializeEncoder()
22+
{
23+
av_opt_set(codec_context_->priv_data, "preset", preset_.c_str(), 0);
24+
av_opt_set(codec_context_->priv_data, "tune", "zerolatency", 0);
25+
av_opt_set_int(codec_context_->priv_data, "crf", 20, 0);
26+
av_opt_set_int(codec_context_->priv_data, "bufsize", 100, 0);
27+
av_opt_set_int(codec_context_->priv_data, "keyint", 30, 0);
28+
av_opt_set_int(codec_context_->priv_data, "g", 1, 0);
29+
30+
// container format options
31+
if (!strcmp(format_context_->oformat->name, "mp4")) {
32+
// set up mp4 for streaming (instead of seekable file output)
33+
av_dict_set(&opt_, "movflags", "+frag_keyframe+empty_moov+faststart", 0);
34+
}
35+
}
36+
37+
H264StreamerType::H264StreamerType() :
38+
LibavStreamerType("mp4", "libx264", "video/mp4")
39+
{
40+
}
41+
42+
boost::shared_ptr<ImageStreamer> H264StreamerType::create_streamer(const async_web_server_cpp::HttpRequest& request,
43+
async_web_server_cpp::HttpConnectionPtr connection,
44+
ros::NodeHandle& nh)
45+
{
46+
return boost::shared_ptr<ImageStreamer>(new H264Streamer(request, connection, nh));
47+
}
48+
49+
}

‎src/libav_streamer.cpp

Lines changed: 45 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ LibavStreamer::LibavStreamer(const async_web_server_cpp::HttpRequest &request,
5050
const std::string &content_type) :
5151
ImageTransportImageStreamer(request, connection, nh), output_format_(0), format_context_(0), codec_(0), codec_context_(0), video_stream_(
5252
0), frame_(0), sws_context_(0), first_image_timestamp_(0), format_name_(
53-
format_name), codec_name_(codec_name), content_type_(content_type)
53+
format_name), codec_name_(codec_name), content_type_(content_type), opt_(0), io_buffer_(0)
5454
{
5555

5656
bitrate_ = request.get_query_param_value_or_default<int>("bitrate", 100000);
@@ -75,12 +75,26 @@ LibavStreamer::~LibavStreamer()
7575
av_frame_free(&frame_);
7676
#endif
7777
}
78+
if (io_buffer_)
79+
delete io_buffer_;
80+
if (format_context_->pb)
81+
av_free(format_context_->pb);
7882
if (format_context_)
7983
avformat_free_context(format_context_);
8084
if (sws_context_)
8185
sws_freeContext(sws_context_);
8286
}
8387

88+
// output callback for ffmpeg IO context
89+
static int dispatch_output_packet(void* opaque, uint8_t* buffer, int buffer_size)
90+
{
91+
async_web_server_cpp::HttpConnectionPtr connection = *((async_web_server_cpp::HttpConnectionPtr*) opaque);
92+
std::vector<uint8_t> encoded_frame;
93+
encoded_frame.assign(buffer, buffer + buffer_size);
94+
connection->write_and_clear(encoded_frame);
95+
return 0; // TODO: can this fail?
96+
}
97+
8498
void LibavStreamer::initialize(const cv::Mat &img)
8599
{
86100
// Load format
@@ -102,6 +116,22 @@ void LibavStreamer::initialize(const cv::Mat &img)
102116
}
103117
format_context_->oformat = output_format_;
104118

119+
// Set up custom IO callback.
120+
size_t io_buffer_size = 3 * 1024; // 3M seen elsewhere and adjudged good
121+
io_buffer_ = new unsigned char[io_buffer_size];
122+
AVIOContext* io_ctx = avio_alloc_context(io_buffer_, io_buffer_size, AVIO_FLAG_WRITE, &connection_, NULL, dispatch_output_packet, NULL);
123+
if (!io_ctx)
124+
{
125+
async_web_server_cpp::HttpReply::stock_reply(async_web_server_cpp::HttpReply::internal_server_error)(request_,
126+
connection_,
127+
NULL, NULL);
128+
throw std::runtime_error("Error setting up IO context");
129+
}
130+
io_ctx->seekable = 0; // no seeking, it's a stream
131+
format_context_->pb = io_ctx;
132+
output_format_->flags |= AVFMT_FLAG_CUSTOM_IO;
133+
output_format_->flags |= AVFMT_NOFILE;
134+
105135
// Load codec
106136
if (codec_name_.empty()) // use default codec if none specified
107137
codec_ = avcodec_find_encoder(output_format_->video_codec);
@@ -127,7 +157,7 @@ void LibavStreamer::initialize(const cv::Mat &img)
127157
// Set options
128158
avcodec_get_context_defaults3(codec_context_, codec_);
129159

130-
codec_context_->codec_id = output_format_->video_codec;
160+
codec_context_->codec_id = codec_->id;
131161
codec_context_->bit_rate = bitrate_;
132162

133163
codec_context_->width = output_width_;
@@ -171,7 +201,9 @@ void LibavStreamer::initialize(const cv::Mat &img)
171201
av_image_alloc(frame_->data, frame_->linesize, output_width_, output_height_,
172202
codec_context_->pix_fmt, 1);
173203

174-
204+
frame_->width = output_width_;
205+
frame_->height = output_height_;
206+
frame_->format = codec_context_->pix_fmt;
175207
output_format_->flags |= AVFMT_NOFILE;
176208

177209
// Generate header
@@ -182,24 +214,6 @@ void LibavStreamer::initialize(const cv::Mat &img)
182214
av_dict_set(&format_context_->metadata, "author", "ROS web_video_server", 0);
183215
av_dict_set(&format_context_->metadata, "title", topic_.c_str(), 0);
184216

185-
if (avio_open_dyn_buf(&format_context_->pb) >= 0)
186-
{
187-
if (avformat_write_header(format_context_, NULL) < 0)
188-
{
189-
async_web_server_cpp::HttpReply::stock_reply(async_web_server_cpp::HttpReply::internal_server_error)(request_,
190-
connection_,
191-
NULL, NULL);
192-
throw std::runtime_error("Error openning dynamic buffer");
193-
}
194-
header_size = avio_close_dyn_buf(format_context_->pb, &header_raw_buffer);
195-
196-
// copy header buffer to vector
197-
header_buffer.resize(header_size);
198-
memcpy(&header_buffer[0], header_raw_buffer, header_size);
199-
200-
av_free(header_raw_buffer);
201-
}
202-
203217
// Send response headers
204218
async_web_server_cpp::HttpReply::builder(async_web_server_cpp::HttpReply::ok).header("Connection", "close").header(
205219
"Server", "web_video_server").header("Cache-Control",
@@ -208,7 +222,13 @@ void LibavStreamer::initialize(const cv::Mat &img)
208222
"Content-type", content_type_).header("Access-Control-Allow-Origin", "*").write(connection_);
209223

210224
// Send video stream header
211-
connection_->write_and_clear(header_buffer);
225+
if (avformat_write_header(format_context_, &opt_) < 0)
226+
{
227+
async_web_server_cpp::HttpReply::stock_reply(async_web_server_cpp::HttpReply::internal_server_error)(request_,
228+
connection_,
229+
NULL, NULL);
230+
throw std::runtime_error("Error openning dynamic buffer");
231+
}
212232
}
213233

214234
void LibavStreamer::initializeEncoder()
@@ -235,7 +255,7 @@ void LibavStreamer::sendImage(const cv::Mat &img, const ros::Time &time)
235255
#else
236256
AVFrame *raw_frame = av_frame_alloc();
237257
av_image_fill_arrays(raw_frame->data, raw_frame->linesize,
238-
img.data, input_coding_format, output_width_, output_height_, 0);
258+
img.data, input_coding_format, output_width_, output_height_, 1);
239259
#endif
240260

241261

@@ -310,18 +330,9 @@ void LibavStreamer::sendImage(const cv::Mat &img, const ros::Time &time)
310330

311331
pkt.stream_index = video_stream_->index;
312332

313-
if (avio_open_dyn_buf(&format_context_->pb) >= 0)
333+
if (av_write_frame(format_context_, &pkt))
314334
{
315-
if (av_write_frame(format_context_, &pkt))
316-
{
317-
throw std::runtime_error("Error when writing frame");
318-
}
319-
size = avio_close_dyn_buf(format_context_->pb, &output_buf);
320-
321-
encoded_frame.resize(size);
322-
memcpy(&encoded_frame[0], output_buf, size);
323-
324-
av_free(output_buf);
335+
throw std::runtime_error("Error when writing frame");
325336
}
326337
}
327338
else

‎src/vp9_streamer.cpp

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#include "web_video_server/vp9_streamer.h"
2+
3+
namespace web_video_server
4+
{
5+
6+
Vp9Streamer::Vp9Streamer(const async_web_server_cpp::HttpRequest& request,
7+
async_web_server_cpp::HttpConnectionPtr connection, ros::NodeHandle& nh) :
8+
LibavStreamer(request, connection, nh, "webm", "libvpx-vp9", "video/webm")
9+
{
10+
}
11+
Vp9Streamer::~Vp9Streamer()
12+
{
13+
}
14+
15+
void Vp9Streamer::initializeEncoder()
16+
{
17+
18+
// codec options set up to provide somehow reasonable performance in cost of poor quality
19+
// should be updated as soon as VP9 encoding matures
20+
av_opt_set_int(codec_context_->priv_data, "pass", 1, 0);
21+
av_opt_set_int(codec_context_->priv_data, "speed", 8, 0);
22+
av_opt_set_int(codec_context_->priv_data, "cpu-used", 4, 0); // 8 is max
23+
av_opt_set_int(codec_context_->priv_data, "crf", 20, 0); // 0..63 (higher is lower quality)
24+
}
25+
26+
Vp9StreamerType::Vp9StreamerType() :
27+
LibavStreamerType("webm", "libvpx-vp9", "video/webm")
28+
{
29+
}
30+
31+
boost::shared_ptr<ImageStreamer> Vp9StreamerType::create_streamer(const async_web_server_cpp::HttpRequest& request,
32+
async_web_server_cpp::HttpConnectionPtr connection,
33+
ros::NodeHandle& nh)
34+
{
35+
return boost::shared_ptr<ImageStreamer>(new Vp9Streamer(request, connection, nh));
36+
}
37+
38+
}

‎src/web_video_server.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
#include "web_video_server/ros_compressed_streamer.h"
1010
#include "web_video_server/jpeg_streamers.h"
1111
#include "web_video_server/vp8_streamer.h"
12+
#include "web_video_server/h264_streamer.h"
13+
#include "web_video_server/vp9_streamer.h"
1214
#include "async_web_server_cpp/http_reply.hpp"
1315

1416
namespace web_video_server
@@ -57,6 +59,8 @@ WebVideoServer::WebVideoServer(ros::NodeHandle &nh, ros::NodeHandle &private_nh)
5759
stream_types_["mjpeg"] = boost::shared_ptr<ImageStreamerType>(new MjpegStreamerType());
5860
stream_types_["ros_compressed"] = boost::shared_ptr<ImageStreamerType>(new RosCompressedStreamerType());
5961
stream_types_["vp8"] = boost::shared_ptr<ImageStreamerType>(new Vp8StreamerType());
62+
stream_types_["h264"] = boost::shared_ptr<ImageStreamerType>(new H264StreamerType());
63+
stream_types_["vp9"] = boost::shared_ptr<ImageStreamerType>(new Vp9StreamerType());
6064

6165
handler_group_.addHandlerForPath("/", boost::bind(&WebVideoServer::handle_list_streams, this, _1, _2, _3, _4));
6266
handler_group_.addHandlerForPath("/stream", boost::bind(&WebVideoServer::handle_stream, this, _1, _2, _3, _4));

0 commit comments

Comments
 (0)