Skip to content

Commit c553d8a

Browse files
talajtalaj
committed
Chunked image encoding
1 parent 13005d4 commit c553d8a

File tree

4 files changed

+282
-0
lines changed

4 files changed

+282
-0
lines changed

‎src/callback_streambuf.hpp‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#ifndef CALLBACK_STREAMBUF_H
2+
#define CALLBACK_STREAMBUF_H
3+
4+
#include <array>
5+
#include <iostream>
6+
7+
template<typename Callback, std::size_t Size, class Char = char>
8+
class callback_streambuf : public std::basic_streambuf<Char>
9+
{
10+
public:
11+
using base_type = std::streambuf;
12+
using char_type = typename base_type::char_type;
13+
using int_type = typename base_type::int_type;
14+
15+
callback_streambuf(Callback callback)
16+
: buffer_{},
17+
callback_(callback)
18+
{
19+
base_type::setp(buffer_.begin(), buffer_.end());
20+
}
21+
22+
protected:
23+
int sync()
24+
{
25+
bool ok = callback_(base_type::pbase(),
26+
base_type::pptr() - base_type::pbase());
27+
base_type::setp(buffer_.begin(), buffer_.end());
28+
return ok ? 0 : -1;
29+
}
30+
31+
int_type overflow(int_type ch)
32+
{
33+
int ret = sync();
34+
if (ch == base_type::traits_type::eof())
35+
{
36+
return ch;
37+
}
38+
base_type::sputc(ch);
39+
return ret == 0 ? 0 : base_type::traits_type::eof();
40+
}
41+
42+
private:
43+
std::array<char_type, Size> buffer_;
44+
Callback callback_;
45+
};
46+
47+
#endif

‎src/mapnik_image.cpp‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
#include "mapnik_color.hpp"
2929

3030
#include "utils.hpp"
31+
#include "callback_streambuf.hpp"
3132

3233
#include "agg_rasterizer_scanline_aa.h"
3334
#include "agg_basics.h"
@@ -82,6 +83,7 @@ void Image::Initialize(v8::Local<v8::Object> target) {
8283
Nan::SetPrototypeMethod(lcons, "setPixel", setPixel);
8384
Nan::SetPrototypeMethod(lcons, "encodeSync", encodeSync);
8485
Nan::SetPrototypeMethod(lcons, "encode", encode);
86+
Nan::SetPrototypeMethod(lcons, "encodeChunked", encodeChunked);
8587
Nan::SetPrototypeMethod(lcons, "view", view);
8688
Nan::SetPrototypeMethod(lcons, "saveSync", saveSync);
8789
Nan::SetPrototypeMethod(lcons, "save", save);
@@ -3736,6 +3738,188 @@ void Image::EIO_AfterEncode(uv_work_t* req)
37363738
delete closure;
37373739
}
37383740

3741+
struct chunked_encode_image_baton_t
3742+
{
3743+
encode_image_baton_t image_baton;
3744+
3745+
uv_async_t async;
3746+
uv_mutex_t mutex;
3747+
3748+
using char_type = char;
3749+
using buffer_type = std::vector<char_type>;
3750+
using buffer_list_type = std::vector<buffer_type>;
3751+
buffer_list_type buffers;
3752+
3753+
chunked_encode_image_baton_t()
3754+
{
3755+
if (int e = uv_async_init(uv_default_loop(), &async, yield_chunk))
3756+
{
3757+
throw std::runtime_error("Cannot create async handler");
3758+
}
3759+
3760+
if (int e = uv_mutex_init(&mutex))
3761+
{
3762+
uv_close(reinterpret_cast<uv_handle_t*>(&async), NULL);
3763+
throw std::runtime_error("Cannot create mutex");
3764+
}
3765+
3766+
async.data = this;
3767+
}
3768+
3769+
~chunked_encode_image_baton_t()
3770+
{
3771+
uv_close(reinterpret_cast<uv_handle_t*>(&async), NULL);
3772+
uv_mutex_destroy(&mutex);
3773+
}
3774+
3775+
template<class Char, class Size>
3776+
bool operator()(const Char* buffer, Size size)
3777+
{
3778+
uv_mutex_lock(&mutex);
3779+
buffers.emplace_back(buffer, buffer + size);
3780+
uv_mutex_unlock(&mutex);
3781+
3782+
if (int e = uv_async_send(&async))
3783+
{
3784+
image_baton.error = true;
3785+
image_baton.error_name = "Cannot call async callback";
3786+
return false;
3787+
}
3788+
3789+
return true;
3790+
}
3791+
3792+
static void yield_chunk(uv_async_t* handle)
3793+
{
3794+
using closure_type = chunked_encode_image_baton_t;
3795+
closure_type & closure = *reinterpret_cast<closure_type*>(handle->data);
3796+
3797+
buffer_list_type local_buffers;
3798+
3799+
uv_mutex_lock(&closure.mutex);
3800+
closure.buffers.swap(local_buffers);
3801+
uv_mutex_unlock(&closure.mutex);
3802+
3803+
Nan::HandleScope scope;
3804+
3805+
for (auto const & buffer : local_buffers)
3806+
{
3807+
v8::Local<v8::Value> argv[2] = {
3808+
Nan::Null(), Nan::CopyBuffer(buffer.data(),
3809+
buffer.size()).ToLocalChecked() };
3810+
Nan::MakeCallback(Nan::GetCurrentContext()->Global(),
3811+
Nan::New(closure.image_baton.cb), 2, argv);
3812+
}
3813+
}
3814+
};
3815+
3816+
void Image::EIO_EncodeChunked(uv_work_t* work)
3817+
{
3818+
using closure_type = chunked_encode_image_baton_t;
3819+
closure_type & closure = *reinterpret_cast<closure_type*>(work->data);
3820+
try
3821+
{
3822+
callback_streambuf<closure_type&, 1024> streambuf(closure);
3823+
std::ostream stream(&streambuf);
3824+
save_to_stream(*closure.image_baton.im->this_,
3825+
stream, closure.image_baton.format);
3826+
stream.flush();
3827+
3828+
uv_mutex_lock(&closure.mutex);
3829+
closure.buffers.emplace_back(); // Signalize end of stream
3830+
uv_mutex_unlock(&closure.mutex);
3831+
}
3832+
catch (std::exception const& ex)
3833+
{
3834+
closure.image_baton.error = true;
3835+
closure.image_baton.error_name = ex.what();
3836+
}
3837+
}
3838+
3839+
void Image::EIO_AfterEncodeChunked(uv_work_t* work, int status)
3840+
{
3841+
using closure_type = chunked_encode_image_baton_t;
3842+
closure_type & closure = *reinterpret_cast<closure_type*>(work->data);
3843+
3844+
if (closure.image_baton.error)
3845+
{
3846+
v8::Local<v8::Value> argv[1] = {
3847+
Nan::Error(closure.image_baton.error_name.c_str()) };
3848+
Nan::MakeCallback(Nan::GetCurrentContext()->Global(),
3849+
Nan::New(closure.image_baton.cb), 1, argv);
3850+
}
3851+
else
3852+
{
3853+
closure_type::yield_chunk(&closure.async);
3854+
}
3855+
3856+
closure.image_baton.im->Unref();
3857+
closure.image_baton.cb.Reset();
3858+
delete &closure;
3859+
}
3860+
3861+
NAN_METHOD(Image::encodeChunked)
3862+
{
3863+
Image* im = Nan::ObjectWrap::Unwrap<Image>(info.Holder());
3864+
3865+
std::string format = "png";
3866+
palette_ptr palette;
3867+
3868+
// accept custom format
3869+
if (info.Length() >= 1){
3870+
if (!info[0]->IsString()) {
3871+
Nan::ThrowTypeError("first arg, 'format' must be a string");
3872+
return;
3873+
}
3874+
format = TOSTR(info[0]);
3875+
}
3876+
3877+
// options hash
3878+
if (info.Length() >= 2) {
3879+
if (!info[1]->IsObject()) {
3880+
Nan::ThrowTypeError("optional second arg must be an options object");
3881+
return;
3882+
}
3883+
3884+
v8::Local<v8::Object> options = info[1].As<v8::Object>();
3885+
3886+
if (options->Has(Nan::New("palette").ToLocalChecked()))
3887+
{
3888+
v8::Local<v8::Value> format_opt = options->Get(Nan::New("palette").ToLocalChecked());
3889+
if (!format_opt->IsObject()) {
3890+
Nan::ThrowTypeError("'palette' must be an object");
3891+
return;
3892+
}
3893+
3894+
v8::Local<v8::Object> obj = format_opt.As<v8::Object>();
3895+
if (obj->IsNull() || obj->IsUndefined() || !Nan::New(Palette::constructor)->HasInstance(obj)) {
3896+
Nan::ThrowTypeError("mapnik.Palette expected as second arg");
3897+
return;
3898+
}
3899+
3900+
palette = Nan::ObjectWrap::Unwrap<Palette>(obj)->palette();
3901+
}
3902+
}
3903+
3904+
// ensure callback is a function
3905+
v8::Local<v8::Value> callback = info[info.Length() - 1];
3906+
if (!callback->IsFunction()) {
3907+
Nan::ThrowTypeError("last argument must be a callback function");
3908+
return;
3909+
}
3910+
3911+
chunked_encode_image_baton_t *closure = new chunked_encode_image_baton_t();
3912+
closure->image_baton.request.data = closure;
3913+
closure->image_baton.im = im;
3914+
closure->image_baton.format = format;
3915+
closure->image_baton.palette = palette;
3916+
closure->image_baton.error = false;
3917+
closure->image_baton.cb.Reset(callback.As<v8::Function>());
3918+
3919+
uv_queue_work(uv_default_loop(), &closure->image_baton.request, EIO_EncodeChunked, EIO_AfterEncodeChunked);
3920+
im->Ref();
3921+
}
3922+
37393923
/**
37403924
* Get a constrained view of this image given x, y, width, height parameters.
37413925
* @memberof Image

‎src/mapnik_image.hpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,11 @@ class Image: public Nan::ObjectWrap {
2929
static NAN_METHOD(setPixel);
3030
static NAN_METHOD(encodeSync);
3131
static NAN_METHOD(encode);
32+
static NAN_METHOD(encodeChunked);
3233
static void EIO_Encode(uv_work_t* req);
3334
static void EIO_AfterEncode(uv_work_t* req);
35+
static void EIO_EncodeChunked(uv_work_t* req);
36+
static void EIO_AfterEncodeChunked(uv_work_t* req, int status);
3437

3538
static NAN_METHOD(setGrayScaleToAlpha);
3639
static NAN_METHOD(width);

‎test/image.test.js‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,54 @@ describe('mapnik.Image ', function() {
146146
assert.equal(im.encodeSync().length, im2.encodeSync().length);
147147
});
148148

149+
it('should be able to encode by chunks', function(done) {
150+
var im = new mapnik.Image(256, 256);
151+
assert.ok(im instanceof mapnik.Image);
152+
153+
assert.equal(im.width(), 256);
154+
assert.equal(im.height(), 256);
155+
assert.throws(function() { im.view(); });
156+
157+
var actual_length = 0;
158+
var chunk_count = 0;
159+
160+
im.encodeChunked('png32', {}, function(err, result) {
161+
if (err) throw err;
162+
if (result.length == 0) {
163+
assert.equal(1, chunk_count);
164+
assert.equal(im.encodeSync('png32').length, actual_length);
165+
done();
166+
} else {
167+
chunk_count++;
168+
actual_length += result.length;
169+
}
170+
});
171+
});
172+
173+
it('should be able to encode by chunks - multiple chunks', function(done) {
174+
var im = new mapnik.Image.openSync('./test/data/images/sat_image.png');
175+
assert.ok(im instanceof mapnik.Image);
176+
177+
assert.equal(im.width(), 75);
178+
assert.equal(im.height(), 75);
179+
assert.throws(function() { im.view(); });
180+
181+
var actual_length = 0;
182+
var chunk_count = 0;
183+
184+
im.encodeChunked('png32', {}, function(err, result) {
185+
if (err) throw err;
186+
if (result.length == 0) {
187+
assert.equal(16, chunk_count);
188+
assert.equal(im.encodeSync('png32').length, actual_length);
189+
done();
190+
} else {
191+
chunk_count++;
192+
actual_length += result.length;
193+
}
194+
});
195+
});
196+
149197
it('should be able to open via byte stream', function(done) {
150198
var im = new mapnik.Image(256, 256);
151199
// png

0 commit comments

Comments
 (0)