Skip to content

Commit 0a753d2

Browse files
SpringMTSpringMT
committed
feat: Standardized dictionary handling
Feature * Zstd.compress supports :level and :dict keyward args * Zstd.decompress supports :dict keyward args * Zstd::StreamingCompress.new supports :level and :dict keyward args * Zstd::StreamingDecompress.new supports :dict keyward args Breaking Change * Zstd.compress uses ZSTD_compressStream2 instead of ZSTD_compress * Zstd.decompress uses ZSTD_decompressDCtx instead of ZSTD_decompress Deprecation * Zstd.compress_using_dict adds deprecation warning * use Zstd.compress with :dict keyward args * Zstd.decompress_using_dict adds deprecation warning * use Zstd.decompress with :dict keyward args * Zstd.compress with level args add deprecation warning * use Zstd.compress with :level keyward args * Zstd::StreamingCompress.new with level args add deprecation warning * use Zstd::StreamingCompress.new with :level keyward args
1 parent cf9b448 commit 0a753d2

16 files changed

+262
-95
lines changed

‎benchmarks/results/city.json.gzip‎

0 Bytes
Binary file not shown.

‎examples/sinatra/Gemfile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
source "https://rubygems.org"
2+
3+
gem "sinatra"
4+
gem "rackup"
5+
gem "zstd-ruby", path: "../../"

‎examples/sinatra/Gemfile.lock‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
PATH
2+
remote: ../..
3+
specs:
4+
zstd-ruby (1.5.6.1)
5+
6+
GEM
7+
remote: https://rubygems.org/
8+
specs:
9+
base64 (0.2.0)
10+
mustermann (3.0.0)
11+
ruby2_keywords (~> 0.0.1)
12+
rack (3.0.10)
13+
rack-protection (4.0.0)
14+
base64 (>= 0.1.0)
15+
rack (>= 3.0.0, < 4)
16+
rack-session (2.0.0)
17+
rack (>= 3.0.0)
18+
rackup (2.1.0)
19+
rack (>= 3)
20+
webrick (~> 1.8)
21+
ruby2_keywords (0.0.5)
22+
sinatra (4.0.0)
23+
mustermann (~> 3.0)
24+
rack (>= 3.0.0, < 4)
25+
rack-protection (= 4.0.0)
26+
rack-session (>= 2.0.0, < 3)
27+
tilt (~> 2.0)
28+
tilt (2.3.0)
29+
webrick (1.8.1)
30+
31+
PLATFORMS
32+
arm64-darwin-21
33+
ruby
34+
35+
DEPENDENCIES
36+
rackup
37+
sinatra
38+
zstd-ruby!
39+
40+
BUNDLED WITH
41+
2.5.7

‎examples/sinatra/app.rb‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
require 'sinatra'
2+
require 'zstd-ruby'
3+
4+
get '/' do
5+
headers["Content-Encoding"] = "zstd"
6+
Zstd.compress('Hello world!')
7+
end

‎ext/zstdruby/common.h‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#ifndef ZSTD_RUBY_H
22
#define ZSTD_RUBY_H 1
33

4-
#include "ruby.h"
4+
#include <ruby.h>
55
#include "./libzstd/zstd.h"
66

77
static int convert_compression_level(VALUE compression_level_value)
@@ -12,4 +12,55 @@ static int convert_compression_level(VALUE compression_level_value)
1212
return NUM2INT(compression_level_value);
1313
}
1414

15+
static size_t zstd_compress(ZSTD_CCtx* const ctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input, ZSTD_EndDirective endOp)
16+
{
17+
return ZSTD_compressStream2(ctx, output, input, endOp);
18+
}
19+
20+
static void set_compress_params(ZSTD_CCtx* const ctx, VALUE level_from_args, VALUE kwargs)
21+
{
22+
ID kwargs_keys[2];
23+
kwargs_keys[0] = rb_intern("level");
24+
kwargs_keys[1] = rb_intern("dict");
25+
VALUE kwargs_values[2];
26+
rb_get_kwargs(kwargs, kwargs_keys, 0, 2, kwargs_values);
27+
28+
int compression_level = ZSTD_CLEVEL_DEFAULT;
29+
if (kwargs_values[0] != Qundef && kwargs_values[0] != Qnil) {
30+
compression_level = convert_compression_level(kwargs_values[0]);
31+
} else if (!NIL_P(level_from_args)) {
32+
rb_warn("`level` in args is deprecated; use keyword args `level:` instead.");
33+
compression_level = convert_compression_level(level_from_args);
34+
}
35+
ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, compression_level);
36+
37+
if (kwargs_values[1] != Qundef && kwargs_values[1] != Qnil) {
38+
char* dict_buffer = RSTRING_PTR(kwargs_values[1]);
39+
size_t dict_size = RSTRING_LEN(kwargs_values[1]);
40+
size_t load_dict_ret = ZSTD_CCtx_loadDictionary(ctx, dict_buffer, dict_size);
41+
if (ZSTD_isError(load_dict_ret)) {
42+
ZSTD_freeCCtx(ctx);
43+
rb_raise(rb_eRuntimeError, "%s", "ZSTD_CCtx_loadDictionary failed");
44+
}
45+
}
46+
}
47+
48+
static void set_decompress_params(ZSTD_DCtx* const dctx, VALUE kwargs)
49+
{
50+
ID kwargs_keys[1];
51+
kwargs_keys[0] = rb_intern("dict");
52+
VALUE kwargs_values[1];
53+
rb_get_kwargs(kwargs, kwargs_keys, 0, 1, kwargs_values);
54+
55+
if (kwargs_values[0] != Qundef && kwargs_values[0] != Qnil) {
56+
char* dict_buffer = RSTRING_PTR(kwargs_values[0]);
57+
size_t dict_size = RSTRING_LEN(kwargs_values[0]);
58+
size_t load_dict_ret = ZSTD_DCtx_loadDictionary(dctx, dict_buffer, dict_size);
59+
if (ZSTD_isError(load_dict_ret)) {
60+
ZSTD_freeDCtx(dctx);
61+
rb_raise(rb_eRuntimeError, "%s", "ZSTD_CCtx_loadDictionary failed");
62+
}
63+
}
64+
}
65+
1566
#endif /* ZSTD_RUBY_H */

‎ext/zstdruby/main.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
#include <common.h>
1+
#include "common.h"
2+
23
VALUE rb_mZstd;
34
void zstd_ruby_init(void);
45
void zstd_ruby_skippable_frame_init(void);

‎ext/zstdruby/skippable_frame.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#include <common.h>
1+
#include "common.h"
22

33
extern VALUE rb_mZstd;
44

‎ext/zstdruby/streaming_compress.c‎

Lines changed: 8 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
1-
#include <common.h>
2-
#include <streaming_compress.h>
1+
#include "common.h"
32

43
struct streaming_compress_t {
54
ZSTD_CCtx* ctx;
@@ -71,14 +70,9 @@ rb_streaming_compress_allocate(VALUE klass)
7170
static VALUE
7271
rb_streaming_compress_initialize(int argc, VALUE *argv, VALUE obj)
7372
{
73+
VALUE kwargs;
7474
VALUE compression_level_value;
75-
rb_scan_args(argc, argv, "01", &compression_level_value);
76-
int compression_level = convert_compression_level(compression_level_value);
77-
78-
ID kwargs_keys[1];
79-
kwargs_keys[0] = rb_intern("dict");
80-
VALUE kwargs_values[1];
81-
rb_get_kwargs(kwargs, kwargs_keys, 0, 1, kwargs_values);
75+
rb_scan_args(argc, argv, "01:", &compression_level_value, &kwargs);
8276

8377
struct streaming_compress_t* sc;
8478
TypedData_Get_Struct(obj, struct streaming_compress_t, &streaming_compress_type, sc);
@@ -88,15 +82,8 @@ rb_streaming_compress_initialize(int argc, VALUE *argv, VALUE obj)
8882
if (ctx == NULL) {
8983
rb_raise(rb_eRuntimeError, "%s", "ZSTD_createCCtx error");
9084
}
91-
if (kwargs_values[0] != Qundef && kwargs_values[0] != Qnil) {
92-
char* dict_buffer = RSTRING_PTR(kwargs_values[0]);
93-
size_t dict_size = RSTRING_LEN(kwargs_values[0]);
94-
size_t load_dict_ret = ZSTD_CCtx_loadDictionary(ctx, dict_buffer, dict_size);
95-
if (ZSTD_isError(load_dict_ret)) {
96-
rb_raise(rb_eRuntimeError, "%s", "ZSTD_CCtx_loadDictionary failed");
97-
}
98-
}
99-
ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, compression_level);
85+
set_compress_params(ctx, compression_level_value, kwargs);
86+
10087
sc->ctx = ctx;
10188
sc->buf = rb_str_new(NULL, buffOutSize);
10289
sc->buf_size = buffOutSize;
@@ -119,7 +106,7 @@ no_compress(struct streaming_compress_t* sc, ZSTD_EndDirective endOp)
119106
do {
120107
ZSTD_outBuffer output = { (void*)output_data, sc->buf_size, 0 };
121108

122-
size_t const ret = ZSTD_compressStream2(sc->ctx, &output, &input, endOp);
109+
size_t const ret = zstd_compress(sc->ctx, &output, &input, endOp);
123110
if (ZSTD_isError(ret)) {
124111
rb_raise(rb_eRuntimeError, "flush error error code: %s", ZSTD_getErrorName(ret));
125112
}
@@ -143,7 +130,7 @@ rb_streaming_compress_compress(VALUE obj, VALUE src)
143130
VALUE result = rb_str_new(0, 0);
144131
while (input.pos < input.size) {
145132
ZSTD_outBuffer output = { (void*)output_data, sc->buf_size, 0 };
146-
size_t const ret = ZSTD_compressStream2(sc->ctx, &output, &input, ZSTD_e_continue);
133+
size_t const ret = zstd_compress(sc->ctx, &output, &input, ZSTD_e_continue);
147134
if (ZSTD_isError(ret)) {
148135
rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(ret));
149136
}
@@ -170,7 +157,7 @@ rb_streaming_compress_write(int argc, VALUE *argv, VALUE obj)
170157
ZSTD_inBuffer input = { input_data, input_size, 0 };
171158

172159
while (input.pos < input.size) {
173-
size_t const ret = ZSTD_compressStream2(sc->ctx, &output, &input, ZSTD_e_continue);
160+
size_t const ret = zstd_compress(sc->ctx, &output, &input, ZSTD_e_continue);
174161
if (ZSTD_isError(ret)) {
175162
rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(ret));
176163
}

‎ext/zstdruby/streaming_compress.h‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎ext/zstdruby/streaming_decompress.c‎

Lines changed: 14 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
#include <common.h>
1+
#include "common.h"
22

33
struct streaming_decompress_t {
4-
ZSTD_DCtx* ctx;
4+
ZSTD_DCtx* dctx;
55
VALUE buf;
66
size_t buf_size;
77
};
@@ -21,9 +21,9 @@ static void
2121
streaming_decompress_free(void *p)
2222
{
2323
struct streaming_decompress_t *sd = p;
24-
ZSTD_DCtx* ctx = sd->ctx;
25-
if (ctx != NULL) {
26-
ZSTD_freeDCtx(ctx);
24+
ZSTD_DCtx* dctx = sd->dctx;
25+
if (dctx != NULL) {
26+
ZSTD_freeDCtx(dctx);
2727
}
2828
xfree(sd);
2929
}
@@ -61,40 +61,29 @@ rb_streaming_decompress_allocate(VALUE klass)
6161
{
6262
struct streaming_decompress_t* sd;
6363
VALUE obj = TypedData_Make_Struct(klass, struct streaming_decompress_t, &streaming_decompress_type, sd);
64-
sd->ctx = NULL;
64+
sd->dctx = NULL;
6565
sd->buf = Qnil;
6666
sd->buf_size = 0;
6767
return obj;
6868
}
6969

7070
static VALUE
71-
rb_streaming_decompress_initialize(VALUE obj)
71+
rb_streaming_decompress_initialize(int argc, VALUE *argv, VALUE obj)
7272
{
7373
VALUE kwargs;
7474
rb_scan_args(argc, argv, "00:", &kwargs);
7575

76-
ID kwargs_keys[1];
77-
kwargs_keys[0] = rb_intern("dict");
78-
VALUE kwargs_values[1];
79-
rb_get_kwargs(kwargs, kwargs_keys, 0, 1, kwargs_values);
80-
8176
struct streaming_decompress_t* sd;
8277
TypedData_Get_Struct(obj, struct streaming_decompress_t, &streaming_decompress_type, sd);
8378
size_t const buffOutSize = ZSTD_DStreamOutSize();
8479

85-
ZSTD_DCtx* ctx = ZSTD_createDCtx();
86-
if (ctx == NULL) {
80+
ZSTD_DCtx* dctx = ZSTD_createDCtx();
81+
if (dctx == NULL) {
8782
rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDCtx error");
8883
}
89-
if (kwargs_values[0] != Qundef && kwargs_values[0] != Qnil) {
90-
char* dict_buffer = RSTRING_PTR(kwargs_values[0]);
91-
size_t dict_size = RSTRING_LEN(kwargs_values[0]);
92-
size_t load_dict_ret = ZSTD_DCtx_loadDictionary(ctx, dict_buffer, dict_size);
93-
if (ZSTD_isError(load_dict_ret)) {
94-
rb_raise(rb_eRuntimeError, "%s", "ZSTD_DCtx_loadDictionary failed");
95-
}
96-
}
97-
sd->ctx = ctx;
84+
set_decompress_params(dctx, kwargs);
85+
86+
sd->dctx = dctx;
9887
sd->buf = rb_str_new(NULL, buffOutSize);
9988
sd->buf_size = buffOutSize;
10089

@@ -115,7 +104,7 @@ rb_streaming_decompress_decompress(VALUE obj, VALUE src)
115104
VALUE result = rb_str_new(0, 0);
116105
while (input.pos < input.size) {
117106
ZSTD_outBuffer output = { (void*)output_data, sd->buf_size, 0 };
118-
size_t const ret = ZSTD_decompressStream(sd->ctx, &output, &input);
107+
size_t const ret = ZSTD_decompressStream(sd->dctx, &output, &input);
119108
if (ZSTD_isError(ret)) {
120109
rb_raise(rb_eRuntimeError, "decompress error error code: %s", ZSTD_getErrorName(ret));
121110
}
@@ -130,6 +119,6 @@ zstd_ruby_streaming_decompress_init(void)
130119
{
131120
VALUE cStreamingDecompress = rb_define_class_under(rb_mZstd, "StreamingDecompress", rb_cObject);
132121
rb_define_alloc_func(cStreamingDecompress, rb_streaming_decompress_allocate);
133-
rb_define_method(cStreamingDecompress, "initialize", rb_streaming_decompress_initialize, 0);
122+
rb_define_method(cStreamingDecompress, "initialize", rb_streaming_decompress_initialize, -1);
134123
rb_define_method(cStreamingDecompress, "decompress", rb_streaming_decompress_decompress, 1);
135124
}

0 commit comments

Comments
 (0)