Skip to content

Commit 69a079b

Browse files
kojidevkojidev
committed
add zstd_stream_init, zstd_stream_add
1 parent 58ffb27 commit 69a079b

File tree

7 files changed

+141
-0
lines changed

7 files changed

+141
-0
lines changed

‎builtin-functions/kphp-full/_functions.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,10 @@ final class DeflateContext {
110110
private function __construct() ::: DeflateContext;
111111
}
112112

113+
final class ZstdCtx {
114+
private function __construct() ::: ZstdCtx;
115+
}
116+
113117
/** @var mixed $_SERVER */
114118
global $_SERVER;
115119
/** @var mixed $_GET */
@@ -1453,6 +1457,8 @@ function is_kphp_job_workers_enabled() ::: bool;
14531457
function get_job_workers_number() ::: int;
14541458

14551459
// zstd api
1460+
function zstd_stream_init(array $options = []) ::: ?ZstdCtx;
1461+
function zstd_stream_add(ZstdCtx $context, string $data, bool $end) ::: string | false;
14561462
function zstd_compress(string $data, int $level = 3) ::: string | false;
14571463
function zstd_uncompress(string $data) ::: string | false;
14581464
function zstd_compress_dict(string $data, string $dict) ::: string | false;

‎builtin-functions/kphp-light/string.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
<?php
22

3+
final class ZstdCtx {
4+
private function __construct() ::: ZstdCtx;
5+
}
6+
37
function addcslashes ($str ::: string, $what ::: string) ::: string;
48
function addslashes ($str ::: string) ::: string;
59

@@ -130,6 +134,10 @@ function base_convert ($number ::: string, $frombase ::: int, $tobase ::: int) :
130134
function zstd_compress(string $data, int $level = 3) ::: string | false;
131135
/** @kphp-extern-func-info generate-stub */
132136
function zstd_uncompress(string $data) ::: string | false;
137+
/** @kphp-extern-func-info generate-stub */
138+
function zstd_stream_init(array $options = []) ::: ?ZstdCtx;
139+
/** @kphp-extern-func-info generate-stub */
140+
function zstd_stream_add(ZstdCtx $context, string $data, bool $end) ::: string | false;
133141

134142
/** @kphp-extern-func-info generate-stub */
135143
function getimagesize ($name ::: string) ::: mixed;

‎runtime/zstd.cpp‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,3 +151,62 @@ Optional<string> f$zstd_compress_dict(const string &data, const string &dict) no
151151
Optional<string> f$zstd_uncompress_dict(const string &data, const string &dict) noexcept {
152152
return zstd_uncompress_impl(data, dict);
153153
}
154+
155+
class_instance<C$ZstdCtx> f$zstd_stream_init(const array<mixed> &options) noexcept {
156+
ZSTD_CCtxPtr zctx{ZSTD_createCCtx_advanced(make_custom_alloc())};
157+
if (!zctx) {
158+
php_warning("zstd_compress: can not create context");
159+
return {};
160+
}
161+
162+
int lbound = ZSTD_minCLevel();
163+
int ubound = ZSTD_maxCLevel();
164+
int level = ZSTD_CLEVEL_DEFAULT;
165+
for (const auto &option : options) {
166+
if (!option.is_string_key()) {
167+
php_warning("zstd_stream_init() : unsupported option");
168+
return {};
169+
}
170+
if (option.get_string_key() == string("level")) {
171+
mixed value = option.get_value();
172+
if (value.is_int() && value.as_int() >= lbound && value.as_int() <= ubound) {
173+
level = value.as_int();
174+
} else {
175+
php_warning("zstd_stream_init() : option %s should be number between %d..%d", option.get_string_key().c_str(), lbound, ubound);
176+
return {};
177+
}
178+
}
179+
}
180+
181+
size_t result = ZSTD_CCtx_setParameter(zctx.get(), ZSTD_c_compressionLevel, static_cast<int>(level));
182+
if (ZSTD_isError(result)) {
183+
php_warning("zstd_stream_init: can not init context: %s", ZSTD_getErrorName(result));
184+
return {};
185+
}
186+
187+
class_instance<C$ZstdCtx> ctx;
188+
ctx.alloc();
189+
ctx.get()->ctx = zctx.get();
190+
191+
return ctx;
192+
}
193+
194+
Optional<string> f$zstd_stream_add(const class_instance<C$ZstdCtx> &ctx, const string &data, bool end) noexcept {
195+
php_assert(ZSTD_CStreamOutSize() <= StringLibContext::STATIC_BUFFER_LENGTH);
196+
ZSTD_outBuffer out{StringLibContext::get().static_buf.data(), StringLibContext::STATIC_BUFFER_LENGTH, 0};
197+
ZSTD_inBuffer in{data.c_str(), data.size(), 0};
198+
199+
size_t result;
200+
ZSTD_EndDirective mode = end ? ZSTD_e_end : ZSTD_e_flush;
201+
string encoded_string;
202+
do {
203+
result = ZSTD_compressStream2(ctx->ctx, &out, &in, mode);
204+
if (ZSTD_isError(result)) {
205+
php_warning("zstd_compress: got zstd stream compression error: %s", ZSTD_getErrorName(result));
206+
return false;
207+
}
208+
encoded_string.append(static_cast<char *>(out.dst), out.pos);
209+
out.pos = 0;
210+
} while (result);
211+
return encoded_string;
212+
}

‎runtime/zstd.h‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,10 @@
44

55
#pragma once
66

7+
#include <zstd.h>
8+
79
#include "runtime-common/core/runtime-core.h"
10+
#include "runtime/dummy-visitor-methods.h"
811

912
constexpr int DEFAULT_COMPRESS_LEVEL = 3;
1013

@@ -15,3 +18,19 @@ Optional<string> f$zstd_uncompress(const string &data) noexcept;
1518
Optional<string> f$zstd_compress_dict(const string &data, const string &dict) noexcept;
1619

1720
Optional<string> f$zstd_uncompress_dict(const string &data, const string &dict) noexcept;
21+
22+
struct C$ZstdCtx : public refcountable_php_classes<C$ZstdCtx>, private DummyVisitorMethods {
23+
C$ZstdCtx() = default;
24+
using DummyVisitorMethods::accept;
25+
26+
~C$ZstdCtx() {
27+
dl::CriticalSectionGuard guard;
28+
// TODO
29+
}
30+
31+
ZSTD_CStream *ctx;
32+
};
33+
34+
class_instance<C$ZstdCtx> f$zstd_stream_init(const array<mixed> &options = {}) noexcept;
35+
36+
Optional<string> f$zstd_stream_add(const class_instance<C$ZstdCtx> &ctx, const string &data, bool end) noexcept;

‎tests/cpp/runtime/zstd-test.cpp‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22

33
#include <zstd.h>
44

5+
#include "runtime-common/core/runtime-core.h"
56
#include "runtime-common/stdlib/string/string-context.h"
7+
#include "runtime/zstd.h"
68

79
TEST(zstd_test, test_bounds) {
810
ASSERT_LE(ZSTD_CStreamOutSize(), StringLibContext::STATIC_BUFFER_LENGTH);
@@ -11,3 +13,20 @@ TEST(zstd_test, test_bounds) {
1113
ASSERT_LE(ZSTD_DStreamOutSize(), StringLibContext::STATIC_BUFFER_LENGTH);
1214
ASSERT_LE(ZSTD_DStreamInSize(), StringLibContext::STATIC_BUFFER_LENGTH);
1315
}
16+
17+
TEST(zstd_test, test_compress_decompress) {
18+
const char *encoded_string = f$zstd_uncompress(f$zstd_compress(string("sample string"), 3).val()).val().c_str();
19+
ASSERT_STREQ("sample string", encoded_string);
20+
}
21+
22+
TEST(zstd_test, test_stream_compression) {
23+
class_instance<C$ZstdCtx> ctx = f$zstd_stream_init();
24+
string first_chunk = string("consequat omittam consequat suavitate vocibus contentiones dolores causae appetere eius dicit vis penatibus postulant egestas sanctus mollis verear faucibus salutatus");
25+
string second_chunk = string("tacimates imperdiet augue equidem oporteat mentitum sapien aliquam primis assueverit pretium maiorum constituto novum taciti expetenda mauris an euripidis quam");
26+
string first_chunk_compressed = f$zstd_stream_add(ctx, first_chunk, false).val();
27+
string second_chunk_compressed = f$zstd_stream_add(ctx, second_chunk, true).val();
28+
29+
string whole = first_chunk_compressed.copy_and_make_not_shared().append(second_chunk_compressed);
30+
ASSERT_STREQ(f$zstd_uncompress(first_chunk_compressed).val().c_str(), first_chunk.c_str());
31+
ASSERT_STREQ(f$zstd_uncompress(whole).val().c_str(), first_chunk.append(second_chunk).c_str());
32+
}

‎tests/python/tests/http_server/php/index.php‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,18 @@ public function work(string $output) {
178178

179179
break;
180180
}
181+
} else if ($_SERVER["PHP_SELF"] === "/test_streaming_zstd") {
182+
header('Content-Encoding: zstd');
183+
184+
$ctx = zstd_stream_init();
185+
186+
$chunk_1 = "justo mea orcidignissim corrumpit iuvaret suscipit ocurreret donec turpis eget eros sonet habitasse vix reprimique litora porro nostra nec vidisse pretium facilis referrentur in fusce sententiae appareat quidam repudiandae doctus periculis hac quem suspendisse facilis habemus nibh nisi ex dignissim sociis cursus habitant habitant taciti auctor graeco no facilis persius erat venenatis mandamus graeci falli ipsum elit tation dico nam donec luctus alia vim mucius urna ceteros dicta ad eius postea sea atomorum tale eros comprehensam detracto scripserit non natoque ligula velit eleifend iaculis donec ridiculus ignota quaestio equidem appareat consequat eros percipit mei molestie appetere patrioque dicta mnesarchum<br />";
187+
$chunk_1 = (string)zstd_stream_add($ctx, $chunk_1, false);
188+
print $chunk_1;
189+
190+
$chunk_2 = "libris error cum mollis consectetur usu adversarium vulputate te tractatos gloriatur hinc massa an ponderum delenit persius partiendo elementum per nullam omnesque dolorum sodales lorem commodo magnis maecenas nobis consequat curabitur vidisse potenti eloquentiam risus pretium corrumpit postulant dictumst conclusionemque affert ceteros atqui mutat tantas splendide docendi dolor euismod placerat rhoncus sonet aliquid movet nonumes eius qui ei velit nisl mauris vocent doming maximus ignota definiebas has duo posuere repudiandae nobis ex prompta dicit conubia dolore sed neglegentur graeci ludus suscipiantur cetero suspendisse phasellus molestiae vitae labores ferri equidem autem corrumpit brute finibus tellus risus petentium partiendo epicuri aliquet hendrerit";
191+
$chunk_2 = (string)zstd_stream_add($ctx, $chunk_2, true);
192+
print $chunk_2;
181193
} else if ($_SERVER["PHP_SELF"] === "/test_ignore_user_abort") {
182194
register_shutdown_function('shutdown_function');
183195
/** @var I */

‎tests/python/tests/http_server/test_streaming_zstd.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import zstandard
2+
3+
from python.lib.testcase import KphpServerAutoTestCase
4+
5+
class TestStreamingZstd(KphpServerAutoTestCase):
6+
def test_streaming_zstd(self):
7+
cmp = zstandard.ZstdDecompressor()
8+
9+
url = "/test_streaming_zstd"
10+
response = self.kphp_server.http_get(url, headers={
11+
"Host": "localhost",
12+
"Accept-Encoding": "zstd"
13+
})
14+
self.assertEqual(response.headers["Content-Encoding"], "zstd")
15+
stream_reader = cmp.stream_reader(response.content)
16+
uncompressed_body = stream_reader.read().decode('utf-8')
17+
stream_reader.close()
18+
self.assertEqual(uncompressed_body, """justo mea orcidignissim corrumpit iuvaret suscipit ocurreret donec turpis eget eros sonet habitasse vix reprimique litora porro nostra nec vidisse pretium facilis referrentur in fusce sententiae appareat quidam repudiandae doctus periculis hac quem suspendisse facilis habemus nibh nisi ex dignissim sociis cursus habitant habitant taciti auctor graeco no facilis persius erat venenatis mandamus graeci falli ipsum elit tation dico nam donec luctus alia vim mucius urna ceteros dicta ad eius postea sea atomorum tale eros comprehensam detracto scripserit non natoque ligula velit eleifend iaculis donec ridiculus ignota quaestio equidem appareat consequat eros percipit mei molestie appetere patrioque dicta mnesarchum<br />libris error cum mollis consectetur usu adversarium vulputate te tractatos gloriatur hinc massa an ponderum delenit persius partiendo elementum per nullam omnesque dolorum sodales lorem commodo magnis maecenas nobis consequat curabitur vidisse potenti eloquentiam risus pretium corrumpit postulant dictumst conclusionemque affert ceteros atqui mutat tantas splendide docendi dolor euismod placerat rhoncus sonet aliquid movet nonumes eius qui ei velit nisl mauris vocent doming maximus ignota definiebas has duo posuere repudiandae nobis ex prompta dicit conubia dolore sed neglegentur graeci ludus suscipiantur cetero suspendisse phasellus molestiae vitae labores ferri equidem autem corrumpit brute finibus tellus risus petentium partiendo epicuri aliquet hendrerit""")

0 commit comments

Comments
 (0)