Skip to content

Commit 31a1c8e

Browse files
vlad-dargelvlad-dargel
authored
Merge pull request #1899 from RedisInsight/feature/RI-4141_decompressors
Feature/ri 4141 decompressors
2 parents def4850 + b599bd0 commit 31a1c8e

File tree

10 files changed

+146
-11
lines changed

10 files changed

+146
-11
lines changed

‎package.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@
128128
"@types/jsonpath": "^0.2.0",
129129
"@types/lodash": "^4.14.171",
130130
"@types/node": "14.14.10",
131+
"@types/pako": "^2.0.0",
131132
"@types/react": "^18.0.20",
132133
"@types/react-dom": "^18.0.5",
133134
"@types/react-redux": "^7.1.12",
@@ -222,6 +223,7 @@
222223
"@reduxjs/toolkit": "^1.6.2",
223224
"@stablelib/snappy": "^1.0.2",
224225
"axios": "^0.25.0",
226+
"brotli-unicode": "^1.0.2",
225227
"buffer": "^6.0.3",
226228
"classnames": "^2.3.1",
227229
"connection-string": "^4.3.2",
@@ -243,6 +245,7 @@
243245
"jsonpath": "^1.1.1",
244246
"lodash": "^4.17.21",
245247
"lz4js": "^0.2.0",
248+
"pako": "^2.1.0",
246249
"php-serialize": "^4.0.2",
247250
"rawproto": "^0.7.6",
248251
"react": "^18.2.0",

‎redisinsight/api/src/modules/database/entities/database.entity.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ export enum Compressor {
3131
ZSTD = 'ZSTD',
3232
LZ4 = 'LZ4',
3333
SNAPPY = 'SNAPPY',
34+
Brotli = 'Brotli',
35+
PHPGZCompress = 'PHPGZCompress',
3436
}
3537

3638
@Entity('database_instance')

‎redisinsight/ui/src/constants/keys.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -186,15 +186,17 @@ export enum KeyValueCompressor {
186186
ZSTD = 'ZSTD',
187187
LZ4 = 'LZ4',
188188
SNAPPY = 'SNAPPY',
189-
// Brotli = 'Brotli',
190-
// PHPGZCompress = 'PHPGZCompress',
189+
Brotli = 'Brotli',
190+
PHPGZCompress = 'PHPGZCompress',
191191
}
192192

193193
export const COMPRESSOR_MAGIC_SYMBOLS: ICompressorMagicSymbols = Object.freeze({
194194
[KeyValueCompressor.GZIP]: '31,139', // 1f 8b hex
195195
[KeyValueCompressor.ZSTD]: '40,181,47,253', // 28 b5 2f fd hex
196196
[KeyValueCompressor.LZ4]: '4,34,77,24', // 04 22 4d 18 hex
197197
[KeyValueCompressor.SNAPPY]: '', // no magic symbols
198+
[KeyValueCompressor.Brotli]: '', // no magic symbols
199+
[KeyValueCompressor.PHPGZCompress]: '', // no magic symbols
198200
})
199201

200202
export type ICompressorMagicSymbols = {

‎redisinsight/ui/src/pages/home/components/AddInstanceForm/InstanceForm/form-components/DbCompressor.tsx‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,14 @@ const DbCompressor = (props: Props) => {
4646
value: KeyValueCompressor.ZSTD,
4747
inputDisplay: 'ZSTD',
4848
},
49+
{
50+
value: KeyValueCompressor.Brotli,
51+
inputDisplay: 'Brotli'
52+
},
53+
{
54+
value: KeyValueCompressor.PHPGZCompress,
55+
inputDisplay: 'PHP GZCompress'
56+
},
4957
]
5058

5159
const handleChangeDbCompressorCheckbox = (e: ChangeEvent<HTMLInputElement>): void => {

‎redisinsight/ui/src/utils/decompressors/decompressors.ts‎

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,23 @@
11
import { forIn } from 'lodash'
22
import { unzip } from 'gzip-js'
33
import { decompress as decompressFzstd } from 'fzstd'
4+
// @ts-ignore
45
import { decompress as decompressLz4 } from 'lz4js'
56
import { decompress as decompressSnappy } from '@stablelib/snappy'
7+
// @ts-ignore
8+
import { decompress as decompressBrotli } from 'brotli-unicode/js'
9+
import { inflate } from 'pako'
610
import { COMPRESSOR_MAGIC_SYMBOLS, ICompressorMagicSymbols, KeyValueCompressor } from 'uiSrc/constants'
711
import { RedisResponseBuffer, RedisString } from 'uiSrc/slices/interfaces'
8-
import { anyToBuffer, isEqualBuffers, Nullable } from 'uiSrc/utils'
12+
import { anyToBuffer, bufferToString, bufferToUint8Array, isEqualBuffers, Nullable } from 'uiSrc/utils'
913

1014
const decompressingBuffer = (
1115
reply: RedisResponseBuffer,
1216
compressorInit: Nullable<KeyValueCompressor> = null,
1317
): { value: RedisString, compressor: Nullable<KeyValueCompressor>, isCompressed: boolean } => {
1418
const compressorByValue: Nullable<KeyValueCompressor> = getCompressor(reply)
1519
const compressor = compressorInit === compressorByValue
16-
|| (!compressorByValue && compressorInit === KeyValueCompressor.SNAPPY)
20+
|| (!compressorByValue && compressorInit)
1721
? compressorInit
1822
: null
1923

@@ -56,6 +60,26 @@ const decompressingBuffer = (
5660
isCompressed: !isEqualBuffers(value, reply),
5761
}
5862
}
63+
case KeyValueCompressor.Brotli: {
64+
const value = anyToBuffer(decompressBrotli(bufferToString(reply)))
65+
66+
return {
67+
value,
68+
compressor,
69+
isCompressed: !isEqualBuffers(value, reply),
70+
}
71+
}
72+
case KeyValueCompressor.PHPGZCompress: {
73+
const decompressedValue = inflate(bufferToUint8Array(reply))
74+
if (!decompressedValue) return { value: reply, compressor: null, isCompressed: false }
75+
76+
const value = anyToBuffer(decompressedValue)
77+
return {
78+
value,
79+
compressor,
80+
isCompressed: !isEqualBuffers(value, reply),
81+
}
82+
}
5983
default: {
6084
return { value: reply, compressor: null, isCompressed: false }
6185
}
@@ -73,10 +97,9 @@ const getCompressor = (reply: RedisResponseBuffer): Nullable<KeyValueCompressor>
7397
COMPRESSOR_MAGIC_SYMBOLS,
7498
(magicSymbols: string, compressorName: string) => {
7599
if (
76-
replyStart.startsWith(magicSymbols)
100+
magicSymbols
101+
&& replyStart.startsWith(magicSymbols)
77102
&& replyStart.length > magicSymbols.length
78-
// no magic symbols for SNAPPY
79-
&& compressorName !== KeyValueCompressor.SNAPPY
80103
) {
81104
compressor = compressorName as KeyValueCompressor
82105
return false // break loop

‎redisinsight/ui/src/utils/formatters/bufferFormatters.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,8 @@ const ASCIIToBuffer = (strInit: string) => {
106106
return anyToBuffer(Array.from(Buffer.from(result, 'hex')))
107107
}
108108

109-
const bufferToUTF8 = (reply: RedisResponseBuffer): string => decoder.decode(new Uint8Array(reply.data))
109+
const bufferToUint8Array = (reply: RedisResponseBuffer): Uint8Array => new Uint8Array(reply.data)
110+
const bufferToUTF8 = (reply: RedisResponseBuffer): string => decoder.decode(bufferToUint8Array(reply))
110111

111112
const UintArrayToString = (reply: UintArray): string => decoder.decode(new Uint8Array(reply))
112113

@@ -138,7 +139,7 @@ const hexToBuffer = (data: string): RedisResponseBuffer => {
138139
}
139140

140141
const bufferToJava = (reply: RedisResponseBuffer) => {
141-
const stream = new ObjectInputStream(new Uint8Array(reply.data))
142+
const stream = new ObjectInputStream(bufferToUint8Array(reply))
142143
const decoded = stream.readObject()
143144
const { fields } = decoded
144145
const fieldsArray = Array.from(fields, ([key, value]) => ({ [key]: value }))
@@ -172,6 +173,7 @@ export {
172173
ASCIIToBuffer,
173174
isEqualBuffers,
174175
stringToBuffer,
176+
bufferToUint8Array,
175177
bufferToString,
176178
UintArrayToString,
177179
hexToBuffer,

‎redisinsight/ui/src/utils/tests/decompressors/constants.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,9 @@ export const LZ4_COMPRESSED_VALUE_2 = [4, 34, 77, 24, 64, 112, 223, 1, 0, 0, 128
1818

1919
export const SNAPPY_COMPRESSED_VALUE_1 = [1, 0, 49]
2020
export const SNAPPY_COMPRESSED_VALUE_2 = [1, 0, 50]
21+
22+
export const BROTLI_COMPRESSED_VALUE_1 = [49, 65, 76, 231, 187, 141, 68]
23+
export const BROTLI_COMPRESSED_VALUE_2 = [49, 65, 76, 231, 191, 141, 68]
24+
25+
export const PHPGZCOMPRESS_COMPRESSED_VALUE_1 = [120, 156, 51, 4, 0, 0, 50, 0, 50]
26+
export const PHPGZCOMPRESS_COMPRESSED_VALUE_2 = [120, 156, 51, 2, 0, 0, 51, 0, 51]

‎redisinsight/ui/src/utils/tests/decompressors/decompressors.spec.ts‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@ import {
1616
LZ4_COMPRESSED_VALUE_2,
1717
SNAPPY_COMPRESSED_VALUE_2,
1818
SNAPPY_COMPRESSED_VALUE_1,
19+
BROTLI_COMPRESSED_VALUE_1,
20+
BROTLI_COMPRESSED_VALUE_2,
21+
PHPGZCOMPRESS_COMPRESSED_VALUE_1,
22+
PHPGZCOMPRESS_COMPRESSED_VALUE_2,
1923
} from './constants'
2024

2125
const defaultValues = [
@@ -111,6 +115,38 @@ const defaultValues = [
111115
outputStr: DECOMPRESSED_VALUE_STR_1,
112116
isCompressed: false,
113117
},
118+
{
119+
input: BROTLI_COMPRESSED_VALUE_1,
120+
compressor: KeyValueCompressor.Brotli,
121+
compressorInit: KeyValueCompressor.Brotli,
122+
output: DECOMPRESSED_VALUE_1,
123+
outputStr: DECOMPRESSED_VALUE_STR_1,
124+
isCompressed: true,
125+
},
126+
{
127+
input: BROTLI_COMPRESSED_VALUE_2,
128+
compressor: KeyValueCompressor.Brotli,
129+
compressorInit: KeyValueCompressor.Brotli,
130+
output: DECOMPRESSED_VALUE_2,
131+
outputStr: DECOMPRESSED_VALUE_STR_2,
132+
isCompressed: true,
133+
},
134+
{
135+
input: PHPGZCOMPRESS_COMPRESSED_VALUE_1,
136+
compressor: KeyValueCompressor.PHPGZCompress,
137+
compressorInit: KeyValueCompressor.PHPGZCompress,
138+
output: DECOMPRESSED_VALUE_1,
139+
outputStr: DECOMPRESSED_VALUE_STR_1,
140+
isCompressed: true,
141+
},
142+
{
143+
input: PHPGZCOMPRESS_COMPRESSED_VALUE_2,
144+
compressor: KeyValueCompressor.PHPGZCompress,
145+
compressorInit: KeyValueCompressor.PHPGZCompress,
146+
output: DECOMPRESSED_VALUE_2,
147+
outputStr: DECOMPRESSED_VALUE_STR_2,
148+
isCompressed: true,
149+
},
114150
].map((value) => ({
115151
...value,
116152
input: anyToBuffer(value.input)
@@ -121,7 +157,10 @@ describe('getCompressor', () => {
121157
let expected = compressorByValue || compressor
122158

123159
// SNAPPY doesn't have magic symbols
124-
if (compressor === KeyValueCompressor.SNAPPY) {
160+
if (compressor === KeyValueCompressor.SNAPPY
161+
|| compressor === KeyValueCompressor.Brotli
162+
|| compressor === KeyValueCompressor.PHPGZCompress
163+
) {
125164
expected = null
126165
}
127166

‎redisinsight/ui/src/utils/tests/formatters/bufferFormatters.spec.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ import {
1111
bufferToHex,
1212
bufferToBinary,
1313
binaryToBuffer,
14-
bufferToJava
14+
bufferToJava,
15+
bufferToUint8Array,
1516
} from 'uiSrc/utils'
1617

1718
const defaultValues = [
@@ -141,3 +142,9 @@ describe('bufferToJava', () => {
141142
expect(bufferToJava(input)).toEqual(expected)
142143
})
143144
})
145+
146+
describe('bufferToUint8Array', () => {
147+
test.each(javaValues)('%o', ({ uint8Array }) => {
148+
expect(bufferToUint8Array(anyToBuffer(uint8Array))).toEqual(new Uint8Array(uint8Array))
149+
})
150+
})

‎yarn.lock‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2659,6 +2659,11 @@
26592659
resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.43.tgz#555e5a743f76b6b897d47f945305b618525ddbe6"
26602660
integrity sha512-GqWykok+3uocgfAJM8imbozrqLnPyTrpFlrryURQlw1EesPUCx5XxTiucWDSFF9/NUEXDuD4bnvHm8xfVGWTpQ==
26612661

2662+
"@types/node@^17.0.40":
2663+
version "17.0.45"
2664+
resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.45.tgz#2c0fafd78705e7a18b7906b5201a522719dc5190"
2665+
integrity sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==
2666+
26622667
"@types/normalize-package-data@^2.4.0":
26632668
version "2.4.1"
26642669
resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301"
@@ -2669,6 +2674,11 @@
26692674
resolved "https://registry.yarnpkg.com/@types/numeral/-/numeral-0.0.28.tgz#e43928f0bda10b169b6f7ecf99e3ddf836b8ebe4"
26702675
integrity sha512-Sjsy10w6XFHDktJJdXzBJmoondAKW+LcGpRFH+9+zXEDj0cOH8BxJuZA9vUDSMAzU1YRJlsPKmZEEiTYDlICLw==
26712676

2677+
"@types/pako@^2.0.0":
2678+
version "2.0.0"
2679+
resolved "https://registry.yarnpkg.com/@types/pako/-/pako-2.0.0.tgz#12ab4c19107528452e73ac99132c875ccd43bdfb"
2680+
integrity sha512-10+iaz93qR5WYxTo+PMifD5TSxiOtdRaxBf7INGGXMQgTCu8Z/7GYWYFUOS3q/G0nE5boj1r4FEB+WSy7s5gbA==
2681+
26722682
"@types/parse-json@^4.0.0":
26732683
version "4.0.0"
26742684
resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0"
@@ -4169,6 +4179,13 @@ balanced-match@^1.0.0:
41694179
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
41704180
integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
41714181

4182+
base-unicode@^1.0.0:
4183+
version "1.0.0"
4184+
resolved "https://registry.yarnpkg.com/base-unicode/-/base-unicode-1.0.0.tgz#44b61fd4460b18f6d47ae6f5ea95a45b16a4885a"
4185+
integrity sha512-9y/0iVN7k3NyH0E4oYD/1qZceUVE180YcRRqhsQVNzazPcuRjr67SjUzl/wdhLaVmzjIMEjE5pDH7xb6X9kBFA==
4186+
dependencies:
4187+
buffer "^6.0.3"
4188+
41724189
base64-arraybuffer@~1.0.1:
41734190
version "1.0.1"
41744191
resolved "https://registry.yarnpkg.com/base64-arraybuffer/-/base64-arraybuffer-1.0.1.tgz#87bd13525626db4a9838e00a508c2b73efcf348c"
@@ -4343,6 +4360,27 @@ brorand@^1.0.1, brorand@^1.1.0:
43434360
resolved "https://registry.yarnpkg.com/brorand/-/brorand-1.1.0.tgz#12c25efe40a45e3c323eb8675a0a0ce57b22371f"
43444361
integrity sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=
43454362

4363+
brotli-compress@^1.3.3:
4364+
version "1.3.3"
4365+
resolved "https://registry.yarnpkg.com/brotli-compress/-/brotli-compress-1.3.3.tgz#fe3b50b6234bfbc8e26a137bffff2a07e2a4efe9"
4366+
integrity sha512-cwKOmzEuKqUmRxXDdZimiNoXRRr7AQKMSubJSbYA9FXk+LTPT3fBGpHU8VZRZZctAJ5OCeXGK9PzPpZ1vD0pDA==
4367+
dependencies:
4368+
"@types/node" "^17.0.40"
4369+
brotli-wasm "1.2.0"
4370+
4371+
brotli-unicode@^1.0.2:
4372+
version "1.0.2"
4373+
resolved "https://registry.yarnpkg.com/brotli-unicode/-/brotli-unicode-1.0.2.tgz#25e53a7fdc543337768e90fd59a94b6eb55ea6cc"
4374+
integrity sha512-eqO2kauz48Raqi0I8c4WOhvfr5GAtpHhCvegHRTY00zyn6pEP8jo5nHrwtAJKO68j1ZFi4zel6JrwBz60Z5d3g==
4375+
dependencies:
4376+
base-unicode "^1.0.0"
4377+
brotli-compress "^1.3.3"
4378+
4379+
[email protected]:
4380+
version "1.2.0"
4381+
resolved "https://registry.yarnpkg.com/brotli-wasm/-/brotli-wasm-1.2.0.tgz#0f99b97b0020c8152308c277388aecf2a06b6e32"
4382+
integrity sha512-PdDi7awF36zFujZyFJb9UNrP1l+If7iCgXhLKE1SpwqFQSK2yc7w2dysOmME7p325yQaZNvae7ruzypB3YhFxA==
4383+
43464384
browser-process-hrtime@^1.0.0:
43474385
version "1.0.0"
43484386
resolved "https://registry.yarnpkg.com/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz#3c9b4b7d782c8121e56f10106d84c0d0ffc94626"
@@ -12357,6 +12395,11 @@ p-try@^2.0.0:
1235712395
resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6"
1235812396
integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==
1235912397

12398+
pako@^2.1.0:
12399+
version "2.1.0"
12400+
resolved "https://registry.yarnpkg.com/pako/-/pako-2.1.0.tgz#266cc37f98c7d883545d11335c00fbd4062c9a86"
12401+
integrity sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==
12402+
1236012403
pako@~1.0.2, pako@~1.0.5:
1236112404
version "1.0.11"
1236212405
resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf"

0 commit comments

Comments
 (0)