Skip to content

Commit 2263ab2

Browse files
mariasergeenkomariasergeenko
authored
Merge pull request #3885 from RedisInsight/e2e/feature/decompressors
add formatters
2 parents 0a82817 + fb77390 commit 2263ab2

12 files changed

+1831
-3
lines changed

‎tests/e2e/helpers/decompressors/base-decompressors-populator.ts‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
import { createClient } from 'redis';
2+
3+
export abstract class BaseDatabasePopulator {
4+
private client: ReturnType<typeof createClient>;
5+
6+
protected abstract createCompressedKeys(): Promise<void>;
7+
8+
constructor(private host: string, private port: string) {
9+
const dbConf = { port: Number.parseInt(port), host, username: 'default' };
10+
this.client = createClient(dbConf);
11+
12+
this.client.on('error', (error: string) => {
13+
throw new Error(`Redis connection error: ${error}`);
14+
});
15+
}
16+
17+
/**
18+
* Populate db with compressed keys
19+
*/
20+
public async populateDB(): Promise<void> {
21+
this.client.on('connect', async () => {
22+
console.log('Connected to Redis');
23+
try {
24+
await this.createCompressedKeys();
25+
} catch (error) {
26+
console.error('Error during key creation:', error);
27+
} finally {
28+
await this.client.quit();
29+
}
30+
});
31+
}
32+
33+
/**
34+
* create a hash
35+
* @param prefix prefix of the key name
36+
* @param values values of the hash
37+
*/
38+
protected async createHash(
39+
prefix: string,
40+
values: Buffer[]
41+
): Promise<void> {
42+
let fields: string[] = [];
43+
44+
const randomNumber = Array.from({ length: 5 }).map(() => Math.random());
45+
46+
values.forEach((value) => {
47+
const field = `${value.toString()}:${randomNumber.toString()}`;
48+
const fieldValue = `${value.toString()}:${randomNumber.toString()}`;
49+
fields.push(field, fieldValue);
50+
});
51+
52+
try {
53+
await this.client.hset(`${prefix}:hash`, ...fields);
54+
console.log(`Hash created with prefix: ${prefix}`);
55+
} catch (error) {
56+
console.error(`Error creating hash with prefix ${prefix}:`, error);
57+
throw error;
58+
}
59+
}
60+
61+
/**
62+
* create a string
63+
* @param prefix prefix of the key name
64+
* @param value values of the string
65+
*/
66+
protected async createString(prefix: string, value: Buffer): Promise<void> {
67+
this.client.set(`${prefix}:string`, value, (error: Error | null) => {
68+
if (error) {
69+
console.error(`Error saving key ${prefix}:`, error);
70+
throw error;
71+
}
72+
console.log(`Key ${prefix} successfully saved.`);
73+
});
74+
}
75+
}

‎tests/e2e/helpers/decompressors/brotli-database-populator.ts‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
import * as proto from 'protobufjs';
2+
import { pack as msgpackrPack } from 'msgpackr';
3+
import * as brotli from 'brotli-unicode';
4+
import * as fflate from 'fflate';
5+
import * as fs from 'fs';
6+
import { BaseDatabasePopulator } from './base-decompressors-populator';
7+
8+
const COMPRESSED_PREFIX = 'Comp';
9+
const BROTLI_PREFIX = 'BROTLI';
10+
11+
export class BrotliDatabasePopulator extends BaseDatabasePopulator {
12+
13+
/**
14+
* Create keys with all types of Bolti compression
15+
*/
16+
protected async createCompressedKeys(): Promise<void> {
17+
await this.createBrotliUnicodeKeys();
18+
await this.createBrotliASCIIKeys();
19+
await this.createBrotliVectorKeys();
20+
await this.createBrotliJSONKeys();
21+
await this.createBrotliPHPUnserializedJSONKeys();
22+
await this.createBrotliJavaSerializedObjectKeys();
23+
await this.createBrotliMsgpackKeys();
24+
await this.createBrotliProtobufKeys();
25+
await this.createBrotliPickleKeys();
26+
}
27+
28+
private async createBrotliUnicodeKeys() {
29+
const encoder = new TextEncoder();
30+
const prefix = `${COMPRESSED_PREFIX}:${BROTLI_PREFIX}:Unicode`;
31+
const rawValue = '漢字';
32+
const buf = encoder.encode(rawValue);
33+
const value = Buffer.from(await brotli.compress(buf));
34+
await this.createString(prefix, value);
35+
await this.createHash(prefix, [value]);
36+
}
37+
38+
private async createBrotliASCIIKeys() {
39+
const prefix = `${COMPRESSED_PREFIX}:${BROTLI_PREFIX}:ASCII`;
40+
const rawValue = '\xac\xed\x00\x05t\x0a4102';
41+
const buf = fflate.strToU8(rawValue);
42+
const value = Buffer.from(await brotli.compress(buf));
43+
await this.createString(prefix, value);
44+
await this.createHash(prefix, [value]);
45+
}
46+
47+
private async createBrotliVectorKeys() {
48+
const prefix = `${COMPRESSED_PREFIX}:${BROTLI_PREFIX}:Vector`;
49+
const rawValue = JSON.parse(fs.readFileSync('./test-data/decompressors/vector.json', 'utf8'));
50+
const buf = fflate.strToU8(rawValue);
51+
const value = Buffer.from(await brotli.compress(buf));
52+
await this.createString(prefix, value);
53+
await this.createHash(prefix, [value]);
54+
}
55+
56+
private async createBrotliJSONKeys() {
57+
const prefix = `${COMPRESSED_PREFIX}:${BROTLI_PREFIX}:JSON`;
58+
const rawValue = '{"test":"test"}';
59+
const buf = fflate.strToU8(rawValue);
60+
const value = Buffer.from(await brotli.compress(buf));
61+
await this.createString(prefix, value);
62+
await this.createHash(prefix, [value]);
63+
}
64+
65+
private async createBrotliPHPUnserializedJSONKeys() {
66+
const prefix = `${COMPRESSED_PREFIX}:${BROTLI_PREFIX}:PHP`;
67+
const rawValue = 'a:2:{i:0;s:12:"Sample array";i:1;a:2:{i:0;s:5:"Apple";i:1;s:6:"Orange";}}';
68+
const buf = fflate.strToU8(rawValue);
69+
const value = Buffer.from(await brotli.compress(buf));
70+
await this.createString(prefix, value);
71+
await this.createHash(prefix, [value]);
72+
}
73+
74+
private async createBrotliPickleKeys() {
75+
const prefix = `${COMPRESSED_PREFIX}:${BROTLI_PREFIX}:Pickle`;
76+
const rawValue = fs.readFileSync('./test-data/decompressors/pickleFile1.pickle');
77+
const value = Buffer.from(await brotli.compress(rawValue));
78+
await this.createString(prefix, value);
79+
await this.createHash(prefix, [value]);
80+
};
81+
82+
private async createBrotliJavaSerializedObjectKeys() {
83+
const prefix = `${COMPRESSED_PREFIX}:${BROTLI_PREFIX}:Java`;
84+
const rawValue = fs.readFileSync('./test-data/decompressors/test_serialised_obj.ser');
85+
const rawValue2 = fs.readFileSync('./test-data/decompressors/test_annotated_obj.ser');
86+
87+
const value = Buffer.from(await brotli.compress(rawValue));
88+
const value2 = Buffer.from(await brotli.compress(rawValue2));
89+
90+
await this.createString(prefix, value);
91+
await this.createString(prefix, value2);
92+
await this.createHash(prefix, [value,value2]);
93+
}
94+
95+
private async createBrotliMsgpackKeys(): Promise<void> {
96+
const prefix = `${COMPRESSED_PREFIX}:${BROTLI_PREFIX}:Msgpack`;
97+
const rawValue = msgpackrPack({
98+
hello: 'World',
99+
array: [1, 2],
100+
obj: {test: 'test'},
101+
boolean: false,
102+
});
103+
const value = Buffer.from(await brotli.compress(rawValue));
104+
await this.createString(prefix, value);
105+
await this.createHash(prefix, [value]);
106+
}
107+
108+
private createBrotliProtobufKeys(): Promise<void> {
109+
return new Promise((resolve, reject) => {
110+
const prefix = `${COMPRESSED_PREFIX}:${BROTLI_PREFIX}:Proto`;
111+
proto.load('./test-data/decompressors/awesome.proto', async (err, root) => {
112+
if (err || !root) {
113+
console.error('Error loading protobuf:', err);
114+
return reject(err);
115+
}
116+
117+
try {
118+
const Book = root.lookupType('com.book.BookStore');
119+
const payload = {name: 'Test name', books: {0: 'book 1', 1: 'book 2'}};
120+
const message = Book.create(payload);
121+
const rawValue = Book.encode(message).finish();
122+
123+
const value = Buffer.from(await brotli.compress(rawValue));
124+
await this.createString(prefix, value);
125+
await this.createHash(prefix, [value]);
126+
resolve();
127+
} catch (error) {
128+
reject(error);
129+
}
130+
});
131+
});
132+
}
133+
}
134+

‎tests/e2e/helpers/decompressors/gzip-database-populator.ts‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
import { pack as msgpackrPack } from 'msgpackr';
2+
import * as fs from 'fs';
3+
import * as fflate from 'fflate';
4+
import * as proto from 'protobufjs';
5+
import { BaseDatabasePopulator } from './base-decompressors-populator';
6+
7+
8+
const COMPRESSED_PREFIX = 'Comp';
9+
const GZIP_PREFIX = 'GZIP';
10+
11+
export class GzipDatabasePopulator extends BaseDatabasePopulator {
12+
13+
/**
14+
* Create keys with all types of Gzip compression
15+
*/
16+
protected async createCompressedKeys(): Promise<void> {
17+
await this.createGZIPUnicodeKeys();
18+
await this.createGZIPASCIIKeys();
19+
await this.createGZIPJSONKeys();
20+
await this.createGZIPPHPUnserializedJSONKeys();
21+
await this.createGZIPMsgpackKeys();
22+
await this.createGZIPProtobufKeys();
23+
await this.createGZIPPickleKeys();
24+
await this.createGZIPJavaSerializedObjectKeys();
25+
await this.createGZIPVectorKeys();
26+
}
27+
28+
private async createGZIPUnicodeKeys(): Promise<void> {
29+
const prefix = `${COMPRESSED_PREFIX}:${GZIP_PREFIX}:Unicode`;
30+
const rawValue = '漢字';
31+
const buf = fflate.strToU8(rawValue);
32+
const value = Buffer.from(fflate.compressSync(buf));
33+
34+
await this.createString(prefix, value);
35+
await this.createHash(prefix, [value]);
36+
}
37+
38+
private async createGZIPASCIIKeys(): Promise<void> {
39+
const prefix = `${COMPRESSED_PREFIX}:${GZIP_PREFIX}:ASCII`;
40+
const rawValue = '\xac\xed\x00\x05t\x0a4102';
41+
const buf = fflate.strToU8(rawValue);
42+
const value = Buffer.from(fflate.compressSync(buf));
43+
44+
await this.createString(prefix, value);
45+
await this.createHash(prefix, [value]);
46+
}
47+
48+
private async createGZIPJSONKeys(): Promise<void> {
49+
const prefix = `${COMPRESSED_PREFIX}:${GZIP_PREFIX}:JSON`;
50+
const rawValue = '{"test":"test"}';
51+
const buf = fflate.strToU8(rawValue);
52+
const value = Buffer.from(fflate.compressSync(buf));
53+
54+
await this.createString(prefix, value);
55+
await this.createHash(prefix, [value]);
56+
}
57+
58+
private async createGZIPPHPUnserializedJSONKeys(): Promise<void> {
59+
const prefix = `${COMPRESSED_PREFIX}:${GZIP_PREFIX}:PHP`;
60+
const rawValue =
61+
'a:2:{i:0;s:12:"Sample array";i:1;a:2:{i:0;s:5:"Apple";i:1;s:6:"Orange";}}';
62+
const buf = fflate.strToU8(rawValue);
63+
const value = Buffer.from(fflate.compressSync(buf));
64+
65+
await this.createString(prefix, value);
66+
await this.createHash(prefix, [value]);
67+
}
68+
69+
private async createGZIPJavaSerializedObjectKeys(): Promise<void> {
70+
const prefix = `${COMPRESSED_PREFIX}:${GZIP_PREFIX}:Java`;
71+
const rawValue = fs.readFileSync('./test-data/decompressors/test_serialised_obj.ser');
72+
const rawValue2 = fs.readFileSync('./test-data/decompressors/test_annotated_obj.ser');
73+
74+
const value = Buffer.from(fflate.compressSync(rawValue));
75+
const value2 = Buffer.from(fflate.compressSync(rawValue2));
76+
77+
await this.createString(prefix, value);
78+
await this.createString(prefix, value2);
79+
await this.createHash(prefix, [value,value2]);
80+
}
81+
82+
private async createGZIPMsgpackKeys(): Promise<void> {
83+
const prefix = `${COMPRESSED_PREFIX}:${GZIP_PREFIX}:Msgpack`;
84+
const rawValue = msgpackrPack({
85+
hello: 'World',
86+
array: [1, 2],
87+
obj: {test: 'test'},
88+
boolean: false,
89+
});
90+
91+
const value = Buffer.from(fflate.compressSync(rawValue));
92+
93+
await this.createString(prefix, value);
94+
await this.createHash(prefix, [value]);
95+
}
96+
97+
private async createGZIPVectorKeys(): Promise<void> {
98+
const prefix = `${COMPRESSED_PREFIX}:${GZIP_PREFIX}:Vector`;
99+
const rawValue = JSON.parse(fs.readFileSync('./test-data/decompressors/vector.json', 'utf8'));
100+
const value = Buffer.from(fflate.compressSync(rawValue));
101+
102+
await this.createString(prefix, value);
103+
await this.createHash(prefix, [value]);
104+
}
105+
106+
private createGZIPProtobufKeys(): Promise<void> {
107+
const prefix = `${COMPRESSED_PREFIX}:${GZIP_PREFIX}:Proto`;
108+
109+
return new Promise((resolve, reject) => {
110+
proto.load('./test-data/decompressors/awesome.proto', async (err, root) => {
111+
if (err || !root) {
112+
console.error('Error loading protobuf:', err);
113+
return reject(err);
114+
}
115+
116+
const Book = root.lookupType('com.book.BookStore');
117+
const payloadBookStore = {
118+
name: 'Test name',
119+
books: {0: 'book 1', 1: 'book 2'},
120+
};
121+
const message = Book.create(payloadBookStore);
122+
const rawValue = Book.encode(message).finish();
123+
const value = Buffer.from(fflate.compressSync(rawValue));
124+
125+
await this.createString(prefix, value);
126+
await this.createHash(prefix, [value]);
127+
128+
resolve();
129+
});
130+
});
131+
}
132+
133+
private async createGZIPPickleKeys(): Promise<void> {
134+
const prefix = `${COMPRESSED_PREFIX}:${GZIP_PREFIX}:Pickle`;
135+
const rawValue = fs.readFileSync('./test-data/decompressors/pickleFile1.pickle');
136+
const value = Buffer.from(fflate.compressSync(rawValue));
137+
138+
await this.createString(prefix, value);
139+
await this.createHash(prefix, [value]);
140+
}
141+
}

0 commit comments

Comments
 (0)