Skip to content

Commit 1fcdefd

Browse files
author
Georg Traar
committed
Add request zlib encoding
1 parent c1f9159 commit 1fcdefd

File tree

6 files changed

+323
-111
lines changed

6 files changed

+323
-111
lines changed

README.md

Lines changed: 27 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -76,20 +76,22 @@ The `CrateDBClient` can be configured with either environment variables or direc
7676

7777
#### Configuration Options
7878

79-
| Option | Type | Default Value | Description |
80-
| ------------------ | ----------------------- | ----------------------------------------------------- | ------------------------------------------------------------ |
81-
| `user` | `string` | `'crate'` or `process.env.CRATEDB_USER` | Database user. |
82-
| `password` | `string` or `null` | `''` or `process.env.CRATEDB_PASSWORD` | Database password. |
83-
| `jwt` | `string \| null` | `null` | JWT token for Bearer authentication. |
84-
| `host` | `string` | `'localhost'` or `process.env.CRATEDB_HOST` | Database host. |
85-
| `port` | `number` | `4200` or `process.env.CRATEDB_PORT` | Database port. |
86-
| `defaultSchema` | `string` | `null` or `process.env.CRATEDB_DEFAULT_SCHEMA` | Default schema for queries. |
87-
| `connectionString` | `string` | `null` | Connection string, e.g., `https://user:password@host:port/`. |
88-
| `ssl` | `object` or `null` | `null` | SSL configuration; |
89-
| `keepAlive` | `boolean` | `true` | Enables HTTP keep-alive for persistent connections. |
90-
| `maxConnections` | `number` | `20` | Limits the maximum number of concurrent connections. |
91-
| `deserialization` | `DeserializationConfig` | `{ long: 'number', timestamp: 'date', date: 'date' }` | Controls deserialization behaviour |
92-
| `rowMode` | `'array' \| 'object'` | `'array'` | Controls the format of returned rows. |
79+
| Option | Type | Default Value | Description |
80+
| ---------------------- | ----------------------- | ----------------------------------------------------- | ------------------------------------------------------------ |
81+
| `user` | `string` | `'crate'` or `process.env.CRATEDB_USER` | Database user. |
82+
| `password` | `string` or `null` | `''` or `process.env.CRATEDB_PASSWORD` | Database password. |
83+
| `jwt` | `string \| null` | `null` | JWT token for Bearer authentication. |
84+
| `host` | `string` | `'localhost'` or `process.env.CRATEDB_HOST` | Database host. |
85+
| `port` | `number` | `4200` or `process.env.CRATEDB_PORT` | Database port. |
86+
| `defaultSchema` | `string` | `null` or `process.env.CRATEDB_DEFAULT_SCHEMA` | Default schema for queries. |
87+
| `connectionString` | `string` | `null` | Connection string, e.g., `https://user:password@host:port/`. |
88+
| `ssl` | `object` or `null` | `null` | SSL configuration; |
89+
| `keepAlive` | `boolean` | `true` | Enables HTTP keep-alive for persistent connections. |
90+
| `maxConnections` | `number` | `20` | Limits the maximum number of concurrent connections. |
91+
| `deserialization` | `DeserializationConfig` | `{ long: 'number', timestamp: 'date', date: 'date' }` | Controls deserialization behaviour |
92+
| `rowMode` | `'array' \| 'object'` | `'array'` | Controls the format of returned rows. |
93+
| `enableCompression` | `boolean` | `true` | Enables GZIP compression for large requests. |
94+
| `compressionThreshold` | `number` | `1024` | Minimum size in bytes before compression is applied. |
9395

9496
#### Environment Variables
9597

@@ -103,6 +105,17 @@ export CRATEDB_PORT=4200
103105
export CRATEDB_DEFAULT_SCHEMA=doc
104106
```
105107

108+
#### RequestCompression
109+
110+
The client supports automatic GZIP compression for large requests to improve network efficiency. When enabled, any request larger than the compression threshold will be automatically compressed.
111+
112+
```typescript
113+
const client = new CrateDBClient({
114+
enableCompression: true,
115+
compressionThreshold: 1024, // Default to 1KB
116+
});
117+
```
118+
106119
---
107120

108121
## Usage

examples/simple.js

Lines changed: 16 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { CrateDBClient } from '../dist/CrateDBClient.js';
1+
import { CrateDBClient } from '@proddata/node-cratedb';
22

33
(async () => {
44
const client = new CrateDBClient({
@@ -12,13 +12,11 @@ import { CrateDBClient } from '../dist/CrateDBClient.js';
1212
try {
1313
// --- Example 1: Creating a Table ---
1414
console.log('Creating a table...');
15-
await client.createTable({
16-
locations: {
17-
id: 'INT PRIMARY KEY',
18-
name: 'TEXT',
19-
kind: 'TEXT',
20-
description: 'TEXT',
21-
},
15+
await client.createTable('locations', {
16+
id: { type: 'INTEGER', primaryKey: true },
17+
name: { type: 'TEXT' },
18+
kind: { type: 'TEXT' },
19+
description: { type: 'TEXT' },
2220
});
2321
console.log('Table "locations" created.');
2422

@@ -53,34 +51,18 @@ import { CrateDBClient } from '../dist/CrateDBClient.js';
5351

5452
await client.refresh('locations');
5553

56-
// --- Example 5: Querying with a Cursor ---
57-
console.log('Querying data using a cursor...');
58-
const cursor = client.createCursor('SELECT * FROM locations ORDER BY id');
59-
await cursor.open();
54+
// --- Example 5: Streaming Query Results ---
55+
console.log('Streaming query results...');
56+
for await (const row of client.streamQuery('SELECT * FROM locations ORDER BY id')) {
57+
console.log('Record:', row);
58+
}
6059

61-
console.log('First record:', await cursor.fetchone()); // Fetch one record
62-
console.log('Next two records:', await cursor.fetchmany(2)); // Fetch 2 records
63-
console.log('All remaining records:', await cursor.fetchall()); // Fetch all remaining records
60+
// --- Example 6: Getting Primary Keys ---
61+
console.log('Getting primary keys...');
62+
const primaryKeys = await client.getPrimaryKeys('locations');
63+
console.log('Primary keys:', primaryKeys);
6464

65-
await cursor.close(); // Close the cursor and commit the transaction
66-
67-
// --- Example 6: Updating Data ---
68-
console.log('Updating a record...');
69-
await client.update('locations', { description: 'Blue and beautiful.' }, 'id = 1');
70-
await client.refresh('locations');
71-
72-
const updatedResult = await client.execute('SELECT * FROM locations WHERE id = 1');
73-
console.log('Updated record:', updatedResult.rows);
74-
75-
// --- Example 7: Deleting Data ---
76-
console.log('Deleting a record...');
77-
await client.delete('locations', 'id = 4');
78-
await client.refresh('locations');
79-
80-
const remainingResult = await client.execute('SELECT * FROM locations');
81-
console.log('Remaining records:', remainingResult.rows);
82-
83-
// --- Example 8: Dropping the Table ---
65+
// --- Example 7: Dropping the Table ---
8466
console.log('Dropping the table...');
8567
await client.drop('locations');
8668
console.log('Table "locations" dropped.');

examples/test.js

Lines changed: 108 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,123 @@
11
import http from 'http';
22
import zlib from 'zlib';
33

4+
// Configurable number of requests (default to 100)
5+
const numRequests = process.argv[2] ? parseInt(process.argv[2], 10) : 100;
6+
47
const requestData = JSON.stringify({
58
stmt: 'SELECT * FROM sys.summits',
69
});
710

8-
// Compress request data using gzip
9-
zlib.deflate(requestData, (err, compressedData) => {
10-
if (err) {
11-
console.error('Compression failed:', err);
12-
return;
13-
}
11+
// Pre-compress the request data using deflate once (since it's the same for every request)
12+
const compressedData = zlib.deflateSync(requestData);
13+
14+
/**
15+
* Formats a number of bytes into a human-readable string.
16+
* @param {number} bytes - The number of bytes.
17+
* @returns {string} The formatted string.
18+
*/
19+
function formatBytes(bytes) {
20+
if (bytes < 1024) return bytes + ' B';
21+
const kb = bytes / 1024;
22+
if (kb < 1024) return kb.toFixed(2) + ' KB';
23+
const mb = kb / 1024;
24+
if (mb < 1024) return mb.toFixed(2) + ' MB';
25+
const gb = mb / 1024;
26+
return gb.toFixed(2) + ' GB';
27+
}
1428

15-
const options = {
16-
hostname: 'localhost',
17-
port: 4200,
18-
path: '/_sql',
19-
method: 'POST',
20-
headers: {
21-
'Content-Type': 'application/json',
22-
'Content-Encoding': 'deflate', // Tell CrateDB the request is compressed
23-
Accept: 'application/json',
24-
'Accept-Encoding': 'gzip',
25-
},
26-
};
27-
28-
const req = http.request(options, (res) => {
29-
let responseData = '';
30-
31-
res.on('data', (chunk) => {
32-
responseData += chunk;
29+
/**
30+
* Performs a single HTTP request using the given Accept-Encoding header.
31+
* @param {string|undefined} acceptEncoding - The Accept-Encoding header value (or undefined for no header).
32+
* @returns {Promise<number>} A promise that resolves with the response size in bytes.
33+
*/
34+
function performRequest(acceptEncoding) {
35+
return new Promise((resolve, reject) => {
36+
const options = {
37+
hostname: 'localhost',
38+
port: 4200,
39+
path: '/_sql',
40+
method: 'POST',
41+
headers: {
42+
'Content-Type': 'application/json',
43+
'Content-Encoding': 'deflate', // The request body is compressed with deflate
44+
Accept: 'application/json',
45+
},
46+
};
47+
48+
if (acceptEncoding) {
49+
options.headers['Accept-Encoding'] = acceptEncoding;
50+
}
51+
52+
const req = http.request(options, (res) => {
53+
let responseSize = 0;
54+
res.on('data', (chunk) => {
55+
responseSize += chunk.length;
56+
});
57+
res.on('end', () => {
58+
resolve(responseSize);
59+
});
3360
});
3461

35-
res.on('end', () => {
36-
console.log('Response:', responseData);
62+
req.on('error', (error) => {
63+
reject(error);
3764
});
38-
});
3965

40-
req.on('error', (error) => {
41-
console.error('Request error:', error);
66+
req.write(compressedData);
67+
req.end();
4268
});
69+
}
70+
71+
/**
72+
* Runs a batch of requests.
73+
* @param {string|undefined} acceptEncoding - The Accept-Encoding header value for the batch.
74+
* @param {number} count - Number of requests to perform.
75+
* @returns {Promise<{elapsedTime: number, totalResponseSize: number}>}
76+
*/
77+
async function runBatch(acceptEncoding, count) {
78+
const startTime = Date.now();
79+
let totalResponseSize = 0;
80+
for (let i = 0; i < count; i++) {
81+
try {
82+
const size = await performRequest(acceptEncoding);
83+
totalResponseSize += size;
84+
} catch (error) {
85+
console.error('Request error:', error);
86+
}
87+
}
88+
const elapsedTime = Date.now() - startTime;
89+
return { elapsedTime, totalResponseSize };
90+
}
91+
92+
/**
93+
* Runs all test batches and logs the aggregated metrics.
94+
*/
95+
async function runTests() {
96+
console.log(`Running ${numRequests} requests with Accept-Encoding: gzip`);
97+
const gzipResults = await runBatch('gzip', numRequests);
98+
console.log(
99+
`GZIP - Total time: ${gzipResults.elapsedTime} ms, ` +
100+
`Average time: ${(gzipResults.elapsedTime / numRequests).toFixed(2)} ms, ` +
101+
`Total response size: ${formatBytes(gzipResults.totalResponseSize)}`
102+
);
103+
104+
console.log(`Running ${numRequests} requests with Accept-Encoding: deflate`);
105+
const deflateResults = await runBatch('deflate', numRequests);
106+
console.log(
107+
`DEFLATE - Total time: ${deflateResults.elapsedTime} ms, ` +
108+
`Average time: ${(deflateResults.elapsedTime / numRequests).toFixed(2)} ms, ` +
109+
`Total response size: ${formatBytes(deflateResults.totalResponseSize)}`
110+
);
111+
112+
console.log(`Running ${numRequests} requests without Accept-Encoding header`);
113+
const noHeaderResults = await runBatch(undefined, numRequests);
114+
console.log(
115+
`No Accept-Encoding - Total time: ${noHeaderResults.elapsedTime} ms, ` +
116+
`Average time: ${(noHeaderResults.elapsedTime / numRequests).toFixed(2)} ms, ` +
117+
`Total response size: ${formatBytes(noHeaderResults.totalResponseSize)}`
118+
);
119+
}
43120

44-
req.write(compressedData); // Send gzipped request body
45-
req.end();
121+
runTests().catch((error) => {
122+
console.error('Error during tests:', error);
46123
});

0 commit comments

Comments
 (0)