Make lz4 module optional (#246)

* Make `lz4` module optional

Signed-off-by: Levko Kravets <[email protected]>

* Fix lint errors

Signed-off-by: Levko Kravets <[email protected]>

---------

Signed-off-by: Levko Kravets <[email protected]>

Author: kravets-levko

.eslintrc

@@ -15,7 +15,14 @@
         "@typescript-eslint/no-throw-literal": "off",
         "no-restricted-syntax": "off",
         "no-case-declarations": "off",
-        "max-classes-per-file": "off"
+        "max-classes-per-file": "off",
+        "import/no-extraneous-dependencies": [
+          "error",
+          {
+            "devDependencies": true,
+            "optionalDependencies": true
+          }
+        ]
       }
     }
   ]

lib/DBSQLSession.ts

@@ -29,7 +29,7 @@ import IOperation from './contracts/IOperation';
 import DBSQLOperation from './DBSQLOperation';
 import Status from './dto/Status';
 import InfoValue from './dto/InfoValue';
-import { definedOrError } from './utils';
+import { definedOrError, LZ4 } from './utils';
 import CloseableCollection from './utils/CloseableCollection';
 import { LogLevel } from './contracts/IDBSQLLogger';
 import HiveDriverError from './errors/HiveDriverError';
@@ -190,7 +190,7 @@ export default class DBSQLSession implements IDBSQLSession {
       ...getArrowOptions(clientConfig),
       canDownloadResult: options.useCloudFetch ?? clientConfig.useCloudFetch,
       parameters: getQueryParameters(this.sessionHandle, options.namedParameters, options.ordinalParameters),
-      canDecompressLZ4Result: clientConfig.useLZ4Compression,
+      canDecompressLZ4Result: clientConfig.useLZ4Compression && Boolean(LZ4),
     });
     const response = await this.handleResponse(operationPromise);
     const operation = this.createOperation(response);

lib/result/ArrowResultHandler.ts

@@ -1,8 +1,9 @@
-import LZ4 from 'lz4';
 import { TGetResultSetMetadataResp, TRowSet } from '../../thrift/TCLIService_types';
+import HiveDriverError from '../errors/HiveDriverError';
 import IClientContext from '../contracts/IClientContext';
 import IResultsProvider, { ResultsProviderFetchNextOptions } from './IResultsProvider';
 import { ArrowBatch, hiveSchemaToArrowSchema } from './utils';
+import { LZ4 } from '../utils';
 
 export default class ArrowResultHandler implements IResultsProvider<ArrowBatch> {
   protected readonly context: IClientContext;
@@ -24,6 +25,10 @@ export default class ArrowResultHandler implements IResultsProvider<ArrowBatch>
     // so it's possible to infer Arrow schema from Hive schema ignoring `useArrowNativeTypes` option
     this.arrowSchema = arrowSchema ?? hiveSchemaToArrowSchema(schema);
     this.isLZ4Compressed = lz4Compressed ?? false;
+
+    if (this.isLZ4Compressed && !LZ4) {
+      throw new HiveDriverError('Cannot handle LZ4 compressed result: module `lz4` not installed');
+    }
   }
 
   public async hasMore() {
@@ -47,7 +52,7 @@ export default class ArrowResultHandler implements IResultsProvider<ArrowBatch>
     let totalRowCount = 0;
     rowSet?.arrowBatches?.forEach(({ batch, rowCount }) => {
       if (batch) {
-        batches.push(this.isLZ4Compressed ? LZ4.decode(batch) : batch);
+        batches.push(this.isLZ4Compressed ? LZ4!.decode(batch) : batch);
         totalRowCount += rowCount.toNumber(true);
       }
     });

lib/result/CloudFetchResultHandler.ts

@@ -1,9 +1,10 @@
-import LZ4 from 'lz4';
 import fetch, { RequestInfo, RequestInit, Request } from 'node-fetch';
 import { TGetResultSetMetadataResp, TRowSet, TSparkArrowResultLink } from '../../thrift/TCLIService_types';
+import HiveDriverError from '../errors/HiveDriverError';
 import IClientContext from '../contracts/IClientContext';
 import IResultsProvider, { ResultsProviderFetchNextOptions } from './IResultsProvider';
 import { ArrowBatch } from './utils';
+import { LZ4 } from '../utils';
 
 export default class CloudFetchResultHandler implements IResultsProvider<ArrowBatch> {
   protected readonly context: IClientContext;
@@ -24,6 +25,10 @@ export default class CloudFetchResultHandler implements IResultsProvider<ArrowBa
     this.context = context;
     this.source = source;
     this.isLZ4Compressed = lz4Compressed ?? false;
+
+    if (this.isLZ4Compressed && !LZ4) {
+      throw new HiveDriverError('Cannot handle LZ4 compressed result: module `lz4` not installed');
+    }
   }
 
   public async hasMore() {
@@ -58,7 +63,7 @@ export default class CloudFetchResultHandler implements IResultsProvider<ArrowBa
     }
 
     if (this.isLZ4Compressed) {
-      batch.batches = batch.batches.map((buffer) => LZ4.decode(buffer));
+      batch.batches = batch.batches.map((buffer) => LZ4!.decode(buffer));
     }
     return batch;
   }

lib/utils/index.ts

@@ -1,5 +1,6 @@
 import definedOrError from './definedOrError';
 import buildUserAgentString from './buildUserAgentString';
 import formatProgress, { ProgressUpdateTransformer } from './formatProgress';
+import LZ4 from './lz4';
 
-export { definedOrError, buildUserAgentString, formatProgress, ProgressUpdateTransformer };
+export { definedOrError, buildUserAgentString, formatProgress, ProgressUpdateTransformer, LZ4 };

lib/utils/lz4.ts

@@ -0,0 +1,16 @@
+import type LZ4Namespace from 'lz4';
+
+type LZ4Module = typeof LZ4Namespace;
+
+function tryLoadLZ4Module(): LZ4Module | undefined {
+  try {
+    return require('lz4'); // eslint-disable-line global-require
+  } catch (err) {
+    const isModuleNotFoundError = err instanceof Error && 'code' in err && err.code === 'MODULE_NOT_FOUND';
+    if (!isModuleNotFoundError) {
+      throw err;
+    }
+  }
+}
+
+export default tryLoadLZ4Module();

package-lock.json

@@ -74,7 +74,6 @@
   "dependencies": {
     "apache-arrow": "^13.0.0",
     "commander": "^9.3.0",
-    "lz4": "^0.6.5",
     "node-fetch": "^2.6.12",
     "node-int64": "^0.4.0",
     "open": "^8.4.2",
@@ -83,5 +82,8 @@
     "thrift": "^0.16.0",
     "uuid": "^9.0.0",
     "winston": "^3.8.2"
+  },
+  "optionalDependencies": {
+    "lz4": "^0.6.5"
   }
 }