fix(parquet): Types to fix parquet module build

Author: felixpalmer

modules/parquet/src/parquetjs/codecs/plain.ts

@@ -194,7 +194,7 @@ function encodeValues_BYTE_ARRAY(values: Buffer[]): Buffer {
   let buf_pos = 0;
   for (let i = 0; i < values.length; i++) {
     buf.writeUInt32LE(values[i].length, buf_pos);
-    values[i].copy(buf, buf_pos + 4);
+    values[i].copy(buf as Uint8Array, buf_pos + 4);
     buf_pos += 4 + values[i].length;
   }
   return buf;
@@ -222,7 +222,7 @@ function encodeValues_FIXED_LEN_BYTE_ARRAY(values: Buffer[], opts: ParquetCodecO
       throw new Error(`invalid value for FIXED_LEN_BYTE_ARRAY: ${values[i]}`);
     }
   }
-  return Buffer.concat(values);
+  return Buffer.concat(values as Uint8Array[]);
 }
 
 function decodeValues_FIXED_LEN_BYTE_ARRAY(

modules/parquet/src/parquetjs/codecs/rle.ts

@@ -40,7 +40,7 @@ export function encodeValues(
     if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {
       // If we have any data in runs we need to encode them
       if (run.length) {
-        buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
+        buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)] as Uint8Array[]);
         run = [];
       }
       repeats = 1;
@@ -49,17 +49,17 @@ export function encodeValues(
     } else {
       // If values changes we need to post any previous repeated values
       if (repeats) {
-        buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);
+        buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)] as Uint8Array[]);
         repeats = 0;
       }
       run.push(values[i]);
     }
   }
 
   if (repeats) {
-    buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);
+    buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)] as Uint8Array[]);
   } else if (run.length) {
-    buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
+    buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)] as Uint8Array[]);
   }
 
   if (opts.disableEnvelope) {
@@ -70,7 +70,7 @@ export function encodeValues(
 
   // @ts-ignore buffer polyfill
   envelope.writeUInt32LE(buf.length, undefined);
-  buf.copy(envelope, 4);
+  buf.copy(envelope as Uint8Array, 4);
 
   return envelope;
 }
@@ -175,7 +175,7 @@ function encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer
     }
   }
 
-  return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);
+  return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf] as Uint8Array[]);
 }
 
 function encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {
@@ -190,5 +190,5 @@ function encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptio
     value >> 8; //  TODO - this looks wrong
   }
 
-  return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);
+  return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf] as Uint8Array[]);
 }

modules/parquet/src/parquetjs/encoder/parquet-encoder.ts

@@ -376,15 +376,15 @@ async function encodeDataPage(
   page: Buffer;
 }> {
   /* encode repetition and definition levels */
-  let rLevelsBuf: Buffer<ArrayBufferLike> = Buffer.alloc(0);
+  let rLevelsBuf: Buffer = Buffer.alloc(0);
   if (column.rLevelMax > 0) {
     rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
       bitWidth: getBitWidth(column.rLevelMax)
       // disableEnvelope: false
     });
   }
 
-  let dLevelsBuf: Buffer<ArrayBufferLike> = Buffer.alloc(0);
+  let dLevelsBuf: Buffer = Buffer.alloc(0);
   if (column.dLevelMax > 0) {
     dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
       bitWidth: getBitWidth(column.dLevelMax)
@@ -398,7 +398,7 @@ async function encodeDataPage(
     bitWidth: column.typeLength
   });
 
-  const dataBuf = Buffer.concat([rLevelsBuf, dLevelsBuf, valuesBuf]);
+  const dataBuf = Buffer.concat([rLevelsBuf, dLevelsBuf, valuesBuf] as Uint8Array[]);
 
   // compression = column.compression === 'UNCOMPRESSED' ? (compression || 'UNCOMPRESSED') : column.compression;
   const compressedBuf = await Compression.deflate(column.compression!, dataBuf);
@@ -418,7 +418,7 @@ async function encodeDataPage(
 
   /* concat page header, repetition and definition levels and values */
   const headerBuf = serializeThrift(header);
-  const page = Buffer.concat([headerBuf, compressedBuf]);
+  const page = Buffer.concat([headerBuf, compressedBuf] as Uint8Array[]);
 
   return {header, headerSize: headerBuf.length, page};
 }
@@ -445,15 +445,15 @@ async function encodeDataPageV2(
   const compressedBuf = await Compression.deflate(column.compression!, valuesBuf);
 
   /* encode repetition and definition levels */
-  let rLevelsBuf: Buffer<ArrayBufferLike> = Buffer.alloc(0);
+  let rLevelsBuf: Buffer = Buffer.alloc(0);
   if (column.rLevelMax > 0) {
     rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
       bitWidth: getBitWidth(column.rLevelMax),
       disableEnvelope: true
     });
   }
 
-  let dLevelsBuf: Buffer<ArrayBufferLike> = Buffer.alloc(0);
+  let dLevelsBuf: Buffer = Buffer.alloc(0);
   if (column.dLevelMax > 0) {
     dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
       bitWidth: getBitWidth(column.dLevelMax),
@@ -479,7 +479,7 @@ async function encodeDataPageV2(
 
   /* concat page header, repetition and definition levels and values */
   const headerBuf = serializeThrift(header);
-  const page = Buffer.concat([headerBuf, rLevelsBuf, dLevelsBuf, compressedBuf]);
+  const page = Buffer.concat([headerBuf, rLevelsBuf, dLevelsBuf, compressedBuf] as Uint8Array[]);
   return {header, headerSize: headerBuf.length, page};
 }
 
@@ -536,7 +536,7 @@ async function encodeColumnChunk(
 
   /* concat metadata header and data pages */
   const metadataOffset = baseOffset + pageBuf.length;
-  const body = Buffer.concat([pageBuf, serializeThrift(metadata)]);
+  const body = Buffer.concat([pageBuf, serializeThrift(metadata)] as Uint8Array[]);
   return {body, metadata, metadataOffset};
 }
 
@@ -573,7 +573,7 @@ async function encodeRowGroup(
     metadata.columns.push(cchunk);
     metadata.total_byte_size = new Int64(Number(metadata.total_byte_size) + cchunkData.body.length);
 
-    body = Buffer.concat([body, cchunkData.body]);
+    body = Buffer.concat([body, cchunkData.body] as Uint8Array[]);
   }
 
   return {body, metadata};
@@ -638,7 +638,7 @@ function encodeFooter(
   const metadataEncoded = serializeThrift(metadata);
   const footerEncoded = Buffer.alloc(metadataEncoded.length + 8);
 
-  metadataEncoded.copy(footerEncoded);
+  metadataEncoded.copy(footerEncoded as Uint8Array);
   footerEncoded.writeUInt32LE(metadataEncoded.length, metadataEncoded.length);
   footerEncoded.write(PARQUET_MAGIC, metadataEncoded.length + 4);
   return footerEncoded;

modules/parquet/src/parquetjs/parser/decoders.ts

@@ -440,7 +440,7 @@ async function decodeDictionaryPage(
     );
 
     dictCursor = {
-      buffer: valuesBuf as unknown as Buffer<ArrayBuffer>,
+      buffer: valuesBuf as unknown as Buffer,
       offset: 0,
       size: valuesBuf.length
     };

modules/parquet/src/parquetjs/utils/read-utils.ts

@@ -30,7 +30,7 @@ export function serializeThrift(obj: any): Buffer {
   obj.write(protocol);
   transport.flush();
 
-  return Buffer.concat(output);
+  return Buffer.concat(output as Uint8Array[]);
 }
 
 export function decodeThrift(obj: any, buf: Buffer, offset?: number) {