#795 Use big-endian RDWs when generating VRL file.

Author: yruslan

cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/processor/impl/CobolProcessorToRdw.scala

@@ -26,7 +26,7 @@ import java.io.OutputStream
 /**
   * Implementation of the CobolProcessor trait, responsible for processing COBOL data streams
   * by extracting records and applying a user-defined raw record processor. This processor
-  * converts the input format to the variable record length format with little-endian RDW records.
+  * converts the input format to the variable record length format with big-endian RDW records.
   *
   * Please, do not use this class directly. Use `CobolProcessor.builder()` instead.
   *

cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/processor/impl/StreamProcessor.scala

@@ -70,7 +70,7 @@ object StreamProcessor {
   }
 
   /**
-    * Processes a stream of COBOL raw records and writes it back as a variable length format with little-endian RDW headers.
+    * Processes a stream of COBOL raw records and writes it back as a variable length format with big-endian RDW headers.
     *
     * @param copybook        the COBOL copybook that describes the schema of the records.
     * @param options         arbitrary options used for splitting input data into records (same as 'spark-cobol' options).
@@ -96,7 +96,7 @@ object StreamProcessor {
 
       val updatedRecord = recordProcessor.processRecord(record, ctx)
 
-      val rdw = Array[Byte](0, 0, ((updatedRecord.length) & 0xFF).toByte, (((updatedRecord.length) >> 8) & 0xFF).toByte)
+      val rdw = Array[Byte](((updatedRecord.length >> 8) & 0xFF).toByte, ((updatedRecord.length) & 0xFF).toByte, 0, 0)
 
       outputStream.write(rdw)
       outputStream.write(updatedRecord)

cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/processor/impl/CobolProcessorToRdwSuite.scala

@@ -53,7 +53,7 @@ class CobolProcessorToRdwSuite extends AnyWordSpec {
       assert(count == 4)
       assert(outputArray.length == 24)
       assert(outputArray.sameElements(
-        Array(0, 0, 2, 0, -16, -16, 0, 0, 2, 0, -15, -15, 0, 0, 2, 0, -14, -14, 0, 0, 2, 0, -13, -13)
+        Array(0, 2, 0, 0, -16, -16, 0, 2, 0, 0, -15, -15, 0, 2, 0, 0, -14, -14, 0, 2, 0, 0, -13, -13)
       ))
     }
   }

spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/SparkCobolProcessorSuite.scala

@@ -130,13 +130,15 @@ class SparkCobolProcessorSuite extends AnyWordSpec with SparkTestBase with Binar
         val outputData = readBinaryFile(outputFile)
 
         assert(outputData.sameElements(
-          Array(0, 0, 1, 0, -16, 0, 0, 1, 0, -15, 0, 0, 1, 0, -14, 0, 0, 1, 0, -13).map(_.toByte)
+          Array(0, 1, 0, 0, -16, 0, 1, 0, 0, -15, 0, 1, 0, 0, -14, 0, 1, 0, 0, -13).map(_.toByte)
         ))
 
         val actual = spark.read
           .format("cobol")
           .option("copybook_contents", copybook)
           .option("record_format", "V")
+          .option("is_rdw_big_endian", "true")
+          .option("pedantic", "true")
           .load(outputFile)
           .toJSON
           .collect()