From c0c831bdad96d321a123cc7b0f0d0b063ed857c1 Mon Sep 17 00:00:00 2001 From: Ruslan Iushchenko Date: Tue, 9 Sep 2025 08:15:51 +0200 Subject: [PATCH 1/8] #782 Add encoders for numbers having DISPLAY format. --- .../parser/encoding/DisplayEncoders.scala | 103 ++++++ .../encoding/DisplayEncodersSuite.scala | 304 ++++++++++++++++++ .../cobol/testutils/ComparisonUtils.scala | 6 +- 3 files changed, 410 insertions(+), 3 deletions(-) create mode 100644 cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncoders.scala create mode 100644 cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala diff --git a/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncoders.scala b/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncoders.scala new file mode 100644 index 00000000..e651bf13 --- /dev/null +++ b/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncoders.scala @@ -0,0 +1,103 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.cobrix.cobol.parser.encoding + +import java.math.RoundingMode + +object DisplayEncoders { + def encodeDisplayNumber(number: java.math.BigDecimal, + isSigned: Boolean, + outputSize: Int, + precision: Int, + scale: Int, + scaleFactor: Int, + explicitDecimalPoint: Boolean): Array[Byte] = { + val bytes = new Array[Byte](outputSize) + + if (number == null || precision < 1 || scale < 0 || outputSize < 1 || (scaleFactor > 0 && scale > 0)) + return bytes + + val num = if (explicitDecimalPoint) { + val shift = scaleFactor + + val bigDecimal = if (shift == 0) + number.setScale(scale, RoundingMode.HALF_EVEN) + else + number.movePointLeft(shift).setScale(scale, RoundingMode.HALF_EVEN) + + val bigDecimalValue1 = bigDecimal.toString + + val bigDecimalValue = if (bigDecimalValue1.startsWith("0.")) + bigDecimalValue1.drop(1) + else if (bigDecimalValue1.startsWith("-0.")) + "-" + bigDecimalValue1.drop(2) + else + bigDecimalValue1 + + val bigDecimalValueLen = bigDecimalValue.length + + if (bigDecimalValueLen > outputSize || (!isSigned && bigDecimal.signum() < 0)) + return bytes + + bigDecimalValue + } else { + val shift = scaleFactor - scale + + val bigInt = if (shift == 0) + number.setScale(0, RoundingMode.HALF_EVEN).toBigIntegerExact + else + number.movePointLeft(shift).setScale(0, RoundingMode.HALF_EVEN).toBigIntegerExact + + val intValue = bigInt.toString + val intValueLen = intValue.length + + if (intValueLen > outputSize || (!isSigned && bigInt.signum() < 0)) + return bytes + + intValue + } + setPaddedEbcdicNumber(num, bytes) + bytes + } + + def setPaddedEbcdicNumber(num: String, array: Array[Byte]): Unit = { + val numLen = num.length + val arLen = array.length + + if (numLen > arLen) + return + + var i = 0 + while (i < arLen) { + var ebcdic = 0x40.toByte + + if (i < numLen) { + val c = num(numLen - i - 1) + if (c >= '0' && c <= '9') { + ebcdic = ((c - '0') + 0xF0).toByte + } else if (c == '.' || c == ',') { + ebcdic = 0x4B + } else if (c == '-') { + ebcdic = 0x60 + } + } + + array(arLen - i - 1) = ebcdic + i += 1 + } + } +} diff --git a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala new file mode 100644 index 00000000..20953a74 --- /dev/null +++ b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala @@ -0,0 +1,304 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.cobrix.cobol.parser.encoding + +import org.scalatest.wordspec.AnyWordSpec +import za.co.absa.cobrix.cobol.testutils.ComparisonUtils.assertArraysEqual + +class DisplayEncodersSuite extends AnyWordSpec { + "encodeDisplayNumber" should { + "integral number" when { + "encode a number" in { + val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(12345), isSigned = true, 6, 5, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a number with an even precision" in { + val expected = Array(0xF1, 0xF2, 0xF3, 0xF4).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(1234), isSigned = true, 4, 4, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a small number" in { + val expected = Array(0x40, 0x40, 0x40, 0x40, 0x40, 0xF5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(5), isSigned = true, 6, 5, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode an unsigned number" in { + val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(12345), isSigned = false, 6, 5, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a negative number" in { + val expected = Array(0x60, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-12345), isSigned = true, 6, 5, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a small negative number" in { + val expected = Array(0x40, 0x40, 0x40, 0x40, 0x60, 0xF7).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-7), isSigned = true, 6, 5, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a too big number" ignore { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(123456), isSigned = true, 6, 5, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a too big negative number" in { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-123456), isSigned = true, 6, 5, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a number with negative scale" in { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(12345), isSigned = true, 6, 5, -1, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "attempt to encode a signed number when unsigned is expected" in { + val expected = Array[Byte](0x00, 0x00, 0x00, 0x00, 0x00, 0x00) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-12345), isSigned = false, 6, 5, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "attempt to encode a number with an incorrect precision" ignore { + val expected = Array[Byte](0x00, 0x00, 0x00, 0x00, 0x00) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(12345), isSigned = false, 5, 4, 0, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "attempt to encode a number with zero precision" in { + val expected = Array[Byte](0x00) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(12345), isSigned = false, 1, 0, 0, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + } + + "decimal number" when { + "encode a number" in { + val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(123.45), isSigned = true, 6, 5, 2, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a number with explicit decimal" in { + val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0x4B, 0xF4, 0xF5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(123.45), isSigned = true, 7, 5, 2, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a small number 1" in { + val expected = Array(0x40, 0x40, 0x40, 0x40, 0x40, 0xF5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(0.05), isSigned = true, 6, 5, 2, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a small number 2" in { + val expected = Array(0x40, 0x40, 0x40, 0x40, 0xF5, 0xF0).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(0.5), isSigned = true, 6, 5, 2, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a small number 3" in { + val expected = Array(0x40, 0x40, 0x40, 0x40, 0x40, 0xF1).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(0.005), isSigned = true, 6, 5, 2, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a small number 1 with explicit decimal point" in { + val expected = Array(0x40, 0x40, 0x40, 0x4B, 0xF0, 0xF5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(0.05), isSigned = true, 6, 5, 2, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a small number 2 with explicit decimal point" in { + val expected = Array(0x40, 0x40, 0x40, 0x4B, 0xF5, 0xF0).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(0.5), isSigned = true, 6, 5, 2, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a small number 3 with explicit decimal point" in { + val expected = Array(0x40, 0x40, 0x40, 0x4B, 0xF0, 0xF1).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(0.005), isSigned = true, 6, 5, 2, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode an unsigned number" in { + val expected = Array(0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF0).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(1234.5), isSigned = false, 6, 5, 2, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode an unsigned number with explicit decimal point" in { + val expected = Array(0xF1, 0xF2, 0xF3, 0xF4, 0x4B, 0xF5, 0xF0).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(1234.5), isSigned = false, 7, 5, 2, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a negative number" in { + val expected = Array(0x60, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-12.345), isSigned = true, 6, 5, 3, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a negative number with explicit decimal point" in { + val expected = Array(0x60, 0xF1, 0xF2, 0x4B, 0xF3, 0xF4, 0xF5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-12.345), isSigned = true, 7, 5, 3, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a small negative number" in { + val expected = Array(0x40, 0x40, 0x40, 0x60, 0xF7).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-0.00007), isSigned = true, 5, 4, 5, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a small negative number with explicit decimal point" in { + val expected = Array(0x40, 0x60, 0x4B, 0xF0, 0xF0, 0xF0, 0xF0, 0xF7).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-0.00007), isSigned = true, 8, 4, 5, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a too precise number" in { + val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xF6).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(123.456), isSigned = false, 6, 5, 2, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a too precise number with explicit decimal point" in { + val expected = Array(0xF1, 0xF2, 0xF3, 0x4B, 0xF4, 0xF6).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(123.456), isSigned = false, 6, 5, 2, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a too big number" ignore { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(1234.56), isSigned = false, 6, 5, 2, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a too big number with explicit decimal point" ignore { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(1234.56), isSigned = false, 7, 5, 2, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a too big negative number" in { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-1234.56), isSigned = true, 6, 5, 2, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a too big negative number with explicit decimal point" in { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-1234.56), isSigned = true, 7, 5, 2, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a number with positive scale factor" in { + val expected = Array(0x40, 0x40, 0x40, 0xF1, 0xF2, 0xF3).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(12300), isSigned = true, 6, 5, 0, 2, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a number with positive scale factor with explicit decimal point" in { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(123400), isSigned = true, 6, 5, 1, 2, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a number with negative scale factor" in { + val expected = Array(0x40, 0x40, 0x40, 0xF1, 0xF2, 0xF3).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(1.23), isSigned = true, 6, 5, 0, -2, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a number with negative scale factor with explicit decimal point" in { + val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0x4B, 0xF4).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(1.234), isSigned = true, 6, 5, 1, -2, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a small number with negative scale factor" in { + val expected = Array(0x40, 0x40, 0xF1, 0xF2, 0xF0).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(0.00012), isSigned = false, 5, 4, 3, -3, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a small number with negative scale factor with explicit decimal point" in { + val expected = Array(0x4B, 0xF1, 0xF2, 0xF0).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(0.00012), isSigned = false, 4, 4, 3, -3, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a small negative number with negative scale factor with explicit decimal point" in { + val expected = Array(0x60, 0x4B, 0xF1, 0xF2, 0xF0).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-0.00012), isSigned = true, 5, 4, 3, -3, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + } + } + + +} diff --git a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/testutils/ComparisonUtils.scala b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/testutils/ComparisonUtils.scala index 43fd5301..32a59987 100644 --- a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/testutils/ComparisonUtils.scala +++ b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/testutils/ComparisonUtils.scala @@ -22,9 +22,9 @@ import org.scalatest.Assertions.{fail, succeed} object ComparisonUtils { def assertArraysEqual(actual: Array[Byte], expected: Array[Byte]): Assertion = { if (!actual.sameElements(expected)) { - val actualHex = actual.map(b => f"$b%02X").mkString(" ") - val expectedHex = expected.map(b => f"$b%02X").mkString(" ") - fail(s"Actual: $actualHex\nExpected: $expectedHex") + val actualHex = actual.map(b => f"0x$b%02X").mkString(", ") + val expectedHex = expected.map(b => f"0x$b%02X").mkString(", ") + fail(s"Actual: $actualHex\nExpected: $expectedHex") } else { succeed } From 29c95bc24784ae36b3ae6877dc3f216f5af8add3 Mon Sep 17 00:00:00 2001 From: Ruslan Iushchenko Date: Tue, 9 Sep 2025 09:31:42 +0200 Subject: [PATCH 2/8] #782 Add display encoders to the encoder selector for the spark-cobol writer. --- .../parser/encoding/DisplayEncoders.scala | 118 +++++- .../parser/encoding/EncoderSelector.scala | 31 ++ .../encoding/DisplayEncodersSuite.scala | 375 ++++++++++++++++-- .../writer/FixedLengthEbcdicWriterSuite.scala | 81 +++- 4 files changed, 555 insertions(+), 50 deletions(-) diff --git a/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncoders.scala b/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncoders.scala index e651bf13..b22ef72e 100644 --- a/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncoders.scala +++ b/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncoders.scala @@ -16,16 +16,20 @@ package za.co.absa.cobrix.cobol.parser.encoding +import za.co.absa.cobrix.cobol.parser.position.Position + import java.math.RoundingMode object DisplayEncoders { - def encodeDisplayNumber(number: java.math.BigDecimal, - isSigned: Boolean, - outputSize: Int, - precision: Int, - scale: Int, - scaleFactor: Int, - explicitDecimalPoint: Boolean): Array[Byte] = { + def encodeDisplayNumberSignSeparate(number: java.math.BigDecimal, + signPosition: Option[Position], + outputSize: Int, + precision: Int, + scale: Int, + scaleFactor: Int, + explicitDecimalPoint: Boolean, + ): Array[Byte] = { + val isSigned = signPosition.isDefined val bytes = new Array[Byte](outputSize) if (number == null || precision < 1 || scale < 0 || outputSize < 1 || (scaleFactor > 0 && scale > 0)) @@ -70,11 +74,107 @@ object DisplayEncoders { intValue } - setPaddedEbcdicNumber(num, bytes) + setPaddedEbcdicNumberWithSignSeparate(num, bytes) bytes } - def setPaddedEbcdicNumber(num: String, array: Array[Byte]): Unit = { + def encodeDisplayNumberSignOverpunched(number: java.math.BigDecimal, + signPosition: Option[Position], + outputSize: Int, + precision: Int, + scale: Int, + scaleFactor: Int, + explicitDecimalPoint: Boolean): Array[Byte] = { + val isSigned = signPosition.isDefined + val isNegative = number.signum() < 0 + val bytes = new Array[Byte](outputSize) + + + if (number == null || precision < 1 || scale < 0 || outputSize < 1 || (scaleFactor > 0 && scale > 0)) + return bytes + + val num = if (explicitDecimalPoint) { + val shift = scaleFactor + + val bigDecimal = if (shift == 0) + number.abs().setScale(scale, RoundingMode.HALF_EVEN) + else + number.abs().movePointLeft(shift).setScale(scale, RoundingMode.HALF_EVEN) + + val bigDecimalValue1 = bigDecimal.toString + + val bigDecimalValue = if (bigDecimalValue1.startsWith("0.")) + bigDecimalValue1.drop(1) + else + bigDecimalValue1 + + val bigDecimalValueLen = bigDecimalValue.length + + if (bigDecimalValueLen > outputSize || (!isSigned && isNegative)) + return bytes + + bigDecimalValue + } else { + val shift = scaleFactor - scale + + val bigInt = if (shift == 0) + number.abs().setScale(0, RoundingMode.HALF_EVEN).toBigIntegerExact + else + number.abs().movePointLeft(shift).setScale(0, RoundingMode.HALF_EVEN).toBigIntegerExact + + val intValue = bigInt.toString + val intValueLen = intValue.length + + if (intValueLen > outputSize || (!isSigned && isNegative)) + return bytes + + intValue + } + setPaddedEbcdicNumberWithSignOverpunched(num, isSigned, isNegative, bytes) + bytes + } + + def setPaddedEbcdicNumberWithSignOverpunched(num: String, isSigned: Boolean, isNegative: Boolean, array: Array[Byte]): Unit = { + val numLen = num.length + val arLen = array.length + + if (numLen > arLen) + return + + var i = 0 + while (i < arLen) { + var ebcdic = 0x40.toByte + + if (i == 0) { + // Signal overpunching + val c = num(numLen - i - 1) + if (c >= '0' && c <= '9') { + val digit = c - '0' + val zone = if (!isSigned) { + 0xF + } else if (isNegative) { + 0xD + } else { + 0xC + } + + ebcdic = ((zone << 4) | digit).toByte + } + } else if (i < numLen) { + val c = num(numLen - i - 1) + if (c >= '0' && c <= '9') { + ebcdic = ((c - '0') + 0xF0).toByte + } else if (c == '.' || c == ',') { + ebcdic = 0x4B + } + } + + array(arLen - i - 1) = ebcdic + i += 1 + } + } + + def setPaddedEbcdicNumberWithSignSeparate(num: String, array: Array[Byte]): Unit = { val numLen = num.length val arLen = array.length diff --git a/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/EncoderSelector.scala b/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/EncoderSelector.scala index 56962c99..ddce9a04 100644 --- a/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/EncoderSelector.scala +++ b/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/EncoderSelector.scala @@ -19,6 +19,7 @@ package za.co.absa.cobrix.cobol.parser.encoding import za.co.absa.cobrix.cobol.parser.ast.datatype.{AlphaNumeric, COMP3, COMP3U, COMP4, COMP9, CobolType, Decimal, Integral, Usage} import za.co.absa.cobrix.cobol.parser.decoders.BinaryUtils import za.co.absa.cobrix.cobol.parser.encoding.codepage.{CodePage, CodePageCommon} +import za.co.absa.cobrix.cobol.parser.position.Position import java.nio.charset.{Charset, StandardCharsets} import java.util @@ -48,6 +49,10 @@ object EncoderSelector { Option(getBinaryEncoder(decimalBinary.compact, decimalBinary.precision, decimalBinary.scale, decimalBinary.scaleFactor, decimalBinary.signPosition.isDefined, isBigEndian = true)) case decimalBinary: Decimal if decimalBinary.compact.exists(_.isInstanceOf[COMP9]) => Option(getBinaryEncoder(decimalBinary.compact, decimalBinary.precision, decimalBinary.scale, decimalBinary.scaleFactor, decimalBinary.signPosition.isDefined, isBigEndian = false)) + case integralDisplay: Integral if integralDisplay.compact.isEmpty => + Option(getDisplayEncoder(integralDisplay.precision, 0, 0, integralDisplay.signPosition, isExplicitDecimalPt = false, isSignSeparate = integralDisplay.isSignSeparate)) + case decimalDisplay: Decimal if decimalDisplay.compact.isEmpty => + Option(getDisplayEncoder(decimalDisplay.precision, decimalDisplay.scale, decimalDisplay.scaleFactor, decimalDisplay.signPosition, decimalDisplay.explicitDecimal, decimalDisplay.isSignSeparate)) case _ => None } @@ -140,4 +145,30 @@ object EncoderSelector { } } + def getDisplayEncoder(precision: Int, + scale: Int, + scaleFactor: Int, + signPosition: Option[Position], + isExplicitDecimalPt: Boolean, + isSignSeparate: Boolean): Encoder = { + val isSigned = signPosition.isDefined + val numBytes = BinaryUtils.getBytesCount(None, precision, isSigned, isExplicitDecimalPt = isExplicitDecimalPt, isSignSeparate = isSignSeparate) + (a: Any) => { + val number = a match { + case null => null + case d: java.math.BigDecimal => d + case n: java.math.BigInteger => new java.math.BigDecimal(n) + case n: Byte => new java.math.BigDecimal(n) + case n: Int => new java.math.BigDecimal(n) + case n: Long => new java.math.BigDecimal(n) + case x => new java.math.BigDecimal(x.toString) + } + if (isSignSeparate) { + DisplayEncoders.encodeDisplayNumberSignSeparate(number, signPosition, numBytes, precision, scale, scaleFactor, explicitDecimalPoint = isExplicitDecimalPt) + } else { + DisplayEncoders.encodeDisplayNumberSignOverpunched(number, signPosition, numBytes, precision, scale, scaleFactor, explicitDecimalPoint = isExplicitDecimalPt) + } + } + } + } diff --git a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala index 20953a74..c27fda4d 100644 --- a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala +++ b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala @@ -17,91 +17,379 @@ package za.co.absa.cobrix.cobol.parser.encoding import org.scalatest.wordspec.AnyWordSpec +import za.co.absa.cobrix.cobol.parser.position._ import za.co.absa.cobrix.cobol.testutils.ComparisonUtils.assertArraysEqual class DisplayEncodersSuite extends AnyWordSpec { - "encodeDisplayNumber" should { + "encodeDisplayNumberSignOverpunched" should { + "integral number" when { + "encode a number" in { + val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xC5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(12345), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a number with an even precision" in { + val expected = Array(0xF1, 0xF2, 0xF3, 0xC4).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(1234), signPosition = Some(Left), 4, 4, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a small number" in { + val expected = Array(0x40, 0x40, 0x40, 0x40, 0x40, 0xC5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(5), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode an unsigned number" in { + val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(12345), signPosition = None, 6, 5, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a negative number" in { + val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xD5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-12345), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a small negative number" in { + val expected = Array(0x40, 0x40, 0x40, 0x40, 0x40, 0xD7).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-7), signPosition = Some(Right), 6, 5, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a too big number" ignore { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(123456), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a too big negative number" in { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-123456), signPosition = Some(Left), 5, 5, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a number with negative scale" in { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(12345), signPosition = Some(Left), 6, 5, -1, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "attempt to encode a signed number when unsigned is expected" in { + val expected = Array[Byte](0x00, 0x00, 0x00, 0x00, 0x00, 0x00) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-12345), signPosition = None, 6, 5, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "attempt to encode a number with an incorrect precision" ignore { + val expected = Array[Byte](0x00, 0x00, 0x00, 0x00, 0x00) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(12345), signPosition = None, 5, 4, 0, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "attempt to encode a number with zero precision" in { + val expected = Array[Byte](0x00) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(12345), signPosition = None, 1, 0, 0, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + } + + "decimal number" when { + "encode a number" in { + val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xC5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(123.45), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a number with explicit decimal" in { + val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0x4B, 0xF4, 0xC5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(123.45), signPosition = Some(Left), 7, 5, 2, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a small number 1" in { + val expected = Array(0x40, 0x40, 0x40, 0x40, 0x40, 0xC5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(0.05), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a small number 2" in { + val expected = Array(0x40, 0x40, 0x40, 0x40, 0xF5, 0xC0).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(0.5), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a small number 3" in { + val expected = Array(0x40, 0x40, 0x40, 0x40, 0x40, 0xC1).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(0.005), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a small number 1 with explicit decimal point" in { + val expected = Array(0x40, 0x40, 0x40, 0x4B, 0xF0, 0xC5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(0.05), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a small number 2 with explicit decimal point" in { + val expected = Array(0x40, 0x40, 0x40, 0x4B, 0xF5, 0xC0).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(0.5), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a small number 3 with explicit decimal point" in { + val expected = Array(0x40, 0x40, 0x40, 0x4B, 0xF0, 0xC1).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(0.005), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode an unsigned number" in { + val expected = Array(0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF0).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234.5), signPosition = None, 6, 5, 2, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode an unsigned number with explicit decimal point" in { + val expected = Array(0xF1, 0xF2, 0xF3, 0xF4, 0x4B, 0xF5, 0xF0).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(1234.5), signPosition = None, 7, 5, 2, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a negative number" in { + val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xD5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-12.345), signPosition = Some(Left), 6, 5, 3, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a negative number with explicit decimal point" in { + val expected = Array(0x40, 0xF1, 0xF2, 0x4B, 0xF3, 0xF4, 0xD5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-12.345), signPosition = Some(Right), 7, 5, 3, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a small negative number" in { + val expected = Array(0x40, 0x40, 0x40, 0x40, 0xD7).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-0.00007), signPosition = Some(Right), 5, 4, 5, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a small negative number with explicit decimal point" in { + val expected = Array(0x40, 0x40, 0x4B, 0xF0, 0xF0, 0xF0, 0xF0, 0xD7).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-0.00007), signPosition = Some(Left), 8, 4, 5, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a too precise number" in { + val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xF6).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(123.456), signPosition = None, 6, 5, 2, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a too precise number with explicit decimal point" in { + val expected = Array(0xF1, 0xF2, 0xF3, 0x4B, 0xF4, 0xF6).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(123.456), signPosition = None, 6, 5, 2, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a too big number" ignore { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(1234.56), signPosition = None, 6, 5, 2, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a too big number with explicit decimal point" ignore { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(1234.56), signPosition = None, 7, 5, 2, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a too big negative number" in { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-1234.56), signPosition = Some(Right), 5, 5, 2, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a too big negative number with explicit decimal point" in { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-1234.56), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a number with positive scale factor" in { + val expected = Array(0x40, 0x40, 0x40, 0xF1, 0xF2, 0xC3).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(12300), signPosition = Some(Left), 6, 5, 0, 2, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a number with positive scale factor with explicit decimal point" in { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(123400), signPosition = Some(Left), 6, 5, 1, 2, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a number with negative scale factor" in { + val expected = Array(0x40, 0x40, 0x40, 0xF1, 0xF2, 0xC3).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(1.23), signPosition = Some(Left), 6, 5, 0, -2, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a number with negative scale factor with explicit decimal point" in { + val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0x4B, 0xC4).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(1.234), signPosition = Some(Left), 6, 5, 1, -2, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a small number with negative scale factor" in { + val expected = Array(0x40, 0x40, 0xF1, 0xF2, 0xF0).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(0.00012), signPosition = None, 5, 4, 3, -3, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a small number with negative scale factor with explicit decimal point" in { + val expected = Array(0x4B, 0xF1, 0xF2, 0xF0).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(0.00012), signPosition = None, 4, 4, 3, -3, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a small negative number with negative scale factor with explicit decimal point" in { + val expected = Array(0x4B, 0xF1, 0xF2, 0xD0).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-0.00012), signPosition = Some(Left), 4, 4, 3, -3, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a small negative number with negative scale factor with explicit decimal point and sign from right side" in { + val expected = Array(0x4B, 0xF1, 0xF2, 0xD0).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-0.00012), signPosition = Some(Right), 4, 4, 3, -3, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + } + } + + + "encodeDisplayNumberSignSeparate" should { "integral number" when { "encode a number" in { val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(12345), isSigned = true, 6, 5, 0, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(12345), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a number with an even precision" in { val expected = Array(0xF1, 0xF2, 0xF3, 0xF4).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(1234), isSigned = true, 4, 4, 0, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234), signPosition = Some(Left), 4, 4, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a small number" in { val expected = Array(0x40, 0x40, 0x40, 0x40, 0x40, 0xF5).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(5), isSigned = true, 6, 5, 0, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(5), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode an unsigned number" in { val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(12345), isSigned = false, 6, 5, 0, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(12345), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a negative number" in { val expected = Array(0x60, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-12345), isSigned = true, 6, 5, 0, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-12345), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a small negative number" in { val expected = Array(0x40, 0x40, 0x40, 0x40, 0x60, 0xF7).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-7), isSigned = true, 6, 5, 0, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-7), signPosition = Some(Right), 6, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a too big number" ignore { val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(123456), isSigned = true, 6, 5, 0, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(123456), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a too big negative number" in { val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-123456), isSigned = true, 6, 5, 0, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-123456), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a number with negative scale" in { val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(12345), isSigned = true, 6, 5, -1, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(12345), signPosition = Some(Left), 6, 5, -1, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "attempt to encode a signed number when unsigned is expected" in { val expected = Array[Byte](0x00, 0x00, 0x00, 0x00, 0x00, 0x00) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-12345), isSigned = false, 6, 5, 0, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-12345), signPosition = None, 6, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "attempt to encode a number with an incorrect precision" ignore { val expected = Array[Byte](0x00, 0x00, 0x00, 0x00, 0x00) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(12345), isSigned = false, 5, 4, 0, 0, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(12345), signPosition = None, 5, 4, 0, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "attempt to encode a number with zero precision" in { val expected = Array[Byte](0x00) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(12345), isSigned = false, 1, 0, 0, 0, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(12345), signPosition = None, 1, 0, 0, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } @@ -110,189 +398,196 @@ class DisplayEncodersSuite extends AnyWordSpec { "decimal number" when { "encode a number" in { val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(123.45), isSigned = true, 6, 5, 2, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(123.45), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a number with explicit decimal" in { val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0x4B, 0xF4, 0xF5).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(123.45), isSigned = true, 7, 5, 2, 0, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(123.45), signPosition = Some(Left), 7, 5, 2, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a small number 1" in { val expected = Array(0x40, 0x40, 0x40, 0x40, 0x40, 0xF5).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(0.05), isSigned = true, 6, 5, 2, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(0.05), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a small number 2" in { val expected = Array(0x40, 0x40, 0x40, 0x40, 0xF5, 0xF0).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(0.5), isSigned = true, 6, 5, 2, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(0.5), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a small number 3" in { val expected = Array(0x40, 0x40, 0x40, 0x40, 0x40, 0xF1).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(0.005), isSigned = true, 6, 5, 2, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(0.005), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a small number 1 with explicit decimal point" in { val expected = Array(0x40, 0x40, 0x40, 0x4B, 0xF0, 0xF5).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(0.05), isSigned = true, 6, 5, 2, 0, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(0.05), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a small number 2 with explicit decimal point" in { val expected = Array(0x40, 0x40, 0x40, 0x4B, 0xF5, 0xF0).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(0.5), isSigned = true, 6, 5, 2, 0, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(0.5), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a small number 3 with explicit decimal point" in { val expected = Array(0x40, 0x40, 0x40, 0x4B, 0xF0, 0xF1).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(0.005), isSigned = true, 6, 5, 2, 0, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(0.005), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode an unsigned number" in { val expected = Array(0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF0).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(1234.5), isSigned = false, 6, 5, 2, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234.5), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode an unsigned number with explicit decimal point" in { val expected = Array(0xF1, 0xF2, 0xF3, 0xF4, 0x4B, 0xF5, 0xF0).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(1234.5), isSigned = false, 7, 5, 2, 0, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234.5), signPosition = None, 7, 5, 2, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a negative number" in { val expected = Array(0x60, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-12.345), isSigned = true, 6, 5, 3, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-12.345), signPosition = Some(Left), 6, 5, 3, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a negative number with explicit decimal point" in { val expected = Array(0x60, 0xF1, 0xF2, 0x4B, 0xF3, 0xF4, 0xF5).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-12.345), isSigned = true, 7, 5, 3, 0, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-12.345), signPosition = Some(Right), 7, 5, 3, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a small negative number" in { val expected = Array(0x40, 0x40, 0x40, 0x60, 0xF7).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-0.00007), isSigned = true, 5, 4, 5, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-0.00007), signPosition = Some(Right), 5, 4, 5, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a small negative number with explicit decimal point" in { val expected = Array(0x40, 0x60, 0x4B, 0xF0, 0xF0, 0xF0, 0xF0, 0xF7).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-0.00007), isSigned = true, 8, 4, 5, 0, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-0.00007), signPosition = Some(Left), 8, 4, 5, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a too precise number" in { val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xF6).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(123.456), isSigned = false, 6, 5, 2, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(123.456), signPosition = None, 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a too precise number with explicit decimal point" in { val expected = Array(0xF1, 0xF2, 0xF3, 0x4B, 0xF4, 0xF6).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(123.456), isSigned = false, 6, 5, 2, 0, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(123.456), signPosition = None, 6, 5, 2, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a too big number" ignore { val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(1234.56), isSigned = false, 6, 5, 2, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234.56), signPosition = None, 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a too big number with explicit decimal point" ignore { val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(1234.56), isSigned = false, 7, 5, 2, 0, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234.56), signPosition = None, 7, 5, 2, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a too big negative number" in { val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-1234.56), isSigned = true, 6, 5, 2, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-1234.56), signPosition = Some(Right), 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a too big negative number with explicit decimal point" in { val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-1234.56), isSigned = true, 7, 5, 2, 0, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-1234.56), signPosition = Some(Left), 7, 5, 2, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a number with positive scale factor" in { val expected = Array(0x40, 0x40, 0x40, 0xF1, 0xF2, 0xF3).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(12300), isSigned = true, 6, 5, 0, 2, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(12300), signPosition = Some(Left), 6, 5, 0, 2, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a number with positive scale factor with explicit decimal point" in { val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(123400), isSigned = true, 6, 5, 1, 2, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(123400), signPosition = Some(Left), 6, 5, 1, 2, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a number with negative scale factor" in { val expected = Array(0x40, 0x40, 0x40, 0xF1, 0xF2, 0xF3).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(1.23), isSigned = true, 6, 5, 0, -2, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1.23), signPosition = Some(Left), 6, 5, 0, -2, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a number with negative scale factor with explicit decimal point" in { val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0x4B, 0xF4).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(1.234), isSigned = true, 6, 5, 1, -2, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1.234), signPosition = Some(Left), 6, 5, 1, -2, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a small number with negative scale factor" in { val expected = Array(0x40, 0x40, 0xF1, 0xF2, 0xF0).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(0.00012), isSigned = false, 5, 4, 3, -3, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(0.00012), signPosition = None, 5, 4, 3, -3, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a small number with negative scale factor with explicit decimal point" in { val expected = Array(0x4B, 0xF1, 0xF2, 0xF0).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(0.00012), isSigned = false, 4, 4, 3, -3, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(0.00012), signPosition = None, 4, 4, 3, -3, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a small negative number with negative scale factor with explicit decimal point" in { val expected = Array(0x60, 0x4B, 0xF1, 0xF2, 0xF0).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumber(new java.math.BigDecimal(-0.00012), isSigned = true, 5, 4, 3, -3, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-0.00012), signPosition = Some(Left), 5, 4, 3, -3, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + + "encode a small negative number with negative scale factor with explicit decimal point and sign from right side" in { + val expected = Array(0x60, 0x4B, 0xF1, 0xF2, 0xF0).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-0.00012), signPosition = Some(Right), 5, 4, 3, -3, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } diff --git a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala index ef425e5b..0008bada 100644 --- a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala +++ b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala @@ -18,6 +18,7 @@ package za.co.absa.cobrix.spark.cobol.writer import org.apache.hadoop.fs.Path import org.apache.spark.sql.SaveMode +import org.scalatest.Assertion import org.scalatest.wordspec.AnyWordSpec import za.co.absa.cobrix.spark.cobol.source.base.SparkTestBase import za.co.absa.cobrix.spark.cobol.source.fixtures.BinaryFileFixture @@ -258,6 +259,76 @@ class FixedLengthEbcdicWriterSuite extends AnyWordSpec with SparkTestBase with B } } + "write data frames with DISPLAY fields" in { + withTempDirectory("cobol_writer1") { tempDir => + val df = List( + (-1, 100.5, new java.math.BigDecimal(10.23), 1, 10050, new java.math.BigDecimal(10.12)), + (2, 800.4, new java.math.BigDecimal(30), 2, 80040, new java.math.BigDecimal(30)), + (3, 22.33, new java.math.BigDecimal(-20), 3, -2233, new java.math.BigDecimal(-20)) + ).toDF("A", "B", "C", "D", "E", "F") + + val path = new Path(tempDir, "writer1") + + val copybookContentsWithBinFields = + """ 01 RECORD. + 05 A PIC S9(1). + 05 B PIC 9(4)V9(2). + 05 C PIC S9(2).9(2). + 05 D PIC 9(1). + 05 E PIC S9(6) SIGN IS LEADING SEPARATE. + 05 F PIC S9(2).9(2) SIGN IS TRAILING SEPARATE. + """ + + df.coalesce(1) + .orderBy("A") + .write + .format("cobol") + .mode(SaveMode.Overwrite) + .option("copybook_contents", copybookContentsWithBinFields) + .save(path.toString) + + val fs = path.getFileSystem(spark.sparkContext.hadoopConfiguration) + + assert(fs.exists(path), "Output directory should exist") + val files = fs.listStatus(path) + .filter(_.getPath.getName.startsWith("part-")) + + assert(files.nonEmpty, "Output directory should contain part files") + + val partFile = files.head.getPath + val data = fs.open(partFile) + val bytes = new Array[Byte](files.head.getLen.toInt) + data.readFully(bytes) + data.close() + + // Expected EBCDIC data for sample test data + val expected = Array( + 0xD1, // -1 PIC S9(1). + 0x40, 0xF1, 0xF0, 0xF0, 0xF5, 0xF0, // 100.5 PIC 9(4)V9(2) + 0xF1, 0xF0, 0x4B, 0xF2, 0xC3, // 10.23 PIC S9(2).9(2) + 0xF1, // 1 9(1) + 0x40, 0x40, 0xF1, 0xF0, 0xF0, 0xF5, 0xF0, // 10050 S9(6) SIGN IS LEADING SEPARATE. + 0x40, 0xF1, 0xF0, 0x4B, 0xF1, 0xF2, // 10.12 S9(2).9(2) SIGN IS TRAILING SEPARATE + + 0xC2, + 0x40, 0xF8, 0xF0, 0xF0, 0xF4, 0xF0, + 0xF3, 0xF0, 0x4B, 0xF0, 0xC0, + 0xF2, + 0x40, 0x40, 0xF8, 0xF0, 0xF0, 0xF4, 0xF0, + 0x40, 0xF3, 0xF0, 0x4B, 0xF0, 0xF0, + + 0xC3, + 0x40, 0x40, 0xF2, 0xF2, 0xF3, 0xF3, + 0xF2, 0xF0, 0x4B, 0xF0, 0xD0, + 0xF3, + 0x40, 0x40, 0x60, 0xF2, 0xF2, 0xF3, 0xF3, + 0x60, 0xF2, 0xF0, 0x4B, 0xF0, 0xF0 + ).map(_.toByte) + + assertArraysEqual(bytes, expected) + } + } + "write should successfully append" in { withTempDirectory("cobol_writer3") { tempDir => val df = List(("A", "First"), ("B", "Scnd"), ("C", "Last")).toDF("A", "B") @@ -330,7 +401,15 @@ class FixedLengthEbcdicWriterSuite extends AnyWordSpec with SparkTestBase with B assert(fs.exists(path), "Output directory should exist") } } - } + def assertArraysEqual(actual: Array[Byte], expected: Array[Byte]): Assertion = { + if (!actual.sameElements(expected)) { + val actualHex = actual.map(b => f"0x$b%02X").mkString(", ") + val expectedHex = expected.map(b => f"0x$b%02X").mkString(", ") + fail(s"Actual: $actualHex\nExpected: $expectedHex") + } else { + succeed + } + } } From 128b79b7bfa529d5a77e5f62baade00a8cf622d3 Mon Sep 17 00:00:00 2001 From: Ruslan Iushchenko Date: Tue, 9 Sep 2025 17:03:57 +0200 Subject: [PATCH 3/8] Update sbt and Maven dependencies. --- pom.xml | 8 +++----- project/Dependencies.scala | 6 +++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index 631e0443..d7edc415 100644 --- a/pom.xml +++ b/pom.xml @@ -94,13 +94,11 @@ yyyy-MM-dd'T'HH:mm:ssX 3.10.1 - 1.8 - 1.8 3.2.1 2.3 3.0.1 2.19.1 - 1.0 + 2.2.0 2.4.2 1.8.1 1.3.0 @@ -110,8 +108,8 @@ 2.12.20 2.12 - 3.5.2 - 3.2.14 + 3.5.6 + 3.2.19 2.4.16 15.0 2.13.1 diff --git a/project/Dependencies.scala b/project/Dependencies.scala index a8697936..17a8920d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -23,12 +23,12 @@ object Dependencies { private val slf4jVersion = "1.7.25" private val jacksonVersion = "2.13.0" - private val scalatestVersion = "3.2.14" + private val scalatestVersion = "3.2.19" private val mockitoVersion = "4.11.0" private val defaultSparkVersionForScala211 = "2.4.8" private val defaultSparkVersionForScala212 = "3.4.4" - private val defaultSparkVersionForScala213 = "3.5.5" + private val defaultSparkVersionForScala213 = "3.5.6" def sparkFallbackVersion(scalaVersion: String): String = { if (scalaVersion.startsWith("2.11.")) { @@ -87,7 +87,7 @@ object Dependencies { val CobolConvertersDependencies: Seq[ModuleID] = Seq( // compile - "org.slf4j" % "slf4j-api" % slf4jVersion, + "org.slf4j" % "slf4j-api" % slf4jVersion, "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion, "com.fasterxml.jackson.dataformat" % "jackson-dataformat-xml" % jacksonVersion, "com.fasterxml.jackson.dataformat" % "jackson-dataformat-csv" % jacksonVersion, From a328804451e3fc7058ec9c1553a7ce868150bb93 Mon Sep 17 00:00:00 2001 From: Ruslan Iushchenko Date: Wed, 10 Sep 2025 08:50:24 +0200 Subject: [PATCH 4/8] #782 spark-cobol writer: Add support for sign position and ignore redefines when parsing the schema. --- README.md | 2 +- .../parser/encoding/DisplayEncoders.scala | 41 ++++++++----- .../encoding/DisplayEncodersSuite.scala | 58 +++++++++---------- .../cobol/writer/BasicRecordCombiner.scala | 7 ++- .../writer/FixedLengthEbcdicWriterSuite.scala | 29 +++++----- 5 files changed, 76 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index e4f7b9de..685bb4ee 100644 --- a/README.md +++ b/README.md @@ -1686,7 +1686,7 @@ The writer is still in its early stages and has several limitations: ``` - Supported types: - `PIC X(n)` alphanumeric. - - `PIC S9(n)` numeric (integral and decimal) with `COMP`/`COMP-4`/`COMP-5` (big-endian), `COMP-3`, and + - `PIC S9(n)` numeric (integral and decimal) with `DISPLAY`, `COMP`/`COMP-4`/`COMP-5` (big-endian), `COMP-3`, and `COMP-9` (Cobrix little-endian). - Only fixed record length output is supported (`record_format = F`). - `REDEFINES` and `OCCURS` are not supported. diff --git a/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncoders.scala b/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncoders.scala index b22ef72e..9f45c1cf 100644 --- a/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncoders.scala +++ b/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncoders.scala @@ -27,9 +27,11 @@ object DisplayEncoders { precision: Int, scale: Int, scaleFactor: Int, - explicitDecimalPoint: Boolean, - ): Array[Byte] = { + explicitDecimalPoint: Boolean): Array[Byte] = { val isSigned = signPosition.isDefined + val lengthAdjustment = if (isSigned) 1 else 0 + val isSignPositionRight = signPosition.contains(za.co.absa.cobrix.cobol.parser.position.Right) + val isNegative = number.signum() < 0 val bytes = new Array[Byte](outputSize) if (number == null || precision < 1 || scale < 0 || outputSize < 1 || (scaleFactor > 0 && scale > 0)) @@ -39,22 +41,20 @@ object DisplayEncoders { val shift = scaleFactor val bigDecimal = if (shift == 0) - number.setScale(scale, RoundingMode.HALF_EVEN) + number.abs().setScale(scale, RoundingMode.HALF_EVEN) else - number.movePointLeft(shift).setScale(scale, RoundingMode.HALF_EVEN) + number.abs().movePointLeft(shift).setScale(scale, RoundingMode.HALF_EVEN) val bigDecimalValue1 = bigDecimal.toString val bigDecimalValue = if (bigDecimalValue1.startsWith("0.")) bigDecimalValue1.drop(1) - else if (bigDecimalValue1.startsWith("-0.")) - "-" + bigDecimalValue1.drop(2) else bigDecimalValue1 - val bigDecimalValueLen = bigDecimalValue.length + val bigDecimalValueLen = bigDecimalValue.length + lengthAdjustment - if (bigDecimalValueLen > outputSize || (!isSigned && bigDecimal.signum() < 0)) + if (bigDecimalValueLen > outputSize || (!isSigned && isNegative)) return bytes bigDecimalValue @@ -62,19 +62,19 @@ object DisplayEncoders { val shift = scaleFactor - scale val bigInt = if (shift == 0) - number.setScale(0, RoundingMode.HALF_EVEN).toBigIntegerExact + number.abs().setScale(0, RoundingMode.HALF_EVEN).toBigIntegerExact else - number.movePointLeft(shift).setScale(0, RoundingMode.HALF_EVEN).toBigIntegerExact + number.abs().movePointLeft(shift).setScale(0, RoundingMode.HALF_EVEN).toBigIntegerExact val intValue = bigInt.toString - val intValueLen = intValue.length + val intValueLen = intValue.length + lengthAdjustment - if (intValueLen > outputSize || (!isSigned && bigInt.signum() < 0)) + if (intValueLen > outputSize || (!isSigned && isNegative)) return bytes intValue } - setPaddedEbcdicNumberWithSignSeparate(num, bytes) + setPaddedEbcdicNumberWithSignSeparate(num, isSigned, isNegative, isSignPositionRight, bytes) bytes } @@ -174,7 +174,20 @@ object DisplayEncoders { } } - def setPaddedEbcdicNumberWithSignSeparate(num: String, array: Array[Byte]): Unit = { + def setPaddedEbcdicNumberWithSignSeparate(absNum: String, isSigned: Boolean, isNegative: Boolean, isSignPositionRight: Boolean, array: Array[Byte]): Unit = { + val num = if (isSigned) { + if (isNegative) { + if (isSignPositionRight) { + s"$absNum-" + } else { + s"-$absNum" + } + } else { + absNum + } + } else { + absNum + } val numLen = num.length val arLen = array.length diff --git a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala index c27fda4d..40a02748 100644 --- a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala +++ b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala @@ -59,15 +59,15 @@ class DisplayEncodersSuite extends AnyWordSpec { } "encode a small negative number" in { - val expected = Array(0x40, 0x40, 0x40, 0x40, 0x40, 0xD7).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-7), signPosition = Some(Right), 6, 5, 0, 0, explicitDecimalPoint = false) + val expected = Array( 0x40, 0x40, 0x40, 0x40, 0xD7).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-7), signPosition = Some(Right), 5, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } - "encode a too big number" ignore { - val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(123456), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) + "encode a too big number" in { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(123456), signPosition = Some(Left), 5, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } @@ -93,9 +93,9 @@ class DisplayEncodersSuite extends AnyWordSpec { assertArraysEqual(actual, expected) } - "attempt to encode a number with an incorrect precision" ignore { - val expected = Array[Byte](0x00, 0x00, 0x00, 0x00, 0x00) - val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(12345), signPosition = None, 5, 4, 0, 0, explicitDecimalPoint = true) + "attempt to encode a number with an incorrect precision" in { + val expected = Array[Byte](0x00, 0x00, 0x00, 0x00) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(12345), signPosition = None, 4, 4, 0, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } @@ -221,16 +221,16 @@ class DisplayEncodersSuite extends AnyWordSpec { assertArraysEqual(actual, expected) } - "encode a too big number" ignore { - val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(1234.56), signPosition = None, 6, 5, 2, 0, explicitDecimalPoint = false) + "encode a too big number" in { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(1234.56), signPosition = None, 5, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } - "encode a too big number with explicit decimal point" ignore { - val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(1234.56), signPosition = None, 7, 5, 2, 0, explicitDecimalPoint = true) + "encode a too big number with explicit decimal point" in { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(1234.56), signPosition = None, 6, 5, 2, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } @@ -318,8 +318,8 @@ class DisplayEncodersSuite extends AnyWordSpec { } "encode a number with an even precision" in { - val expected = Array(0xF1, 0xF2, 0xF3, 0xF4).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234), signPosition = Some(Left), 4, 4, 0, 0, explicitDecimalPoint = false) + val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234), signPosition = Some(Left), 5, 4, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } @@ -346,13 +346,13 @@ class DisplayEncodersSuite extends AnyWordSpec { } "encode a small negative number" in { - val expected = Array(0x40, 0x40, 0x40, 0x40, 0x60, 0xF7).map(_.toByte) + val expected = Array(0x40, 0x40, 0x40, 0x40, 0xF7, 0x60).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-7), signPosition = Some(Right), 6, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } - "encode a too big number" ignore { + "encode a too big number" in { val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(123456), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) @@ -380,9 +380,9 @@ class DisplayEncodersSuite extends AnyWordSpec { assertArraysEqual(actual, expected) } - "attempt to encode a number with an incorrect precision" ignore { + "attempt to encode a number with an incorrect precision" in { val expected = Array[Byte](0x00, 0x00, 0x00, 0x00, 0x00) - val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(12345), signPosition = None, 5, 4, 0, 0, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(12345), signPosition = Some(Left), 5, 4, 0, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } @@ -453,8 +453,8 @@ class DisplayEncodersSuite extends AnyWordSpec { } "encode an unsigned number" in { - val expected = Array(0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF0).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234.5), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) + val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF0).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234.5), signPosition = Some(Left), 7, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } @@ -475,13 +475,13 @@ class DisplayEncodersSuite extends AnyWordSpec { "encode a negative number with explicit decimal point" in { val expected = Array(0x60, 0xF1, 0xF2, 0x4B, 0xF3, 0xF4, 0xF5).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-12.345), signPosition = Some(Right), 7, 5, 3, 0, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-12.345), signPosition = Some(Left), 7, 5, 3, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a small negative number" in { - val expected = Array(0x40, 0x40, 0x40, 0x60, 0xF7).map(_.toByte) + val expected = Array(0x40, 0x40, 0x40, 0xF7, 0x60).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-0.00007), signPosition = Some(Right), 5, 4, 5, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) @@ -508,16 +508,16 @@ class DisplayEncodersSuite extends AnyWordSpec { assertArraysEqual(actual, expected) } - "encode a too big number" ignore { + "encode a too big number" in { val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234.56), signPosition = None, 6, 5, 2, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234.56), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } - "encode a too big number with explicit decimal point" ignore { + "encode a too big number with explicit decimal point" in { val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234.56), signPosition = None, 7, 5, 2, 0, explicitDecimalPoint = true) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234.56), signPosition = Some(Left), 7, 5, 2, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } @@ -586,7 +586,7 @@ class DisplayEncodersSuite extends AnyWordSpec { } "encode a small negative number with negative scale factor with explicit decimal point and sign from right side" in { - val expected = Array(0x60, 0x4B, 0xF1, 0xF2, 0xF0).map(_.toByte) + val expected = Array(0x4B, 0xF1, 0xF2, 0xF0, 0x60).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-0.00012), signPosition = Some(Right), 5, 4, 3, -3, explicitDecimalPoint = true) assertArraysEqual(actual, expected) diff --git a/spark-cobol/src/main/scala/za/co/absa/cobrix/spark/cobol/writer/BasicRecordCombiner.scala b/spark-cobol/src/main/scala/za/co/absa/cobrix/spark/cobol/writer/BasicRecordCombiner.scala index 56fb1c5d..4521d16a 100644 --- a/spark-cobol/src/main/scala/za/co/absa/cobrix/spark/cobol/writer/BasicRecordCombiner.scala +++ b/spark-cobol/src/main/scala/za/co/absa/cobrix/spark/cobol/writer/BasicRecordCombiner.scala @@ -31,9 +31,10 @@ class BasicRecordCombiner extends RecordCombiner { override def combine(df: DataFrame, cobolSchema: CobolSchema, readerParameters: ReaderParameters): RDD[Array[Byte]] = { val ast = getAst(cobolSchema) val copybookFields = ast.children.filter { - case p: Primitive => !p.isFiller - case g: Group => !g.isFiller - case _ => true + case f if f.redefines.nonEmpty => false + case p: Primitive => !p.isFiller + case g: Group => !g.isFiller + case _ => true } validateSchema(df, copybookFields.toSeq) diff --git a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala index 0008bada..71ca5c4d 100644 --- a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala +++ b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala @@ -264,7 +264,7 @@ class FixedLengthEbcdicWriterSuite extends AnyWordSpec with SparkTestBase with B val df = List( (-1, 100.5, new java.math.BigDecimal(10.23), 1, 10050, new java.math.BigDecimal(10.12)), (2, 800.4, new java.math.BigDecimal(30), 2, 80040, new java.math.BigDecimal(30)), - (3, 22.33, new java.math.BigDecimal(-20), 3, -2233, new java.math.BigDecimal(-20)) + (3, 22.33, new java.math.BigDecimal(-20), -3, -2233, new java.math.BigDecimal(-20)) ).toDF("A", "B", "C", "D", "E", "F") val path = new Path(tempDir, "writer1") @@ -274,6 +274,7 @@ class FixedLengthEbcdicWriterSuite extends AnyWordSpec with SparkTestBase with B 05 A PIC S9(1). 05 B PIC 9(4)V9(2). 05 C PIC S9(2).9(2). + 05 C1 PIC X(5) REDEFINES C. 05 D PIC 9(1). 05 E PIC S9(6) SIGN IS LEADING SEPARATE. 05 F PIC S9(2).9(2) SIGN IS TRAILING SEPARATE. @@ -310,19 +311,19 @@ class FixedLengthEbcdicWriterSuite extends AnyWordSpec with SparkTestBase with B 0x40, 0x40, 0xF1, 0xF0, 0xF0, 0xF5, 0xF0, // 10050 S9(6) SIGN IS LEADING SEPARATE. 0x40, 0xF1, 0xF0, 0x4B, 0xF1, 0xF2, // 10.12 S9(2).9(2) SIGN IS TRAILING SEPARATE - 0xC2, - 0x40, 0xF8, 0xF0, 0xF0, 0xF4, 0xF0, - 0xF3, 0xF0, 0x4B, 0xF0, 0xC0, - 0xF2, - 0x40, 0x40, 0xF8, 0xF0, 0xF0, 0xF4, 0xF0, - 0x40, 0xF3, 0xF0, 0x4B, 0xF0, 0xF0, - - 0xC3, - 0x40, 0x40, 0xF2, 0xF2, 0xF3, 0xF3, - 0xF2, 0xF0, 0x4B, 0xF0, 0xD0, - 0xF3, - 0x40, 0x40, 0x60, 0xF2, 0xF2, 0xF3, 0xF3, - 0x60, 0xF2, 0xF0, 0x4B, 0xF0, 0xF0 + 0xC2, // 2 PIC S9(1). + 0x40, 0xF8, 0xF0, 0xF0, 0xF4, 0xF0, // 800.4 PIC 9(4)V9(2) + 0xF3, 0xF0, 0x4B, 0xF0, 0xC0, // 30 PIC S9(2).9(2) + 0xF2, // 2 9(1) + 0x40, 0x40, 0xF8, 0xF0, 0xF0, 0xF4, 0xF0, // 80040 S9(6) SIGN IS LEADING SEPARATE. + 0x40, 0xF3, 0xF0, 0x4B, 0xF0, 0xF0, // 30 S9(2).9(2) SIGN IS TRAILING SEPARATE + + 0xC3, // 3 PIC S9(1). + 0x40, 0x40, 0xF2, 0xF2, 0xF3, 0xF3, // 22.33 PIC 9(4)V9(2) + 0xF2, 0xF0, 0x4B, 0xF0, 0xD0, // -20 PIC S9(2).9(2) + 0x00, // null PIC 9(1) (because a negative value cannot be converted to this PIC) + 0x40, 0x40, 0x60, 0xF2, 0xF2, 0xF3, 0xF3, // -2233 S9(6) SIGN IS LEADING SEPARATE. + 0xF2, 0xF0, 0x4B, 0xF0, 0xF0, 0x60 // -20 S9(2).9(2) SIGN IS TRAILING SEPARATE ).map(_.toByte) assertArraysEqual(bytes, expected) From 41566c4773e2b48ed7e838489ca331c2373d8d0a Mon Sep 17 00:00:00 2001 From: Ruslan Iushchenko Date: Wed, 10 Sep 2025 10:04:33 +0200 Subject: [PATCH 5/8] #782 spark-cobol writer: DISPLAY numbers with sign separate always puts the sign, '+' and '-'. --- .../parser/encoding/DisplayEncoders.scala | 52 +++++------ .../encoding/DisplayEncodersSuite.scala | 92 +++++++++---------- .../writer/FixedLengthEbcdicWriterSuite.scala | 60 +++++++++--- 3 files changed, 118 insertions(+), 86 deletions(-) diff --git a/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncoders.scala b/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncoders.scala index 9f45c1cf..0404dc81 100644 --- a/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncoders.scala +++ b/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncoders.scala @@ -31,12 +31,13 @@ object DisplayEncoders { val isSigned = signPosition.isDefined val lengthAdjustment = if (isSigned) 1 else 0 val isSignPositionRight = signPosition.contains(za.co.absa.cobrix.cobol.parser.position.Right) - val isNegative = number.signum() < 0 val bytes = new Array[Byte](outputSize) if (number == null || precision < 1 || scale < 0 || outputSize < 1 || (scaleFactor > 0 && scale > 0)) return bytes + val isNegative = number.signum() < 0 + val num = if (explicitDecimalPoint) { val shift = scaleFactor @@ -45,7 +46,7 @@ object DisplayEncoders { else number.abs().movePointLeft(shift).setScale(scale, RoundingMode.HALF_EVEN) - val bigDecimalValue1 = bigDecimal.toString + val bigDecimalValue1 = bigDecimal.toPlainString val bigDecimalValue = if (bigDecimalValue1.startsWith("0.")) bigDecimalValue1.drop(1) @@ -86,13 +87,13 @@ object DisplayEncoders { scaleFactor: Int, explicitDecimalPoint: Boolean): Array[Byte] = { val isSigned = signPosition.isDefined - val isNegative = number.signum() < 0 val bytes = new Array[Byte](outputSize) - if (number == null || precision < 1 || scale < 0 || outputSize < 1 || (scaleFactor > 0 && scale > 0)) return bytes + val isNegative = number.signum() < 0 + val num = if (explicitDecimalPoint) { val shift = scaleFactor @@ -101,7 +102,7 @@ object DisplayEncoders { else number.abs().movePointLeft(shift).setScale(scale, RoundingMode.HALF_EVEN) - val bigDecimalValue1 = bigDecimal.toString + val bigDecimalValue1 = bigDecimal.toPlainString val bigDecimalValue = if (bigDecimalValue1.startsWith("0.")) bigDecimalValue1.drop(1) @@ -143,7 +144,7 @@ object DisplayEncoders { var i = 0 while (i < arLen) { - var ebcdic = 0x40.toByte + var ebcdic = 0xF0.toByte if (i == 0) { // Signal overpunching @@ -164,7 +165,7 @@ object DisplayEncoders { val c = num(numLen - i - 1) if (c >= '0' && c <= '9') { ebcdic = ((c - '0') + 0xF0).toByte - } else if (c == '.' || c == ',') { + } else if (c == '.') { ebcdic = 0x4B } } @@ -174,43 +175,38 @@ object DisplayEncoders { } } - def setPaddedEbcdicNumberWithSignSeparate(absNum: String, isSigned: Boolean, isNegative: Boolean, isSignPositionRight: Boolean, array: Array[Byte]): Unit = { - val num = if (isSigned) { - if (isNegative) { - if (isSignPositionRight) { - s"$absNum-" - } else { - s"-$absNum" - } - } else { - absNum - } - } else { - absNum - } + def setPaddedEbcdicNumberWithSignSeparate(num: String, isSigned: Boolean, isNegative: Boolean, isSignPositionRight: Boolean, array: Array[Byte]): Unit = { val numLen = num.length val arLen = array.length + val fullNumLength = if (isSigned) numLen + 1 else numLen - if (numLen > arLen) + if (fullNumLength > arLen) return + val shift = if (isSigned && isSignPositionRight) 1 else 0 var i = 0 - while (i < arLen) { - var ebcdic = 0x40.toByte + while (i < arLen - shift) { + var ebcdic = 0xF0.toByte if (i < numLen) { val c = num(numLen - i - 1) if (c >= '0' && c <= '9') { ebcdic = ((c - '0') + 0xF0).toByte - } else if (c == '.' || c == ',') { + } else if (c == '.') { ebcdic = 0x4B - } else if (c == '-') { - ebcdic = 0x60 } } - array(arLen - i - 1) = ebcdic + array(arLen - i - shift - 1) = ebcdic i += 1 } + + if (isSigned) { + if (isNegative) { + if (isSignPositionRight) array(arLen - 1) = 0x60 else array(0) = 0x60 + } else { + if (isSignPositionRight) array(arLen - 1) = 0x4E else array(0) = 0x4E + } + } } } diff --git a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala index 40a02748..3c3f0cd5 100644 --- a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala +++ b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala @@ -24,7 +24,7 @@ class DisplayEncodersSuite extends AnyWordSpec { "encodeDisplayNumberSignOverpunched" should { "integral number" when { "encode a number" in { - val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xC5).map(_.toByte) + val expected = Array(0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xC5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(12345), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) @@ -38,28 +38,28 @@ class DisplayEncodersSuite extends AnyWordSpec { } "encode a small number" in { - val expected = Array(0x40, 0x40, 0x40, 0x40, 0x40, 0xC5).map(_.toByte) + val expected = Array(0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xC5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(5), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode an unsigned number" in { - val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) + val expected = Array(0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(12345), signPosition = None, 6, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a negative number" in { - val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xD5).map(_.toByte) + val expected = Array(0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xD5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-12345), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a small negative number" in { - val expected = Array( 0x40, 0x40, 0x40, 0x40, 0xD7).map(_.toByte) + val expected = Array( 0xF0, 0xF0, 0xF0, 0xF0, 0xD7).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-7), signPosition = Some(Right), 5, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) @@ -110,56 +110,56 @@ class DisplayEncodersSuite extends AnyWordSpec { "decimal number" when { "encode a number" in { - val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xC5).map(_.toByte) + val expected = Array(0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xC5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(123.45), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a number with explicit decimal" in { - val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0x4B, 0xF4, 0xC5).map(_.toByte) + val expected = Array(0xF0, 0xF1, 0xF2, 0xF3, 0x4B, 0xF4, 0xC5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(123.45), signPosition = Some(Left), 7, 5, 2, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a small number 1" in { - val expected = Array(0x40, 0x40, 0x40, 0x40, 0x40, 0xC5).map(_.toByte) + val expected = Array(0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xC5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(0.05), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a small number 2" in { - val expected = Array(0x40, 0x40, 0x40, 0x40, 0xF5, 0xC0).map(_.toByte) + val expected = Array(0xF0, 0xF0, 0xF0, 0xF0, 0xF5, 0xC0).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(0.5), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a small number 3" in { - val expected = Array(0x40, 0x40, 0x40, 0x40, 0x40, 0xC1).map(_.toByte) + val expected = Array(0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xC1).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(0.005), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a small number 1 with explicit decimal point" in { - val expected = Array(0x40, 0x40, 0x40, 0x4B, 0xF0, 0xC5).map(_.toByte) + val expected = Array(0xF0, 0xF0, 0xF0, 0x4B, 0xF0, 0xC5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(0.05), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a small number 2 with explicit decimal point" in { - val expected = Array(0x40, 0x40, 0x40, 0x4B, 0xF5, 0xC0).map(_.toByte) + val expected = Array(0xF0, 0xF0, 0xF0, 0x4B, 0xF5, 0xC0).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(0.5), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a small number 3 with explicit decimal point" in { - val expected = Array(0x40, 0x40, 0x40, 0x4B, 0xF0, 0xC1).map(_.toByte) + val expected = Array(0xF0, 0xF0, 0xF0, 0x4B, 0xF0, 0xC1).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(0.005), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) @@ -167,7 +167,7 @@ class DisplayEncodersSuite extends AnyWordSpec { "encode an unsigned number" in { val expected = Array(0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF0).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234.5), signPosition = None, 6, 5, 2, 0, explicitDecimalPoint = false) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(1234.5), signPosition = None, 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } @@ -180,36 +180,36 @@ class DisplayEncodersSuite extends AnyWordSpec { } "encode a negative number" in { - val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xD5).map(_.toByte) + val expected = Array(0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xD5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-12.345), signPosition = Some(Left), 6, 5, 3, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a negative number with explicit decimal point" in { - val expected = Array(0x40, 0xF1, 0xF2, 0x4B, 0xF3, 0xF4, 0xD5).map(_.toByte) + val expected = Array(0xF0, 0xF1, 0xF2, 0x4B, 0xF3, 0xF4, 0xD5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-12.345), signPosition = Some(Right), 7, 5, 3, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a small negative number" in { - val expected = Array(0x40, 0x40, 0x40, 0x40, 0xD7).map(_.toByte) + val expected = Array(0xF0, 0xF0, 0xF0, 0xF0, 0xD7).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-0.00007), signPosition = Some(Right), 5, 4, 5, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a small negative number with explicit decimal point" in { - val expected = Array(0x40, 0x40, 0x4B, 0xF0, 0xF0, 0xF0, 0xF0, 0xD7).map(_.toByte) + val expected = Array(0xF0, 0xF0, 0x4B, 0xF0, 0xF0, 0xF0, 0xF0, 0xD7).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(-0.00007), signPosition = Some(Left), 8, 4, 5, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a too precise number" in { - val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xF6).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(123.456), signPosition = None, 6, 5, 2, 0, explicitDecimalPoint = false) + val expected = Array(0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF6).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(123.456), signPosition = None, 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } @@ -250,7 +250,7 @@ class DisplayEncodersSuite extends AnyWordSpec { } "encode a number with positive scale factor" in { - val expected = Array(0x40, 0x40, 0x40, 0xF1, 0xF2, 0xC3).map(_.toByte) + val expected = Array(0xF0, 0xF0, 0xF0, 0xF1, 0xF2, 0xC3).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(12300), signPosition = Some(Left), 6, 5, 0, 2, explicitDecimalPoint = false) assertArraysEqual(actual, expected) @@ -264,21 +264,21 @@ class DisplayEncodersSuite extends AnyWordSpec { } "encode a number with negative scale factor" in { - val expected = Array(0x40, 0x40, 0x40, 0xF1, 0xF2, 0xC3).map(_.toByte) + val expected = Array(0xF0, 0xF0, 0xF0, 0xF1, 0xF2, 0xC3).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(1.23), signPosition = Some(Left), 6, 5, 0, -2, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a number with negative scale factor with explicit decimal point" in { - val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0x4B, 0xC4).map(_.toByte) + val expected = Array(0xF0, 0xF1, 0xF2, 0xF3, 0x4B, 0xC4).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(1.234), signPosition = Some(Left), 6, 5, 1, -2, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a small number with negative scale factor" in { - val expected = Array(0x40, 0x40, 0xF1, 0xF2, 0xF0).map(_.toByte) + val expected = Array(0xF0, 0xF0, 0xF1, 0xF2, 0xF0).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(0.00012), signPosition = None, 5, 4, 3, -3, explicitDecimalPoint = false) assertArraysEqual(actual, expected) @@ -311,29 +311,29 @@ class DisplayEncodersSuite extends AnyWordSpec { "encodeDisplayNumberSignSeparate" should { "integral number" when { "encode a number" in { - val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) + val expected = Array(0x4E, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(12345), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a number with an even precision" in { - val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4).map(_.toByte) + val expected = Array(0x4E, 0xF1, 0xF2, 0xF3, 0xF4).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234), signPosition = Some(Left), 5, 4, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a small number" in { - val expected = Array(0x40, 0x40, 0x40, 0x40, 0x40, 0xF5).map(_.toByte) + val expected = Array(0x4E, 0xF0, 0xF0, 0xF0, 0xF0, 0xF5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(5), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode an unsigned number" in { - val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(12345), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) + val expected = Array(0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(12345), signPosition = None, 6, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } @@ -346,7 +346,7 @@ class DisplayEncodersSuite extends AnyWordSpec { } "encode a small negative number" in { - val expected = Array(0x40, 0x40, 0x40, 0x40, 0xF7, 0x60).map(_.toByte) + val expected = Array(0xF0, 0xF0, 0xF0, 0xF0, 0xF7, 0x60).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-7), signPosition = Some(Right), 6, 5, 0, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) @@ -397,63 +397,63 @@ class DisplayEncodersSuite extends AnyWordSpec { "decimal number" when { "encode a number" in { - val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) + val expected = Array(0x4E, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(123.45), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a number with explicit decimal" in { - val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0x4B, 0xF4, 0xF5).map(_.toByte) + val expected = Array(0x4E, 0xF1, 0xF2, 0xF3, 0x4B, 0xF4, 0xF5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(123.45), signPosition = Some(Left), 7, 5, 2, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a small number 1" in { - val expected = Array(0x40, 0x40, 0x40, 0x40, 0x40, 0xF5).map(_.toByte) + val expected = Array(0x4E, 0xF0, 0xF0, 0xF0, 0xF0, 0xF5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(0.05), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a small number 2" in { - val expected = Array(0x40, 0x40, 0x40, 0x40, 0xF5, 0xF0).map(_.toByte) + val expected = Array(0x4E, 0xF0, 0xF0, 0xF0, 0xF5, 0xF0).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(0.5), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a small number 3" in { - val expected = Array(0x40, 0x40, 0x40, 0x40, 0x40, 0xF1).map(_.toByte) + val expected = Array(0x4E, 0xF0, 0xF0, 0xF0, 0xF0, 0xF1).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(0.005), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a small number 1 with explicit decimal point" in { - val expected = Array(0x40, 0x40, 0x40, 0x4B, 0xF0, 0xF5).map(_.toByte) + val expected = Array(0x4E, 0xF0, 0xF0, 0x4B, 0xF0, 0xF5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(0.05), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a small number 2 with explicit decimal point" in { - val expected = Array(0x40, 0x40, 0x40, 0x4B, 0xF5, 0xF0).map(_.toByte) + val expected = Array(0x4E, 0xF0, 0xF0, 0x4B, 0xF5, 0xF0).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(0.5), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a small number 3 with explicit decimal point" in { - val expected = Array(0x40, 0x40, 0x40, 0x4B, 0xF0, 0xF1).map(_.toByte) + val expected = Array(0x4E, 0xF0, 0xF0, 0x4B, 0xF0, 0xF1).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(0.005), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode an unsigned number" in { - val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF0).map(_.toByte) + val expected = Array(0x4E, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF0).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234.5), signPosition = Some(Left), 7, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) @@ -481,21 +481,21 @@ class DisplayEncodersSuite extends AnyWordSpec { } "encode a small negative number" in { - val expected = Array(0x40, 0x40, 0x40, 0xF7, 0x60).map(_.toByte) + val expected = Array(0xF0, 0xF0, 0xF0, 0xF7, 0x60).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-0.00007), signPosition = Some(Right), 5, 4, 5, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a small negative number with explicit decimal point" in { - val expected = Array(0x40, 0x60, 0x4B, 0xF0, 0xF0, 0xF0, 0xF0, 0xF7).map(_.toByte) + val expected = Array(0x60, 0xF0, 0x4B, 0xF0, 0xF0, 0xF0, 0xF0, 0xF7).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(-0.00007), signPosition = Some(Left), 8, 4, 5, 0, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a too precise number" in { - val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0xF4, 0xF6).map(_.toByte) + val expected = Array(0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF6).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(123.456), signPosition = None, 6, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) @@ -537,7 +537,7 @@ class DisplayEncodersSuite extends AnyWordSpec { } "encode a number with positive scale factor" in { - val expected = Array(0x40, 0x40, 0x40, 0xF1, 0xF2, 0xF3).map(_.toByte) + val expected = Array(0x4E, 0xF0, 0xF0, 0xF1, 0xF2, 0xF3).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(12300), signPosition = Some(Left), 6, 5, 0, 2, explicitDecimalPoint = false) assertArraysEqual(actual, expected) @@ -551,21 +551,21 @@ class DisplayEncodersSuite extends AnyWordSpec { } "encode a number with negative scale factor" in { - val expected = Array(0x40, 0x40, 0x40, 0xF1, 0xF2, 0xF3).map(_.toByte) + val expected = Array(0x4E, 0xF0, 0xF0, 0xF1, 0xF2, 0xF3).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1.23), signPosition = Some(Left), 6, 5, 0, -2, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } "encode a number with negative scale factor with explicit decimal point" in { - val expected = Array(0x40, 0xF1, 0xF2, 0xF3, 0x4B, 0xF4).map(_.toByte) + val expected = Array(0x4E, 0xF1, 0xF2, 0xF3, 0x4B, 0xF4).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1.234), signPosition = Some(Left), 6, 5, 1, -2, explicitDecimalPoint = true) assertArraysEqual(actual, expected) } "encode a small number with negative scale factor" in { - val expected = Array(0x40, 0x40, 0xF1, 0xF2, 0xF0).map(_.toByte) + val expected = Array(0xF0, 0xF0, 0xF1, 0xF2, 0xF0).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(0.00012), signPosition = None, 5, 4, 3, -3, explicitDecimalPoint = false) assertArraysEqual(actual, expected) diff --git a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala index 71ca5c4d..d5411882 100644 --- a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala +++ b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala @@ -21,9 +21,10 @@ import org.apache.spark.sql.SaveMode import org.scalatest.Assertion import org.scalatest.wordspec.AnyWordSpec import za.co.absa.cobrix.spark.cobol.source.base.SparkTestBase -import za.co.absa.cobrix.spark.cobol.source.fixtures.BinaryFileFixture +import za.co.absa.cobrix.spark.cobol.source.fixtures.{BinaryFileFixture, TextComparisonFixture} +import za.co.absa.cobrix.spark.cobol.utils.SparkUtils -class FixedLengthEbcdicWriterSuite extends AnyWordSpec with SparkTestBase with BinaryFileFixture { +class FixedLengthEbcdicWriterSuite extends AnyWordSpec with SparkTestBase with BinaryFileFixture with TextComparisonFixture { import spark.implicits._ @@ -264,7 +265,7 @@ class FixedLengthEbcdicWriterSuite extends AnyWordSpec with SparkTestBase with B val df = List( (-1, 100.5, new java.math.BigDecimal(10.23), 1, 10050, new java.math.BigDecimal(10.12)), (2, 800.4, new java.math.BigDecimal(30), 2, 80040, new java.math.BigDecimal(30)), - (3, 22.33, new java.math.BigDecimal(-20), -3, -2233, new java.math.BigDecimal(-20)) + (3, 22.33, new java.math.BigDecimal(-20), -3, -2233, new java.math.BigDecimal(-20.456)) ).toDF("A", "B", "C", "D", "E", "F") val path = new Path(tempDir, "writer1") @@ -305,28 +306,63 @@ class FixedLengthEbcdicWriterSuite extends AnyWordSpec with SparkTestBase with B // Expected EBCDIC data for sample test data val expected = Array( 0xD1, // -1 PIC S9(1). - 0x40, 0xF1, 0xF0, 0xF0, 0xF5, 0xF0, // 100.5 PIC 9(4)V9(2) + 0xF0, 0xF1, 0xF0, 0xF0, 0xF5, 0xF0, // 100.5 PIC 9(4)V9(2) 0xF1, 0xF0, 0x4B, 0xF2, 0xC3, // 10.23 PIC S9(2).9(2) 0xF1, // 1 9(1) - 0x40, 0x40, 0xF1, 0xF0, 0xF0, 0xF5, 0xF0, // 10050 S9(6) SIGN IS LEADING SEPARATE. - 0x40, 0xF1, 0xF0, 0x4B, 0xF1, 0xF2, // 10.12 S9(2).9(2) SIGN IS TRAILING SEPARATE + 0x4E, 0xF0, 0xF1, 0xF0, 0xF0, 0xF5, 0xF0, // 10050 S9(6) SIGN IS LEADING SEPARATE. + 0xF1, 0xF0, 0x4B, 0xF1, 0xF2, 0x4E, // 10.12 S9(2).9(2) SIGN IS TRAILING SEPARATE 0xC2, // 2 PIC S9(1). - 0x40, 0xF8, 0xF0, 0xF0, 0xF4, 0xF0, // 800.4 PIC 9(4)V9(2) + 0xF0, 0xF8, 0xF0, 0xF0, 0xF4, 0xF0, // 800.4 PIC 9(4)V9(2) 0xF3, 0xF0, 0x4B, 0xF0, 0xC0, // 30 PIC S9(2).9(2) 0xF2, // 2 9(1) - 0x40, 0x40, 0xF8, 0xF0, 0xF0, 0xF4, 0xF0, // 80040 S9(6) SIGN IS LEADING SEPARATE. - 0x40, 0xF3, 0xF0, 0x4B, 0xF0, 0xF0, // 30 S9(2).9(2) SIGN IS TRAILING SEPARATE + 0x4E, 0xF0, 0xF8, 0xF0, 0xF0, 0xF4, 0xF0, // 80040 S9(6) SIGN IS LEADING SEPARATE. + 0xF3, 0xF0, 0x4B, 0xF0, 0xF0, 0x4E, // 30 S9(2).9(2) SIGN IS TRAILING SEPARATE 0xC3, // 3 PIC S9(1). - 0x40, 0x40, 0xF2, 0xF2, 0xF3, 0xF3, // 22.33 PIC 9(4)V9(2) + 0xF0, 0xF0, 0xF2, 0xF2, 0xF3, 0xF3, // 22.33 PIC 9(4)V9(2) 0xF2, 0xF0, 0x4B, 0xF0, 0xD0, // -20 PIC S9(2).9(2) 0x00, // null PIC 9(1) (because a negative value cannot be converted to this PIC) - 0x40, 0x40, 0x60, 0xF2, 0xF2, 0xF3, 0xF3, // -2233 S9(6) SIGN IS LEADING SEPARATE. - 0xF2, 0xF0, 0x4B, 0xF0, 0xF0, 0x60 // -20 S9(2).9(2) SIGN IS TRAILING SEPARATE + 0x60, 0xF0, 0xF0, 0xF2, 0xF2, 0xF3, 0xF3, // -2233 S9(6) SIGN IS LEADING SEPARATE. + 0xF2, 0xF0, 0x4B, 0xF4, 0xF6, 0x60 // -20 S9(2).9(2) SIGN IS TRAILING SEPARATE ).map(_.toByte) assertArraysEqual(bytes, expected) + + val df2 = spark.read.format("cobol") + .option("copybook_contents", copybookContentsWithBinFields) + .load(path.toString) + .orderBy("A") + + val expectedJson = + """[ { + | "A" : -1, + | "B" : 100.5, + | "C" : 10.23, + | "C1" : "10.2C", + | "D" : 1, + | "E" : 10050, + | "F" : 10.12 + |}, { + | "A" : 2, + | "B" : 800.4, + | "C" : 30.0, + | "C1" : "30.0{", + | "D" : 2, + | "E" : 80040, + | "F" : 30.0 + |}, { + | "A" : 3, + | "B" : 22.33, + | "C" : -20.0, + | "C1" : "20.0}", + | "E" : -2233, + | "F" : -20.46 + |} ]""".stripMargin + + val actualJson = SparkUtils.convertDataFrameToPrettyJSON(df2) + + compareText(actualJson, expectedJson) } } From 236b55577354d91ef215780a4a5845c2f3a7ed09 Mon Sep 17 00:00:00 2001 From: Ruslan Iushchenko Date: Wed, 10 Sep 2025 10:06:00 +0200 Subject: [PATCH 6/8] #782 spark-cobol writer: DISPLAY numbers: fixed a unit test for unsigned numbers. --- .../cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala index 3c3f0cd5..fd70548d 100644 --- a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala +++ b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala @@ -453,8 +453,8 @@ class DisplayEncodersSuite extends AnyWordSpec { } "encode an unsigned number" in { - val expected = Array(0x4E, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF0).map(_.toByte) - val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234.5), signPosition = Some(Left), 7, 5, 2, 0, explicitDecimalPoint = false) + val expected = Array(0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF0).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(1234.5), signPosition = None, 7, 5, 2, 0, explicitDecimalPoint = false) assertArraysEqual(actual, expected) } From 13445ff5f49621fc31a7b4ea37a6169788351348 Mon Sep 17 00:00:00 2001 From: Ruslan Iushchenko Date: Wed, 10 Sep 2025 11:12:06 +0200 Subject: [PATCH 7/8] #782 spark-cobol writer: DISPLAY numbers: handle nulls properly. --- .../encoding/BCDNumberEncodersSuite.scala | 15 +++++++ .../parser/encoding/BinaryEncodersSuite.scala | 7 ++++ .../encoding/DisplayEncodersSuite.scala | 42 +++++++++++++++++++ .../writer/FixedLengthEbcdicWriterSuite.scala | 15 ++++++- 4 files changed, 77 insertions(+), 2 deletions(-) diff --git a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/BCDNumberEncodersSuite.scala b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/BCDNumberEncodersSuite.scala index e3331d48..67905be0 100644 --- a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/BCDNumberEncodersSuite.scala +++ b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/BCDNumberEncodersSuite.scala @@ -17,11 +17,19 @@ package za.co.absa.cobrix.cobol.parser.encoding import org.scalatest.wordspec.AnyWordSpec +import za.co.absa.cobrix.cobol.parser.position.Left import za.co.absa.cobrix.cobol.testutils.ComparisonUtils._ class BCDNumberEncodersSuite extends AnyWordSpec { "encodeBCDNumber" should { "integral number" when { + "encode a null" in { + val expected = Array(0x00, 0x00).map(_.toByte) + val actual = BCDNumberEncoders.encodeBCDNumber(null: java.math.BigDecimal, 2, 0, 0, signed = true, mandatorySignNibble = true) + + assertArraysEqual(actual, expected) + } + "encode a number" in { val expected = Array[Byte](0x12, 0x34, 0x5C) val actual = BCDNumberEncoders.encodeBCDNumber(new java.math.BigDecimal(12345), 5, 0, 0, signed = true, mandatorySignNibble = true) @@ -133,6 +141,13 @@ class BCDNumberEncodersSuite extends AnyWordSpec { } "decimal number" when { + "encode a null" in { + val expected = Array(0x00, 0x00).map(_.toByte) + val actual = BCDNumberEncoders.encodeBCDNumber(null: java.math.BigDecimal, 2, 1, 0, signed = true, mandatorySignNibble = true) + + assertArraysEqual(actual, expected) + } + "encode a number" in { val expected = Array[Byte](0x12, 0x34, 0x5C) val actual = BCDNumberEncoders.encodeBCDNumber(java.math.BigDecimal.valueOf(123.45), 5, 2, 0, signed = true, mandatorySignNibble = true) diff --git a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/BinaryEncodersSuite.scala b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/BinaryEncodersSuite.scala index cbf12745..5e7894a1 100644 --- a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/BinaryEncodersSuite.scala +++ b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/BinaryEncodersSuite.scala @@ -21,6 +21,13 @@ import za.co.absa.cobrix.cobol.testutils.ComparisonUtils._ class BinaryEncodersSuite extends AnyWordSpec { "encodeBinaryNumber" should { + "encode a null" in { + val expected = Array(0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = BinaryEncoders.encodeBinaryNumber(null: java.math.BigDecimal, isSigned = true, outputSize = 4, bigEndian = true, precision = 5, scale = 0, scaleFactor = 0) + + assertArraysEqual(actual, expected) + } + "encode a positive integer in big-endian format" in { val expected = Array(0x00, 0x00, 0x30, 0x39).map(_.toByte) // 12345 in hex val actual = BinaryEncoders.encodeBinaryNumber(new java.math.BigDecimal(12345), isSigned = true, outputSize = 4, bigEndian = true, precision = 5, scale = 0, scaleFactor = 0) diff --git a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala index fd70548d..359458a2 100644 --- a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala +++ b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala @@ -23,6 +23,13 @@ import za.co.absa.cobrix.cobol.testutils.ComparisonUtils.assertArraysEqual class DisplayEncodersSuite extends AnyWordSpec { "encodeDisplayNumberSignOverpunched" should { "integral number" when { + "encode a null" in { + val expected = Array(0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(null: java.math.BigDecimal, signPosition = Some(Left), 3, 2, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + "encode a number" in { val expected = Array(0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xC5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(12345), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) @@ -109,6 +116,20 @@ class DisplayEncodersSuite extends AnyWordSpec { } "decimal number" when { + "encode a null" in { + val expected = Array(0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(null: java.math.BigDecimal, signPosition = Some(Left), 3, 2, 1, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a null and separate decimal point" in { + val expected = Array(0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(null: java.math.BigDecimal, signPosition = Some(Left), 4, 2, 1, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + "encode a number" in { val expected = Array(0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xC5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(123.45), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) @@ -310,6 +331,13 @@ class DisplayEncodersSuite extends AnyWordSpec { "encodeDisplayNumberSignSeparate" should { "integral number" when { + "encode a null" in { + val expected = Array(0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(null: java.math.BigDecimal, signPosition = Some(Left), 3, 2, 0, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + "encode a number" in { val expected = Array(0x4E, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(12345), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false) @@ -396,6 +424,20 @@ class DisplayEncodersSuite extends AnyWordSpec { } "decimal number" when { + "encode a null" in { + val expected = Array(0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(null: java.math.BigDecimal, signPosition = Some(Left), 4, 2, 1, 0, explicitDecimalPoint = false) + + assertArraysEqual(actual, expected) + } + + "encode a null and separate decimal point" in { + val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte) + val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(null: java.math.BigDecimal, signPosition = Some(Left), 5, 2, 1, 0, explicitDecimalPoint = true) + + assertArraysEqual(actual, expected) + } + "encode a number" in { val expected = Array(0x4E, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte) val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(123.45), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false) diff --git a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala index d5411882..103a2af7 100644 --- a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala +++ b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala @@ -262,10 +262,12 @@ class FixedLengthEbcdicWriterSuite extends AnyWordSpec with SparkTestBase with B "write data frames with DISPLAY fields" in { withTempDirectory("cobol_writer1") { tempDir => + val bigDecimalNull = null: java.math.BigDecimal val df = List( (-1, 100.5, new java.math.BigDecimal(10.23), 1, 10050, new java.math.BigDecimal(10.12)), (2, 800.4, new java.math.BigDecimal(30), 2, 80040, new java.math.BigDecimal(30)), - (3, 22.33, new java.math.BigDecimal(-20), -3, -2233, new java.math.BigDecimal(-20.456)) + (3, 22.33, new java.math.BigDecimal(-20), -3, -2233, new java.math.BigDecimal(-20.456)), + (4, -1.0, bigDecimalNull, 400, 1000000, bigDecimalNull) ).toDF("A", "B", "C", "D", "E", "F") val path = new Path(tempDir, "writer1") @@ -324,7 +326,14 @@ class FixedLengthEbcdicWriterSuite extends AnyWordSpec with SparkTestBase with B 0xF2, 0xF0, 0x4B, 0xF0, 0xD0, // -20 PIC S9(2).9(2) 0x00, // null PIC 9(1) (because a negative value cannot be converted to this PIC) 0x60, 0xF0, 0xF0, 0xF2, 0xF2, 0xF3, 0xF3, // -2233 S9(6) SIGN IS LEADING SEPARATE. - 0xF2, 0xF0, 0x4B, 0xF4, 0xF6, 0x60 // -20 S9(2).9(2) SIGN IS TRAILING SEPARATE + 0xF2, 0xF0, 0x4B, 0xF4, 0xF6, 0x60, // -20 S9(2).9(2) SIGN IS TRAILING SEPARATE + + 0xC4, // 4 PIC S9(1). + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // nulls + 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 ).map(_.toByte) assertArraysEqual(bytes, expected) @@ -358,6 +367,8 @@ class FixedLengthEbcdicWriterSuite extends AnyWordSpec with SparkTestBase with B | "C1" : "20.0}", | "E" : -2233, | "F" : -20.46 + |}, { + | "A" : 4 |} ]""".stripMargin val actualJson = SparkUtils.convertDataFrameToPrettyJSON(df2) From e4b1633212eded1516ac9e24d3b141ed32dace86 Mon Sep 17 00:00:00 2001 From: Ruslan Iushchenko Date: Wed, 10 Sep 2025 11:42:56 +0200 Subject: [PATCH 8/8] #782 spark-cobol writer: DISPLAY numbers: fix a nitpick PR comment regarding the variable name. --- .../spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala index 103a2af7..6e618425 100644 --- a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala +++ b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala @@ -272,7 +272,7 @@ class FixedLengthEbcdicWriterSuite extends AnyWordSpec with SparkTestBase with B val path = new Path(tempDir, "writer1") - val copybookContentsWithBinFields = + val copybookContentsWithDisplayFields = """ 01 RECORD. 05 A PIC S9(1). 05 B PIC 9(4)V9(2). @@ -288,7 +288,7 @@ class FixedLengthEbcdicWriterSuite extends AnyWordSpec with SparkTestBase with B .write .format("cobol") .mode(SaveMode.Overwrite) - .option("copybook_contents", copybookContentsWithBinFields) + .option("copybook_contents", copybookContentsWithDisplayFields) .save(path.toString) val fs = path.getFileSystem(spark.sparkContext.hadoopConfiguration) @@ -339,7 +339,7 @@ class FixedLengthEbcdicWriterSuite extends AnyWordSpec with SparkTestBase with B assertArraysEqual(bytes, expected) val df2 = spark.read.format("cobol") - .option("copybook_contents", copybookContentsWithBinFields) + .option("copybook_contents", copybookContentsWithDisplayFields) .load(path.toString) .orderBy("A")