Skip to content

Commit 13445ff

Browse files
yruslanyruslan
committed
#782 spark-cobol writer: DISPLAY numbers: handle nulls properly.
1 parent 236b555 commit 13445ff

File tree

4 files changed

+77
-2
lines changed

4 files changed

+77
-2
lines changed

‎cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/BCDNumberEncodersSuite.scala‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,19 @@
1717
package za.co.absa.cobrix.cobol.parser.encoding
1818

1919
import org.scalatest.wordspec.AnyWordSpec
20+
import za.co.absa.cobrix.cobol.parser.position.Left
2021
import za.co.absa.cobrix.cobol.testutils.ComparisonUtils._
2122

2223
class BCDNumberEncodersSuite extends AnyWordSpec {
2324
"encodeBCDNumber" should {
2425
"integral number" when {
26+
"encode a null" in {
27+
val expected = Array(0x00, 0x00).map(_.toByte)
28+
val actual = BCDNumberEncoders.encodeBCDNumber(null: java.math.BigDecimal, 2, 0, 0, signed = true, mandatorySignNibble = true)
29+
30+
assertArraysEqual(actual, expected)
31+
}
32+
2533
"encode a number" in {
2634
val expected = Array[Byte](0x12, 0x34, 0x5C)
2735
val actual = BCDNumberEncoders.encodeBCDNumber(new java.math.BigDecimal(12345), 5, 0, 0, signed = true, mandatorySignNibble = true)
@@ -133,6 +141,13 @@ class BCDNumberEncodersSuite extends AnyWordSpec {
133141
}
134142

135143
"decimal number" when {
144+
"encode a null" in {
145+
val expected = Array(0x00, 0x00).map(_.toByte)
146+
val actual = BCDNumberEncoders.encodeBCDNumber(null: java.math.BigDecimal, 2, 1, 0, signed = true, mandatorySignNibble = true)
147+
148+
assertArraysEqual(actual, expected)
149+
}
150+
136151
"encode a number" in {
137152
val expected = Array[Byte](0x12, 0x34, 0x5C)
138153
val actual = BCDNumberEncoders.encodeBCDNumber(java.math.BigDecimal.valueOf(123.45), 5, 2, 0, signed = true, mandatorySignNibble = true)

‎cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/BinaryEncodersSuite.scala‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,13 @@ import za.co.absa.cobrix.cobol.testutils.ComparisonUtils._
2121

2222
class BinaryEncodersSuite extends AnyWordSpec {
2323
"encodeBinaryNumber" should {
24+
"encode a null" in {
25+
val expected = Array(0x00, 0x00, 0x00, 0x00).map(_.toByte)
26+
val actual = BinaryEncoders.encodeBinaryNumber(null: java.math.BigDecimal, isSigned = true, outputSize = 4, bigEndian = true, precision = 5, scale = 0, scaleFactor = 0)
27+
28+
assertArraysEqual(actual, expected)
29+
}
30+
2431
"encode a positive integer in big-endian format" in {
2532
val expected = Array(0x00, 0x00, 0x30, 0x39).map(_.toByte) // 12345 in hex
2633
val actual = BinaryEncoders.encodeBinaryNumber(new java.math.BigDecimal(12345), isSigned = true, outputSize = 4, bigEndian = true, precision = 5, scale = 0, scaleFactor = 0)

‎cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/encoding/DisplayEncodersSuite.scala‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,13 @@ import za.co.absa.cobrix.cobol.testutils.ComparisonUtils.assertArraysEqual
2323
class DisplayEncodersSuite extends AnyWordSpec {
2424
"encodeDisplayNumberSignOverpunched" should {
2525
"integral number" when {
26+
"encode a null" in {
27+
val expected = Array(0x00, 0x00, 0x00).map(_.toByte)
28+
val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(null: java.math.BigDecimal, signPosition = Some(Left), 3, 2, 0, 0, explicitDecimalPoint = false)
29+
30+
assertArraysEqual(actual, expected)
31+
}
32+
2633
"encode a number" in {
2734
val expected = Array(0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xC5).map(_.toByte)
2835
val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(12345), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false)
@@ -109,6 +116,20 @@ class DisplayEncodersSuite extends AnyWordSpec {
109116
}
110117

111118
"decimal number" when {
119+
"encode a null" in {
120+
val expected = Array(0x00, 0x00, 0x00).map(_.toByte)
121+
val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(null: java.math.BigDecimal, signPosition = Some(Left), 3, 2, 1, 0, explicitDecimalPoint = false)
122+
123+
assertArraysEqual(actual, expected)
124+
}
125+
126+
"encode a null and separate decimal point" in {
127+
val expected = Array(0x00, 0x00, 0x00, 0x00).map(_.toByte)
128+
val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(null: java.math.BigDecimal, signPosition = Some(Left), 4, 2, 1, 0, explicitDecimalPoint = true)
129+
130+
assertArraysEqual(actual, expected)
131+
}
132+
112133
"encode a number" in {
113134
val expected = Array(0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xC5).map(_.toByte)
114135
val actual = DisplayEncoders.encodeDisplayNumberSignOverpunched(new java.math.BigDecimal(123.45), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false)
@@ -310,6 +331,13 @@ class DisplayEncodersSuite extends AnyWordSpec {
310331

311332
"encodeDisplayNumberSignSeparate" should {
312333
"integral number" when {
334+
"encode a null" in {
335+
val expected = Array(0x00, 0x00, 0x00).map(_.toByte)
336+
val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(null: java.math.BigDecimal, signPosition = Some(Left), 3, 2, 0, 0, explicitDecimalPoint = false)
337+
338+
assertArraysEqual(actual, expected)
339+
}
340+
313341
"encode a number" in {
314342
val expected = Array(0x4E, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte)
315343
val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(12345), signPosition = Some(Left), 6, 5, 0, 0, explicitDecimalPoint = false)
@@ -396,6 +424,20 @@ class DisplayEncodersSuite extends AnyWordSpec {
396424
}
397425

398426
"decimal number" when {
427+
"encode a null" in {
428+
val expected = Array(0x00, 0x00, 0x00, 0x00).map(_.toByte)
429+
val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(null: java.math.BigDecimal, signPosition = Some(Left), 4, 2, 1, 0, explicitDecimalPoint = false)
430+
431+
assertArraysEqual(actual, expected)
432+
}
433+
434+
"encode a null and separate decimal point" in {
435+
val expected = Array(0x00, 0x00, 0x00, 0x00, 0x00).map(_.toByte)
436+
val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(null: java.math.BigDecimal, signPosition = Some(Left), 5, 2, 1, 0, explicitDecimalPoint = true)
437+
438+
assertArraysEqual(actual, expected)
439+
}
440+
399441
"encode a number" in {
400442
val expected = Array(0x4E, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5).map(_.toByte)
401443
val actual = DisplayEncoders.encodeDisplayNumberSignSeparate(new java.math.BigDecimal(123.45), signPosition = Some(Left), 6, 5, 2, 0, explicitDecimalPoint = false)

‎spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/writer/FixedLengthEbcdicWriterSuite.scala‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -262,10 +262,12 @@ class FixedLengthEbcdicWriterSuite extends AnyWordSpec with SparkTestBase with B
262262

263263
"write data frames with DISPLAY fields" in {
264264
withTempDirectory("cobol_writer1") { tempDir =>
265+
val bigDecimalNull = null: java.math.BigDecimal
265266
val df = List(
266267
(-1, 100.5, new java.math.BigDecimal(10.23), 1, 10050, new java.math.BigDecimal(10.12)),
267268
(2, 800.4, new java.math.BigDecimal(30), 2, 80040, new java.math.BigDecimal(30)),
268-
(3, 22.33, new java.math.BigDecimal(-20), -3, -2233, new java.math.BigDecimal(-20.456))
269+
(3, 22.33, new java.math.BigDecimal(-20), -3, -2233, new java.math.BigDecimal(-20.456)),
270+
(4, -1.0, bigDecimalNull, 400, 1000000, bigDecimalNull)
269271
).toDF("A", "B", "C", "D", "E", "F")
270272

271273
val path = new Path(tempDir, "writer1")
@@ -324,7 +326,14 @@ class FixedLengthEbcdicWriterSuite extends AnyWordSpec with SparkTestBase with B
324326
0xF2, 0xF0, 0x4B, 0xF0, 0xD0, // -20 PIC S9(2).9(2)
325327
0x00, // null PIC 9(1) (because a negative value cannot be converted to this PIC)
326328
0x60, 0xF0, 0xF0, 0xF2, 0xF2, 0xF3, 0xF3, // -2233 S9(6) SIGN IS LEADING SEPARATE.
327-
0xF2, 0xF0, 0x4B, 0xF4, 0xF6, 0x60 // -20 S9(2).9(2) SIGN IS TRAILING SEPARATE
329+
0xF2, 0xF0, 0x4B, 0xF4, 0xF6, 0x60, // -20 S9(2).9(2) SIGN IS TRAILING SEPARATE
330+
331+
0xC4, // 4 PIC S9(1).
332+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // nulls
333+
0x00, 0x00, 0x00, 0x00, 0x00,
334+
0x00,
335+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
336+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00
328337
).map(_.toByte)
329338

330339
assertArraysEqual(bytes, expected)
@@ -358,6 +367,8 @@ class FixedLengthEbcdicWriterSuite extends AnyWordSpec with SparkTestBase with B
358367
| "C1" : "20.0}",
359368
| "E" : -2233,
360369
| "F" : -20.46
370+
|}, {
371+
| "A" : 4
361372
|} ]""".stripMargin
362373

363374
val actualJson = SparkUtils.convertDataFrameToPrettyJSON(df2)

0 commit comments

Comments
 (0)