Make explicit fallback to zero more focused when NaN is attempted to be written

This is a fixup for a 765865d a few commits back. Original commit comments:

This was already how Java behaved in non-high-precision cases. However NaN processing would produce corrupted PDF in Java for highPrecision case and also caused exceptions in .NET.

DEVSIX-2883

Author: yulian-gaponenko

io/src/main/java/com/itextpdf/io/source/ByteUtils.java

@@ -128,11 +128,6 @@ static byte[] getIsoBytes(double d, ByteBuffer buffer) {
     }
 
     static byte[] getIsoBytes(double d, ByteBuffer buffer, boolean highPrecision) {
-        if (Double.isNaN(d)) {
-            Logger logger = LoggerFactory.getLogger(ByteUtils.class);
-            logger.error(LogMessageConstant.ATTEMPT_PROCESS_NAN);
-            d = 0;
-        }
         if (highPrecision) {
             if (Math.abs(d) < 0.000001) {
                 if (buffer != null) {
@@ -142,6 +137,11 @@ static byte[] getIsoBytes(double d, ByteBuffer buffer, boolean highPrecision) {
                     return zero;
                 }
             }
+            if (Double.isNaN(d)) {
+                Logger logger = LoggerFactory.getLogger(ByteUtils.class);
+                logger.error(LogMessageConstant.ATTEMPT_PROCESS_NAN);
+                d = 0;
+            }
             byte[] result = DecimalFormatUtil.formatNumber(d, "0.######").getBytes(StandardCharsets.ISO_8859_1);
             if (buffer != null) {
                 buffer.prepend(result);
@@ -240,9 +240,15 @@ static byte[] getIsoBytes(double d, ByteBuffer buffer, boolean highPrecision) {
             d += 0.5;
             long v;
             if (d > Long.MAX_VALUE) {
-                //by default cast logic do the same, but not in .NET
+                // by default cast logic do the same, but not in .NET
                 v = Long.MAX_VALUE;
             } else {
+                if (Double.isNaN(d)) {
+                    Logger logger = LoggerFactory.getLogger(ByteUtils.class);
+                    logger.error(LogMessageConstant.ATTEMPT_PROCESS_NAN);
+                    // in java NaN casted to long results in 0, but in .NET it results in long.MIN_VALUE
+                    d = 0;
+                }
                 v = (long) d;
             }
             int intLen = longSize(v);