Some further fixes

Author: NWilson

.github/workflows/dev.yml

@@ -440,6 +440,7 @@ jobs:
         run: bazelisk test //... --enable_runfiles --incompatible_strict_action_env --test_output=all
 
   # XXX Maybe this should just go in the ManyConfig tests, actually??
+  # XXX JIT IS TURNED OFF - apparently it needs some fixes (not surprising)
   ebcdic:
     # Tests the full support for EBCDIC on a non-EBCDIC platform, using a
     # hardcoded EBCDIC-1047 codepage.

Makefile.am

@@ -781,7 +781,7 @@ EXTRA_DIST += \
   testdata/testinput25 \
   testdata/testinput26 \
   testdata/testinput27 \
-  testdata/testinputEBC \
+  testdata/testinput28 \
   testdata/testinputheap \
   testdata/testoutput1 \
   testdata/testoutput2 \
@@ -823,7 +823,7 @@ EXTRA_DIST += \
   testdata/testoutput25 \
   testdata/testoutput26 \
   testdata/testoutput27 \
-  testdata/testoutputEBC \
+  testdata/testoutput28 \
   testdata/testoutputheap-16 \
   testdata/testoutputheap-32 \
   testdata/testoutputheap-8 \

RunTest

@@ -130,7 +130,7 @@ fi
 # have a diff that lacks a -u option. Try to deal with this.
 
 cf="diff"
-diff -u /dev/null /dev/null 2>/dev/null && cf="diff -U15 --color=always" # XXX TEST HACK
+diff -u /dev/null /dev/null 2>/dev/null && cf="diff -u"
 
 # Find the test data
 
@@ -374,8 +374,14 @@ if [ $ebcdic -eq 1 ]; then
   # modified input file. That's it.
   filterebcdic()
     {
-    filter_cmd="$1"
     filter_args=
+    while [ "$1" != "$pcre2test" ] ; do
+      # Shift off any Valgrind args
+      filter_args="$filter_args $1"
+      shift
+    done
+    # Shift off the $pcre2test arg
+    filter_args="$filter_args $1"
     shift
     while [ $# -gt 0 ] ; do
       case "$1" in
@@ -393,9 +399,9 @@ if [ $ebcdic -eq 1 ]; then
     if [ $# -gt 0 ] ; then
       input="$1"
       shift
-      sed '/^# if !EBCDIC/,/^# endif/d' "$input" | "$filter_cmd" $filter_args "/dev/stdin" "$@"
+      sed '/^# if !EBCDIC/,/^# endif/d' "$input" | $filter_args "/dev/stdin" "$@"
     else
-      "$filter_cmd" $filter_args "$@"
+      $filter_args "$@"
     fi
     }
   sim="filterebcdic $sim"

maint/ManyConfigTests

@@ -20,6 +20,7 @@
 # -nomainvalgrind skip the main (non-JIT) valgrind tests
 # -notmp          skip the tests in a temporary directory
 # -notmpjit       skip the JIT test in a temporary directory
+# -noebcdic       skip the EBCDIC tests
 # -novalgrind     skip all the valgrind tests
 
 # Alternatively, if any of those names are given with '+' instead of '-no',
@@ -38,6 +39,7 @@ usemain=1
 usemainvalgrind=1
 usetmp=1
 usetmpjit=1
+useebcdic=0 # XXX Set to 1 after fixed on all plats
 usevalgrind=1
 
 dummy=0
@@ -55,8 +57,9 @@ while [ $# -gt 0 ] ; do
           usemain=0
           usemainvalgrind=0
           usetmp=0
-          usetmpjit=0 
-          usevalgrind=0 
+          usetmpjit=0
+          useebcdic=0
+          usevalgrind=0
           seenplus=1
         fi;;
   esac
@@ -74,6 +77,7 @@ while [ $# -gt 0 ] ; do
     -nomainvalgrind) usemainvalgrind=0;;
     -notmp)          usetmp=0; usetmpjit=0;;
     -notmpjit)       usetmpjit=0;;
+    -noebcdic)       useebcdic=0;;
     -novalgrind)     usevalgrind=0;;
     +asan)           useasan=1;;
     +usan)           useusan=1;;
@@ -85,6 +89,7 @@ while [ $# -gt 0 ] ; do
     +mainvalgrind)   usemainvalgrind=1;;
     +tmp)            usetmp=1;;
     +tmpjit)         usetmpjit=1;;
+    +ebcdic)         useebcdic=1;;
     +valgrind)       usevalgrind=1; usejitvalgrind=1; usemainvalgrind=1;;
     *)               echo "Unknown option '$1'"; exit 1;;
   esac
@@ -221,7 +226,11 @@ runtest()
     exit 1
   fi
 
-  if [ $pcre2_8 -gt 0 ]; then
+  if [ "x$withebcdic" != x ]; then
+    echo "Skipping pcre2grep and pcre2posix tests: not supported on EBCDIC"
+  elif [ $pcre2_8 -eq 0 ]; then
+    echo "Skipping pcre2grep and pcre2posix tests: 8-bit library not compiled"
+  else
     echo "Running pcre2grep tests $withvalgrind"
     $srcdir/RunGrepTest $valgrind >teststdoutM 2>teststderrM
     if [ $? -ne 0 -o -s teststderrM ]; then
@@ -239,8 +248,6 @@ runtest()
       echo "**** Test failed ****"
       exit 1
     fi
-  else
-    echo "Skipping pcre2grep and pcre2posix tests: 8-bit library not compiled"
   fi
 
   if [ "$jit" -gt 0 ]; then
@@ -268,6 +275,7 @@ testtotal=`expr 17 \* $usemain + \
   1 \* $ISGCC \* $usemain \* $useasan + \
   1 \* $ISGCC \* $usemain \* $useusan + \
   13 \* $usejit + \
+  2 \* $useebcdic + \
   2 \* $usemainvalgrind + \
   2 \* $usejitvalgrind`
 
@@ -388,6 +396,22 @@ if [ $usejit -ne 0 ]; then
   done
 fi
 
+# Now run some tests with EBCDIC enabled
+
+if [ $useebcdic -ne 0 ]; then
+  echo "---------- EBCDIC tests in the current directory ----------"
+  withebcdic="with EBCDIC"
+
+  for opts in \
+    "--disable-unicode --enable-ebcdic --enable-ebcdic-ignoring-compiler" \
+    "--disable-unicode --enable-ebcdic --enable-ebcdic-ignoring-compiler --enable-ebcdic-nl25"
+  do
+    runtest
+  done
+fi
+
+withebcdic=
+
 # Now re-run some of the tests under valgrind.
 
 if [ $usevalgrind -ne 0 ]; then
@@ -464,8 +488,8 @@ if [ $usetmp -ne 0 -o $usetmpjit -ne 0 ]; then
     do
       runtest
     done
-  fi   
-  
+  fi
+
   if [ $usetmpjit -ne 0 ]; then
     echo "---------- JIT tests in the $tmp directory ----------"
     for opts in \

src/pcre2_error.c

@@ -58,15 +58,7 @@ performance issue because these strings are used only when there is an error.
 
 Each substring ends with \0 to insert a null character. This includes the final
 substring, so that the whole string ends with \0\0, which can be detected when
-counting through.
-
-In the rare configuration of EBCDIC-with-ASCII-compiler, we currently output
-ASCII strings for the error messages, which is unlikely to cause complaints if
-some client does want to use PCRE2 on Linux or Windows to process their EBCDIC
-files.
-
-XXX No! Let's just make it 100% EBCDIC.
-*/
+counting through. */
 
 static const unsigned char compile_error_texts[] =
   "no error\0"
@@ -330,7 +322,7 @@ pcre2_get_error_message(int enumber, PCRE2_UCHAR *buffer, PCRE2_SIZE size)
 {
 const unsigned char *message;
 PCRE2_SIZE i;
-int n;
+int n, rc = 0;
 
 if (size == 0) return PCRE2_ERROR_NOMEMORY;
 
@@ -360,14 +352,23 @@ for (i = 0; *message != 0; i++)
   {
   if (i >= size - 1)
     {
-    buffer[i] = 0;     /* Terminate partial message */
-    return PCRE2_ERROR_NOMEMORY;
+    rc = PCRE2_ERROR_NOMEMORY;
+    break;
     }
   buffer[i] = *message++;
   }
 
-buffer[i] = 0;
-return (int)i;
+#if defined EBCDIC && 'a' != 0x81
+/* If compiling for EBCDIC, but the compiler's string literals are not EBCDIC,
+then we are in the "force EBCDIC 1047" mode. I have chosen to add a few lines
+here to translate the error strings on the fly, rather than require the string
+literals above to be written out arduously using the "STR_XYZ" macros. */
+for (PCRE2_SIZE j = 0; j < i; ++j)
+  buffer[j] = PRIV(ascii_to_ebcdic_1047)[buffer[j]];
+#endif
+
+buffer[i] = 0;     /* Terminate message, even if truncated. */
+return rc? rc : (int)i;
 }
 
 /* End of pcre2_error.c */

src/pcre2_internal.h

@@ -2268,6 +2268,8 @@ extern const uint8_t          PRIV(utf8_table4)[];
 #define _pcre2_utt                     PCRE2_SUFFIX(_pcre2_utt_)
 #define _pcre2_utt_names               PCRE2_SUFFIX(_pcre2_utt_names_)
 #define _pcre2_utt_size                PCRE2_SUFFIX(_pcre2_utt_size_)
+#define _pcre2_ebcdic_1047_to_ascii    PCRE2_SUFFIX(_pcre2_ebcdic_1047_to_ascii_)
+#define _pcre2_ascii_to_ebcdic_1047    PCRE2_SUFFIX(_pcre2_ascii_to_ebcdic_1047_)
 
 extern const uint8_t                   PRIV(OP_lengths)[];
 extern const uint32_t                  PRIV(callout_end_delims)[];
@@ -2300,6 +2302,8 @@ extern const char                     *PRIV(unicode_version);
 extern const ucp_type_table            PRIV(utt)[];
 extern const char                      PRIV(utt_names)[];
 extern const size_t                    PRIV(utt_size);
+extern const uint8_t                   PRIV(ebcdic_1047_to_ascii)[];
+extern const uint8_t                   PRIV(ascii_to_ebcdic_1047)[];
 
 /* Mode-dependent macros and hidden and private structures are defined in a
 separate file so that pcre2test can include them at all supported widths. When

src/pcre2_tables.c

@@ -238,9 +238,10 @@ files. */
 *          Tables for EBCDIC support             *
 *************************************************/
 
-#if defined(EBCDIC) && (defined(PCRE2_PCRE2TEST) || defined(PCRE2_DFTABLES))
+#if defined(EBCDIC) && \
+  (defined(PCRE2_PCRE2TEST) || defined(PCRE2_DFTABLES) || 'a' != 0x81)
 
-const uint8_t ebcdic_1047_to_ascii[256] = {
+const uint8_t PRIV(ebcdic_1047_to_ascii)[256] = {
   0x00,0x01,0x02,0x03,0x9c,0x09,0x86,0x7f,0x97,0x8d,0x8e,0x0b,0x0c,0x0d,0x0e,0x0f,
 #ifdef EBCDIC_NL25
   0x10,0x11,0x12,0x13,0x9d,0x85,0x08,0x87,0x18,0x19,0x92,0x8f,0x1c,0x1d,0x1e,0x1f,
@@ -264,7 +265,7 @@ const uint8_t ebcdic_1047_to_ascii[256] = {
   0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0xb3,0xdb,0xdc,0xd9,0xda,0x9f,
 };
 
-const uint8_t ascii_to_ebcdic_1047[256] = {
+const uint8_t PRIV(ascii_to_ebcdic_1047)[256] = {
 #ifdef EBCDIC_NL25
   0x00,0x01,0x02,0x03,0x37,0x2d,0x2e,0x2f,0x16,0x05,0x25,0x0b,0x0c,0x0d,0x0e,0x0f,
 #else

src/pcre2test.c

@@ -315,9 +315,11 @@ previous definition of PRIV avoids name clashes. */
 
 /* Forward-declarations for PRINTABLE(). */
 
+#if defined(EBCDIC) && !EBCDIC_IO
 static void ascii_to_ebcdic_str(uint8_t *buf, size_t len);
 static uint32_t ascii_to_ebcdic(uint32_t c);
 static uint32_t ebcdic_to_ascii(uint32_t c);
+#endif
 
 /* 32-bit integer values in the input are read by strtoul() or strtol(). The
 check needed for overflow depends on whether long ints are in fact longer than
@@ -4610,21 +4612,7 @@ if (len < 0)
 else
   {
   fprintf(outfile, "%s", before);
-
-  /* In the rare configuration of EBCDIC-with-ASCII-compiler, we currently
-  output ASCII strings for the error messages, so we do special filtering for
-  that here. */
-
-#ifdef SUPPORT_PCRE2_16
-  if (test_mode == PCRE16_MODE)
-    for (int i = 0; i <= len; i++) pbuffer8[i] = (uint8_t)pbuffer16[i];
-#endif
-#ifdef SUPPORT_PCRE2_32
-  if (test_mode == PCRE32_MODE)
-    for (int i = 0; i <= len; i++) pbuffer8[i] = (uint8_t)pbuffer32[i];
-#endif
-  fprintf(outfile, "%s", pbuffer8);
-
+  PCHARSV(CASTVAR(void *, pbuffer), 0, len, FALSE, outfile);
   fprintf(outfile, "%s", after);
   }
 return len >= 0;
@@ -9921,15 +9909,7 @@ least 128 code units, because it is used for retrieving error messages. */
       }
     else
       {
-#ifdef SUPPORT_PCRE2_16
-      if (test_mode == PCRE16_MODE)
-        for (int i = 0; i <= len; i++) pbuffer8[i] = (uint8_t)pbuffer16[i];
-#endif
-#ifdef SUPPORT_PCRE2_32
-      if (test_mode == PCRE32_MODE)
-        for (int i = 0; i <= len; i++) pbuffer8[i] = (uint8_t)pbuffer32[i];
-#endif
-      printf("%s", pbuffer8);
+      PCHARSV(CASTVAR(void *, pbuffer), 0, len, FALSE, stdout);
       }
     printf("\n");
     if (*endptr == 0) goto EXIT;