A few more fixes, after further testing

Author: NWilson

.github/workflows/build.yml

@@ -281,6 +281,55 @@ jobs:
             ../maint/RunManifestTest install-dir ../maint/manifest-cmakeinstall-solaris
             ../maint/RunSymbolTest install-dir/lib/ ../maint/
 
+  zos:
+    name: z/OS
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    concurrency:
+      group: zos-ssh-build
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+
+      - name: Prepare
+        run: ./autogen.sh
+
+      - name: Build & test
+        env:
+          ZOS_HOST: ${{ secrets.ZOS_HOST }}
+          ZOS_PORT: ${{ secrets.ZOS_PORT }}
+          ZOS_PRIVATE_KEY: ${{ secrets.ZOS_PRIVATE_KEY }}
+          ZOS_KNOWN_HOSTS: ${{ secrets.ZOS_KNOWN_HOSTS }}
+        run: |
+          (umask 0077 && printenv ZOS_PRIVATE_KEY > id_rsa_zos)
+          mkdir -p ~/.ssh
+          printenv ZOS_KNOWN_HOSTS > ~/.ssh/known_hosts
+
+          tar czf ../pcre2-build.tar.gz --exclude=.git .
+          mv ../pcre2-build.tar.gz .
+
+          scp -i id_rsa_zos -P "$ZOS_PORT" pcre2-build.tar.gz "$ZOS_HOST:/data/"
+          ssh -i id_rsa_zos -p "$ZOS_PORT" "$ZOS_HOST" /data/zopen/usr/local/bin/bash -c \
+            'export _BPXK_AUTOCVT=ON;
+            export _CEE_RUNOPTS="FILETAG(AUTOCVT,AUTOTAG) POSIX(ON)";
+            export _TAG_REDIR_ERR=txt;
+            export _TAG_REDIR_IN=txt;
+            export _TAG_REDIR_OUT=txt;
+            . /data/zopen/etc/zopen-config;
+            set -e;
+            set -x;
+            cd /data;
+            rm -rf pcre2-build;
+            mkdir pcre2-build;
+            gtar xzf pcre2-build.tar.gz -C pcre2-build;
+            cd pcre2-build;
+            chtag -R -tc ISO8859-1 .;
+            MAKE=gmake CC=xlc ./configure --enable-ebcdic --disable-unicode;
+            gmake;
+            gmake check'
+
   distcheck:
     name: Build & verify distribution
     runs-on: ubuntu-latest

Makefile.am

@@ -235,6 +235,7 @@ lib_LTLIBRARIES =
 # TESTS is for binary unit tests, check_SCRIPTS for script-based tests
 
 TESTS =
+XFAIL_TESTS =
 check_SCRIPTS =
 dist_noinst_SCRIPTS =
 
@@ -724,6 +725,9 @@ if WITH_PCRE2_8
 TESTS += RunGrepTest
 EXTRA_DIST += RunGrepTest.bat
 dist_noinst_SCRIPTS += RunGrepTest
+if WITH_EBCDIC
+XFAIL_TESTS += RunGrepTest
+endif # WITH_EBCDIC
 endif # WITH_PCRE2_8
 
 ## Distribute all the test data files

RunTest

@@ -130,9 +130,11 @@ fi
 
 # Set up a suitable "diff" command for comparison. Some systems
 # have a diff that lacks a -u option. Try to deal with this.
+# Use gdiff if available.
 
 cf="diff"
-diff -u /dev/null /dev/null 2>/dev/null && cf="diff -u"
+gdiff /dev/null /dev/null 2>/dev/null && cf="gdiff"
+$cf -u /dev/null /dev/null 2>/dev/null && cf="$cf -u"
 
 # Find the test data
 

src/pcre2_compile.c

@@ -55,11 +55,14 @@ POSSIBILITY OF SUCH DAMAGE.
 #else
 #define PRINTABLE(c) ((c) >= 32 && (c) < 127)
 #endif
-#define CHAR_OUTPUT(c)  (c)
-#define CHAR_INPUT(c)   (c)
+#define CHAR_OUTPUT(c)      (c)
+#define CHAR_OUTPUT_HEX(c)  (c)
+#define CHAR_INPUT(c)       (c)
+#define CHAR_INPUT_HEX(c)   (c)
 #include "pcre2_printint.c"
 #undef PRINTABLE
 #undef CHAR_OUTPUT
+#undef CHAR_OUTPUT_HEX
 #undef CHAR_INPUT
 #define DEBUG_CALL_PRINTINT
 #endif

src/pcre2_compile_class.c

@@ -1285,7 +1285,7 @@ while (TRUE)
       uint8_t posix_vertical[4] = { CHAR_LF, CHAR_VT, CHAR_FF, CHAR_CR };
       uint8_t posix_underscore = CHAR_UNDERSCORE;
       uint8_t *chars = NULL;
-      uint n = 0;
+      int n = 0;
 
       if (tabopt == 1) { chars = posix_vertical; n = 4; }
       else if (tabopt == 2) { chars = &posix_underscore; n = 1; }

src/pcre2_printint.c

@@ -120,10 +120,10 @@ if (utf)
 if (one_code_unit)
   {
   if (PRINTABLE(c))
-    fprintf(f, "%c", (char)CHAR_OUTPUT(c));
+    fprintf(f, "%c", CHAR_OUTPUT(c));
   else
     {
-    c = CHAR_OUTPUT(c);
+    c = CHAR_OUTPUT_HEX(c);
     if (c < 0x80) fprintf(f, "\\x%02x", c);
     else fprintf(f, "\\x{%02x}", c);
     }
@@ -219,7 +219,7 @@ while (*ptr != '\0')
   {
   uint32_t c = *ptr++;
   if (PRINTABLE(c)) fprintf(f, "%c", CHAR_OUTPUT(c));
-  else fprintf(f, "\\x{%x}", CHAR_OUTPUT(c));
+  else fprintf(f, "\\x{%x}", CHAR_OUTPUT_HEX(c));
   }
 }
 
@@ -230,7 +230,7 @@ for (; len > 0; len--)
   {
   uint32_t c = *ptr++;
   if (PRINTABLE(c)) fprintf(f, "%c", CHAR_OUTPUT(c));
-  else fprintf(f, "\\x{%x}", CHAR_OUTPUT(c));
+  else fprintf(f, "\\x{%x}", CHAR_OUTPUT_HEX(c));
   }
 }
 
@@ -481,22 +481,22 @@ if (negated)
 
 for (input = 0; input < 256; input++)
   {
-  i = CHAR_INPUT(input);
+  i = CHAR_INPUT_HEX(input);
   if ((map[i/8] & (1u << (i&7))) != 0)
     {
     int j, jinput;
     for (jinput = input; jinput+1 < 256; jinput++)
       {
-      j = CHAR_INPUT(jinput+1);
+      j = CHAR_INPUT_HEX(jinput+1);
       if ((map[j/8] & (1u << (j&7))) == 0) break;
       }
-    j = CHAR_INPUT(jinput);
+    j = CHAR_INPUT_HEX(jinput);
     if (i == CHAR_MINUS || i == CHAR_BACKSLASH ||
         i == CHAR_RIGHT_SQUARE_BRACKET ||
         (first && i == CHAR_CIRCUMFLEX_ACCENT))
       fprintf(f, "\\");
     if (PRINTABLE(i)) fprintf(f, "%c", CHAR_OUTPUT(i));
-    else fprintf(f, "\\x%02x", CHAR_OUTPUT(i));
+    else fprintf(f, "\\x%02x", CHAR_OUTPUT_HEX(i));
     first = FALSE;
     if (jinput > input)
       {
@@ -505,7 +505,7 @@ for (input = 0; input < 256; input++)
           j == CHAR_RIGHT_SQUARE_BRACKET)
         fprintf(f, "\\");
       if (PRINTABLE(j)) fprintf(f, "%c", CHAR_OUTPUT(j));
-      else fprintf(f, "\\x%02x", CHAR_OUTPUT(j));
+      else fprintf(f, "\\x%02x", CHAR_OUTPUT_HEX(j));
       }
     input = jinput;
     }

src/pcre2test.c

@@ -262,10 +262,8 @@ as a hex value when showing compiled patterns. We use it in cases when the
 locale has not been explicitly changed, so as to get consistent output from
 systems that differ in their output from isprint() even in the "C" locale. */
 
-#if defined(EBCDIC) && !EBCDIC_IO
-#define PRINTABLE(c) (ebcdic_to_ascii(c) >= 32 && ebcdic_to_ascii(c) < 127)
-#elif defined(EBCDIC)
-#define PRINTABLE(c) ((c) >= 64 && (c) < 255)
+#if defined(EBCDIC)
+#define PRINTABLE(c) printable(c)
 #else
 #define PRINTABLE(c) ((c) >= 32 && (c) < 127)
 #endif
@@ -275,11 +273,20 @@ format. The input character is encoded in PCRE2's native codepage (EBCDIC, if
 enabled), but the output may differ in the case where pcre2test uses ASCII input
 and output. */
 #if defined(EBCDIC) && !EBCDIC_IO
-#define CHAR_OUTPUT(c)  ebcdic_to_ascii(c)
-#define CHAR_INPUT(c)   ascii_to_ebcdic(c)
+#define CHAR_OUTPUT(c)      ebcdic_to_ascii(c)
+#define CHAR_OUTPUT_HEX(c)  CHAR_OUTPUT(c)
+#define CHAR_INPUT(c)       ascii_to_ebcdic(c)
+#define CHAR_INPUT_HEX(c)   CHAR_INPUT(c)
+#elif defined(EBCDIC)
+#define CHAR_OUTPUT(c)      (c)
+#define CHAR_OUTPUT_HEX(c)  ebcdic_to_ascii(c)
+#define CHAR_INPUT(c)       (c)
+#define CHAR_INPUT_HEX(c)   ascii_to_ebcdic(c)
 #else
-#define CHAR_OUTPUT(c)  (c)
-#define CHAR_INPUT(c)   (c)
+#define CHAR_OUTPUT(c)      (c)
+#define CHAR_OUTPUT_HEX(c)  CHAR_OUTPUT(c)
+#define CHAR_INPUT(c)       (c)
+#define CHAR_INPUT_HEX(c)   CHAR_INPUT(c)
 #endif
 
 /* We have to include some of the library source files because we need
@@ -313,10 +320,15 @@ previous definition of PRIV avoids name clashes. */
 #include "pcre2_tables.c"
 #include "pcre2_ucd.c"
 
-/* Forward-declarations for PRINTABLE(). */
+/* Forward-declarations for PRINTABLE(), etc. */
 
+#if defined(EBCDIC)
+static BOOL printable(uint32_t c);
+#endif
 #if defined(EBCDIC) && !EBCDIC_IO
 static void ascii_to_ebcdic_str(uint8_t *buf, size_t len);
+#endif
+#if defined(EBCDIC)
 static uint32_t ascii_to_ebcdic(uint32_t c);
 static uint32_t ebcdic_to_ascii(uint32_t c);
 #endif
@@ -3045,14 +3057,71 @@ return (PCRE2_JIT_STACK *)arg;
 *         EBCDIC support functions               *
 *************************************************/
 
+#if defined(EBCDIC)
+static BOOL
+printable(uint32_t c)
+{
+if ((c >= CHAR_a && c <= CHAR_i) ||
+    (c >= CHAR_j && c <= CHAR_r) ||
+    (c >= CHAR_s && c <= CHAR_z) ||
+    (c >= CHAR_A && c <= CHAR_I) ||
+    (c >= CHAR_J && c <= CHAR_R) ||
+    (c >= CHAR_S && c <= CHAR_Z) ||
+    (c >= CHAR_0 && c <= CHAR_9))
+  return TRUE;
+
+switch (c)
+  {
+  case CHAR_SPACE:
+  case CHAR_EXCLAMATION_MARK:
+  case CHAR_QUOTATION_MARK:
+  case CHAR_NUMBER_SIGN:
+  case CHAR_DOLLAR_SIGN:
+  case CHAR_PERCENT_SIGN:
+  case CHAR_AMPERSAND:
+  case CHAR_APOSTROPHE:
+  case CHAR_LEFT_PARENTHESIS:
+  case CHAR_RIGHT_PARENTHESIS:
+  case CHAR_ASTERISK:
+  case CHAR_PLUS:
+  case CHAR_COMMA:
+  case CHAR_MINUS:
+  case CHAR_DOT:
+  case CHAR_SLASH:
+  case CHAR_COLON:
+  case CHAR_SEMICOLON:
+  case CHAR_LESS_THAN_SIGN:
+  case CHAR_EQUALS_SIGN:
+  case CHAR_GREATER_THAN_SIGN:
+  case CHAR_QUESTION_MARK:
+  case CHAR_COMMERCIAL_AT:
+  case CHAR_LEFT_SQUARE_BRACKET:
+  case CHAR_BACKSLASH:
+  case CHAR_RIGHT_SQUARE_BRACKET:
+  case CHAR_CIRCUMFLEX_ACCENT:
+  case CHAR_UNDERSCORE:
+  case CHAR_GRAVE_ACCENT:
+  case CHAR_LEFT_CURLY_BRACKET:
+  case CHAR_VERTICAL_LINE:
+  case CHAR_RIGHT_CURLY_BRACKET:
+  case CHAR_TILDE:
+  return TRUE;
+  }
+
+return FALSE;
+}
+#endif
+
 #if defined(EBCDIC) && !EBCDIC_IO
 static void
 ascii_to_ebcdic_str(uint8_t *buf, size_t len)
 {
 for (size_t i = 0; i < len; ++i)
   buf[i] = ascii_to_ebcdic_1047[buf[i]];
 }
+#endif
 
+#if defined(EBCDIC)
 static uint32_t
 ascii_to_ebcdic(uint32_t c)
 {
@@ -3161,7 +3230,7 @@ if (PRINTABLE(c))
   return 1;
   }
 
-c = CHAR_OUTPUT(c);
+c = CHAR_OUTPUT_HEX(c);
 
 if (c < 0x100)
   {
@@ -4983,7 +5052,7 @@ if ((pat_patctl.control & CTL_INFO) != 0)
     fprintf(outfile, "Starting code units:");
     for (input = 0; input < 256; input++)
       {
-      int i = CHAR_INPUT(input);
+      int i = CHAR_INPUT_HEX(input);
       if ((start_bits[i/8] & (1u << (i&7))) != 0)
         {
         if (c > 75)
@@ -4998,7 +5067,7 @@ if ((pat_patctl.control & CTL_INFO) != 0)
           }
         else
           {
-          fprintf(outfile, " \\x%02x", CHAR_OUTPUT(i));
+          fprintf(outfile, " \\x%02x", CHAR_OUTPUT_HEX(i));
           c += 5;
           }
         }
@@ -5645,8 +5714,9 @@ if ((pat_patctl.control & CTL_HEXPAT) != 0)
         }
       c = toupper(c);
       d = toupper(d);
-      *pt++ = ((isdigit(c)? (c - '0') : (c - 'A' + 10)) << 4) +
-               (isdigit(d)? (d - '0') : (d - 'A' + 10));
+      c = isdigit(c)? (c - '0') : (c - 'A' + 10);
+      d = isdigit(d)? (d - '0') : (d - 'A' + 10);
+      *pt++ = CHAR_OUTPUT(CHAR_INPUT_HEX((c << 4) + d));
       }
     }
   *pt = 0;
@@ -7554,6 +7624,7 @@ while ((c = *p++) != 0)
     c -= '0';
     while (i++ < 2 && isdigit(*p) && *p < '8')
       c = c * 8 + (*p++ - '0');
+    c = CHAR_OUTPUT(CHAR_INPUT_HEX(c));
 
     encoding = (utf && c > 255)? FORCE_UTF : FORCE_RAW;
     break;
@@ -7572,6 +7643,7 @@ while ((c = *p++) != 0)
           }
         else c = c * 8 + (*pt - '0');
         }
+      c = CHAR_OUTPUT(CHAR_INPUT_HEX(c));
       if (i == 0 || *pt != '}')
         {
         fprintf(outfile, "** Malformed \\o{ escape\n");
@@ -7603,6 +7675,7 @@ while ((c = *p++) != 0)
           }
         else c = c * 16 + (tolower(*pt) - (isdigit(*pt)? '0' : 'a' - 10));
         }
+      c = CHAR_OUTPUT(CHAR_INPUT_HEX(c));
       if (i == 0 || *pt != '}')
         {
         fprintf(outfile, "** Malformed \\x{ escape\n");
@@ -7622,6 +7695,7 @@ while ((c = *p++) != 0)
         c = c * 16 + (tolower(*p) - (isdigit(*p)? '0' : 'a' - 10));
         p++;
         }
+      c = CHAR_OUTPUT(CHAR_INPUT_HEX(c));
 #if defined SUPPORT_PCRE2_8
       if (utf && (test_mode == PCRE8_MODE)) encoding = FORCE_RAW;
 #endif

testdata/testinput2

@@ -6202,8 +6202,6 @@ a)"xI
 /\sxxx\s/tables=2
     AB\x{85}xxx\x{a0}XYZ
 
-#endif
-
 /^\w+/tables=2
     École
 
@@ -6215,6 +6213,8 @@ a)"xI
 /^\w+/tables=3
     École
 
+#endif
+
 /"(*MARK:>" 00 "<).."/hex,mark,no_start_optimize
     AB
     A\=ph