Last-ish round of fixes

Author: NWilson

.github/workflows/dev.yml

@@ -439,28 +439,6 @@ jobs:
       - name: Test
         run: bazelisk test //... --enable_runfiles --incompatible_strict_action_env --test_output=all
 
-  # XXX Maybe this should just go in the ManyConfig tests, actually??
-  # XXX JIT IS TURNED OFF - apparently it needs some fixes (not surprising)
-  ebcdic:
-    # Tests the full support for EBCDIC on a non-EBCDIC platform, using a
-    # hardcoded EBCDIC-1047 codepage.
-    name: EBCDIC
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          submodules: true
-
-      - name: Configure
-        run: cmake -DPCRE2_SUPPORT_JIT=OFF -DPCRE2_SUPPORT_UNICODE=OFF -DPCRE2_EBCDIC=ON -DPCRE2_EBCDIC_IGNORING_COMPILER=ON -DPCRE2_DEBUG=ON -DCMAKE_C_FLAGS="$CFLAGS_GCC_STYLE" -DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DCMAKE_BUILD_TYPE=Release -B build
-
-      - name: Build
-        run: cd build && make -j3
-
-      - name: Test
-        run: cd build && ../RunTest
-
   heron:
     # Job to verify that the tasks performed by PrepareRelease have been done. It is
     # the committer's responsibility (currently) to run PrepareRelease themselves when

Makefile.am

@@ -782,6 +782,7 @@ EXTRA_DIST += \
   testdata/testinput26 \
   testdata/testinput27 \
   testdata/testinput28 \
+  testdata/testinput29 \
   testdata/testinputheap \
   testdata/testoutput1 \
   testdata/testoutput2 \
@@ -824,6 +825,7 @@ EXTRA_DIST += \
   testdata/testoutput26 \
   testdata/testoutput27 \
   testdata/testoutput28 \
+  testdata/testoutput29 \
   testdata/testoutputheap-16 \
   testdata/testoutputheap-32 \
   testdata/testoutputheap-8 \

RunTest

@@ -85,7 +85,8 @@ title25="Test 25: UTF pattern conversion tests"
 title26="Test 26: Unicode property tests (compatible with Perl >= 5.38)"
 title27="Test 27: Auto-generated unicode property tests"
 title28="Test 28: EBCDIC-specific tests"
-maxtest=28
+title29="Test 29: EBCDIC-specific tests (for NL=0x25)"
+maxtest=29
 titleheap="Test 'heap': Environment-specific heap tests"
 
 if [ $# -eq 1 -a "$1" = "list" ]; then
@@ -118,6 +119,7 @@ if [ $# -eq 1 -a "$1" = "list" ]; then
   echo $title26
   echo $title27
   echo $title28
+  echo $title29
   echo ""
   echo $titleheap
   echo ""
@@ -169,7 +171,10 @@ checkresult()
   esac
   cf_out="$testdata/testoutput$2"
   if [ $ebcdic -eq 1 ] ; then
-    sed '/^# if !EBCDIC/,/^# endif/d' "$cf_out" >testtry2
+    # We currently only use the #if ... #endif support in pcre2test for EBCDIC
+    # testing. Run in "preprocess-only" mode (-E) on the testoutput file to trim
+    # the output lines matching the input lines which are discarded.
+    $sim $pcre2test -q -E "$cf_out" >testtry2
     cf_out=testtry2
   fi
   $cf "$cf_out" testtry
@@ -249,6 +254,7 @@ do25=no
 do26=no
 do27=no
 do28=no
+do29=no
 doheap=no
 
 while [ $# -gt 0 ] ; do
@@ -282,6 +288,7 @@ while [ $# -gt 0 ] ; do
    26) do26=yes;;
    27) do27=yes;;
    28) do28=yes;;
+   29) do29=yes;;
    heap) doheap=yes;;
   -8) arg8=yes;;
   -16) arg16=yes;;
@@ -358,53 +365,25 @@ support32=$?
 $sim $pcre2test -C backslash-C >/dev/null
 supportBSC=$?
 
-# Check if compiled in EBCDIC mode, and whether we have EBCDIC I/O
+# Check if compiled in EBCDIC mode, and whether we have EBCDIC I/O and NL=0x25
 $sim $pcre2test -C ebcdic >/dev/null
 ebcdic=$?
 $sim $pcre2test -C ebcdic-io >/dev/null
 ebcdic_io=$?
+$sim $pcre2test -C ebcdic-nl25 >/dev/null
+ebcdic_nl25=$?
+
 if [ $ebcdic -eq 1 ]; then
-  # XXX This is a bit unholy. How about I just bite the bullet and implement
-  # the "#if" stuff directly inside pcre2test? It wouldn't be totally crazy.
-  # This does seem to work fine here though.
-  #
-  # Purpose of code: sneakily parse the pcre2test commandline, to find the
-  # filename of the input file. Run that through sed, to strip out the chunks
-  # blocked out by "# if !EBCDIC...#endif", and then invoke pcre2test with the
-  # modified input file. That's it.
-  filterebcdic()
-    {
-    filter_args=
-    while [ "$1" != "$pcre2test" ] ; do
-      # Shift off any Valgrind args
-      filter_args="$filter_args $1"
-      shift
-    done
-    # Shift off the $pcre2test arg
-    filter_args="$filter_args $1"
-    shift
-    while [ $# -gt 0 ] ; do
-      case "$1" in
-        -C|-error|-pattern|-S|-subject|-t|-tm|-T|-TM)
-          if [ $# -gt 1 ] ; then
-            filter_args="$filter_args $1 $2"; shift; shift
-          else
-            filter_args="$filter_args $1"; shift
-          fi
-          ;;
-        -*) filter_args="$filter_args $1"; shift;;
-        *) break;;
-      esac
-    done
-    if [ $# -gt 0 ] ; then
-      input="$1"
-      shift
-      sed '/^# if !EBCDIC/,/^# endif/d' "$input" | $filter_args "/dev/stdin" "$@"
-    else
-      $filter_args "$@"
-    fi
-    }
-  sim="filterebcdic $sim"
+  if [ $ebcdic_io -eq 0 ]; then
+    echo "Running tests in EBCDIC mode, and expecting ASCII test data"
+  else
+    echo "Running tests in EBCDIC mode, and expecting EBCDIC test data"
+    echo "If you are on an EBCDIC machine, you will need to convert the PCRE2"
+    echo "testdata/ directory from ISO8859-1, so the data match the EBCDIC"
+    echo "codepage that your C compiler is using for C character literals."
+    echo "For example:"
+    echo "  iconv -f ISO8859-1 -t IBM-1047 ..."
+  fi
 fi
 
 # Initialize all bitsizes skipped
@@ -483,7 +462,7 @@ if [ $do0  = no -a $do1  = no -a $do2  = no -a $do3  = no -a \
      $do16 = no -a $do17 = no -a $do18 = no -a $do19 = no -a \
      $do20 = no -a $do21 = no -a $do22 = no -a $do23 = no -a \
      $do24 = no -a $do25 = no -a $do26 = no -a $do27 = no -a \
-     $do28 = no -a $doheap = no \
+     $do28 = no -a $do29 = no -a $doheap = no \
    ]; then
   do0=yes
   do1=yes
@@ -514,6 +493,7 @@ if [ $do0  = no -a $do1  = no -a $do2  = no -a $do3  = no -a \
   do26=yes
   do27=yes
   do28=yes
+  do29=yes
 fi
 
 # Handle any explicit skips at this stage, so that an argument list may consist
@@ -980,33 +960,26 @@ for bmode in "$test8" "$test16" "$test32"; do
     if [ $ebcdic -eq 0 ] ; then
       echo "  Skipped when not targetting EBCDIC"
     else
-      if [ $ebcdic_io -eq 0 ] ; then
-        # Our testdata files are in ASCII, and the pcre2test program is using
-        # ASCII input: all easy.
-        for opt in "" "-dfa"; do
-          $sim $valgrind $pcre2test -q $setstack $bmode $opt $testdata/testinput28 testtry
-          checkresult $? 28 "$opt"
-        done
-      else
-        echo "Cannot run EBCDIC tests:"
-        echo "  Ironically this particular test script does not automatically"
-        echo "  run the tests on an actual EBCDIC system. The testdata files"
-        echo "  shipped with PCRE2 are in ASCII."
-        echo "  You may be able to run the tests manually if you know which"
-        echo "  EBCDIC codepage you used when compiling PCRE2, and then convert"
-        echo "  the testdata to match. For example, if the C compiler used to build"
-        echo "  PCRE2 was using IBM-1047:"
-        echo ""
-        echo "  iconv -f ISO8859-1 -t IBM-1047 <testdata/testinputEBC >testinputEBC-native"
-        echo "  pcre2test -q -$bmode testinputEBC-native >testoutputEBC-native"
-        echo "  [ $? -eq 0 ] || echo 'pcre2test failed'"
-        echo "  iconv -f IBM-1047 -t ISO8859-1 <testoutputEBC-native >testoutputEBC-ascii"
-        echo "  $cf testdata/testoutputEBC testoutputEBC-ascii"
-        echo ""
-        echo "This is speculative. The PCRE2 maintainers do not have access to an"
-        echo "EBCDIC system to test this. Please report back if you try it."
-        exit 1
-      fi
+      for opt in "" $jitopt "-dfa"; do
+        $sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput28 testtry
+        checkresult $? 28 "$opt"
+      done
+    fi
+  fi
+
+  # EBCDIC tests (for NL=0x25)
+
+  if [ $do29 = yes ] ; then
+    echo $title29
+    if [ $ebcdic -eq 0 ] ; then
+      echo "  Skipped when not targetting EBCDIC"
+    elif [ $ebcdic_nl25 -eq 0 ] ; then
+      echo "  Skipped because EBCDIC newline is not 0x25"
+    else
+      for opt in "" $jitopt "-dfa"; do
+        $sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput29 testtry
+        checkresult $? 29 "$opt"
+      done
     fi
   fi
 

RunTest.bat

@@ -71,6 +71,10 @@ set unicode=%ERRORLEVEL%
 set jit=%ERRORLEVEL%
 %pcre2test% -C backslash-C >NUL
 set supportBSC=%ERRORLEVEL%
+%pcre2test% -C ebcdic >NUL
+set ebcdic=%ERRORLEVEL%
+%pcre2test% -C ebcdic-nl25 >NUL
+set ebcdic_nl25=%ERRORLEVEL%
 
 if %support8% EQU 1 (
 if not exist testout8 md testout8
@@ -114,19 +118,20 @@ set do24=no
 set do25=no
 set do26=no
 set do27=no
-@rem XXX ADD TEST 28
+set do28=no
+set do29=no
 set all=yes
 
 for %%a in (%*) do (
   set valid=no
-  for %%v in (1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27) do if %%v == %%a set valid=yes
+  for %%v in (1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29) do if %%v == %%a set valid=yes
   if "!valid!" == "yes" (
     set do%%a=yes
     set all=no
   ) else (
     echo Invalid test number - %%a!
     echo Usage %0 [ test_number ] ...
-    echo Where test_number is one or more optional test numbers 1 through 27, default is all tests.
+    echo Where test_number is one or more optional test numbers 1 through 29, default is all tests.
     exit /b 1
   )
 )
@@ -160,6 +165,8 @@ if "%all%" == "yes" (
   set do25=yes
   set do26=yes
   set do27=yes
+  set do28=yes
+  set do29=yes
 )
 
 @echo RunTest.bat's pcre2test output is written to newly created subfolders
@@ -215,6 +222,8 @@ if "%do24%" == "yes" call :do24
 if "%do25%" == "yes" call :do25
 if "%do26%" == "yes" call :do26
 if "%do27%" == "yes" call :do27
+if "%do28%" == "yes" call :do28
+if "%do29%" == "yes" call :do29
 :modeSkip
 if "%mode%" == "" (
   set mode=-16
@@ -293,18 +302,28 @@ if errorlevel 1 (
   %pcre2test% %mode% %4 %5 %6 %7 %8 %9 -error -80,-62,-2,-1,0,100,101,191,300 >>%2%bits%\%testoutput%
 )
 
-fc /n %srcdir%\testdata\%testoutput% %2%bits%\%testoutput% >NUL
+set testexpected=%srcdir%\testdata\%testoutput%
+
+if %ebcdic% EQU 1 (
+  @rem We currently only use the #if ... #endif support in pcre2test for EBCDIC
+  @rem testing. Run in "preprocess-only" mode (-E) on the testoutput file to trim
+  @rem the output lines matching the input lines which are discarded.
+  %pcre2test% -q -E %testexpected% >%2%bits%\%testoutput%-trimmed
+  set testexpected=%2%bits%\%testoutput%-trimmed
+)
+
+fc /n %testexpected% %2%bits%\%testoutput% >NUL
 
 if errorlevel 1 (
-  echo.          failed comparison: fc /n %srcdir%\testdata\%testoutput% %2%bits%\%testoutput%
+  echo.          failed comparison: fc /n %testexpected% %2%bits%\%testoutput%
   if [%1]==[3] (
     echo.
     echo ** Test 3 failure usually means french locale is not
     echo ** available on the system, rather than a bug or problem with PCRE2.
     echo.
     goto :eof
 )
-  fc /n %srcdir%\testdata\%testoutput% %2%bits%\%testoutput%
+  fc /n %testexpected% %2%bits%\%testoutput%
 
   set failed="yes"
   goto :eof
@@ -544,6 +563,30 @@ if %unicode% EQU 0 (
   if %jit% EQU 1 call :runsub 27 testoutjit "Test with JIT Override" -q -jit
 goto :eof
 
+:do28
+if %ebcdic% EQU 0 (
+  echo Test 28 Skipped when not targetting EBCDIC.
+  goto :eof
+)
+  call :runsub 28 testout "EBCDIC-specific tests" -q
+  call :runsub 28 testout "EBCDIC-specific tests (DFA)" -q -dfa
+  if %jit% EQU 1 call :runsub 28 testoutjit "Test with JIT Override" -q -jit
+goto :eof
+
+:do29
+if %ebcdic% EQU 0 (
+  echo Test 29 Skipped when not targetting EBCDIC.
+  goto :eof
+)
+if %ebcdic_nl25% EQU 0 (
+  echo Test 29 Skipped because EBCDIC newline is not 0x25.
+  goto :eof
+)
+  call :runsub 29 testout "EBCDIC-specific tests for NL=0x25" -q
+  call :runsub 29 testout "EBCDIC-specific tests for NL=0x25 (DFA)" -q -dfa
+  if %jit% EQU 1 call :runsub 29 testoutjit "Test with JIT Override" -q -jit
+goto :eof
+
 :conferror
 @echo.
 @echo Either your build is incomplete or you have a configuration error.

configure.ac

@@ -1169,7 +1169,7 @@ AC_CONFIG_COMMANDS([script-chmod], [chmod a+x pcre2-config])
 
 # Make sure that pcre2_chartables.c is removed in case the method for
 # creating it was changed by reconfiguration.
-AC_CONFIG_COMMANDS([delete-old-chartables], [rm -f pcre2_chartables.c])
+AC_CONFIG_COMMANDS([delete-old-chartables], [rm -f src/pcre2_chartables.c])
 
 # Handle --enable-Werror/errwarn. This must come last, so that we don't mess
 # with any of the library-detection tests (or similar).