Add an option to build translation tables

ddeclerck · ddeclerck · commit a2b5ed29f02a · 2025-03-10T19:10:06.000+01:00
diff --git a/.github/workflows/windows-msys2.yml b/.github/workflows/windows-msys2.yml
@@ -43,8 +43,10 @@ jobs:
           mkdir _build && cd _build
           # this is a "prepare" only step for this workflow only, so build
           # without any optional packages as one-time build speeding up this part
+          #../configure --without-db --without-curses --without-xml2 --without-json \
+          #             --without-iconv --disable-dependency-tracking
           ../configure --without-db --without-curses --without-xml2 --without-json \
-                       --without-iconv --disable-dependency-tracking
+                       --disable-dependency-tracking
 
       - name: Build GnuCOBOL Source Distribution
         shell: msys2 {0}
diff --git a/build_windows/vs2005/cobc.vcproj b/build_windows/vs2005/cobc.vcproj
@@ -212,6 +212,10 @@
 				RelativePath="..\..\cobc\field.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\cobc\gentable.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\cobc\help.c"
 				>
diff --git a/build_windows/vs2008/cobc.vcproj b/build_windows/vs2008/cobc.vcproj
@@ -207,6 +207,10 @@
 				RelativePath="..\..\cobc\field.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\cobc\gentable.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\cobc\help.c"
 				>
diff --git a/build_windows/vs2010/cobc.vcxproj b/build_windows/vs2010/cobc.vcxproj
@@ -192,6 +192,7 @@
     <ClCompile Include="..\..\cobc\config.c" />
     <ClCompile Include="..\..\cobc\error.c" />
     <ClCompile Include="..\..\cobc\field.c" />
+    <ClCompile Include="..\..\cobc\gentable.c" />
     <ClCompile Include="..\..\cobc\help.c" />
     <ClCompile Include="..\..\cobc\replace.c" />
     <ClCompile Include="..\..\cobc\reserved.c" />
@@ -354,4 +355,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>
diff --git a/build_windows/vs2012/cobc.vcxproj b/build_windows/vs2012/cobc.vcxproj
@@ -196,6 +196,7 @@
     <ClCompile Include="..\..\cobc\config.c" />
     <ClCompile Include="..\..\cobc\error.c" />
     <ClCompile Include="..\..\cobc\field.c" />
+    <ClCompile Include="..\..\cobc\gentable.c" />
     <ClCompile Include="..\..\cobc\help.c" />
     <ClCompile Include="..\..\cobc\replace.c" />
     <ClCompile Include="..\..\cobc\reserved.c" />
@@ -358,4 +359,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>
diff --git a/build_windows/vs2013/cobc.vcxproj b/build_windows/vs2013/cobc.vcxproj
@@ -198,6 +198,7 @@
     <ClCompile Include="..\..\cobc\config.c" />
     <ClCompile Include="..\..\cobc\error.c" />
     <ClCompile Include="..\..\cobc\field.c" />
+    <ClCompile Include="..\..\cobc\gentable.c" />
     <ClCompile Include="..\..\cobc\help.c" />
     <ClCompile Include="..\..\cobc\replace.c" />
     <ClCompile Include="..\..\cobc\reserved.c" />
@@ -360,4 +361,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>
diff --git a/build_windows/vs2015/cobc.vcxproj b/build_windows/vs2015/cobc.vcxproj
@@ -196,6 +196,7 @@
     <ClCompile Include="..\..\cobc\config.c" />
     <ClCompile Include="..\..\cobc\error.c" />
     <ClCompile Include="..\..\cobc\field.c" />
+    <ClCompile Include="..\..\cobc\gentable.c" />
     <ClCompile Include="..\..\cobc\help.c" />
     <ClCompile Include="..\..\cobc\replace.c" />
     <ClCompile Include="..\..\cobc\reserved.c" />
@@ -358,4 +359,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>
diff --git a/build_windows/vs2017/cobc.vcxproj b/build_windows/vs2017/cobc.vcxproj
@@ -198,6 +198,7 @@
     <ClCompile Include="..\..\cobc\config.c" />
     <ClCompile Include="..\..\cobc\error.c" />
     <ClCompile Include="..\..\cobc\field.c" />
+    <ClCompile Include="..\..\cobc\gentable.c" />
     <ClCompile Include="..\..\cobc\help.c" />
     <ClCompile Include="..\..\cobc\replace.c" />
     <ClCompile Include="..\..\cobc\reserved.c" />
@@ -360,4 +361,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>
diff --git a/build_windows/vs2019/cobc.vcxproj b/build_windows/vs2019/cobc.vcxproj
@@ -198,6 +198,7 @@
     <ClCompile Include="..\..\cobc\config.c" />
     <ClCompile Include="..\..\cobc\error.c" />
     <ClCompile Include="..\..\cobc\field.c" />
+    <ClCompile Include="..\..\cobc\gentable.c" />
     <ClCompile Include="..\..\cobc\help.c" />
     <ClCompile Include="..\..\cobc\replace.c" />
     <ClCompile Include="..\..\cobc\reserved.c" />
@@ -360,4 +361,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>
diff --git a/cobc/Makefile.am b/cobc/Makefile.am
@@ -23,7 +23,7 @@ bin_PROGRAMS = cobc
 cobc_SOURCES = cobc.c cobc.h ppparse.y pplex.c parser.y scanner.c config.c \
 	reserved.c error.c tree.c tree.h field.c typeck.c codegen.c help.c \
 	config.def flag.def warning.def codeoptim.def ppparse.def \
-	codeoptim.c replace.c 
+	codeoptim.c replace.c gentable.c
 
 #cobc_SOURCES = cobc.c cobc.h ppparse.y pplex.l parser.y scanner.l config.c
 
diff --git a/cobc/cobc.c b/cobc/cobc.c
@@ -116,6 +116,7 @@ enum compile_level {
 #define CB_FLAG_GETOPT_DEPEND_ADD_PHONY     24
 #define CB_FLAG_GETOPT_DEPEND_KEEP_MISSING  25
 #define CB_FLAG_GETOPT_DEPEND_ON_THE_SIDE   26
+#define CB_FLAG_GETOPT_GENTABLE             27
 
 /* Info display limits */
 #define	CB_IMSG_SIZE		24
@@ -634,6 +635,7 @@ static const struct option long_options[] = {
 	{"MG",			CB_NO_ARG, NULL, CB_FLAG_GETOPT_DEPEND_KEEP_MISSING},
 	{"MD",			CB_NO_ARG, NULL, CB_FLAG_GETOPT_DEPEND_ON_THE_SIDE},
 	{"fcopybook-deps",	CB_NO_ARG, &cb_flag_copybook_deps, 1},
+	{"gentable",		CB_RQ_ARG, NULL, CB_FLAG_GETOPT_GENTABLE},
 	{"coverage",	CB_NO_ARG, &cb_coverage_enabled, 1},
 	{"P",			CB_OP_ARG, NULL, 'P'},
 	{"Xref",		CB_NO_ARG, NULL, 'X'},
@@ -3408,6 +3410,19 @@ process_command_line (const int argc, char **argv)
 			}
 			break;
 
+		case CB_FLAG_GETOPT_GENTABLE: {
+			/* -gentable=<enc-from>,<enc-to> */
+			const char *from, *to;
+			int res;
+			from = strtok (cob_optarg, ",");
+			to = strtok (NULL, "");
+			if ((from == NULL) || (to == NULL)) {
+				cobc_err_exit (COBC_INV_PAR, "-gentable");
+			}
+			res = gentable (stdout, from, to);
+			exit (res ? EXIT_FAILURE : EXIT_SUCCESS);
+		}
+
 		default:
 			/* as we postpone most options simply skip everything other here */
 			break;
diff --git a/cobc/cobc.h b/cobc/cobc.h
@@ -694,4 +694,7 @@ extern int		cb_strcasecmp (const void *, const void *);
 extern unsigned char	cb_toupper (const unsigned char);
 extern unsigned char	cb_tolower (const unsigned char);
 
+/* gentable.c */
+extern int		gentable (FILE *stream, const char *fromcode, const char *tocode);
+
 #endif /* CB_COBC_H */
diff --git a/cobc/gentable.c b/cobc/gentable.c
@@ -0,0 +1,193 @@
+/*
+   Copyright (C) 2025 Free Software Foundation, Inc.
+   Written by David Declerck.
+
+   This file is part of GnuCOBOL.
+
+   The GnuCOBOL compiler is free software: you can redistribute it
+   and/or modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation, either version 3 of the
+   License, or (at your option) any later version.
+
+   GnuCOBOL is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GnuCOBOL.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+#include "config.h"
+#include "cobc.h"
+
+#include <stdio.h>
+
+#ifdef HAVE_ICONV
+
+#include <iconv.h>
+#include <errno.h>
+#include <time.h>
+
+#define TABLE_SIZE 256
+#define LINE_SIZE   16
+
+static int
+current_year (void)
+{
+	time_t t = time (NULL);
+	struct tm *tm = localtime (&t);
+	return tm->tm_year + 1900;
+}
+
+static void
+output_license (FILE *stream)
+{
+	fprintf(stream,
+"# Copyright (C) %04d Free Software Foundation, Inc.\n"
+"# Written by David Declerck.\n"
+"#\n"
+"# This file is part of the GnuCOBOL runtime.\n"
+"#\n"
+"# The GnuCOBOL runtime is free software: you can redistribute it\n"
+"# and/or modify it under the terms of the GNU Lesser General Public License\n"
+"# as published by the Free Software Foundation, either version 3 of the\n"
+"# License, or (at your option) any later version.\n"
+"#\n"
+"# GnuCOBOL is distributed in the hope that it will be useful,\n"
+"# but WITHOUT ANY WARRANTY; without even the implied warranty of\n"
+"# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n"
+"# GNU General Public License for more details.\n"
+"#\n"
+"# You should have received a copy of the GNU Lesser General Public License\n"
+"# along with GnuCOBOL.  If not, see <https://www.gnu.org/licenses/>.\n",
+	current_year ());
+}
+
+static void
+output_table (FILE *stream, const char *table)
+{
+	int i;
+	for (i = 0; i < TABLE_SIZE; ++i) {
+		fprintf (stream, "%02X%c", (unsigned char)table[i],
+				 ((i+1) % LINE_SIZE == 0) ? '\n' : ' ');
+	}
+}
+
+static int
+translate_subst_char (iconv_t ic, char *ebcdic_subst, char *ascii_subst)
+{
+	size_t ebcdic_size = 1, ascii_size = 1;
+	size_t res = iconv (ic, &ebcdic_subst, &ebcdic_size, &ascii_subst, &ascii_size);
+	iconv (ic, NULL, NULL, NULL, NULL);
+	return (res == (size_t)0);
+}
+
+/* We assume fromcode to be an EBCDIC variant and tocode to be an ASCII-based encoding */
+int
+gentable (FILE *stream, const char *fromcode, const char *tocode)
+{
+	char ebcdic[TABLE_SIZE], ascii[TABLE_SIZE] = { 0 };
+	char *ebcdic_ptr = ebcdic, *ascii_ptr = ascii;
+	size_t ebcdic_size = TABLE_SIZE, ascii_size = TABLE_SIZE;
+	char ebcdic_subst = 0x3F, ascii_subst = 0;
+	iconv_t ic;
+	size_t res;
+	int i, nb_reversible = 0;
+
+	for (i = 0; i < TABLE_SIZE; ++i) {
+		ebcdic[i] = (char)i;
+	}
+
+	ic = iconv_open (tocode, fromcode);
+	if (ic == (iconv_t)-1) {
+		fprintf(stderr, "Conversion from %s to %s is not supported by your iconv implementation.\n", fromcode, tocode);
+		return -1;
+	}
+
+	/* Note: POSIX iconv performs an implementation-defined conversion when
+	   encountering an untranslatable character (MUSL uses * and *BSD uses ?)
+	   and returns the number of such non-reversible conversions performed.
+	   GNU iconv instead sets errno to EILSEQ (unless //TRANSLIT is used).
+	   To cope with these differences, we convert the table character by
+	   character and when encountering an untranslatable character, we
+	   "translate" it to the substitution character translated from the
+	   EBCDIC substitution character (0x3F). */
+
+	if (!translate_subst_char (ic, &ebcdic_subst, &ascii_subst)) {
+		fprintf(stderr, "Can not convert the substitution character from %s to %s.\n", fromcode, tocode);
+		return -1;
+	}
+
+	for (i = 0; i < TABLE_SIZE; ++i) {
+		ebcdic_size = 1;
+		res = iconv (ic, &ebcdic_ptr, &ebcdic_size, &ascii_ptr, &ascii_size);
+		if (res == (size_t)-1) {
+			switch (errno) {
+				/* GNU iconv: an untranslatable character was met */
+				case EILSEQ:
+					*ascii_ptr = ascii_subst;
+					++ebcdic_ptr; ++ascii_ptr;
+					++ebcdic_size; ++ascii_size;
+					++nb_reversible;
+					break;
+				case EINVAL:
+				case E2BIG:
+				default:
+					fprintf(stderr, "An error occurred after converting %ld characters.\n", (ebcdic_ptr - ebcdic));
+					iconv_close (ic);
+					return -1;
+			}
+		/* POSIX iconv: a substitution was performed */
+		} else if (res != (size_t)0) {
+			*(ascii_ptr - 1) = ascii_subst;
+			nb_reversible += (int)res;
+		}
+	}
+
+	iconv_close (ic);
+
+	fprintf(stream, "# GnuCOBOL %s/%s translation tables\n\n", fromcode, tocode);
+	output_license (stream);
+
+	fprintf(stream, "\n# %s to %s translation table\n\n", fromcode, tocode);
+	output_table (stream, ascii);
+
+	fprintf(stream, "\n# %s to %s translation table\n\n", tocode, fromcode);
+	if (nb_reversible <= 0) {
+		fprintf(stream, "# This translation being symmetric, the table is built from the previous one.\n\n");
+	} else {
+		for (i = 0; i < TABLE_SIZE; ++i) {
+			ebcdic[(unsigned char)i] = (char)0x3F;
+		}
+		for (i = 0; i < TABLE_SIZE; ++i) {
+			ebcdic[(unsigned char)ascii[(unsigned char)i]] = (char)i;
+		}
+//TODO: renumber substituted characters if needed (should not be in this branch as it make the translation reversible)
+		/* Restore the substitution character, as it is used
+                   several times, we can't just reverse the table */
+		ebcdic[(unsigned char)ascii_subst] = (char)ebcdic_subst;
+		output_table (stream, ebcdic);
+		fprintf(stream, "\n");
+
+		fprintf(stderr, "Warning: %d non-reversible conversions were performed, you might want to review the generated table.\n", nb_reversible);
+	}
+
+	return 0;
+}
+
+#else
+
+int
+gentable (FILE *stream, const char *fromcode, const char *tocode)
+{
+	COB_UNUSED (stream);
+	COB_UNUSED (fromcode);
+	COB_UNUSED (tocode);
+
+	fprintf(stderr, "Error: GnuCOBOL was not compiled with iconv support; translation table generation is not available.\n");
+
+	return -1;
+}
+
+#endif /* HAVE_ICONV */
diff --git a/cobc/help.c b/cobc/help.c
@@ -150,6 +150,7 @@ cobc_print_usage_common_options (void)
 	puts (_("  -MD                   output dependencies in .d files while compiling"));
 	puts (_("  -ext <extension>      add file extension for resolving COPY"));
 	puts (_("  -fcopybook-deps       output copybook names as dependencies"));
+	puts (_("  -gentable=<enc-from>,<enc-to>\tbuild an EBCDIC-to-ASCII translation table between the given encodings"));
 	putchar ('\n');
 }
 
diff --git a/config/ebcdic500_ascii8bit.ttbl b/config/ebcdic500_ascii8bit.ttbl
@@ -26,7 +26,7 @@
 #
 # Note: the only differences with a "plain" EBCDIC 500 to 8-bit ASCII table
 # are that EBCDIC 6A (broken bar) is translated to ASCII 7C (vertical bar)
-# and EBCDIC BB (vertical bar) is translated to ASCII 1A (subtitution char).
+# and EBCDIC BB (vertical bar) is translated to ASCII E3 (unrelated character).
 # This seems to be the "usual" way of translating these characters.
 
 00 01 02 03 9C 09 86 7F 97 8D 8E 0B 0C 0D 0E 0F

Original file line number	Diff line number	Diff line change
`@@ -212,6 +212,10 @@`
`212`	`212`	`RelativePath="..\..\cobc\field.c"`
`213`	`213`	`>`
`214`	`214`	`</File>`
	`215`	`+ <File`
	`216`	`+ RelativePath="..\..\cobc\gentable.c"`
	`217`	`+ >`
	`218`	`+ </File>`
`215`	`219`	`<File`
`216`	`220`	`RelativePath="..\..\cobc\help.c"`
`217`	`221`	`>`
Original file line number	Diff line number	Diff line change
`@@ -207,6 +207,10 @@`
`207`	`207`	`RelativePath="..\..\cobc\field.c"`
`208`	`208`	`>`
`209`	`209`	`</File>`
	`210`	`+ <File`
	`211`	`+ RelativePath="..\..\cobc\gentable.c"`
	`212`	`+ >`
	`213`	`+ </File>`
`210`	`214`	`<File`
`211`	`215`	`RelativePath="..\..\cobc\help.c"`
`212`	`216`	`>`