autotools, CMake: fix issues with snprintf test and sanitizers.

Avoid trying to cast negative values to unsigned types, or doing shifts
of signed types, in order not to have the test program fail if we're
building with undefined-behavior sanitizers enabled.

See the-tcpdump-group/libpcap#1396 for the equivalent libpcap issue.

Author: guyharris

CMakeLists.txt

@@ -542,32 +542,50 @@ if (NOT CMAKE_CROSSCOMPILING)
     #include <inttypes.h>
     #include <sys/types.h>
 
+    #if defined(_WIN32) && !defined(_SSIZE_T_DEFINED)
+    /*
+     * On UN*Xes, this is a signed integer type of the same size as size_t.
+     *
+     * It's not defined by Visual Studio; we assume that ptrdiff_t will
+     * be a type that is a signed integer type of the same size as size_t.
+     */
+    typedef ptrdiff_t ssize_t;
+    #endif
+
+    /*
+     * Avoid trying to cast negative values to unsigned types, or doing
+     * shifts of signed types, in order not to have the test program fail
+     * if we're building with undefined-behavior sanitizers enabled.
+     */
     int main()
     {
       char buf[100];
-      uint64_t t = (uint64_t)1 << 32;
+      unsigned int ui = sizeof(buf);
+      int i = sizeof(buf);
+      int64_t i64 = INT64_C(0x100000000);
+      uint64_t ui64 = UINT64_C(0x100000000);
 
-      snprintf(buf, sizeof(buf), \"%zu\", sizeof(buf));
+      snprintf(buf, sizeof(buf), \"%zu\", (size_t)ui);
       if (strncmp(buf, \"100\", sizeof(buf)))
         return 1;
 
-      snprintf(buf, sizeof(buf), \"%zd\", -sizeof(buf));
+      snprintf(buf, sizeof(buf), \"%zd\", (ssize_t)(-i));
       if (strncmp(buf, \"-100\", sizeof(buf)))
         return 2;
 
-      snprintf(buf, sizeof(buf), \"%\" PRId64, -t);
+      snprintf(buf, sizeof(buf), \"%\" PRId64, -i64);
       if (strncmp(buf, \"-4294967296\", sizeof(buf)))
         return 3;
 
-      snprintf(buf, sizeof(buf), \"0o%\" PRIo64, t);
+      snprintf(buf, sizeof(buf), \"0o%\" PRIo64, ui64);
       if (strncmp(buf, \"0o40000000000\", sizeof(buf)))
         return 4;
 
-      snprintf(buf, sizeof(buf), \"0x%\" PRIx64, t);
+      snprintf(buf, sizeof(buf), \"0x%\" PRIx64, ui64);
       if (strncmp(buf, \"0x100000000\", sizeof(buf)))
         return 5;
 
-      snprintf(buf, sizeof(buf), \"%\" PRIu64, t);
+      snprintf(buf, sizeof(buf), \"%\" PRIu64, ui64);
       if (strncmp(buf, \"4294967296\", sizeof(buf)))
         return 6;
 

configure.ac

@@ -327,32 +327,50 @@ AC_RUN_IFELSE(
 #include <inttypes.h>
 #include <sys/types.h>
 
+#if defined(_WIN32) && !defined(_SSIZE_T_DEFINED)
+/*
+ * On UN*Xes, this is a signed integer type of the same size as size_t.
+ *
+ * It's not defined by Visual Studio; we assume that ptrdiff_t will
+ * be a type that is a signed integer type of the same size as size_t.
+ */
+typedef ptrdiff_t ssize_t;
+#endif
+
+/*
+ * Avoid trying to cast negative values to unsigned types, or doing
+ * shifts of signed types, in order not to have the test program fail
+ * if we're building with undefined-behavior sanitizers enabled.
+ */
 int main()
 {
   char buf[100];
-  uint64_t t = (uint64_t)1 << 32;
+  unsigned int ui = sizeof(buf);
+  int i = sizeof(buf);
+  int64_t i64 = INT64_C(0x100000000);
+  uint64_t ui64 = UINT64_C(0x100000000);
 
-  snprintf(buf, sizeof(buf), "%zu", sizeof(buf));
+  snprintf(buf, sizeof(buf), "%zu", (size_t)ui);
   if (strncmp(buf, "100", sizeof(buf)))
     return 1;
 
-  snprintf(buf, sizeof(buf), "%zd", -sizeof(buf));
+  snprintf(buf, sizeof(buf), "%zd", (ssize_t)(-i));
   if (strncmp(buf, "-100", sizeof(buf)))
     return 2;
 
-  snprintf(buf, sizeof(buf), "%" PRId64, -t);
+  snprintf(buf, sizeof(buf), "%" PRId64, -i64);
   if (strncmp(buf, "-4294967296", sizeof(buf)))
     return 3;
 
-  snprintf(buf, sizeof(buf), "0o%" PRIo64, t);
+  snprintf(buf, sizeof(buf), "0o%" PRIo64, ui64);
   if (strncmp(buf, "0o40000000000", sizeof(buf)))
     return 4;
 
-  snprintf(buf, sizeof(buf), "0x%" PRIx64, t);
+  snprintf(buf, sizeof(buf), "0x%" PRIx64, ui64);
   if (strncmp(buf, "0x100000000", sizeof(buf)))
     return 5;
 
-  snprintf(buf, sizeof(buf), "%" PRIu64, t);
+  snprintf(buf, sizeof(buf), "%" PRIu64, ui64);
   if (strncmp(buf, "4294967296", sizeof(buf)))
     return 6;