Make byteswapping more universal

Now little-endian systems can load big-endian models too.
Model endianness heuristic is based on guessing using
model version field.

Additional fixes for testsuite after removing
capability to write non-native endian files.

Author: AlekseiNikiforovIBM

examples/gguf-split/gguf-split.cpp

@@ -328,14 +328,20 @@ struct split_strategy {
                 const char * t_name = gguf_get_tensor_name(ctx_out, i);
                 struct ggml_tensor * t = ggml_get_tensor(ctx_meta, t_name);
                 auto n_bytes = ggml_nbytes(t);
+                auto n_elements = ggml_nelements(t) / ggml_blck_size(t->type);
                 read_buf.resize(n_bytes);
 
                 // calculate offset
                 auto i_tensor_in = gguf_find_tensor(ctx_gguf, t_name); // idx of tensor in the input file
                 auto offset = gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, i_tensor_in);
 
+                ggml_byteswap_t byteswap_func = nullptr;
+                if (gguf_needs_byteswap(ctx_gguf)) {
+                    byteswap_func = ggml_get_type_traits(t->type)->byteswap;
+                }
+
                 // copy tensor from input to output file
-                copy_file_to_file(f_input, fout, offset, n_bytes);
+                copy_file_to_file(f_input, fout, offset, n_bytes, n_elements, byteswap_func);
                 zeros(fout, GGML_PAD(n_bytes, GGUF_DEFAULT_ALIGNMENT) - n_bytes);
             }
 
@@ -346,13 +352,18 @@ struct split_strategy {
         }
     }
 
-    void copy_file_to_file(std::ifstream & f_in, std::ofstream & f_out, const size_t in_offset, const size_t len) {
+    void copy_file_to_file(std::ifstream & f_in, std::ofstream & f_out, const size_t in_offset, const size_t len, const size_t elements, ggml_byteswap_t byteswap_func) {
         // TODO: detect OS and use copy_file_range() here for better performance
         if (read_buf.size() < len) {
             read_buf.resize(len);
         }
         f_in.seekg(in_offset);
         f_in.read((char *)read_buf.data(), len);
+
+        if (byteswap_func != nullptr) {
+            byteswap_func(read_buf.data(), elements);
+        }
+
         f_out.write((const char *)read_buf.data(), len);
     }
 };
@@ -541,6 +552,13 @@ static void gguf_merge(const split_params & split_params) {
             f_input.seekg(offset);
             f_input.read((char *)read_data.data(), n_bytes);
 
+            if (gguf_needs_byteswap(ctx_gguf)) {
+                auto byteswap = ggml_get_type_traits(t->type)->byteswap;
+                if (byteswap != nullptr) {
+                    byteswap(read_data.data(), ggml_nelements(t) / ggml_blck_size(t->type));
+                }
+            }
+
             // write tensor data + padding
             fout.write((const char *)read_data.data(), n_bytes);
             zeros(fout, GGML_PAD(n_bytes, GGUF_DEFAULT_ALIGNMENT) - n_bytes);

ggml/include/gguf.h

@@ -197,6 +197,9 @@ extern "C" {
     // writes the meta data to pointer "data"
     GGML_API void   gguf_get_meta_data(const struct gguf_context * ctx, void * data);
 
+    // returns true if gguf file needs byteswapping when reading. byteswapping for writing not implemented
+    GGML_API bool gguf_needs_byteswap(const struct gguf_context * ctx);
+
 #ifdef  __cplusplus
 }
 #endif

ggml/src/ggml-impl.h

@@ -29,11 +29,11 @@
 #endif
 
 #if defined(__gnu_linux__)
-#include <endian.h>
+#include <byteswap.h>
 #else // defined(__gnu_linux__)
-#define le64toh(x) (x)
-#define le32toh(x) (x)
-#define le16toh(x) (x)
+#define bswap_16(x) (x)
+#define bswap_32(x) (x)
+#define bswap_64(x) (x)
 #endif // defined(__gnu_linux__)
 
 #ifdef __cplusplus
@@ -562,68 +562,46 @@ static inline ggml_bf16_t ggml_compute_fp32_to_bf16(float s) {
 #define GGML_BF16_TO_FP32(x) ggml_compute_bf16_to_fp32(x)
 
 // endianness conversion
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-#define ggml_convert_from_le16(x) GGML_UNUSED(x)
-#define ggml_convert_from_le32(x) GGML_UNUSED(x)
-#define ggml_convert_from_le64(x) GGML_UNUSED(x)
-#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-static inline void ggml_convert_from_le16(void * value) {
-    *((uint16_t*)value) = le16toh(*((uint16_t*)value));
+static inline void ggml_bswap16(void * value) {
+    *((uint16_t*)value) = bswap_16(*((uint16_t*)value));
 }
 
-static inline void ggml_convert_from_le32(void * value) {
-    *((uint32_t*)value) = le32toh(*((uint32_t*)value));
+static inline void ggml_bswap32(void * value) {
+    *((uint32_t*)value) = bswap_32(*((uint32_t*)value));
 }
 
-static inline void ggml_convert_from_le64(void * value) {
-    *((uint64_t*)value) = le64toh(*((uint64_t*)value));
+static inline void ggml_bswap64(void * value) {
+    *((uint64_t*)value) = bswap_64(*((uint64_t*)value));
 }
-#else // __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-#error Unexpected or undefined __BYTE_ORDER__
-#endif // __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-
-#define ggml_convert_to_le16(x) ggml_convert_from_le16(x)
-#define ggml_convert_to_le32(x) ggml_convert_from_le32(x)
-#define ggml_convert_to_le64(x) ggml_convert_from_le64(x)
 
 #ifdef __cplusplus
 }
 #endif
 
 #ifdef __cplusplus
 #include <vector>
-
-// endianness conversion
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-#define ggml_convert_from_le(x) GGML_UNUSED(x)
-#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
 #include <type_traits>
 
 template <typename T, std::enable_if_t<sizeof(T) == 1, int> = 0>
-static inline void ggml_convert_from_le(T * value)
+static inline void ggml_bswap(T * value)
 {
     GGML_UNUSED(value);
 }
 
 template <typename T, std::enable_if_t<sizeof(T) == 2, int> = 0>
-static inline void ggml_convert_from_le(T * value) {
-    ggml_convert_from_le16(value);
+static inline void ggml_bswap(T * value) {
+    ggml_bswap16(value);
 }
 
 template <typename T, std::enable_if_t<sizeof(T) == 4, int> = 0>
-static inline void ggml_convert_from_le(T * value) {
-    ggml_convert_from_le32(value);
+static inline void ggml_bswap(T * value) {
+    ggml_bswap32(value);
 }
 
 template <typename T, std::enable_if_t<sizeof(T) == 8, int> = 0>
-static inline void ggml_convert_from_le(T * value) {
-    ggml_convert_from_le64(value);
+static inline void ggml_bswap(T * value) {
+    ggml_bswap64(value);
 }
-#else // __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-#error Unexpected or undefined __BYTE_ORDER__
-#endif // __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-
-#define ggml_convert_to_le(x) ggml_convert_from_le(x)
 
 // expose GGUF internals for test code
 GGML_API size_t gguf_type_size(enum gguf_type type);

ggml/src/ggml.c

@@ -6564,179 +6564,179 @@ bool ggml_threadpool_params_match(const struct ggml_threadpool_params * p0, cons
 static void ggml_byteswap_i16(void * restrict buffer, size_t elements) {
     uint16_t *data_ptr = (uint16_t*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(data_ptr + i);
+        ggml_bswap16(data_ptr + i);
     }
 }
 
 static void ggml_byteswap_i32(void * restrict buffer, size_t elements) {
     uint32_t *data_ptr = (uint32_t*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le32(data_ptr + i);
+        ggml_bswap32(data_ptr + i);
     }
 }
 
 static void ggml_byteswap_i64(void * restrict buffer, size_t elements) {
     uint64_t *data_ptr = (uint64_t*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le64(data_ptr + i);
+        ggml_bswap64(data_ptr + i);
     }
 }
 
 static void ggml_byteswap_q4_0(void * restrict buffer, size_t elements) {
     block_q4_0 *data_ptr = (block_q4_0*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].d));
     }
 }
 
 static void ggml_byteswap_q4_1(void * restrict buffer, size_t elements) {
     block_q4_1 *data_ptr = (block_q4_1*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
-        ggml_convert_from_le16(&(data_ptr[i].m));
+        ggml_bswap16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].m));
     }
 }
 
 static void ggml_byteswap_q5_0(void * restrict buffer, size_t elements) {
     block_q5_0 *data_ptr = (block_q5_0*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].d));
     }
 }
 
 static void ggml_byteswap_q5_1(void * restrict buffer, size_t elements) {
     block_q5_1 *data_ptr = (block_q5_1*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
-        ggml_convert_from_le16(&(data_ptr[i].m));
+        ggml_bswap16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].m));
     }
 }
 
 static void ggml_byteswap_q8_0(void * restrict buffer, size_t elements) {
     block_q8_0 *data_ptr = (block_q8_0*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].d));
     }
 }
 
 static void ggml_byteswap_q8_1(void * restrict buffer, size_t elements) {
     block_q8_1 *data_ptr = (block_q8_1*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
-        ggml_convert_from_le16(&(data_ptr[i].s));
+        ggml_bswap16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].s));
     }
 }
 
 static void ggml_byteswap_q2_k(void * restrict buffer, size_t elements) {
     block_q2_K *data_ptr = (block_q2_K*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
-        ggml_convert_from_le16(&(data_ptr[i].dmin));
+        ggml_bswap16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].dmin));
     }
 }
 
 static void ggml_byteswap_q3_k(void * restrict buffer, size_t elements) {
     block_q3_K *data_ptr = (block_q3_K*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].d));
     }
 }
 
 static void ggml_byteswap_q4_k(void * restrict buffer, size_t elements) {
     block_q4_K *data_ptr = (block_q4_K*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
-        ggml_convert_from_le16(&(data_ptr[i].dmin));
+        ggml_bswap16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].dmin));
     }
 }
 
 static void ggml_byteswap_q5_k(void * restrict buffer, size_t elements) {
     block_q5_K *data_ptr = (block_q5_K*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
-        ggml_convert_from_le16(&(data_ptr[i].dmin));
+        ggml_bswap16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].dmin));
     }
 }
 
 static void ggml_byteswap_q6_k(void * restrict buffer, size_t elements) {
     block_q6_K *data_ptr = (block_q6_K*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].d));
     }
 }
 
 static void ggml_byteswap_iq2_xxs(void * restrict buffer, size_t elements) {
     block_iq2_xxs *data_ptr = (block_iq2_xxs*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].d));
         for (size_t j = 0; j < QK_K/8; ++j) {
-            ggml_convert_from_le16(&(data_ptr[i].qs[j]));
+            ggml_bswap16(&(data_ptr[i].qs[j]));
         }
     }
 }
 
 static void ggml_byteswap_iq2_xs(void * restrict buffer, size_t elements) {
     block_iq2_xs *data_ptr = (block_iq2_xs*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].d));
         for (size_t j = 0; j < QK_K/8; ++j) {
-            ggml_convert_from_le16(&(data_ptr[i].qs[j]));
+            ggml_bswap16(&(data_ptr[i].qs[j]));
         }
     }
 }
 
 static void ggml_byteswap_iq3_xxs(void * restrict buffer, size_t elements) {
     block_iq3_xxs *data_ptr = (block_iq3_xxs*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].d));
     }
 }
 
 static void ggml_byteswap_iq3_s(void * restrict buffer, size_t elements) {
     block_iq3_s *data_ptr = (block_iq3_s*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].d));
     }
 }
 
 static void ggml_byteswap_iq2_s(void * restrict buffer, size_t elements) {
     block_iq2_s *data_ptr = (block_iq2_s*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].d));
     }
 }
 
 static void ggml_byteswap_iq1_s(void * restrict buffer, size_t elements) {
     block_iq1_s *data_ptr = (block_iq1_s*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].d));
         for (size_t j = 0; j < QK_K/32; ++j) {
-            ggml_convert_from_le16(&(data_ptr[i].qh[j]));
+            ggml_bswap16(&(data_ptr[i].qh[j]));
         }
     }
 }
 
 static void ggml_byteswap_iq4_nl(void * restrict buffer, size_t elements) {
     block_iq4_nl *data_ptr = (block_iq4_nl*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].d));
     }
 }
 
 static void ggml_byteswap_iq4_xs(void * restrict buffer, size_t elements) {
     block_iq4_xs *data_ptr = (block_iq4_xs*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
-        ggml_convert_from_le16(&(data_ptr[i].scales_h));
+        ggml_bswap16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].scales_h));
     }
 }
 
 static void ggml_byteswap_q8_k(void * restrict buffer, size_t elements) {
     block_q8_K *data_ptr = (block_q8_K*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le32(&(data_ptr[i].d));
+        ggml_bswap32(&(data_ptr[i].d));
         for (size_t j = 0; j < QK_K/16; ++j) {
-            ggml_convert_from_le16(&(data_ptr[i].bsums[j]));
+            ggml_bswap16(&(data_ptr[i].bsums[j]));
         }
     }
 }
@@ -6762,13 +6762,13 @@ static void ggml_byteswap_q4_0_8x8(void * restrict buffer, size_t elements) {
 static void ggml_byteswap_tq1_0(void * restrict buffer, size_t elements) {
     block_tq1_0 *data_ptr = (block_tq1_0*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].d));
     }
 }
 
 static void ggml_byteswap_tq2_0(void * restrict buffer, size_t elements) {
     block_tq2_0 *data_ptr = (block_tq2_0*) buffer;
     for (size_t i = 0; i < elements; ++i) {
-        ggml_convert_from_le16(&(data_ptr[i].d));
+        ggml_bswap16(&(data_ptr[i].d));
     }
 }