enable repacking again [no ci]

The previous commit disabled repacking in the CPU backend because of the
supports check which also checked the buffer types. This has been
relaxed now to allow repacking again.

Author: danbev

ggml/src/ggml-backend.cpp

@@ -1995,11 +1995,7 @@ struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, s
         if (extra_buffer && !ggml_op_is_empty(node->op) && node->src[0]) {
             size_t id = ggml_hash_find(&hash_set, node);
             ggml_status status = ggml_backend_buffer_init_tensor(extra_buffer, node_copies[id]);
-            if (status == GGML_STATUS_SUCCESS) {
-                if (node_copies[id]->extra != nullptr) {
-                    node_copies[id]->buffer = extra_buffer;
-                }
-            } else {
+            if (status != GGML_STATUS_SUCCESS) {
                 GGML_LOG_ERROR("%s: failed to initialize tensor in extra buffer for graph copy\n", __func__);
             }
         }

ggml/src/ggml-cpu/repack.cpp

@@ -1872,9 +1872,7 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons
 static bool supports_tensor(const struct ggml_tensor * op) {
     if (op->op == GGML_OP_MUL_MAT &&
         op->src[0]->buffer &&
-        (ggml_n_dims(op->src[0]) == 2) &&
-         op->src[0]->buffer->buft == ggml_backend_cpu_repack_buffer_type() &&
-         ggml_repack_get_optimal_repack_type(op->src[0])) {
+        (ggml_n_dims(op->src[0]) == 2) && ggml_repack_get_optimal_repack_type(op->src[0])) {
 
         if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) {
             return false;
@@ -1885,9 +1883,7 @@ static bool supports_tensor(const struct ggml_tensor * op) {
         }
 
     } else if (op->op == GGML_OP_MUL_MAT_ID && op->src[0]->buffer &&
-            (ggml_n_dims(op->src[0]) == 3) &&
-            op->src[0]->buffer->buft == ggml_backend_cpu_repack_buffer_type() &&
-            ggml_repack_get_optimal_repack_type(op->src[0])) {
+            (ggml_n_dims(op->src[0]) == 3) && ggml_repack_get_optimal_repack_type(op->src[0])) {
 
         if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) {
             return false;
@@ -1903,10 +1899,12 @@ static bool supports_tensor(const struct ggml_tensor * op) {
 static enum ggml_status ggml_backend_cpu_repack_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {
     if (tensor->op == GGML_OP_NONE) {
         tensor->extra = (void *) const_cast<ggml::cpu::tensor_traits *>(ggml_repack_get_optimal_repack_type(tensor));
+        tensor->buffer = buffer;
     }
 
     if (supports_tensor(tensor)) {
         tensor->src[0]->extra = (void *) const_cast<ggml::cpu::tensor_traits *>(ggml_repack_get_optimal_repack_type(tensor->src[0]));
+        tensor->src[0]->buffer = buffer;
     }
 
     GGML_UNUSED(buffer);
@@ -1955,7 +1953,39 @@ static size_t ggml_backend_cpu_repack_buffer_type_get_alignment(ggml_backend_buf
 namespace ggml::cpu::repack {
 class extra_buffer_type : ggml::cpu::extra_buffer_type {
     bool supports_op(ggml_backend_dev_t, const struct ggml_tensor * op) override {
-        return supports_tensor(op);
+        if (    op->op == GGML_OP_MUL_MAT &&
+                op->src[0]->buffer &&
+                (ggml_n_dims(op->src[0]) == 2) &&
+                op->src[0]->buffer->buft == ggml_backend_cpu_repack_buffer_type() &&
+                ggml_repack_get_optimal_repack_type(op->src[0])
+                ) {
+            if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) {
+                return false;
+            }
+            if (op->src[1]->type == GGML_TYPE_F32) {
+                return true;
+            }
+            //if (op->src[1]->type == GGML_TYPE_Q8_0) {
+            //    return true;
+            //}
+            // may be possible if Q8_0 packed...
+        } else if (op->op == GGML_OP_MUL_MAT_ID
+                && op->src[0]->buffer
+                && (ggml_n_dims(op->src[0]) == 3)
+                && op->src[0]->buffer->buft == ggml_backend_cpu_repack_buffer_type()
+                && ggml_repack_get_optimal_repack_type(op->src[0])
+                ) {
+            if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) {
+                return false;
+            }
+            if (op->src[1]->type == GGML_TYPE_F32) {
+                return true;
+            }
+            //if (op->src[1]->type == GGML_TYPE_Q8_0) {
+            //    return true;
+            //}
+        }
+        return false;
     }
 
     ggml::cpu::tensor_traits * get_tensor_traits(const struct ggml_tensor * op) override {