Skip to content

Commit 86ed72d

Browse files
ggerganovggerganov
committed
ggml : add ggml-metal-impl.h
ggml-ci
1 parent 63bab93 commit 86ed72d

File tree

6 files changed

+269
-255
lines changed

6 files changed

+269
-255
lines changed

‎Makefile‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -901,9 +901,12 @@ ggml/src/ggml-metal.o: \
901901
ifdef GGML_METAL_EMBED_LIBRARY
902902
ggml/src/ggml-metal-embed.o: \
903903
ggml/src/ggml-metal.metal \
904-
ggml/src/ggml-common.h
904+
ggml/src/ggml-common.h \
905+
ggml/src/ggml-metal-impl.h
905906
@echo "Embedding Metal library"
906-
@sed -e '/#include "ggml-common.h"/r ggml/src/ggml-common.h' -e '/#include "ggml-common.h"/d' < ggml/src/ggml-metal.metal > ggml/src/ggml-metal-embed.metal
907+
@sed -e '/#include "ggml-common.h"/r ggml/src/ggml-common.h' -e '/#include "ggml-common.h"/d' < ggml/src/ggml-metal.metal > ggml/src/ggml-metal-embed.metal.tmp
908+
@sed -e '/#include "ggml-metal-impl.h"/r ggml/src/ggml-metal-impl.h' -e '/#include "ggml-metal-impl.h"/d' < ggml/src/ggml-metal-embed.metal.tmp > ggml/src/ggml-metal-embed.metal
909+
907910
$(eval TEMP_ASSEMBLY=$(shell mktemp -d))
908911
@echo ".section __DATA, __ggml_metallib" > $(TEMP_ASSEMBLY)/ggml-metal-embed.s
909912
@echo ".globl _ggml_metallib_start" >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s

‎ggml/src/CMakeLists.txt‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -62,35 +62,39 @@ if (GGML_METAL)
6262
add_compile_definitions(GGML_METAL_USE_BF16)
6363
endif()
6464

65-
# copy ggml-common.h and ggml-metal.metal to bin directory
66-
configure_file(ggml-common.h ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h COPYONLY)
67-
configure_file(ggml-metal.metal ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal COPYONLY)
65+
# copy metal files to bin directory
66+
configure_file(ggml-common.h ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h COPYONLY)
67+
configure_file(ggml-metal-impl.h ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal-impl.h COPYONLY)
68+
configure_file(ggml-metal.metal ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal COPYONLY)
6869

6970
if (GGML_METAL_EMBED_LIBRARY)
7071
enable_language(ASM)
7172

7273
add_compile_definitions(GGML_METAL_EMBED_LIBRARY)
7374

7475
set(METALLIB_COMMON "${CMAKE_CURRENT_SOURCE_DIR}/ggml-common.h")
76+
set(METALLIB_IMPL "${CMAKE_CURRENT_SOURCE_DIR}/ggml-metal-impl.h")
7577
set(METALLIB_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/ggml-metal.metal")
7678

7779
file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/autogenerated")
7880

7981
# merge ggml-common.h and ggml-metal.metal into a single file
80-
set(METALLIB_EMBED_ASM "${CMAKE_BINARY_DIR}/autogenerated/ggml-metal-embed.s")
81-
set(METALLIB_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-metal-embed.metal")
82+
set(METALLIB_EMBED_ASM "${CMAKE_BINARY_DIR}/autogenerated/ggml-metal-embed.s")
83+
set(METALLIB_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-metal-embed.metal")
84+
set(METALLIB_SOURCE_EMBED_TMP "${CMAKE_BINARY_DIR}/autogenerated/ggml-metal-embed.metal.tmp")
8285

8386
add_custom_command(
8487
OUTPUT ${METALLIB_EMBED_ASM}
8588
COMMAND echo "Embedding Metal library"
86-
COMMAND sed -e '/\#include \"ggml-common.h\"/r ${METALLIB_COMMON}' -e '/\#include \"ggml-common.h\"/d' < ${METALLIB_SOURCE} > ${METALLIB_SOURCE_EMBED}
89+
COMMAND sed -e '/\#include \"ggml-common.h\"/r ${METALLIB_COMMON}' -e '/\#include \"ggml-common.h\"/d' < ${METALLIB_SOURCE} > ${METALLIB_SOURCE_EMBED_TMP}
90+
COMMAND sed -e '/\#include \"ggml-metal-impl.h\"/r ${METALLIB_IMPL}' -e '/\#include \"ggml-metal-impl.h\"/d' < ${METALLIB_SOURCE_EMBED_TMP} > ${METALLIB_SOURCE_EMBED}
8791
COMMAND echo ".section __DATA,__ggml_metallib" > ${METALLIB_EMBED_ASM}
8892
COMMAND echo ".globl _ggml_metallib_start" >> ${METALLIB_EMBED_ASM}
8993
COMMAND echo "_ggml_metallib_start:" >> ${METALLIB_EMBED_ASM}
9094
COMMAND echo ".incbin \\\"${METALLIB_SOURCE_EMBED}\\\"" >> ${METALLIB_EMBED_ASM}
9195
COMMAND echo ".globl _ggml_metallib_end" >> ${METALLIB_EMBED_ASM}
9296
COMMAND echo "_ggml_metallib_end:" >> ${METALLIB_EMBED_ASM}
93-
DEPENDS ggml-metal.metal ggml-common.h
97+
DEPENDS ggml-metal.metal ggml-common.h ggml-metal-impl.h
9498
COMMENT "Generate assembly for embedded Metal library"
9599
)
96100

@@ -128,8 +132,9 @@ if (GGML_METAL)
128132
COMMAND xcrun -sdk macosx metallib ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
129133
COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air
130134
COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h
135+
COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal-impl.h
131136
COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal
132-
DEPENDS ggml-metal.metal ggml-common.h
137+
DEPENDS ggml-metal.metal ggml-common.h ggml-metal-impl.h
133138
COMMENT "Compiling Metal kernels"
134139
)
135140

‎ggml/src/ggml-common.h‎

Lines changed: 0 additions & 240 deletions
Original file line numberDiff line numberDiff line change
@@ -418,246 +418,6 @@ typedef struct {
418418
} block_iq4_xs;
419419
static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding");
420420

421-
#if defined(GGML_COMMON_DECL_METAL_KARGS)
422-
typedef struct {
423-
int32_t ne00;
424-
int32_t ne01;
425-
int32_t ne02;
426-
int32_t ne03;
427-
uint64_t nb00;
428-
uint64_t nb01;
429-
uint64_t nb02;
430-
uint64_t nb03;
431-
int32_t ne10;
432-
int32_t ne11;
433-
int32_t ne12;
434-
int32_t ne13;
435-
uint64_t nb10;
436-
uint64_t nb11;
437-
uint64_t nb12;
438-
uint64_t nb13;
439-
int32_t ne0;
440-
int32_t ne1;
441-
int32_t ne2;
442-
int32_t ne3;
443-
uint64_t nb0;
444-
uint64_t nb1;
445-
uint64_t nb2;
446-
uint64_t nb3;
447-
int32_t dim;
448-
} ggml_metal_kargs_concat;
449-
450-
typedef struct {
451-
int32_t ne00;
452-
int32_t ne01;
453-
int32_t ne02;
454-
int32_t ne03;
455-
uint64_t nb00;
456-
uint64_t nb01;
457-
uint64_t nb02;
458-
uint64_t nb03;
459-
int32_t ne10;
460-
int32_t ne11;
461-
int32_t ne12;
462-
int32_t ne13;
463-
uint64_t nb10;
464-
uint64_t nb11;
465-
uint64_t nb12;
466-
uint64_t nb13;
467-
int32_t ne0;
468-
int32_t ne1;
469-
int32_t ne2;
470-
int32_t ne3;
471-
uint64_t nb0;
472-
uint64_t nb1;
473-
uint64_t nb2;
474-
uint64_t nb3;
475-
uint64_t offs;
476-
} ggml_metal_kargs_bin;
477-
478-
typedef struct {
479-
int32_t ne00;
480-
int32_t ne01;
481-
int32_t ne02;
482-
int32_t ne03;
483-
uint64_t nb00;
484-
uint64_t nb01;
485-
uint64_t nb02;
486-
uint64_t nb03;
487-
int32_t ne0;
488-
int32_t ne1;
489-
int32_t ne2;
490-
int32_t ne3;
491-
uint64_t nb0;
492-
uint64_t nb1;
493-
uint64_t nb2;
494-
uint64_t nb3;
495-
} ggml_metal_kargs_repeat;
496-
497-
typedef struct {
498-
int64_t ne00;
499-
int64_t ne01;
500-
int64_t ne02;
501-
int64_t ne03;
502-
uint64_t nb00;
503-
uint64_t nb01;
504-
uint64_t nb02;
505-
uint64_t nb03;
506-
int64_t ne0;
507-
int64_t ne1;
508-
int64_t ne2;
509-
int64_t ne3;
510-
uint64_t nb0;
511-
uint64_t nb1;
512-
uint64_t nb2;
513-
uint64_t nb3;
514-
} ggml_metal_kargs_cpy;
515-
516-
typedef struct {
517-
int32_t ne00;
518-
int32_t ne01;
519-
int32_t ne02;
520-
int32_t ne03;
521-
uint64_t nb00;
522-
uint64_t nb01;
523-
uint64_t nb02;
524-
uint64_t nb03;
525-
int32_t ne0;
526-
int32_t ne1;
527-
int32_t ne2;
528-
int32_t ne3;
529-
uint64_t nb0;
530-
uint64_t nb1;
531-
uint64_t nb2;
532-
uint64_t nb3;
533-
int32_t n_past;
534-
int32_t n_dims;
535-
int32_t n_ctx_orig;
536-
float freq_base;
537-
float freq_scale;
538-
float ext_factor;
539-
float attn_factor;
540-
float beta_fast;
541-
float beta_slow;
542-
} ggml_metal_kargs_rope;
543-
544-
typedef struct {
545-
int32_t ne01;
546-
int32_t ne02;
547-
int32_t ne03;
548-
uint64_t nb01;
549-
uint64_t nb02;
550-
uint64_t nb03;
551-
int32_t ne11;
552-
int32_t ne_12_2; // assume K and V are same shape
553-
int32_t ne_12_3;
554-
uint64_t nb_12_1;
555-
uint64_t nb_12_2;
556-
uint64_t nb_12_3;
557-
uint64_t nb31;
558-
int32_t ne1;
559-
int32_t ne2;
560-
float scale;
561-
float max_bias;
562-
float m0;
563-
float m1;
564-
uint16_t n_head_log2;
565-
float logit_softcap;
566-
} ggml_metal_kargs_flash_attn_ext;
567-
568-
typedef struct {
569-
int32_t ne00;
570-
int32_t ne02;
571-
uint64_t nb01;
572-
uint64_t nb02;
573-
uint64_t nb03;
574-
int32_t ne12;
575-
uint64_t nb10;
576-
uint64_t nb11;
577-
uint64_t nb12;
578-
uint64_t nb13;
579-
int32_t ne0;
580-
int32_t ne1;
581-
int16_t r2;
582-
int16_t r3;
583-
} ggml_metal_kargs_mul_mm;
584-
585-
typedef struct {
586-
int32_t ne00;
587-
int32_t ne01;
588-
int32_t ne02;
589-
uint64_t nb00;
590-
uint64_t nb01;
591-
uint64_t nb02;
592-
uint64_t nb03;
593-
int32_t ne10;
594-
int32_t ne11;
595-
int32_t ne12;
596-
uint64_t nb10;
597-
uint64_t nb11;
598-
uint64_t nb12;
599-
uint64_t nb13;
600-
int32_t ne0;
601-
int32_t ne1;
602-
int16_t r2;
603-
int16_t r3;
604-
} ggml_metal_kargs_mul_mv;
605-
606-
typedef struct {
607-
int32_t nei0;
608-
int32_t nei1;
609-
uint64_t nbi1;
610-
int32_t ne00;
611-
int32_t ne02;
612-
uint64_t nb01;
613-
uint64_t nb02;
614-
int32_t ne11;
615-
int32_t ne12;
616-
int32_t ne13;
617-
uint64_t nb10;
618-
uint64_t nb11;
619-
uint64_t nb12;
620-
int32_t ne0;
621-
int32_t ne1;
622-
} ggml_metal_kargs_mul_mm_id;
623-
624-
typedef struct {
625-
int32_t nei0;
626-
int32_t nei1;
627-
uint64_t nbi1;
628-
int32_t ne00;
629-
int32_t ne01;
630-
int32_t ne02;
631-
uint64_t nb00;
632-
uint64_t nb01;
633-
uint64_t nb02;
634-
int32_t ne10;
635-
int32_t ne11;
636-
int32_t ne12;
637-
int32_t ne13;
638-
uint64_t nb10;
639-
uint64_t nb11;
640-
uint64_t nb12;
641-
int32_t ne0;
642-
int32_t ne1;
643-
uint64_t nb1;
644-
} ggml_metal_kargs_mul_mv_id;
645-
646-
typedef struct {
647-
int32_t ne00;
648-
int32_t ne00_4;
649-
uint64_t nb01;
650-
float eps;
651-
} ggml_metal_kargs_norm;
652-
653-
typedef struct {
654-
int32_t ne00;
655-
int32_t ne00_4;
656-
uint64_t nb01;
657-
float eps;
658-
} ggml_metal_kargs_rms_norm;
659-
#endif
660-
661421
#endif // GGML_COMMON_DECL
662422
#endif // GGML_COMMON_DECL
663423

0 commit comments

Comments
 (0)