Skip to content

Commit 51ccdeb

Browse files
giladgdgiladgd
committed
feat: move GGML_ALIGNED_MALLOC to ggml-backend-impl.h, add support for vm_allocate on macOS
1 parent cec2d4e commit 51ccdeb

File tree

3 files changed

+97
-72
lines changed

3 files changed

+97
-72
lines changed

‎ggml/src/ggml-backend-impl.h‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,25 @@
11
#pragma once
22

3+
#include "ggml-impl.h"
4+
5+
#if defined(_MSC_VER) || defined(__MINGW32__)
6+
#include <malloc.h> // using malloc.h with MSC/MINGW
7+
#elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)
8+
#include <alloca.h>
9+
#endif
10+
11+
#include <errno.h>
12+
13+
#ifdef GGML_USE_CPU_HBM
14+
#include <hbwmalloc.h>
15+
#endif
16+
17+
#if defined(__APPLE__)
18+
#include <unistd.h>
19+
#include <mach/mach.h>
20+
#include <TargetConditionals.h>
21+
#endif
22+
323
// ggml-backend internal header
424

525
#include "ggml-backend.h"
@@ -222,6 +242,75 @@ extern "C" {
222242
// TODO: backends can be loaded as a dynamic library, in which case it needs to export this function
223243
// typedef ggml_backend_register_t * (*ggml_backend_init)(void);
224244

245+
246+
//
247+
// Memory allocation
248+
//
249+
250+
#if defined(_MSC_VER) || defined(__MINGW32__)
251+
#define GGML_ALIGNED_MALLOC(size) _aligned_malloc(size, GGML_MEM_ALIGN)
252+
#define GGML_ALIGNED_FREE(ptr, size) _aligned_free(ptr)
253+
#else
254+
inline static void * ggml_aligned_malloc(size_t size) {
255+
if (size == 0) {
256+
GGML_LOG_WARN("Behavior may be unexpected when allocating 0 bytes for ggml_aligned_malloc!\n");
257+
return NULL;
258+
}
259+
void * aligned_memory = NULL;
260+
#ifdef GGML_USE_CPU_HBM
261+
int result = hbw_posix_memalign(&aligned_memory, 16, size);
262+
#elif TARGET_OS_OSX
263+
kern_return_t alloc_status = vm_allocate((vm_map_t) mach_task_self(), (vm_address_t *) &aligned_memory, size, VM_FLAGS_ANYWHERE);
264+
int result = EFAULT;
265+
switch (alloc_status) {
266+
case KERN_SUCCESS:
267+
result = 0;
268+
break;
269+
270+
case KERN_INVALID_ADDRESS:
271+
result = EINVAL;
272+
break;
273+
274+
case KERN_NO_SPACE:
275+
result = ENOMEM;
276+
break;
277+
278+
default:
279+
result = EFAULT;
280+
break;
281+
}
282+
#elif GGML_USE_METAL
283+
int result = posix_memalign(&aligned_memory, sysconf(_SC_PAGESIZE), size);
284+
#else
285+
int result = posix_memalign(&aligned_memory, GGML_MEM_ALIGN, size);
286+
#endif
287+
if (result != 0) {
288+
// Handle allocation failure
289+
const char *error_desc = "unknown allocation error";
290+
switch (result) {
291+
case EINVAL:
292+
error_desc = "invalid alignment value";
293+
break;
294+
case ENOMEM:
295+
error_desc = "insufficient memory";
296+
break;
297+
}
298+
GGML_LOG_ERROR("%s: %s (attempted to allocate %6.2f MB)\n", __func__, error_desc, size/(1024.0*1024.0));
299+
GGML_ABORT("fatal error");
300+
return NULL;
301+
}
302+
return aligned_memory;
303+
}
304+
#define GGML_ALIGNED_MALLOC(size) ggml_aligned_malloc(size)
305+
#ifdef GGML_USE_CPU_HBM
306+
#define GGML_ALIGNED_FREE(ptr, size) if(NULL != ptr) hbw_free(ptr)
307+
#elif TARGET_OS_OSX
308+
#define GGML_ALIGNED_FREE(ptr, size) if(NULL != ptr) vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)ptr, size)
309+
#else
310+
#define GGML_ALIGNED_FREE(ptr, size) free(ptr)
311+
#endif
312+
#endif
313+
225314
#ifdef __cplusplus
226315
}
227316
#endif

‎ggml/src/ggml-backend.cpp‎

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@
2424
#ifdef __APPLE__
2525
#include <sys/types.h>
2626
#include <sys/sysctl.h>
27-
#include <unistd.h>
28-
#include <TargetConditionals.h>
2927
#endif
3028

3129

@@ -704,7 +702,7 @@ static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {
704702
}
705703

706704
static void ggml_backend_cpu_buffer_free_buffer(ggml_backend_buffer_t buffer) {
707-
free(buffer->context);
705+
GGML_ALIGNED_FREE(buffer->context, buffer->size);
708706
}
709707

710708
static void ggml_backend_cpu_buffer_memset_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
@@ -772,21 +770,12 @@ static const char * ggml_backend_cpu_buffer_type_get_name(ggml_backend_buffer_ty
772770
}
773771

774772
static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
775-
#if defined(GGML_USE_METAL) || defined(TARGET_OS_OSX)
776-
void * data = NULL;
777-
int result = posix_memalign(&data, sysconf(_SC_PAGESIZE), size);
778-
if (result != 0) {
779-
GGML_LOG_ERROR("%s: failed to allocate buffer using posix_memalign with size %zu\n", __func__, size);
780-
return NULL;
781-
}
782-
#else
783-
size += TENSOR_ALIGNMENT; // malloc may return an address that is not aligned
784-
void * data = malloc(size); // TODO: use GGML_ALIGNED_MALLOC (move to ggml-impl.h)
773+
void * data = GGML_ALIGNED_MALLOC(size);
774+
785775
if (data == NULL) {
786776
GGML_LOG_ERROR("%s: failed to allocate buffer of size %zu\n", __func__, size);
787777
return NULL;
788778
}
789-
#endif
790779

791780
return ggml_backend_buffer_init(buft, ggml_backend_cpu_buffer_i, data, size);
792781
}

‎ggml/src/ggml.c‎

Lines changed: 5 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,12 @@
33

44
#include "ggml-backend.h"
55
#include "ggml-impl.h"
6+
#include "ggml-backend-impl.h"
67
#include "ggml-cpu-impl.h"
78
#include "ggml-quants.h"
89
#include "ggml.h"
910
#include "ggml-aarch64.h"
1011

11-
#if defined(_MSC_VER) || defined(__MINGW32__)
12-
#include <malloc.h> // using malloc.h with MSC/MINGW
13-
#elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)
14-
#include <alloca.h>
15-
#endif
16-
1712
#include <assert.h>
1813
#include <errno.h>
1914
#include <time.h>
@@ -35,10 +30,6 @@
3530
#include <omp.h>
3631
#endif
3732

38-
#ifdef GGML_USE_METAL
39-
#include <unistd.h>
40-
#endif
41-
4233
#if defined(__ARM_FEATURE_SVE) || defined(__ARM_FEATURE_MATMUL_INT8)
4334
#undef GGML_USE_LLAMAFILE
4435
#endif
@@ -184,10 +175,6 @@ typedef void * thread_ret_t;
184175

185176
typedef pthread_t ggml_thread_t;
186177

187-
#ifdef GGML_USE_CPU_HBM
188-
#include <hbwmalloc.h>
189-
#endif
190-
191178
#if defined(__APPLE__)
192179
#include <TargetConditionals.h>
193180
#endif
@@ -386,47 +373,6 @@ void ggml_log_callback_default(enum ggml_log_level level, const char * text, voi
386373
//#define GGML_SOFT_MAX_ACCELERATE
387374
#endif
388375

389-
#if defined(_MSC_VER) || defined(__MINGW32__)
390-
#define GGML_ALIGNED_MALLOC(size) _aligned_malloc(size, GGML_MEM_ALIGN)
391-
#define GGML_ALIGNED_FREE(ptr) _aligned_free(ptr)
392-
#else
393-
inline static void * ggml_aligned_malloc(size_t size) {
394-
if (size == 0) {
395-
GGML_LOG_WARN("Behavior may be unexpected when allocating 0 bytes for ggml_aligned_malloc!\n");
396-
return NULL;
397-
}
398-
void * aligned_memory = NULL;
399-
#ifdef GGML_USE_CPU_HBM
400-
int result = hbw_posix_memalign(&aligned_memory, 16, size);
401-
#elif GGML_USE_METAL
402-
int result = posix_memalign(&aligned_memory, sysconf(_SC_PAGESIZE), size);
403-
#else
404-
int result = posix_memalign(&aligned_memory, GGML_MEM_ALIGN, size);
405-
#endif
406-
if (result != 0) {
407-
// Handle allocation failure
408-
const char *error_desc = "unknown allocation error";
409-
switch (result) {
410-
case EINVAL:
411-
error_desc = "invalid alignment value";
412-
break;
413-
case ENOMEM:
414-
error_desc = "insufficient memory";
415-
break;
416-
}
417-
GGML_LOG_ERROR("%s: %s (attempted to allocate %6.2f MB)\n", __func__, error_desc, size/(1024.0*1024.0));
418-
GGML_ABORT("fatal error");
419-
return NULL;
420-
}
421-
return aligned_memory;
422-
}
423-
#define GGML_ALIGNED_MALLOC(size) ggml_aligned_malloc(size)
424-
#ifdef GGML_USE_CPU_HBM
425-
#define GGML_ALIGNED_FREE(ptr) if(NULL != ptr) hbw_free(ptr)
426-
#else
427-
#define GGML_ALIGNED_FREE(ptr) free(ptr)
428-
#endif
429-
#endif
430376

431377
inline static void * ggml_malloc(size_t size) {
432378
if (size == 0) {
@@ -3909,7 +3855,7 @@ void ggml_free(struct ggml_context * ctx) {
39093855
__func__, i, ggml_used_mem(ctx));
39103856

39113857
if (ctx->mem_buffer_owned) {
3912-
GGML_ALIGNED_FREE(ctx->mem_buffer);
3858+
GGML_ALIGNED_FREE(ctx->mem_buffer, ctx->mem_size);
39133859
}
39143860

39153861
found = true;
@@ -19630,8 +19576,9 @@ void ggml_threadpool_free(struct ggml_threadpool* threadpool) {
1963019576
ggml_cond_destroy(&threadpool->cond);
1963119577
#endif // GGML_USE_OPENMP
1963219578

19633-
GGML_ALIGNED_FREE(threadpool->workers);
19634-
GGML_ALIGNED_FREE(threadpool);
19579+
const size_t workers_size = sizeof(struct ggml_compute_state) * n_threads;
19580+
GGML_ALIGNED_FREE(threadpool->workers, workers_size);
19581+
GGML_ALIGNED_FREE(threadpool, sizeof(struct ggml_threadpool));
1963519582
}
1963619583

1963719584
#ifndef GGML_USE_OPENMP

0 commit comments

Comments
 (0)