Skip to content

Commit b97c9ed

Browse files
jeffbolznvjeffbolznv
authored
vulkan: Allow fallback to sysmem memory when vidmem is full (ggml-org#15649)
* vulkan: Allow fallback to sysmem memory when vidmem is full * vulkan: Add env var GGML_VK_ALLOW_SYSMEM_FALLBACK
1 parent 94e82c7 commit b97c9ed

File tree

1 file changed

+53
-50
lines changed

1 file changed

+53
-50
lines changed

‎ggml/src/ggml-vulkan/ggml-vulkan.cpp

Lines changed: 53 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -566,6 +566,7 @@ struct vk_device_struct {
566566

567567
bool disable_fusion;
568568
bool disable_host_visible_vidmem;
569+
bool allow_sysmem_fallback;
569570

570571
#ifdef GGML_VULKAN_MEMORY_DEBUG
571572
std::unique_ptr<vk_memory_logger> memory_logger;
@@ -1808,8 +1809,8 @@ static uint32_t find_properties(const vk::PhysicalDeviceMemoryProperties* mem_pr
18081809
return UINT32_MAX;
18091810
}
18101811

1811-
static vk_buffer ggml_vk_create_buffer(vk_device& device, size_t size, vk::MemoryPropertyFlags req_flags, vk::MemoryPropertyFlags fallback_flags = vk::MemoryPropertyFlags(0)) {
1812-
VK_LOG_DEBUG("ggml_vk_create_buffer(" << device->name << ", " << size << ", " << to_string(req_flags) << ", " << to_string(fallback_flags) << ")");
1812+
static vk_buffer ggml_vk_create_buffer(vk_device& device, size_t size, const std::initializer_list<vk::MemoryPropertyFlags> & req_flags_list) {
1813+
VK_LOG_DEBUG("ggml_vk_create_buffer(" << device->name << ", " << size << ", " << to_string(req_flags_list.begin()[0]) << ", " << to_string(req_flags_list.begin()[req_flags_list.size()-1]) << ")");
18131814
if (size > device->max_memory_allocation_size) {
18141815
throw vk::OutOfDeviceMemoryError("Requested buffer size exceeds device memory allocation limit");
18151816
}
@@ -1836,42 +1837,27 @@ static vk_buffer ggml_vk_create_buffer(vk_device& device, size_t size, vk::Memor
18361837

18371838
vk::PhysicalDeviceMemoryProperties mem_props = device->physical_device.getMemoryProperties();
18381839

1839-
uint32_t memory_type_index = UINT32_MAX;
1840+
for (auto &req_flags : req_flags_list) {
1841+
uint32_t memory_type_index = find_properties(&mem_props, &mem_req, req_flags);
18401842

1841-
memory_type_index = find_properties(&mem_props, &mem_req, req_flags);
1842-
buf->memory_property_flags = req_flags;
1843+
if (memory_type_index == UINT32_MAX) {
1844+
continue;
1845+
}
1846+
buf->memory_property_flags = req_flags;
18431847

1844-
if (memory_type_index == UINT32_MAX && fallback_flags) {
1845-
memory_type_index = find_properties(&mem_props, &mem_req, fallback_flags);
1846-
buf->memory_property_flags = fallback_flags;
1848+
try {
1849+
buf->device_memory = device->device.allocateMemory({ mem_req.size, memory_type_index });
1850+
break;
1851+
} catch (const vk::SystemError& e) {
1852+
// loop and retry
1853+
}
18471854
}
18481855

1849-
if (memory_type_index == UINT32_MAX) {
1856+
if (buf->device_memory == VK_NULL_HANDLE) {
18501857
device->device.destroyBuffer(buf->buffer);
18511858
throw vk::OutOfDeviceMemoryError("No suitable memory type found");
18521859
}
18531860

1854-
try {
1855-
buf->device_memory = device->device.allocateMemory({ mem_req.size, memory_type_index });
1856-
} catch (const vk::SystemError& e) {
1857-
if (buf->memory_property_flags != fallback_flags) {
1858-
// Try again with fallback flags
1859-
memory_type_index = find_properties(&mem_props, &mem_req, fallback_flags);
1860-
buf->memory_property_flags = fallback_flags;
1861-
1862-
try {
1863-
buf->device_memory = device->device.allocateMemory({ mem_req.size, memory_type_index });
1864-
}
1865-
catch (const vk::SystemError& e) {
1866-
device->device.destroyBuffer(buf->buffer);
1867-
throw e;
1868-
}
1869-
} else {
1870-
// Out of Host/Device memory, clean up buffer
1871-
device->device.destroyBuffer(buf->buffer);
1872-
throw e;
1873-
}
1874-
}
18751861
buf->ptr = nullptr;
18761862

18771863
if (buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) {
@@ -1892,7 +1878,7 @@ static vk_buffer ggml_vk_create_buffer(vk_device& device, size_t size, vk::Memor
18921878

18931879
static vk_buffer ggml_vk_create_buffer_check(vk_device& device, size_t size, vk::MemoryPropertyFlags req_flags, vk::MemoryPropertyFlags fallback_flags = vk::MemoryPropertyFlags(0)) {
18941880
try {
1895-
return ggml_vk_create_buffer(device, size, req_flags, fallback_flags);
1881+
return ggml_vk_create_buffer(device, size, {req_flags, fallback_flags});
18961882
} catch (const vk::SystemError& e) {
18971883
std::cerr << "ggml_vulkan: Memory allocation of size " << size << " failed." << std::endl;
18981884
std::cerr << "ggml_vulkan: " << e.what() << std::endl;
@@ -1904,15 +1890,29 @@ static vk_buffer ggml_vk_create_buffer_device(vk_device& device, size_t size) {
19041890
vk_buffer buf;
19051891
try {
19061892
if (device->prefer_host_memory) {
1907-
buf = ggml_vk_create_buffer(device, size, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent, vk::MemoryPropertyFlagBits::eDeviceLocal);
1893+
buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,
1894+
vk::MemoryPropertyFlagBits::eDeviceLocal});
19081895
} else if (device->uma) {
19091896
// Fall back to host memory type
1910-
buf = ggml_vk_create_buffer(device, size, vk::MemoryPropertyFlagBits::eDeviceLocal, vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent);
1897+
buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal,
1898+
vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});
19111899
} else if (device->disable_host_visible_vidmem) {
1912-
buf = ggml_vk_create_buffer(device, size, vk::MemoryPropertyFlagBits::eDeviceLocal, vk::MemoryPropertyFlagBits::eDeviceLocal);
1900+
if (device->allow_sysmem_fallback) {
1901+
buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal,
1902+
vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});
1903+
} else {
1904+
buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal});
1905+
}
19131906
} else {
19141907
// use rebar if available, otherwise fallback to device only visible memory
1915-
buf = ggml_vk_create_buffer(device, size, vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent, vk::MemoryPropertyFlagBits::eDeviceLocal);
1908+
if (device->allow_sysmem_fallback) {
1909+
buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,
1910+
vk::MemoryPropertyFlagBits::eDeviceLocal,
1911+
vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});
1912+
} else {
1913+
buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,
1914+
vk::MemoryPropertyFlagBits::eDeviceLocal});
1915+
}
19161916
}
19171917
} catch (const vk::SystemError& e) {
19181918
std::cerr << "ggml_vulkan: Device memory allocation of size " << size << " failed." << std::endl;
@@ -3437,6 +3437,9 @@ static vk_device ggml_vk_get_device(size_t idx) {
34373437
const char* GGML_VK_DISABLE_HOST_VISIBLE_VIDMEM = getenv("GGML_VK_DISABLE_HOST_VISIBLE_VIDMEM");
34383438
device->disable_host_visible_vidmem = GGML_VK_DISABLE_HOST_VISIBLE_VIDMEM != nullptr;
34393439

3440+
const char* GGML_VK_ALLOW_SYSMEM_FALLBACK = getenv("GGML_VK_ALLOW_SYSMEM_FALLBACK");
3441+
device->allow_sysmem_fallback = GGML_VK_ALLOW_SYSMEM_FALLBACK != nullptr;
3442+
34403443
bool fp16_storage = false;
34413444
bool fp16_compute = false;
34423445
bool maintenance4_support = false;
@@ -4774,8 +4777,8 @@ static vk_buffer ggml_vk_create_buffer_temp(ggml_backend_vk_context * ctx, size_
47744777
static void * ggml_vk_host_malloc(vk_device& device, size_t size) {
47754778
VK_LOG_MEMORY("ggml_vk_host_malloc(" << size << ")");
47764779
vk_buffer buf = ggml_vk_create_buffer(device, size,
4777-
vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached,
4778-
vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent);
4780+
{vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached,
4781+
vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});
47794782

47804783
if(!(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible)) {
47814784
fprintf(stderr, "WARNING: failed to allocate %.2f MB of pinned memory\n",
@@ -9182,7 +9185,7 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t
91829185
if (ctx->prealloc_split_k != nullptr) {
91839186
ggml_vk_destroy_buffer(ctx->prealloc_split_k);
91849187
}
9185-
ctx->prealloc_split_k = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne * split_k, vk::MemoryPropertyFlagBits::eDeviceLocal);
9188+
ctx->prealloc_split_k = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne * split_k, {vk::MemoryPropertyFlagBits::eDeviceLocal});
91869189
}
91879190
}
91889191

@@ -9192,9 +9195,9 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t
91929195

91939196
ggml_pipeline_allocate_descriptor_sets(ctx);
91949197

9195-
vk_buffer d_X = ggml_vk_create_buffer_check(ctx->device, sizeof(X_TYPE) * x_ne, vk::MemoryPropertyFlagBits::eDeviceLocal);
9196-
vk_buffer d_Y = ggml_vk_create_buffer_check(ctx->device, sizeof(Y_TYPE) * y_ne, vk::MemoryPropertyFlagBits::eDeviceLocal);
9197-
vk_buffer d_D = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne, vk::MemoryPropertyFlagBits::eDeviceLocal);
9198+
vk_buffer d_X = ggml_vk_create_buffer_check(ctx->device, sizeof(X_TYPE) * x_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal});
9199+
vk_buffer d_Y = ggml_vk_create_buffer_check(ctx->device, sizeof(Y_TYPE) * y_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal});
9200+
vk_buffer d_D = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal});
91989201

91999202
X_TYPE* x = (X_TYPE *) malloc(sizeof(X_TYPE) * x_ne);
92009203
Y_TYPE* y = (Y_TYPE *) malloc(sizeof(Y_TYPE) * y_ne);
@@ -9420,8 +9423,8 @@ static void ggml_vk_test_dequant(ggml_backend_vk_context * ctx, size_t ne, ggml_
94209423
const size_t qx_sz = ne * ggml_type_size(quant)/ggml_blck_size(quant);
94219424
float * x = (float *) malloc(x_sz);
94229425
void * qx = malloc(qx_sz);
9423-
vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, vk::MemoryPropertyFlagBits::eDeviceLocal);
9424-
vk_buffer x_buf = ggml_vk_create_buffer_check(ctx->device, x_sz_f16, vk::MemoryPropertyFlagBits::eDeviceLocal);
9426+
vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
9427+
vk_buffer x_buf = ggml_vk_create_buffer_check(ctx->device, x_sz_f16, {vk::MemoryPropertyFlagBits::eDeviceLocal});
94259428
float * x_ref = (float *) malloc(x_sz);
94269429
ggml_fp16_t * x_chk = (ggml_fp16_t *) malloc(x_sz_f16);
94279430

@@ -9526,8 +9529,8 @@ static void ggml_vk_test_dequant(ggml_backend_vk_context * ctx, size_t ne, ggml_
95269529
// float * x = (float *) malloc(x_sz);
95279530
// block_q8_1 * qx = (block_q8_1 *)malloc(qx_sz);
95289531
// block_q8_1 * qx_res = (block_q8_1 *)malloc(qx_sz);
9529-
// vk_buffer x_buf = ggml_vk_create_buffer_check(ctx->device, x_sz, vk::MemoryPropertyFlagBits::eDeviceLocal);
9530-
// vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, vk::MemoryPropertyFlagBits::eDeviceLocal);
9532+
// vk_buffer x_buf = ggml_vk_create_buffer_check(ctx->device, x_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
9533+
// vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
95319534
//
95329535
// for (size_t i = 0; i < ne; i++) {
95339536
// x[i] = rand() / (float)RAND_MAX;
@@ -9674,10 +9677,10 @@ static void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m,
96749677
float * x = (float *) malloc(x_sz);
96759678
float * y = (float *) malloc(y_sz);
96769679
void * qx = malloc(qx_sz);
9677-
vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, vk::MemoryPropertyFlagBits::eDeviceLocal);
9678-
vk_buffer y_buf = ggml_vk_create_buffer_check(ctx->device, y_sz, vk::MemoryPropertyFlagBits::eDeviceLocal);
9679-
vk_buffer qy_buf = ggml_vk_create_buffer_check(ctx->device, qy_sz, vk::MemoryPropertyFlagBits::eDeviceLocal);
9680-
vk_buffer d_buf = ggml_vk_create_buffer_check(ctx->device, d_sz, vk::MemoryPropertyFlagBits::eDeviceLocal);
9680+
vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
9681+
vk_buffer y_buf = ggml_vk_create_buffer_check(ctx->device, y_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
9682+
vk_buffer qy_buf = ggml_vk_create_buffer_check(ctx->device, qy_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
9683+
vk_buffer d_buf = ggml_vk_create_buffer_check(ctx->device, d_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});
96819684
float * d = (float *) malloc(d_sz);
96829685
float * d_chk = (float *) malloc(d_sz);
96839686

@@ -9704,7 +9707,7 @@ static void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m,
97049707
if (ctx->prealloc_split_k != nullptr) {
97059708
ggml_vk_destroy_buffer(ctx->prealloc_split_k);
97069709
}
9707-
ctx->prealloc_split_k = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne * split_k, vk::MemoryPropertyFlagBits::eDeviceLocal);
9710+
ctx->prealloc_split_k = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne * split_k, {vk::MemoryPropertyFlagBits::eDeviceLocal});
97089711
}
97099712
}
97109713
if (mmq) {

0 commit comments

Comments
 (0)