Skip to content

Add initial LoRA finetuning support; vulkan OUT_PROD; vulkan cross-entropy-backward #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 17 commits into
base: temp-finetuning
Choose a base branch
from
+1,360 −21
Open

Add initial LoRA finetuning support; vulkan OUT_PROD; vulkan cross-entropy-backward #5

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
f7b0025
Add lora finetuning from adapter
makaveli10 Jul 30, 2025
116f3dd
Add: create new lora adapter for target modules to finetune if no lor…
makaveli10 Aug 1, 2025
9e6d8ce
Fix identical loss over epochs; fix garbage lora initization
makaveli10 Aug 5, 2025
8bb11c0
Remove lora training from finetune.cpp
makaveli10 Aug 6, 2025
486ebc1
Add adapter saving & other lora target modules
makaveli10 Aug 6, 2025
c23ada9
Add finetune-lora for lora finetuning in examples
makaveli10 Aug 6, 2025
3f295e1
Add dequantization to out_prod cuda kernel
makaveli10 Aug 7, 2025
0c1ffd1
Update README with finetune-lora
makaveli10 Aug 8, 2025
e9f5d88
Vulkan: add support for fp32 OUT_PROD op
Aug 1, 2025
fb0e501
CPU: add support for fp16_fp32 OUT_PROD op
Aug 12, 2025
2b0c835
Vulkan: add support for f16_f32 OUT_PROD op
Aug 12, 2025
0aef6c8
Vulkan: Add Q4_0/Q8_0 OUT_PROD Vulkan support
zoq Aug 11, 2025
25c5316
vulkan: Add initial cross entropy loss backward shader
makaveli10 Aug 13, 2025
0721550
vulkan: Fix cross-entropy-loss-back dispatch size and wg denominator
makaveli10 Aug 14, 2025
bc7dd9f
vulkan: Change uint32 cast to int32 for outprod; allows android compi…
makaveli10 Aug 14, 2025
b820541
wip vulkan crash fix
Aug 18, 2025
25dfd75
vulkan: Set out_prod pipeline disable_robustness to true
makaveli10 Aug 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 5 additions & 7 deletions ggml/src/ggml-vulkan/ggml-vulkan.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -574,8 +574,8 @@ struct vk_buffer_struct {
}
VK_LOG_DEBUG("~vk_buffer_struct(" << buffer << ", " << size << ")");

device->device.freeMemory(device_memory);
device->device.destroyBuffer(buffer);
device->device.freeMemory(device_memory);
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The change looks good but the commit msg should be updated to remove wip - would also be good to explain what specific crash this fixes in the commit msg.

}
};

Expand Down Expand Up @@ -2941,13 +2941,11 @@ static void ggml_vk_load_shaders(vk_device& device) {
ggml_vk_create_pipeline(device, device->pipeline_rope_vision_f16, "rope_vision_f16", rope_vision_f16_len, rope_vision_f16_data, "main", 4, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1);
}

// TODO: should we have device->subgroup_size here or 0?
ggml_vk_create_pipeline(device, device->pipeline_out_prod_f32, "out_prod_f32", out_prod_f32_len, out_prod_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, { device->subgroup_size }, 1);
ggml_vk_create_pipeline(device, device->pipeline_out_prod_q4_0, "out_prod_q4_0", out_prod_q4_0_len, out_prod_q4_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, { device->subgroup_size }, 1);
ggml_vk_create_pipeline(device, device->pipeline_out_prod_q8_0, "out_prod_q8_0", out_prod_q8_0_len, out_prod_q8_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, { device->subgroup_size }, 1);
ggml_vk_create_pipeline(device, device->pipeline_out_prod_f32, "out_prod_f32", out_prod_f32_len, out_prod_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, { 0 }, 1);
ggml_vk_create_pipeline(device, device->pipeline_out_prod_q4_0, "out_prod_q4_0", out_prod_q4_0_len, out_prod_q4_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, { 0 }, 1);
ggml_vk_create_pipeline(device, device->pipeline_out_prod_q8_0, "out_prod_q8_0", out_prod_q8_0_len, out_prod_q8_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, { 0 }, 1);

// TODO: should we have device->subgroup_size here or 0?
ggml_vk_create_pipeline(device, device->pipeline_out_prod_f16_f32, "out_prod_f16_f32", out_prod_f16_f32_len, out_prod_f16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, { device->subgroup_size }, 1);
ggml_vk_create_pipeline(device, device->pipeline_out_prod_f16_f32, "out_prod_f16_f32", out_prod_f16_f32_len, out_prod_f16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, { 0 }, 1);

ggml_vk_create_pipeline(device, device->pipeline_argsort_f32, "argsort_f32", argsort_f32_len, argsort_f32_data, "main", 2, sizeof(vk_op_argsort_push_constants), {1024, 1, 1}, {}, 1);

Expand Down
2 changes: 1 addition & 1 deletion ggml/src/ggml-vulkan/vulkan-shaders/out_prod_q8_0.comp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ void main() {

for (uint k = 0; k < p.ne01; k += 1) {
const uint a_block_base = get_aoffset() + (i3 / broadcast3) * p.nb03 + (i2 / broadcast2) * p.nb02 + k * p.nb01;
const uint ib = a_block_base + (i0 / QUANT_K);
const uint ib = a_block_base + (i0 / QUANT_K) * p.nb00;
const uint iqs = (i0 % QUANT_K) / QUANT_R;

const vec2 v = dequantize(ib, iqs, 0);
Expand Down