|
| 1 | +#include "clip.h" |
| 2 | +#include "clip-impl.h" |
| 3 | +#include "llava2.h" |
| 4 | + |
| 5 | +#include "llama.h" |
| 6 | + |
| 7 | +#include <algorithm> |
| 8 | +#include <cerrno> |
| 9 | +#include <cstdio> |
| 10 | +#include <cstdlib> |
| 11 | +#include <cstring> |
| 12 | +#include <limits> |
| 13 | +#include <vector> |
| 14 | + |
| 15 | +static const char * IMG_MARKER = "<image>"; |
| 16 | + |
| 17 | +struct llava2_context { |
| 18 | + struct clip_ctx * ctx_clip; |
| 19 | + const struct llama_model * text_model; |
| 20 | + std::vector<float> image_embd_v; // image embedding vector |
| 21 | + int n_threads; |
| 22 | + |
| 23 | + llava2_context(const char * mmproj_fname, |
| 24 | + const struct llama_model * text_model, |
| 25 | + const struct llava2_context_params & ctx_params) : n_threads(ctx_params.n_threads) { |
| 26 | + clip_context_params ctx_clip_params; |
| 27 | + ctx_clip_params.use_gpu = ctx_params.use_gpu; |
| 28 | + ctx_clip_params.verbosity = ctx_params.verbosity; |
| 29 | + ctx_clip = clip_init(mmproj_fname, ctx_clip_params); |
| 30 | + if (!ctx_clip) { |
| 31 | + throw std::runtime_error(string_format("Failed to load CLIP model from %s\n", mmproj_fname)); |
| 32 | + } |
| 33 | + this->text_model = text_model; |
| 34 | + } |
| 35 | + |
| 36 | + ~llava2_context() { |
| 37 | + clip_free(ctx_clip); |
| 38 | + } |
| 39 | +}; |
| 40 | + |
| 41 | +struct llava2_image_tokens_data { |
| 42 | + clip_image_f32_batch_ptr batch_f32; // preprocessed image patches |
| 43 | +}; |
| 44 | + |
| 45 | +llava2_context_ptr llava2_init_from_file(const char * mmproj_fname, |
| 46 | + const struct llama_model * text_model, |
| 47 | + const struct llava2_context_params ctx_params) { |
| 48 | + try { |
| 49 | + auto ctx = std::make_shared<llava2_context>(mmproj_fname, text_model, ctx_params); |
| 50 | + return ctx; |
| 51 | + } catch (const std::exception & e) { |
| 52 | + LOG_ERR("%s: error: %s\n", __func__, e.what()); |
| 53 | + return nullptr; |
| 54 | + } |
| 55 | +} |
| 56 | + |
| 57 | +int32_t llava2_bitmap_init_from_file(const char * fname, llava2_bitmap & output) { |
| 58 | + clip_image_u8_ptr img_u8(clip_image_u8_init()); |
| 59 | + bool ok = clip_image_load_from_file(fname, img_u8.get()); |
| 60 | + if (!ok) { |
| 61 | + LOG_ERR("Unable to load image %s\n", fname); |
| 62 | + return 1; |
| 63 | + } |
| 64 | + unsigned char * data = clip_image_u8_get_data(img_u8.get(), &output.nx, &output.ny); |
| 65 | + output.data.resize(output.nx * output.ny * 3); |
| 66 | + std::memcpy(output.data.data(), data, output.nx * output.ny * 3); |
| 67 | + return 0; |
| 68 | +} |
| 69 | + |
| 70 | +// copied from common_tokenize |
| 71 | +static std::vector<llama_token> llava2_tokenize_text_internal( |
| 72 | + const struct llama_vocab * vocab, |
| 73 | + const std::string & text, |
| 74 | + bool add_special, |
| 75 | + bool parse_special) { |
| 76 | + // upper limit for the number of tokens |
| 77 | + int n_tokens = text.length() + 2 * add_special; |
| 78 | + std::vector<llama_token> result(n_tokens); |
| 79 | + n_tokens = llama_tokenize(vocab, text.data(), text.length(), result.data(), result.size(), add_special, parse_special); |
| 80 | + if (n_tokens < 0) { |
| 81 | + result.resize(-n_tokens); |
| 82 | + int check = llama_tokenize(vocab, text.data(), text.length(), result.data(), result.size(), add_special, parse_special); |
| 83 | + GGML_ASSERT(check == -n_tokens); |
| 84 | + } else { |
| 85 | + result.resize(n_tokens); |
| 86 | + } |
| 87 | + return result; |
| 88 | +} |
| 89 | + |
| 90 | +int32_t llava2_tokenize(llava2_context_ptr & ctx, |
| 91 | + std::vector<llava2_input_chunk> & output, |
| 92 | + const std::string & prompt, |
| 93 | + bool add_special, |
| 94 | + bool parse_special, |
| 95 | + const std::vector<llava2_bitmap> & bitmaps) { |
| 96 | + auto vocab = llama_model_get_vocab(ctx->text_model); |
| 97 | + |
| 98 | + std::vector<std::string> parts = string_split_str(prompt, IMG_MARKER); |
| 99 | + output.clear(); |
| 100 | + output.reserve(parts.size()); |
| 101 | + |
| 102 | + size_t i_img = 0; |
| 103 | + |
| 104 | + for (const auto & part : parts) { |
| 105 | + //printf("tokenizing part: %s\n", part.c_str()); |
| 106 | + bool add_bos = &parts.front() == ∂ |
| 107 | + auto tokens = llava2_tokenize_text_internal(vocab, part, add_special && add_bos, parse_special); |
| 108 | + if (tokens.empty()) { |
| 109 | + continue; |
| 110 | + } |
| 111 | + output.push_back({ |
| 112 | + LLAVA2_INPUT_CHUNK_TYPE_TEXT, |
| 113 | + std::move(tokens), |
| 114 | + {}, |
| 115 | + }); |
| 116 | + |
| 117 | + if (&parts.back() != &part) { |
| 118 | + // add image token to middle of 2 parts |
| 119 | + |
| 120 | + if (i_img >= bitmaps.size()) { |
| 121 | + LOG_ERR("%s: error: not enough images for %d parts\n", __func__, (int)parts.size()); |
| 122 | + return 2; |
| 123 | + } |
| 124 | + |
| 125 | + // shim layer |
| 126 | + clip_image_u8_ptr img_u8(clip_image_u8_init()); |
| 127 | + img_u8->nx = bitmaps[i_img].nx; |
| 128 | + img_u8->ny = bitmaps[i_img].ny; |
| 129 | + img_u8->buf.resize(bitmaps[i_img].data.size()); |
| 130 | + std::memcpy(img_u8->buf.data(), bitmaps[i_img].data.data(), img_u8->nx * img_u8->ny * 3); |
| 131 | + |
| 132 | + // preprocess image |
| 133 | + clip_image_f32_batch_ptr batch_f32; |
| 134 | + bool ok = clip_image_preprocess(ctx->ctx_clip, img_u8.get(), batch_f32.get()); |
| 135 | + if (!ok) { |
| 136 | + LOG_ERR("Unable to preprocess image\n"); |
| 137 | + return 1; |
| 138 | + } |
| 139 | + |
| 140 | + llava2_image_tokens image_tokens; |
| 141 | + //image_tokens.nx = ...; |
| 142 | + //image_tokens.ny = ...; |
| 143 | + image_tokens.n_tokens = clip_n_patches(ctx->ctx_clip); // TODO @ngxson : use clip_n_patches_by_image |
| 144 | + image_tokens.data = std::unique_ptr<llava2_image_tokens_data>( |
| 145 | + new llava2_image_tokens_data{ |
| 146 | + std::move(batch_f32), |
| 147 | + } |
| 148 | + ); |
| 149 | + |
| 150 | + output.push_back({ |
| 151 | + LLAVA2_INPUT_CHUNK_TYPE_IMAGE, |
| 152 | + {}, |
| 153 | + std::move(image_tokens), |
| 154 | + }); |
| 155 | + i_img++; |
| 156 | + } |
| 157 | + } |
| 158 | + |
| 159 | + return 0; |
| 160 | +} |
| 161 | + |
| 162 | +LLAVA2_API int32_t llava2_encode(llava2_context_ptr & ctx, |
| 163 | + const llava2_image_tokens & image_tokens) { |
| 164 | + ctx->image_embd_v.reserve(image_tokens.n_tokens * clip_n_mmproj_embd(ctx->ctx_clip)); |
| 165 | + return clip_image_batch_encode( |
| 166 | + ctx->ctx_clip, |
| 167 | + ctx->n_threads, |
| 168 | + image_tokens.data->batch_f32.get(), |
| 169 | + ctx->image_embd_v.data()); |
| 170 | +} |
| 171 | + |
| 172 | +LLAVA2_API float * llava2_get_output_embd(llava2_context_ptr & ctx) { |
| 173 | + return ctx->image_embd_v.data(); |
| 174 | +} |
0 commit comments