add-qwen3-vl

Author: sangchengmeng

lightllm/models/llama/model.py

@@ -206,6 +206,7 @@ def _init_to_get_rotary(self, default_base=10000):
             / rope_scaling_factor
         )
         freqs = torch.outer(t, inv_freq)
+
         self._cos_cached = torch.cos(freqs).to(self.data_type).cuda()
         self._sin_cached = torch.sin(freqs).to(self.data_type).cuda()
         return

lightllm/models/qwen2_vl/model.py

@@ -1,5 +1,4 @@
 import json
-from operator import mul
 import numpy as np
 from lightllm.common.basemodel.multimodal_tokenizer import BaseMultiModalTokenizer
 from lightllm.models.qwen_vl.layer_infer.pre_layer_infer import LlamaMultimodalPreLayerInfer

lightllm/models/qwen2_vl/vision_process.py

@@ -275,4 +275,5 @@ def _preprocess_bydevice(self, image, device="cuda") -> Tuple[torch.Tensor, torc
 
         pixel_values = torch.cat(processed_images, dim=0)
         image_grid_thw = torch.as_tensor(processed_grids)
+
         return pixel_values, image_grid_thw

lightllm/server/embed_cache/utils.py

@@ -1,7 +1,6 @@
 import torch
 import numpy as np
 from io import BytesIO
-from typing import List, Optional
 import multiprocessing.shared_memory as shm
 
 

lightllm/server/visualserver/model_infer/model_rpc.py

@@ -19,11 +19,7 @@
 from lightllm.models.qwen2_5_vl.qwen2_5_visual import Qwen2_5_VisionTransformerPretrainedModel
 from lightllm.models.qwen3_vl.qwen3_visual import Qwen3VisionTransformerPretrainedModel
 from lightllm.models.tarsier2.tarsier2_visual import TarsierVisionTransformerPretrainedModel
-from lightllm.server.embed_cache.utils import (
-    tensor2bytes,
-    create_shm,
-    get_shm_name_embed,
-)
+from lightllm.server.embed_cache.utils import tensor2bytes, read_shm, create_shm, get_shm_name_data, get_shm_name_embed
 from lightllm.utils.infer_utils import set_random_seed
 from lightllm.utils.infer_utils import calculate_time, mark_start, mark_end
 from lightllm.utils.dist_utils import init_vision_distributed_env