Support int8 deploy & evaluation

czczup · czczup · commit c225130f191d · 2024-04-28T16:25:43.000+08:00
diff --git a/.gitignore b/.gitignore
@@ -165,7 +165,6 @@ cython_debug/
 data_process/
 internvl_chat/work_dirs/
 internvl_chat/unittest/
-internvl_chat/shell/
 internvl_chat/data/
 Husky2/*
 data_process/
diff --git a/README.md b/README.md
diff --git a/internvl_chat/eval/caption/evaluate_caption.py b/internvl_chat/eval/caption/evaluate_caption.py
@@ -234,6 +234,7 @@ def evaluate_chat_model():
     parser.add_argument('--seed', type=int, default=0)
     parser.add_argument('--dynamic', action='store_true')
     parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
     args = parser.parse_args()
 
     if not os.path.exists(args.out_dir):
@@ -253,7 +254,10 @@ def evaluate_chat_model():
 
     tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, trust_remote_code=True, use_fast=False)
     model = InternVLChatModel.from_pretrained(
-        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16).cuda().eval()
+        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16,
+        load_in_8bit=args.load_in_8bit).eval()
+    if not args.load_in_8bit:
+        model = model.cuda()
     image_size = model.config.force_image_size or model.config.vision_config.image_size
     use_thumbnail = model.config.use_thumbnail
 
diff --git a/internvl_chat/eval/cmmmu/evaluate_cmmmu.py b/internvl_chat/eval/cmmmu/evaluate_cmmmu.py
@@ -154,6 +154,7 @@ def evaluate_chat_model():
     parser.add_argument('--seed', type=int, default=0)
     parser.add_argument('--dynamic', action='store_true')
     parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
     args = parser.parse_args()
 
     if not os.path.exists(args.out_dir):
@@ -165,7 +166,10 @@ def evaluate_chat_model():
 
     tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, trust_remote_code=True, use_fast=False)
     model = InternVLChatModel.from_pretrained(
-        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16).cuda().eval()
+        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16,
+        load_in_8bit=args.load_in_8bit).eval()
+    if not args.load_in_8bit:
+        model = model.cuda()
     image_size = model.config.force_image_size or model.config.vision_config.image_size
     use_thumbnail = model.config.use_thumbnail
 
diff --git a/internvl_chat/eval/llava_bench/evaluate_llava_bench.py b/internvl_chat/eval/llava_bench/evaluate_llava_bench.py
@@ -115,6 +115,7 @@ def evaluate_chat_model():
     parser.add_argument('--seed', type=int, default=0)
     parser.add_argument('--dynamic', action='store_true')
     parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
     args = parser.parse_args()
 
     if not os.path.exists(args.out_dir):
@@ -126,7 +127,10 @@ def evaluate_chat_model():
 
     tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, trust_remote_code=True, use_fast=False)
     model = InternVLChatModel.from_pretrained(
-        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16).cuda().eval()
+        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16,
+        load_in_8bit=args.load_in_8bit).eval()
+    if not args.load_in_8bit:
+        model = model.cuda()
     image_size = model.config.force_image_size or model.config.vision_config.image_size
     use_thumbnail = model.config.use_thumbnail
 
diff --git a/internvl_chat/eval/mathvista/evaluate_mathvista.py b/internvl_chat/eval/mathvista/evaluate_mathvista.py
@@ -183,6 +183,7 @@ def evaluate_chat_model():
     parser.add_argument('--seed', type=int, default=0)
     parser.add_argument('--dynamic', action='store_true')
     parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
     args = parser.parse_args()
 
     if not os.path.exists(args.out_dir):
@@ -202,7 +203,10 @@ def evaluate_chat_model():
 
     tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, trust_remote_code=True, use_fast=False)
     model = InternVLChatModel.from_pretrained(
-        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16).cuda().eval()
+        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16,
+        load_in_8bit=args.load_in_8bit).eval()
+    if not args.load_in_8bit:
+        model = model.cuda()
     image_size = model.config.force_image_size or model.config.vision_config.image_size
     use_thumbnail = model.config.use_thumbnail
 
diff --git a/internvl_chat/eval/mmbench/evaluate_mmbench.py b/internvl_chat/eval/mmbench/evaluate_mmbench.py
@@ -270,6 +270,7 @@ def evaluate_chat_model():
     parser.add_argument('--seed', type=int, default=0)
     parser.add_argument('--dynamic', action='store_true')
     parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
     args = parser.parse_args()
 
     if not os.path.exists(args.out_dir):
@@ -289,7 +290,10 @@ def evaluate_chat_model():
 
     tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, trust_remote_code=True, use_fast=False)
     model = InternVLChatModel.from_pretrained(
-        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16).cuda().eval()
+        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16,
+        load_in_8bit=args.load_in_8bit).eval()
+    if not args.load_in_8bit:
+        model = model.cuda()
     image_size = model.config.force_image_size or model.config.vision_config.image_size
     use_thumbnail = model.config.use_thumbnail
 
diff --git a/internvl_chat/eval/mme/eval.py b/internvl_chat/eval/mme/eval.py
@@ -42,12 +42,16 @@ def post_processing(response):
     parser.add_argument('--sample', type=bool, default=False)
     parser.add_argument('--dynamic', action='store_true')
     parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
     args = parser.parse_args()
 
     prompt = 'Answer the question using a single word or phrase.'
     tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, trust_remote_code=True, use_fast=False)
     model = InternVLChatModel.from_pretrained(
-        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16).cuda().eval()
+        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16,
+        load_in_8bit=args.load_in_8bit).eval()
+    if not args.load_in_8bit:
+        model = model.cuda()
     image_size = model.config.force_image_size or model.config.vision_config.image_size
     use_thumbnail = model.config.use_thumbnail
 
diff --git a/internvl_chat/eval/mmmu/evaluate_mmmu.py b/internvl_chat/eval/mmmu/evaluate_mmmu.py
@@ -260,6 +260,7 @@ def evaluate_chat_model():
     parser.add_argument('--seed', type=int, default=0)
     parser.add_argument('--dynamic', action='store_true')
     parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
     args = parser.parse_args()
 
     if not os.path.exists(args.out_dir):
@@ -279,7 +280,10 @@ def evaluate_chat_model():
 
     tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, trust_remote_code=True, use_fast=False)
     model = InternVLChatModel.from_pretrained(
-        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16).cuda().eval()
+        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16,
+        load_in_8bit=args.load_in_8bit).eval()
+    if not args.load_in_8bit:
+        model = model.cuda()
     image_size = model.config.force_image_size or model.config.vision_config.image_size
     use_thumbnail = model.config.use_thumbnail
 
diff --git a/internvl_chat/eval/mmvet/evaluate_mmvet.py b/internvl_chat/eval/mmvet/evaluate_mmvet.py
@@ -121,6 +121,7 @@ def evaluate_chat_model():
     parser.add_argument('--seed', type=int, default=0)
     parser.add_argument('--dynamic', action='store_true')
     parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
     args = parser.parse_args()
 
     if not os.path.exists(args.out_dir):
@@ -132,7 +133,10 @@ def evaluate_chat_model():
 
     tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, trust_remote_code=True, use_fast=False)
     model = InternVLChatModel.from_pretrained(
-        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16).cuda().eval()
+        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16,
+        load_in_8bit=args.load_in_8bit).eval()
+    if not args.load_in_8bit:
+        model = model.cuda()
     image_size = model.config.force_image_size or model.config.vision_config.image_size
     use_thumbnail = model.config.use_thumbnail
 
diff --git a/internvl_chat/eval/mmvp/evaluate_mmvp.py b/internvl_chat/eval/mmvp/evaluate_mmvp.py
@@ -247,6 +247,7 @@ def evaluate_chat_model():
     parser.add_argument('--seed', type=int, default=0)
     parser.add_argument('--dynamic', action='store_true')
     parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
     args = parser.parse_args()
 
     if not os.path.exists(args.out_dir):
@@ -266,7 +267,10 @@ def evaluate_chat_model():
 
     tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, trust_remote_code=True, use_fast=False)
     model = InternVLChatModel.from_pretrained(
-        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16).cuda().eval()
+        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16,
+        load_in_8bit=args.load_in_8bit).eval()
+    if not args.load_in_8bit:
+        model = model.cuda()
     image_size = model.config.force_image_size or model.config.vision_config.image_size
     use_thumbnail = model.config.use_thumbnail
 
@@ -280,5 +284,6 @@ def evaluate_chat_model():
     print(f'[test] template: {model.config.template}')
     print(f'[test] dynamic_image_size: {args.dynamic}')
     print(f'[test] use_thumbnail: {use_thumbnail}')
+    print(f'[test] max_num: {args.max_num}')
 
     evaluate_chat_model()
diff --git a/internvl_chat/eval/pope/evaluate_pope.py b/internvl_chat/eval/pope/evaluate_pope.py
@@ -186,6 +186,7 @@ def evaluate_chat_model():
     parser.add_argument('--seed', type=int, default=0)
     parser.add_argument('--dynamic', action='store_true')
     parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
     args = parser.parse_args()
 
     if not os.path.exists(args.out_dir):
@@ -205,7 +206,10 @@ def evaluate_chat_model():
 
     tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, trust_remote_code=True, use_fast=False)
     model = InternVLChatModel.from_pretrained(
-        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16).cuda().eval()
+        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16,
+        load_in_8bit=args.load_in_8bit).eval()
+    if not args.load_in_8bit:
+        model = model.cuda()
     image_size = model.config.force_image_size or model.config.vision_config.image_size
     use_thumbnail = model.config.use_thumbnail
 
diff --git a/internvl_chat/eval/refcoco/evaluate_grounding.py b/internvl_chat/eval/refcoco/evaluate_grounding.py
@@ -235,6 +235,7 @@ def evaluate_chat_model():
     parser.add_argument('--seed', type=int, default=0)
     parser.add_argument('--dynamic', action='store_true')
     parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
     args = parser.parse_args()
 
     if not os.path.exists(args.out_dir):
@@ -255,7 +256,10 @@ def evaluate_chat_model():
     PATTERN = re.compile(r'\[*\[(.*?),(.*?),(.*?),(.*?)\]\]*')
     tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, trust_remote_code=True, use_fast=False)
     model = InternVLChatModel.from_pretrained(
-        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16).cuda().eval()
+        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16,
+        load_in_8bit=args.load_in_8bit).eval()
+    if not args.load_in_8bit:
+        model = model.cuda()
     image_size = model.config.force_image_size or model.config.vision_config.image_size
     use_thumbnail = model.config.use_thumbnail
     prompt = 'Please provide the bounding box coordinate of the region this sentence describes: <ref>{}</ref>'
diff --git a/internvl_chat/eval/scienceqa/evaluate_scienceqa.py b/internvl_chat/eval/scienceqa/evaluate_scienceqa.py
@@ -217,6 +217,7 @@ def evaluate_chat_model():
     parser.add_argument('--seed', type=int, default=0)
     parser.add_argument('--dynamic', action='store_true')
     parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
     args = parser.parse_args()
 
     if not os.path.exists(args.out_dir):
@@ -236,7 +237,10 @@ def evaluate_chat_model():
 
     tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, trust_remote_code=True, use_fast=False)
     model = InternVLChatModel.from_pretrained(
-        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16).cuda().eval()
+        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16,
+        load_in_8bit=args.load_in_8bit).eval()
+    if not args.load_in_8bit:
+        model = model.cuda()
     image_size = model.config.force_image_size or model.config.vision_config.image_size
     use_thumbnail = model.config.use_thumbnail
 
diff --git a/internvl_chat/eval/seed/evaluate_seed.py b/internvl_chat/eval/seed/evaluate_seed.py
@@ -204,6 +204,7 @@ def evaluate_chat_model():
     parser.add_argument('--seed', type=int, default=0)
     parser.add_argument('--dynamic', action='store_true')
     parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
     args = parser.parse_args()
 
     if not os.path.exists(args.out_dir):
@@ -223,7 +224,10 @@ def evaluate_chat_model():
 
     tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, trust_remote_code=True, use_fast=False)
     model = InternVLChatModel.from_pretrained(
-        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16).cuda().eval()
+        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16,
+        load_in_8bit=args.load_in_8bit).eval()
+    if not args.load_in_8bit:
+        model = model.cuda()
     image_size = model.config.force_image_size or model.config.vision_config.image_size
     use_thumbnail = model.config.use_thumbnail
 
diff --git a/internvl_chat/eval/tiny_lvlm/evaluate_lvlm.py b/internvl_chat/eval/tiny_lvlm/evaluate_lvlm.py
@@ -192,6 +192,7 @@ def evaluate_chat_model():
     parser.add_argument('--seed', type=int, default=0)
     parser.add_argument('--dynamic', action='store_true')
     parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
     args = parser.parse_args()
 
     if not os.path.exists(args.out_dir):
@@ -211,7 +212,10 @@ def evaluate_chat_model():
 
     tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, trust_remote_code=True, use_fast=False)
     model = InternVLChatModel.from_pretrained(
-        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16).cuda().eval()
+        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16,
+        load_in_8bit=args.load_in_8bit).eval()
+    if not args.load_in_8bit:
+        model = model.cuda()
     image_size = model.config.force_image_size or model.config.vision_config.image_size
     use_thumbnail = model.config.use_thumbnail
 
diff --git a/internvl_chat/eval/vqa/evaluate_vqa.py b/internvl_chat/eval/vqa/evaluate_vqa.py
@@ -503,6 +503,7 @@ def evaluate_chat_model():
     parser.add_argument('--seed', type=int, default=0)
     parser.add_argument('--dynamic', action='store_true')
     parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
     args = parser.parse_args()
 
     if not os.path.exists(args.out_dir):
@@ -522,7 +523,10 @@ def evaluate_chat_model():
 
     tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, trust_remote_code=True, use_fast=False)
     model = InternVLChatModel.from_pretrained(
-        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16).cuda().eval()
+        args.checkpoint, low_cpu_mem_usage=True, torch_dtype=torch.bfloat16,
+        load_in_8bit=args.load_in_8bit).eval()
+    if not args.load_in_8bit:
+        model = model.cuda()
     image_size = model.config.force_image_size or model.config.vision_config.image_size
     use_thumbnail = model.config.use_thumbnail
 
diff --git a/internvl_chat/internvl/serve/model_worker.py b/internvl_chat/internvl/serve/model_worker.py
@@ -67,7 +67,9 @@ def __init__(self, controller_addr, worker_addr,
 
         self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
         self.model = InternVLChatModel.from_pretrained(
-            model_path, load_in_8bit=False, torch_dtype=torch.float16).cuda().eval()
+            model_path, load_in_8bit=load_8bit, torch_dtype=torch.float16).eval()
+        if not load_8bit:
+            self.model = self.model.cuda()
         self.image_size = self.model.config.force_image_size
         self.image_processor = CLIPImageProcessor(
             crop_size=self.image_size, do_center_crop=True, do_normalize=True, do_resize=True,
diff --git a/internvl_chat/tools/convert_to_int8.py b/internvl_chat/tools/convert_to_int8.py
@@ -0,0 +1,16 @@
+from transformers import AutoTokenizer, AutoModel
+import torch
+
+path = "OpenGVLab/InternVL-Chat-V1-5"
+model = AutoModel.from_pretrained(
+    path,
+    torch_dtype=torch.bfloat16,
+    low_cpu_mem_usage=True,
+    trust_remote_code=True,
+    load_in_8bit=True).eval()
+
+tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True)
+
+model.save_pretrained("release/InternVL-Chat-V1-5-Int8")
+tokenizer.save_pretrained("release/InternVL-Chat-V1-5-Int8")
+print("finished")
