Fixes openvino decoder models output (#1308)

* fix inference

* add test

* fix

Author: echarlaix

optimum/intel/openvino/modeling_decoder.py

@@ -553,10 +553,11 @@ def forward(
 
         if self._first_iter_beam_search:
             inputs, duplication_indices = self._deduplicate_inputs(inputs)
+
         # Run inference
         self.request.start_async(inputs, share_inputs=True)
         self.request.wait()
-        logits = torch.from_numpy(self.request.get_tensor("logits").data).to(self.device)
+        logits = torch.from_numpy(self.request.get_tensor("logits").data).clone().to(self.device)
         if self.stateful:
             # Need a marker to differentiate the first generate iteration from the others in
             # the first condition at the function beginning above.
@@ -567,7 +568,9 @@ def forward(
         if not self.stateful:
             if self.use_cache:
                 # Tuple of length equal to : number of layer * number of past_key_value per decoder layer (2 corresponds to the self-attention layer)
-                past_key_values = tuple(self.request.get_tensor(key).data for key in self.key_value_output_names)
+                past_key_values = tuple(
+                    np.copy(self.request.get_tensor(key).data) for key in self.key_value_output_names
+                )
                 if self.config.model_type not in MULTI_QUERY_ATTN_MODELS or (
                     self.config.model_type == "falcon" and self.config.new_decoder_architecture
                 ):

optimum/intel/openvino/modeling_seq2seq.py

@@ -985,15 +985,17 @@ def forward(
         # Run inference
         self.request.start_async(inputs, share_inputs=True)
         self.request.wait()
-        logits = torch.from_numpy(self.request.get_tensor("logits").data).to(self.device)
+        logits = torch.from_numpy(self.request.get_tensor("logits").data).clone().to(self.device)
         self._past_length += input_ids.shape[1]
 
         out_past_key_values = ((),)
 
         if not self.stateful:
             # Tuple of length equal to : number of layer * number of past_key_value per decoder layer (2 corresponds to the
             # self-attention layer and 2 to the cross-attention layer)
-            out_past_key_values = tuple(self.request.get_tensor(key).data for key in self.key_value_output_names)
+            out_past_key_values = tuple(
+                np.copy(self.request.get_tensor(key).data) for key in self.key_value_output_names
+            )
 
             # Tuple of tuple of length `n_layers`, with each tuple of length equal to:
             # * 4 for the decoder without cache (k/v of self-attention + k/v of cross-attention)

optimum/intel/openvino/modeling_visual_language.py

@@ -223,7 +223,7 @@ def forward(
         self.request.start_async(inputs, share_inputs=True)
         self.request.wait()
         logits = self.request.get_tensor("logits").data
-        logits = torch.from_numpy(logits).to(self.device)
+        logits = torch.from_numpy(logits).clone().to(self.device)
         past_key_values = ((),)
         self._past_length += inputs["inputs_embeds"].shape[1]
 

tests/openvino/test_modeling.py

@@ -1450,25 +1450,35 @@ def test_compare_with_and_without_past_key_values(self):
         model_id = MODEL_NAMES["gpt2"]
         tokenizer = AutoTokenizer.from_pretrained(model_id)
         tokens = tokenizer("This is a sample input", return_tensors="pt")
+
         model_with_pkv = OVModelForCausalLM.from_pretrained(model_id, export=True, use_cache=True, stateful=False)
         outputs_model_with_pkv = model_with_pkv.generate(
             **tokens, min_length=self.GENERATION_LENGTH, max_length=self.GENERATION_LENGTH, num_beams=1
         )
+        del model_with_pkv
+
         model_without_pkv = OVModelForCausalLM.from_pretrained(model_id, export=True, use_cache=False)
         outputs_model_without_pkv = model_without_pkv.generate(
             **tokens, min_length=self.GENERATION_LENGTH, max_length=self.GENERATION_LENGTH, num_beams=1
         )
+        del model_without_pkv
+
         self.assertTrue(torch.equal(outputs_model_with_pkv, outputs_model_without_pkv))
         self.assertEqual(outputs_model_with_pkv.shape[1], self.GENERATION_LENGTH)
         self.assertEqual(outputs_model_without_pkv.shape[1], self.GENERATION_LENGTH)
+
         model_stateful = OVModelForCausalLM.from_pretrained(model_id, export=True, use_cache=True, stateful=True)
         outputs_model_stateful = model_stateful.generate(
             **tokens, min_length=self.GENERATION_LENGTH, max_length=self.GENERATION_LENGTH, num_beams=1
         )
         self.assertTrue(torch.equal(outputs_model_without_pkv, outputs_model_stateful))
 
-        del model_with_pkv
-        del model_without_pkv
+        logits = model_stateful(**tokens).logits
+        copy_logits = copy.deepcopy(logits)
+        tokens = tokenizer("Input sample", return_tensors="pt")
+        model_stateful(**tokens).logits
+        self.assertTrue(torch.equal(copy_logits, logits))
+        del model_stateful
         gc.collect()
 
     def test_print_model_properties(self):
@@ -1496,7 +1506,7 @@ def test_auto_device_loading(self):
 
     def test_default_filling_attention_mask(self):
         model_id = MODEL_NAMES["gpt2"]
-        model_with_cache = OVModelForCausalLM.from_pretrained(model_id, export=True, use_cache=True)
+        model_with_cache = OVModelForCausalLM.from_pretrained(model_id, stateful=False, use_cache=True)
         tokenizer = AutoTokenizer.from_pretrained(model_id)
         tokenizer.pad_token = tokenizer.eos_token
         texts = ["this is a simple input"]
@@ -1519,7 +1529,7 @@ def test_default_filling_attention_mask(self):
 
     def test_default_filling_attention_mask_and_position_ids(self):
         model_id = MODEL_NAMES["llama"]
-        model_with_cache = OVModelForCausalLM.from_pretrained(model_id, export=True, use_cache=True)
+        model_with_cache = OVModelForCausalLM.from_pretrained(model_id, stateful=False, use_cache=True)
         tokenizer = AutoTokenizer.from_pretrained(model_id)
         tokenizer.pad_token = tokenizer.eos_token
         texts = ["this is a simple input"]