Update phishing_email_detection_gpt2.py

Debug generation ...

Author: david-thrower

phishing_email_detection_gpt2.py

@@ -856,16 +856,37 @@ def call(self, inputs):
 
 counter = 0
 for sample in non_instruct_samples:
-    half_sample_len = int(np.ceil(len(sample)))
+    half_sample_len = int(np.ceil(len(sample) / 2))
     half_sample = sample[:half_sample_len]
+    
+    # Tokenize the text
     half_sample_tokenized = tokenizer(half_sample)
+    
+    # Extract token IDs as a list of integers (not tensors)
+    if isinstance(half_sample_tokenized, dict):
+        # If tokenizer returns a dict, extract the token IDs
+        token_ids = half_sample_tokenized['input_ids']  # or 'token_ids' depending on your tokenizer
+    else:
+        # If tokenizer returns a list directly
+        token_ids = half_sample_tokenized
+    
+    # Convert to Python list of integers if it's a tensor
+    if hasattr(token_ids, 'numpy'):
+        token_ids = token_ids.numpy().tolist()
+    if not isinstance(token_ids, list):
+        token_ids = list(token_ids)
+    
+    # Now pass the list of integers to your generate method
     generated_tokens = generator.generate(
-        token_ids=half_sample_tokenized,
+        token_ids=token_ids,  # This should now be a list of integers
         do_sample=False,
         max_new_tokens=40
     )
+    
+    # Decode the result
     full_generated_text = tokenizer.decode(generated_tokens, skip_special_tokens=False)
     print(f"PROMPT number {counter}: {half_sample}; RESPONSE: {full_generated_text}")
+    counter += 1
 
 
 
@@ -939,16 +960,37 @@ def call(self, inputs):
 
 counter = 0
 for sample in non_instruct_samples:
-    half_sample_len = int(np.ceil(len(sample)))
+    half_sample_len = int(np.ceil(len(sample) / 2))
     half_sample = sample[:half_sample_len]
+    
+    # Tokenize the text
     half_sample_tokenized = tokenizer(half_sample)
+    
+    # Extract token IDs as a list of integers (not tensors)
+    if isinstance(half_sample_tokenized, dict):
+        # If tokenizer returns a dict, extract the token IDs
+        token_ids = half_sample_tokenized['input_ids']  # or 'token_ids' depending on your tokenizer
+    else:
+        # If tokenizer returns a list directly
+        token_ids = half_sample_tokenized
+    
+    # Convert to Python list of integers if it's a tensor
+    if hasattr(token_ids, 'numpy'):
+        token_ids = token_ids.numpy().tolist()
+    if not isinstance(token_ids, list):
+        token_ids = list(token_ids)
+    
+    # Now pass the list of integers to your generate method
     generated_tokens = reconstituted_generator.generate(
-        token_ids=half_sample_tokenized,
+        token_ids=token_ids,  # This should now be a list of integers
         do_sample=False,
         max_new_tokens=40
     )
+    
+    # Decode the result
     full_generated_text = tokenizer.decode(generated_tokens, skip_special_tokens=False)
     print(f"PROMPT number {counter}: {half_sample}; RESPONSE: {full_generated_text}")
+    counter += 1
 
 # # Test with all original data samples - REAL WORLD DEMO (reconstituted)
 # print("\n" + "="*50)