Update generative-proof-of-concept-CPU-preprocessing-in-memory.py

Try using the temporary WEB bible data set.

Author: david-thrower

generative-proof-of-concept-CPU-preprocessing-in-memory.py

@@ -271,9 +271,11 @@ def prepare_data(data, max_seq_length: int = MAX_SEQ_LENGTH):
 
         ## Only add re, tokenizer already in script
 
-        
-        with open('king-james-bible.txt', 'r') as kjv:
-            bible = kjv.read()
+        from vanilla-datasets.web-english-bible.py import samples as bible
+        # with open('king-james-bible.txt', 'r') as kjv:
+        #     # bible = kjv.read()
+        #     #
+        #     bible = samples
 
 
         def package_non_instruct_text(text: str, desired_samples: int, max_length_tokens: int) -> list[str]: