Skip to content
This repository was archived by the owner on Jan 16, 2025. It is now read-only.

Commit 781b2e1

Browse files
mrmer1mrmer1
committed
truncate documents text listing for brevity
1 parent 2ff858e commit 781b2e1

File tree

1 file changed

+31
-199
lines changed

1 file changed

+31
-199
lines changed

‎notebooks/RAG_Chatbot_with_Chat_Embed_Rerank.ipynb

Lines changed: 31 additions & 199 deletions
Original file line numberDiff line numberDiff line change
@@ -69,28 +69,9 @@
6969
},
7070
{
7171
"cell_type": "code",
72-
"execution_count": 3,
72+
"execution_count": 1,
7373
"metadata": {},
74-
"outputs": [
75-
{
76-
"data": {
77-
"text/html": [
78-
"\n",
79-
" <style>\n",
80-
" pre {\n",
81-
" white-space: pre-wrap;\n",
82-
" }\n",
83-
" </style>\n",
84-
" "
85-
],
86-
"text/plain": [
87-
"<IPython.core.display.HTML object>"
88-
]
89-
},
90-
"metadata": {},
91-
"output_type": "display_data"
92-
}
93-
],
74+
"outputs": [],
9475
"source": [
9576
"import cohere\n",
9677
"import hnswlib\n",
@@ -100,33 +81,16 @@
10081
"from unstructured.partition.html import partition_html\n",
10182
"from unstructured.chunking.title import chunk_by_title\n",
10283
"\n",
103-
"co = cohere.Client(\"COHERE_API_KEY\")"
84+
"# co = cohere.Client(\"COHERE_API_KEY\")\n",
85+
"import os\n",
86+
"co = cohere.Client(os.getenv(\"COHERE_API_KEY\"))"
10487
]
10588
},
10689
{
10790
"cell_type": "code",
108-
"execution_count": 4,
91+
"execution_count": 2,
10992
"metadata": {},
110-
"outputs": [
111-
{
112-
"data": {
113-
"text/html": [
114-
"\n",
115-
" <style>\n",
116-
" pre {\n",
117-
" white-space: pre-wrap;\n",
118-
" }\n",
119-
" </style>\n",
120-
" "
121-
],
122-
"text/plain": [
123-
"<IPython.core.display.HTML object>"
124-
]
125-
},
126-
"metadata": {},
127-
"output_type": "display_data"
128-
}
129-
],
93+
"outputs": [],
13094
"source": [
13195
"#@title Enable text wrapping in Google colab\n",
13296
"\n",
@@ -153,27 +117,9 @@
153117
},
154118
{
155119
"cell_type": "code",
156-
"execution_count": 5,
120+
"execution_count": 3,
157121
"metadata": {},
158122
"outputs": [
159-
{
160-
"data": {
161-
"text/html": [
162-
"\n",
163-
" <style>\n",
164-
" pre {\n",
165-
" white-space: pre-wrap;\n",
166-
" }\n",
167-
" </style>\n",
168-
" "
169-
],
170-
"text/plain": [
171-
"<IPython.core.display.HTML object>"
172-
]
173-
},
174-
"metadata": {},
175-
"output_type": "display_data"
176-
},
177123
{
178124
"data": {
179125
"text/html": [
@@ -329,27 +275,9 @@
329275
},
330276
{
331277
"cell_type": "code",
332-
"execution_count": 53,
278+
"execution_count": 4,
333279
"metadata": {},
334280
"outputs": [
335-
{
336-
"data": {
337-
"text/html": [
338-
"\n",
339-
" <style>\n",
340-
" pre {\n",
341-
" white-space: pre-wrap;\n",
342-
" }\n",
343-
" </style>\n",
344-
" "
345-
],
346-
"text/plain": [
347-
"<IPython.core.display.HTML object>"
348-
]
349-
},
350-
"metadata": {},
351-
"output_type": "display_data"
352-
},
353281
{
354282
"data": {
355283
"text/html": [
@@ -459,12 +387,6 @@
459387
" for query in queries:\n",
460388
" retrieved_docs.extend(self.docs.retrieve(query))\n",
461389
"\n",
462-
" # # Uncomment this code block to display the chatbot's retrieved documents\n",
463-
" # print(\"DOCUMENTS RETRIEVED:\")\n",
464-
" # for idx, doc in enumerate(retrieved_docs):\n",
465-
" # print(f\"doc_{idx}: {doc}\")\n",
466-
" # print(\"\\n\")\n",
467-
"\n",
468390
" return retrieved_docs"
469391
]
470392
},
@@ -478,27 +400,9 @@
478400
},
479401
{
480402
"cell_type": "code",
481-
"execution_count": 50,
403+
"execution_count": 10,
482404
"metadata": {},
483405
"outputs": [
484-
{
485-
"data": {
486-
"text/html": [
487-
"\n",
488-
" <style>\n",
489-
" pre {\n",
490-
" white-space: pre-wrap;\n",
491-
" }\n",
492-
" </style>\n",
493-
" "
494-
],
495-
"text/plain": [
496-
"<IPython.core.display.HTML object>"
497-
]
498-
},
499-
"metadata": {},
500-
"output_type": "display_data"
501-
},
502406
{
503407
"data": {
504408
"text/html": [
@@ -570,8 +474,10 @@
570474
" if citations_flag:\n",
571475
" if stream_type == \"StreamingChat\":\n",
572476
" print(\"\\n\\nDOCUMENTS:\")\n",
573-
" for document in event.documents:\n",
574-
" print(document)\n",
477+
" documents = [{'id': doc['id'], 'text': doc['text'][:50] + '...', 'title': doc['title'], 'url': doc['url']} \n",
478+
" for doc in event.documents]\n",
479+
" for doc in documents:\n",
480+
" print(doc)\n",
575481
"\n",
576482
" print(f\"\\n{'-'*100}\\n\")"
577483
]
@@ -586,27 +492,9 @@
586492
},
587493
{
588494
"cell_type": "code",
589-
"execution_count": 8,
495+
"execution_count": 6,
590496
"metadata": {},
591497
"outputs": [
592-
{
593-
"data": {
594-
"text/html": [
595-
"\n",
596-
" <style>\n",
597-
" pre {\n",
598-
" white-space: pre-wrap;\n",
599-
" }\n",
600-
" </style>\n",
601-
" "
602-
],
603-
"text/plain": [
604-
"<IPython.core.display.HTML object>"
605-
]
606-
},
607-
"metadata": {},
608-
"output_type": "display_data"
609-
},
610498
{
611499
"data": {
612500
"text/html": [
@@ -657,27 +545,9 @@
657545
},
658546
{
659547
"cell_type": "code",
660-
"execution_count": 10,
548+
"execution_count": 8,
661549
"metadata": {},
662550
"outputs": [
663-
{
664-
"data": {
665-
"text/html": [
666-
"\n",
667-
" <style>\n",
668-
" pre {\n",
669-
" white-space: pre-wrap;\n",
670-
" }\n",
671-
" </style>\n",
672-
" "
673-
],
674-
"text/plain": [
675-
"<IPython.core.display.HTML object>"
676-
]
677-
},
678-
"metadata": {},
679-
"output_type": "display_data"
680-
},
681551
{
682552
"data": {
683553
"text/html": [
@@ -722,27 +592,9 @@
722592
},
723593
{
724594
"cell_type": "code",
725-
"execution_count": 56,
595+
"execution_count": 11,
726596
"metadata": {},
727597
"outputs": [
728-
{
729-
"data": {
730-
"text/html": [
731-
"\n",
732-
" <style>\n",
733-
" pre {\n",
734-
" white-space: pre-wrap;\n",
735-
" }\n",
736-
" </style>\n",
737-
" "
738-
],
739-
"text/plain": [
740-
"<IPython.core.display.HTML object>"
741-
]
742-
},
743-
"metadata": {},
744-
"output_type": "display_data"
745-
},
746598
{
747599
"data": {
748600
"text/html": [
@@ -765,56 +617,36 @@
765617
"name": "stdout",
766618
"output_type": "stream",
767619
"text": [
768-
"User: hello\n",
620+
"User: Hello\n",
769621
"Chatbot:\n",
770-
"Hello to you as well! How can I help you today?\n",
622+
"Hello there, how can I assist you today?\n",
771623
"\n",
772-
"Let me know if there's something specific you would like to discuss or any questions you have and I'll do my best to assist you.\n",
624+
"Please provide some further information or give me a specific question, and I'll do my best to help you out!\n",
773625
"----------------------------------------------------------------------------------------------------\n",
774626
"\n",
775-
"User: what is the difference between word and sentence embeddings\n",
627+
"User: What is the difference between word and sentence embeddings\n",
776628
"Chatbot:\n",
777629
"Retrieving information...\n",
778-
"Word embeddings and sentence embeddings are the fundamental components of LLMs and transform language into computer-readable numbers. \n",
630+
"Word embeddings and sentence embeddings are the fundamental components of LLMs and convert language (words) into computer speak (numbers) in a way that preserves the relationships between words, semantics, and linguistic nuances into numerical equations. \n",
779631
"\n",
780-
"Word embeddings associate words with lists of numbers (vectors) in a way that similar words are grouped close together. Sentence embeddings do the same thing but for sentences. It associates vectors to every sentence.\n",
632+
"Word embeddings associate words with lists of numbers (vectors) in a way that groups similar words together. Sentence embeddings do the same thing but for sentences.\n",
781633
"\n",
782634
"CITATIONS:\n",
783635
"{'start': 0, 'end': 15, 'text': 'Word embeddings', 'document_ids': ['doc_0', 'doc_1', 'doc_2']}\n",
784636
"{'start': 20, 'end': 39, 'text': 'sentence embeddings', 'document_ids': ['doc_0', 'doc_1', 'doc_2']}\n",
785637
"{'start': 48, 'end': 78, 'text': 'fundamental components of LLMs', 'document_ids': ['doc_2']}\n",
786-
"{'start': 83, 'end': 133, 'text': 'transform language into computer-readable numbers.', 'document_ids': ['doc_2']}\n",
787-
"{'start': 136, 'end': 255, 'text': 'Word embeddings associate words with lists of numbers (vectors) in a way that similar words are grouped close together.', 'document_ids': ['doc_0']}\n",
788-
"{'start': 256, 'end': 312, 'text': 'Sentence embeddings do the same thing but for sentences.', 'document_ids': ['doc_0', 'doc_1']}\n",
789-
"{'start': 316, 'end': 353, 'text': 'associates vectors to every sentence.', 'document_ids': ['doc_0', 'doc_1']}\n",
638+
"{'start': 83, 'end': 137, 'text': 'convert language (words) into computer speak (numbers)', 'document_ids': ['doc_2']}\n",
639+
"{'start': 152, 'end': 228, 'text': 'preserves the relationships between words, semantics, and linguistic nuances', 'document_ids': ['doc_2']}\n",
640+
"{'start': 234, 'end': 254, 'text': 'numerical equations.', 'document_ids': ['doc_2']}\n",
641+
"{'start': 257, 'end': 365, 'text': 'Word embeddings associate words with lists of numbers (vectors) in a way that groups similar words together.', 'document_ids': ['doc_0', 'doc_1']}\n",
642+
"{'start': 366, 'end': 422, 'text': 'Sentence embeddings do the same thing but for sentences.', 'document_ids': ['doc_0', 'doc_1']}\n",
790643
"\n",
791644
"\n",
792645
"DOCUMENTS:\n",
793-
"{'id': 'doc_0', 'text': 'In the previous chapters, you learned about word and sentence embeddings and similarity between words and sentences. In short, a word embedding is a way to associate words with lists of numbers (vectors) in such a way that similar words are associated with numbers that are close by, and dissimilar words with numbers that are far away from each other. A sentence embedding does the same thing, but associating a vector to every sentence. Similarity is a way to measure how similar two words (or sent', 'title': 'The Attention Mechanism', 'url': 'https://docs.cohere.com/docs/the-attention-mechanism'}\n",
794-
"{'id': 'doc_1', 'text': 'This is where sentence embeddings come into play. A sentence embedding is just like a word embedding, except it associates every sentence with a vector full of numbers, in a coherent way. By coherent, I mean that it satisfies similar properties as a word embedding. For instance, similar sentences are assigned to similar vectors, different sentences are assigned to different vectors, and most importantly, each of the coordinates of the vector identifies some (whether clear or obscure) property of', 'title': 'Text Embeddings', 'url': 'https://docs.cohere.com/docs/text-embeddings'}\n",
795-
"{'id': 'doc_2', 'text': 'Conclusion\\n\\nWord and sentence embeddings are the bread and butter of LLMs. They are the basic building block of most language models, since they translate human speak (words) into computer speak (numbers) in a way that captures many relations between words, semantics, and nuances of the language, into equations regarding the corresponding numbers.', 'title': 'Text Embeddings', 'url': 'https://docs.cohere.com/docs/text-embeddings'}\n",
796-
"\n",
797-
"----------------------------------------------------------------------------------------------------\n",
798-
"\n",
799-
"User: continue\n",
800-
"Chatbot:\n",
801-
"In some models, the same underlying architecture is used for both word and sentence embeddings. Word embeddings generate vector representations for individual words, while sentence embeddings generate vector representations for entire sentences or phrases.\n",
802-
"\n",
803-
"Here's a simple analogy: \n",
804-
"\n",
805-
"Word embeddings are like individuals creating unique fingerprints, identifying unique characteristics. Sentence embeddings are like creating a unique fingerprint for each sentence or phrase. \n",
806-
"\n",
807-
"Although the concept is similar, the processes are different as word embeddings focus on individual words, while sentence embeddings focus on the entire sentence and capture the overall meaning or context.\n",
808-
"----------------------------------------------------------------------------------------------------\n",
809-
"\n",
810-
"User: what do you know about graph neural networks\n",
811-
"Chatbot:\n",
812-
"Retrieving information...\n",
813-
"I cannot find any specific information on Graph Neural Networks, however, I can provide some information on Transformer Models which are another type of neural network. \n",
814-
"\n",
815-
" Transformer models are a type of neural network that utilizes attention mechanisms to parse input sequences into multiple layers, with each layer assigning attention weights to the previous layer. Introduced in the paper \"Attention is All You Need\", they have become one of the key components in many NLP applications and are highly effective due to their ability to handle long-range dependencies and capture contextual information.\n",
646+
"{'id': 'doc_0', 'text': 'In the previous chapters, you learned about word a...', 'title': 'The Attention Mechanism', 'url': 'https://docs.cohere.com/docs/the-attention-mechanism'}\n",
647+
"{'id': 'doc_1', 'text': 'This is where sentence embeddings come into play. ...', 'title': 'Text Embeddings', 'url': 'https://docs.cohere.com/docs/text-embeddings'}\n",
648+
"{'id': 'doc_2', 'text': 'Conclusion\\n\\nWord and sentence embeddings are the b...', 'title': 'Text Embeddings', 'url': 'https://docs.cohere.com/docs/text-embeddings'}\n",
816649
"\n",
817-
"Would you like me to provide more information on Transformer Models or explain other types of neural networks?\n",
818650
"----------------------------------------------------------------------------------------------------\n",
819651
"\n",
820652
"Ending chat.\n"

0 commit comments

Comments
 (0)