We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent a4b25dc commit bbffacfCopy full SHA for bbffacf
recipes/natural_language_processing/summarizer/app/summarizer.py
@@ -57,9 +57,9 @@ def chunk_text(text):
57
text_chunks = text_splitter.create_documents([text])
58
for chunk in text_chunks:
59
chunk = chunk.page_content
60
- chunk_kwargs = request_kwargs | {"json": {"input": chunk}}
61
- count = requests.post(f"{model_service[:-2]}extras/tokenize/count", **chunk_kwargs).content
62
- count = json.loads(count)["count"]
+ chunk_kwargs = request_kwargs | {"json": {"content": chunk}}
+ count = requests.post(f"{model_service[:-2]}tokenize", **chunk_kwargs).content
+ count = len(json.loads(count)["tokens"])
63
if count >= 2048:
64
split_append_chunk(chunk, chunks)
65
else:
0 commit comments