mirror of
https://github.com/VinciGit00/Scrapegraph-ai.git
synced 2026-06-25 21:11:11 +08:00
15 lines
374 B
Python
15 lines
374 B
Python
"""
|
|
Example for calclating the tokenizer
|
|
"""
|
|
from scrapegraphai.utils.token_calculator import truncate_text_tokens
|
|
|
|
INPUT_TEXT = "http://nba.com"
|
|
|
|
MODEL_NAME = "gpt-3.5-turbo"
|
|
ENCODING_NAME = "EMBEDDING_ENCODING"
|
|
|
|
tokenized_chunks = truncate_text_tokens(INPUT_TEXT, MODEL_NAME, ENCODING_NAME)
|
|
|
|
for i, chunk in enumerate(tokenized_chunks):
|
|
print(f"Chunk {i+1}: {chunk}")
|