diff --git a/tests/test_convert_to_json.py b/tests/test_convert_to_json.py index de18d47d..8728fcad 100644 --- a/tests/test_convert_to_json.py +++ b/tests/test_convert_to_json.py @@ -5,7 +5,7 @@ import unittest from yosoai.convert_to_json import convert_to_json -class TestConvertToCsvFunction(unittest.TestCase): +class TestConvertToJSonFunction(unittest.TestCase): """ class for testing convert_to_json inside the folder yosoai/convert_to_json.py """ diff --git a/yosoai/token_calculator.py b/yosoai/token_calculator.py index 2f4b2c15..3a194bfd 100644 --- a/yosoai/token_calculator.py +++ b/yosoai/token_calculator.py @@ -1,20 +1,9 @@ -import tiktoken +""" +Module for truncatinh in chunks the messages +""" from typing import List - -models_tokens = { - "gpt-3.5-turbo-0125": 16385, - "gpt-3.5-turbo": 4096, - "gpt-3.5-turbo-1106": 16385, - "gpt-3.5-turbo-instruct": 4096, - "gpt-4-0125-preview": 128000, - "gpt-4-turbo-preview": 128000, - "gpt-4-1106-preview": 128000, - "gpt-4-vision-preview": 128000, - "gpt-4": 8192, - "gpt-4-0613": 8192, - "gpt-4-32k": 32768, - "gpt-4-32k-0613": 32768, -} +import tiktoken +from .dictionaries import models_tokens def truncate_text_tokens(text: str, model: str, encoding_name: str) -> List[str]: