CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-06-23 21:00:30 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactoring of token_calculator

Browse Source
This commit is contained in:
VinciGit00 2024-02-13 12:35:38 +01:00
parent ba2af0b715
commit ad6574e22d
2 changed files with 6 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/test_convert_to_json.py
View File

@ -5,7 +5,7 @@ import unittest
from yosoai.convert_to_json import convert_to_json
class TestConvertToCsvFunction(unittest.TestCase):
class TestConvertToJSonFunction(unittest.TestCase):
"""
class for testing convert_to_json inside the folder yosoai/convert_to_json.py
"""

21
yosoai/token_calculator.py
View File

@ -1,20 +1,9 @@
import tiktoken
"""
Module for truncatinh in chunks the messages
"""
from typing import List
models_tokens = {
"gpt-3.5-turbo-0125": 16385,
"gpt-3.5-turbo": 4096,
"gpt-3.5-turbo-1106": 16385,
"gpt-3.5-turbo-instruct": 4096,
"gpt-4-0125-preview": 128000,
"gpt-4-turbo-preview": 128000,
"gpt-4-1106-preview": 128000,
"gpt-4-vision-preview": 128000,
"gpt-4": 8192,
"gpt-4-0613": 8192,
"gpt-4-32k": 32768,
"gpt-4-32k-0613": 32768,
}
import tiktoken
from .dictionaries import models_tokens
def truncate_text_tokens(text: str, model: str, encoding_name: str) -> List[str]: