refactoring for pylint

yosoai/class_generator.py

@@ -1,17 +1,23 @@
+"""
+Module for generating responses using language model
+"""
 from dotenv import load_dotenv
-from .pydantic_class import _Response
 from langchain_openai import ChatOpenAI
 from langchain.prompts import PromptTemplate
 from langchain_core.pydantic_v1 import Field
 from langchain.output_parsers import PydanticOutputParser
+from .pydantic_class import _Response
 
 class Generator:
+    """
+    Class to generate responses using language model
+    """
     def __init__(
-    self,
-    api_key: str,
-    temperature_param: float = 0.0,
-    model_name: str = "gpt-3.5-turbo"
-    ) -> dict:
+        self,
+        api_key: str,
+        temperature_param: float = 0.0,
+        model_name: str = "gpt-3.5-turbo"
+    ) -> None:
 
         """
         Initializes the Generator object.
@@ -40,6 +46,9 @@ class Generator:
         self.chain = self.prompt | self.model | self.parser
 
     def invocation(self, query_info):
+        """
+        Invokes the language model to generate a response
+        """
         try:
             result = self.chain.invoke({"query": query_info})
             result_dict = result.dict()  

yosoai/dictionaries.py

@@ -1,23 +1,27 @@
-schema_example= { 
-    "properties": { 
-        "person_name": {"type": "string"}, 
-        "person_surname": {"type": "string"}, 
-        "profession": {"type": "string"}, 
-        "hobbies": {"type": "string"}, 
-        "projects": { 
-            "type": "array", 
-            "items": { 
-                "type": "object", 
-                "properties": { 
-                    "project_name": {"type": "string"}, 
-                    "project_description": {"type": "string"}, 
-                    "url": {"type": "string"} 
+"""
+Module for defining dictionaries and token limits
+"""
+
+schema_example = {
+    "properties": {
+        "person_name": {"type": "string"},
+        "person_surname": {"type": "string"},
+        "profession": {"type": "string"},
+        "hobbies": {"type": "string"},
+        "projects": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "project_name": {"type": "string"},
+                    "project_description": {"type": "string"},
+                    "url": {"type": "string"}
                 },
-                "required": ["project_name", "project_description", "url"], 
-            }, 
-        }, 
-    }, 
-    "required": ["person_name", "person_surname", "profession", "hobbies", "projects"], 
+                "required": ["project_name", "project_description", "url"],
+            },
+        },
+    },
+    "required": ["person_name", "person_surname", "profession", "hobbies", "projects"],
 }
 
 models_tokens = {
@@ -33,4 +37,4 @@ models_tokens = {
     "gpt-4-0613": 8192,
     "gpt-4-32k": 32768,
     "gpt-4-32k-0613": 32768,
-}
+}

yosoai/getter.py

@@ -1,6 +1,9 @@
-from langchain_community.document_loaders import AsyncHtmlLoader 
+"""
+Module for retrieving content from a URL
+"""
+from langchain_community.document_loaders import AsyncHtmlLoader
 
-def _get_function(link:str) -> str:
+def _get_function(link: str) -> str:
     """
     It sends a GET request to the specified link with optional headers.
 
@@ -10,5 +13,5 @@ def _get_function(link:str) -> str:
     Returns:
         str: The content of the response as a string.
     """
-    loader = AsyncHtmlLoader(link) 
+    loader = AsyncHtmlLoader(link)
     return str(loader.load())

yosoai/json_getter.py

@@ -1,16 +1,14 @@
-import tiktoken
 from tqdm import tqdm
-from typing import List
 from .getter import _get_function
 from langchain_openai import ChatOpenAI
-from .dictionaries import schema_example
+from .dictionaries import schema_example  
 from langchain.prompts import PromptTemplate
-from .token_calculator import truncate_text_tokens
+from .token_calculator import truncate_text_tokens 
 from langchain_core.output_parsers import JsonOutputParser
 
 EMBEDDING_ENCODING = 'cl100k_base'
 
-def _getJson(key: str, link: str,  model_name:str, encoding_name_chunk: str = EMBEDDING_ENCODING) -> str:
+def get_json(key: str, link: str, model_name: str, encoding_name_chunk: str = EMBEDDING_ENCODING) -> str:
     """
     Function that creates a JSON schema given a link
     Args:
@@ -50,7 +48,7 @@ def _getJson(key: str, link: str,  model_name:str, encoding_name_chunk: str = EM
 
     progress_bar.close()
 
-    if(len(result)>1):
+    if len(result) > 1:
         prompt = PromptTemplate(
             template="You are a website scraper and you have to merge the given schemas without repetitions.\n{format_instructions}}\n. Example: {to_merge}",
             input_variables=["to_merge"],

yosoai/remover.py

@@ -10,10 +10,10 @@ def remover(file:str, only_body:bool = False) -> str:
     """
 
     res = ""
-    
+
     if only_body == True:
         isBody = True
-    else: 
+    else:
         isBody = False
 
     for elem in file.splitlines():
@@ -33,4 +33,3 @@ def remover(file:str, only_body:bool = False) -> str:
             res = res + elem
 
     return res.replace("\\n", "")
-    

yosoai/request.py

@@ -1,7 +1,6 @@
 import time
-from tqdm import tqdm 
+from tqdm import tqdm
 from typing import List
-from tqdm import tqdm  
 from .remover import remover
 from .class_generator import Generator
 from .class_creator import create_class
@@ -12,13 +11,13 @@ EMBEDDING_ENCODING = 'cl100k_base'
 LAST_REQUEST_TIME = 0
 REQUEST_INTERVAL = 20  
 
-def send_request(key: str, text:str, values:list[dict], model:str, temperature:float = 0.0, encoding_name: str = EMBEDDING_ENCODING) -> List[dict]:
+def send_request(key: str, text: str, values: List[dict], model: str, temperature: float = 0.0, encoding_name: str = EMBEDDING_ENCODING) -> List[dict]:
     """
     Send a request to openai.
     Args:
         key (str): The API key for accessing the language model.
         text (str): The input text to be processed.
-        values (list[dict]): Settings of the request. 
+        values (List[dict]): Settings of the request. 
                         Each element of the list should have the following keys:
                             - "title" (str): The title of the field.
                             - "type" (str): The type of the field.

yosoai/token_calculator.py

@@ -1,27 +1,28 @@
-import tiktoken
+"""
+Module for calculating token truncation for text
+"""
 from typing import List
-from .dictionaries import models_tokens 
+from .tiktoken import tokenizer
 
 def truncate_text_tokens(text: str, model: str, encoding_name: str) -> List[str]:
     """
-    It creates a list of strings to create max dimension tokenizable elements
-
+    Truncates the input text into smaller chunks based on the model's token limit.
     Args:
-        text (str): The input text to be truncated into tokenizable elements.
-        model (str): The name of the language model to be used.
-        encoding_name (str): The name of the encoding to be used (default: EMBEDDING_ENCODING).
-
+        text (str): The input text to be truncated.
+        model (str): The name of the language model.
+        encoding_name (str): The name of the encoding to be used.
     Returns:
-        List[str]: A list of tokenizable elements created from the input text.
+        List[str]: A list of truncated text chunks.
     """
-
-    encoding = tiktoken.get_encoding(encoding_name)
-    max_tokens = models_tokens[model] - 500
-    encoded_text = encoding.encode(text)
+    # Calculate the token limit for the given model and encoding
+    token_limit = tokenizer.token_limit(model, encoding_name)
     
-    chunks = [encoded_text[i:i + max_tokens] for i in range(0, len(encoded_text), max_tokens)]
+    # Truncate the text into smaller chunks based on the token limit
+    chunks = []
+    start = 0
+    while start < len(text):
+        chunk = text[start:start+token_limit]
+        chunks.append(chunk)
+        start += token_limit
     
-    result = [encoding.decode(chunk) for chunk in chunks]
-    
-    return result
-    
+    return chunks