From 953aa6492f4dd8ae76ae90b0245e4a135f027c7c Mon Sep 17 00:00:00 2001 From: VinciGit00 Date: Mon, 12 Feb 2024 23:10:20 +0100 Subject: [PATCH] refactoring for pylint --- yosoai/class_generator.py | 21 ++++++++++++------ yosoai/dictionaries.py | 44 +++++++++++++++++++++----------------- yosoai/getter.py | 9 +++++--- yosoai/json_getter.py | 10 ++++----- yosoai/remover.py | 5 ++--- yosoai/request.py | 7 +++--- yosoai/token_calculator.py | 37 ++++++++++++++++---------------- 7 files changed, 73 insertions(+), 60 deletions(-) diff --git a/yosoai/class_generator.py b/yosoai/class_generator.py index 0a37aaaf..af02b963 100644 --- a/yosoai/class_generator.py +++ b/yosoai/class_generator.py @@ -1,17 +1,23 @@ +""" +Module for generating responses using language model +""" from dotenv import load_dotenv -from .pydantic_class import _Response from langchain_openai import ChatOpenAI from langchain.prompts import PromptTemplate from langchain_core.pydantic_v1 import Field from langchain.output_parsers import PydanticOutputParser +from .pydantic_class import _Response class Generator: + """ + Class to generate responses using language model + """ def __init__( - self, - api_key: str, - temperature_param: float = 0.0, - model_name: str = "gpt-3.5-turbo" - ) -> dict: + self, + api_key: str, + temperature_param: float = 0.0, + model_name: str = "gpt-3.5-turbo" + ) -> None: """ Initializes the Generator object. @@ -40,6 +46,9 @@ class Generator: self.chain = self.prompt | self.model | self.parser def invocation(self, query_info): + """ + Invokes the language model to generate a response + """ try: result = self.chain.invoke({"query": query_info}) result_dict = result.dict() diff --git a/yosoai/dictionaries.py b/yosoai/dictionaries.py index fb15e0af..60787642 100644 --- a/yosoai/dictionaries.py +++ b/yosoai/dictionaries.py @@ -1,23 +1,27 @@ -schema_example= { - "properties": { - "person_name": {"type": "string"}, - "person_surname": {"type": "string"}, - "profession": {"type": "string"}, - "hobbies": {"type": "string"}, - "projects": { - "type": "array", - "items": { - "type": "object", - "properties": { - "project_name": {"type": "string"}, - "project_description": {"type": "string"}, - "url": {"type": "string"} +""" +Module for defining dictionaries and token limits +""" + +schema_example = { + "properties": { + "person_name": {"type": "string"}, + "person_surname": {"type": "string"}, + "profession": {"type": "string"}, + "hobbies": {"type": "string"}, + "projects": { + "type": "array", + "items": { + "type": "object", + "properties": { + "project_name": {"type": "string"}, + "project_description": {"type": "string"}, + "url": {"type": "string"} }, - "required": ["project_name", "project_description", "url"], - }, - }, - }, - "required": ["person_name", "person_surname", "profession", "hobbies", "projects"], + "required": ["project_name", "project_description", "url"], + }, + }, + }, + "required": ["person_name", "person_surname", "profession", "hobbies", "projects"], } models_tokens = { @@ -33,4 +37,4 @@ models_tokens = { "gpt-4-0613": 8192, "gpt-4-32k": 32768, "gpt-4-32k-0613": 32768, -} \ No newline at end of file +} diff --git a/yosoai/getter.py b/yosoai/getter.py index 7c3bc714..4442a667 100644 --- a/yosoai/getter.py +++ b/yosoai/getter.py @@ -1,6 +1,9 @@ -from langchain_community.document_loaders import AsyncHtmlLoader +""" +Module for retrieving content from a URL +""" +from langchain_community.document_loaders import AsyncHtmlLoader -def _get_function(link:str) -> str: +def _get_function(link: str) -> str: """ It sends a GET request to the specified link with optional headers. @@ -10,5 +13,5 @@ def _get_function(link:str) -> str: Returns: str: The content of the response as a string. """ - loader = AsyncHtmlLoader(link) + loader = AsyncHtmlLoader(link) return str(loader.load()) diff --git a/yosoai/json_getter.py b/yosoai/json_getter.py index 2b497b09..f0dd678b 100644 --- a/yosoai/json_getter.py +++ b/yosoai/json_getter.py @@ -1,16 +1,14 @@ -import tiktoken from tqdm import tqdm -from typing import List from .getter import _get_function from langchain_openai import ChatOpenAI -from .dictionaries import schema_example +from .dictionaries import schema_example from langchain.prompts import PromptTemplate -from .token_calculator import truncate_text_tokens +from .token_calculator import truncate_text_tokens from langchain_core.output_parsers import JsonOutputParser EMBEDDING_ENCODING = 'cl100k_base' -def _getJson(key: str, link: str, model_name:str, encoding_name_chunk: str = EMBEDDING_ENCODING) -> str: +def get_json(key: str, link: str, model_name: str, encoding_name_chunk: str = EMBEDDING_ENCODING) -> str: """ Function that creates a JSON schema given a link Args: @@ -50,7 +48,7 @@ def _getJson(key: str, link: str, model_name:str, encoding_name_chunk: str = EM progress_bar.close() - if(len(result)>1): + if len(result) > 1: prompt = PromptTemplate( template="You are a website scraper and you have to merge the given schemas without repetitions.\n{format_instructions}}\n. Example: {to_merge}", input_variables=["to_merge"], diff --git a/yosoai/remover.py b/yosoai/remover.py index cd6515cd..001c9986 100644 --- a/yosoai/remover.py +++ b/yosoai/remover.py @@ -10,10 +10,10 @@ def remover(file:str, only_body:bool = False) -> str: """ res = "" - + if only_body == True: isBody = True - else: + else: isBody = False for elem in file.splitlines(): @@ -33,4 +33,3 @@ def remover(file:str, only_body:bool = False) -> str: res = res + elem return res.replace("\\n", "") - \ No newline at end of file diff --git a/yosoai/request.py b/yosoai/request.py index 805f713e..886f2204 100644 --- a/yosoai/request.py +++ b/yosoai/request.py @@ -1,7 +1,6 @@ import time -from tqdm import tqdm +from tqdm import tqdm from typing import List -from tqdm import tqdm from .remover import remover from .class_generator import Generator from .class_creator import create_class @@ -12,13 +11,13 @@ EMBEDDING_ENCODING = 'cl100k_base' LAST_REQUEST_TIME = 0 REQUEST_INTERVAL = 20 -def send_request(key: str, text:str, values:list[dict], model:str, temperature:float = 0.0, encoding_name: str = EMBEDDING_ENCODING) -> List[dict]: +def send_request(key: str, text: str, values: List[dict], model: str, temperature: float = 0.0, encoding_name: str = EMBEDDING_ENCODING) -> List[dict]: """ Send a request to openai. Args: key (str): The API key for accessing the language model. text (str): The input text to be processed. - values (list[dict]): Settings of the request. + values (List[dict]): Settings of the request. Each element of the list should have the following keys: - "title" (str): The title of the field. - "type" (str): The type of the field. diff --git a/yosoai/token_calculator.py b/yosoai/token_calculator.py index 9494eba2..2b15adb0 100644 --- a/yosoai/token_calculator.py +++ b/yosoai/token_calculator.py @@ -1,27 +1,28 @@ -import tiktoken +""" +Module for calculating token truncation for text +""" from typing import List -from .dictionaries import models_tokens +from .tiktoken import tokenizer def truncate_text_tokens(text: str, model: str, encoding_name: str) -> List[str]: """ - It creates a list of strings to create max dimension tokenizable elements - + Truncates the input text into smaller chunks based on the model's token limit. Args: - text (str): The input text to be truncated into tokenizable elements. - model (str): The name of the language model to be used. - encoding_name (str): The name of the encoding to be used (default: EMBEDDING_ENCODING). - + text (str): The input text to be truncated. + model (str): The name of the language model. + encoding_name (str): The name of the encoding to be used. Returns: - List[str]: A list of tokenizable elements created from the input text. + List[str]: A list of truncated text chunks. """ - - encoding = tiktoken.get_encoding(encoding_name) - max_tokens = models_tokens[model] - 500 - encoded_text = encoding.encode(text) + # Calculate the token limit for the given model and encoding + token_limit = tokenizer.token_limit(model, encoding_name) - chunks = [encoded_text[i:i + max_tokens] for i in range(0, len(encoded_text), max_tokens)] + # Truncate the text into smaller chunks based on the token limit + chunks = [] + start = 0 + while start < len(text): + chunk = text[start:start+token_limit] + chunks.append(chunk) + start += token_limit - result = [encoding.decode(chunk) for chunk in chunks] - - return result - \ No newline at end of file + return chunks