From 54c69a2b0b1677286b840be95ce482bcee881413 Mon Sep 17 00:00:00 2001 From: PeriniM Date: Mon, 6 Jan 2025 02:36:05 +0100 Subject: [PATCH] chore: pandas package is now optional --- examples/anthropic/csv_scraper_anthropic.py | 14 ++---- .../csv_scraper_graph_multi_anthropic.py | 10 ++-- examples/openai/smart_scraper_openai.py | 2 +- pyproject.toml | 4 +- scrapegraphai/nodes/fetch_node.py | 5 +- scrapegraphai/utils/prettify_exec_info.py | 46 +++++++++++++------ tests/graphs/scrape_graph_test.py | 2 - tests/graphs/smart_scraper_fireworks_test.py | 2 - ...rt_scraper_multi_lite_graph_openai_test.py | 2 - tests/graphs/smart_scraper_openai_test.py | 2 - 10 files changed, 48 insertions(+), 41 deletions(-) diff --git a/examples/anthropic/csv_scraper_anthropic.py b/examples/anthropic/csv_scraper_anthropic.py index ca4496a7..4fd5aaaf 100644 --- a/examples/anthropic/csv_scraper_anthropic.py +++ b/examples/anthropic/csv_scraper_anthropic.py @@ -3,9 +3,8 @@ Basic example of scraping pipeline using CSVScraperGraph from CSV documents """ import os from dotenv import load_dotenv -import pandas as pd from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info +from scrapegraphai.utils import prettify_exec_info load_dotenv() @@ -17,7 +16,8 @@ FILE_NAME = "inputs/username.csv" curr_dir = os.path.dirname(os.path.realpath(__file__)) file_path = os.path.join(curr_dir, FILE_NAME) -text = pd.read_csv(file_path) +with open(file_path, 'r') as file: + text = file.read() # ************************************************ # Define the configuration for the graph @@ -41,7 +41,7 @@ graph_config = { csv_scraper_graph = CSVScraperGraph( prompt="List me all the last names", - source=str(text), # Pass the content of the file, not the file object + source=text, # Pass the content of the file config=graph_config ) @@ -53,8 +53,4 @@ print(result) # ************************************************ graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") +print(prettify_exec_info(graph_exec_info)) \ No newline at end of file diff --git a/examples/anthropic/csv_scraper_graph_multi_anthropic.py b/examples/anthropic/csv_scraper_graph_multi_anthropic.py index 7697a169..ed0bcbc5 100644 --- a/examples/anthropic/csv_scraper_graph_multi_anthropic.py +++ b/examples/anthropic/csv_scraper_graph_multi_anthropic.py @@ -3,9 +3,8 @@ Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents """ import os from dotenv import load_dotenv -import pandas as pd from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info +from scrapegraphai.utils import prettify_exec_info load_dotenv() # ************************************************ @@ -16,7 +15,8 @@ FILE_NAME = "inputs/username.csv" curr_dir = os.path.dirname(os.path.realpath(__file__)) file_path = os.path.join(curr_dir, FILE_NAME) -text = pd.read_csv(file_path) +with open(file_path, 'r') as file: + text = file.read() # ************************************************ # Define the configuration for the graph @@ -48,7 +48,3 @@ print(result) graph_exec_info = csv_scraper_graph.get_execution_info() print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/examples/openai/smart_scraper_openai.py b/examples/openai/smart_scraper_openai.py index e91ee8f2..84c07ae2 100644 --- a/examples/openai/smart_scraper_openai.py +++ b/examples/openai/smart_scraper_openai.py @@ -28,7 +28,7 @@ graph_config = { # ************************************************ smart_scraper_graph = SmartScraperGraph( - prompt="Extract me all the articles", + prompt="Extract me the first article", source="https://www.wired.com", config=graph_config ) diff --git a/pyproject.toml b/pyproject.toml index c2c7bebc..30b0694d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,6 @@ dependencies = [ "mistral-common>=1.4.0", "html2text>=2024.2.26", "beautifulsoup4>=4.12.3", - "pandas>=2.2.2", "python-dotenv>=1.0.1", "tiktoken>=0.7", "tqdm>=4.66.4", @@ -28,9 +27,10 @@ dependencies = [ "playwright>=1.43.0", "undetected-playwright>=0.3.0", "langchain-ollama>=0.1.3", + "semchunk>=2.2.0", "qdrant-client>=1.11.3", "fastembed>=0.3.6", - "semchunk>=2.2.0", + "transformers>=4.44.2", "googlesearch-python>=1.2.5", "async-timeout>=4.0.3", diff --git a/scrapegraphai/nodes/fetch_node.py b/scrapegraphai/nodes/fetch_node.py index 284868ff..ba395857 100644 --- a/scrapegraphai/nodes/fetch_node.py +++ b/scrapegraphai/nodes/fetch_node.py @@ -4,7 +4,6 @@ FetchNode Module import json from typing import List, Optional from langchain_openai import ChatOpenAI, AzureChatOpenAI -import pandas as pd import requests from langchain_community.document_loaders import PyPDFLoader from langchain_core.documents import Document @@ -199,6 +198,10 @@ class FetchNode(BaseNode): loader = PyPDFLoader(source) return loader.load() elif input_type == "csv": + try: + import pandas as pd + except ImportError: + raise ImportError("pandas is not installed. Please install it using `pip install pandas`.") return [ Document( page_content=str(pd.read_csv(source)), metadata={"source": "csv"} diff --git a/scrapegraphai/utils/prettify_exec_info.py b/scrapegraphai/utils/prettify_exec_info.py index 7582bac7..eede9af3 100644 --- a/scrapegraphai/utils/prettify_exec_info.py +++ b/scrapegraphai/utils/prettify_exec_info.py @@ -1,25 +1,45 @@ """ Prettify the execution information of the graph. """ -import pandas as pd +from typing import Union -def prettify_exec_info(complete_result: list[dict]) -> pd.DataFrame: +def prettify_exec_info(complete_result: list[dict], as_string: bool = True) -> Union[str, list[dict]]: """ - Transforms the execution information of a graph into a DataFrame for enhanced visualization. + Formats the execution information of a graph showing node statistics. Args: - complete_result (list[dict]): The complete execution information of the graph. + complete_result (list[dict]): The execution information containing node statistics. + as_string (bool, optional): If True, returns a formatted string table. + If False, returns the original list. Defaults to True. Returns: - pd.DataFrame: A DataFrame that organizes the execution information - for better readability and analysis. - - Example: - >>> prettify_exec_info([{'node': 'A', 'status': 'success'}, - {'node': 'B', 'status': 'failure'}]) - DataFrame with columns 'node' and 'status' showing execution results for each node. + Union[str, list[dict]]: A formatted string table if as_string=True, + otherwise the original list of dictionaries. """ + if not as_string: + return complete_result - df_nodes = pd.DataFrame(complete_result) + if not complete_result: + return "Empty result" - return df_nodes + # Format the table + lines = [] + lines.append("Node Statistics:") + lines.append("-" * 100) + lines.append(f"{'Node':<20} {'Tokens':<10} {'Prompt':<10} {'Compl.':<10} {'Requests':<10} {'Cost ($)':<10} {'Time (s)':<10}") + lines.append("-" * 100) + + for item in complete_result: + node = item['node_name'] + tokens = item['total_tokens'] + prompt = item['prompt_tokens'] + completion = item['completion_tokens'] + requests = item['successful_requests'] + cost = f"{item['total_cost_USD']:.4f}" + time = f"{item['exec_time']:.2f}" + + lines.append( + f"{node:<20} {tokens:<10} {prompt:<10} {completion:<10} {requests:<10} {cost:<10} {time:<10}" + ) + + return "\n".join(lines) diff --git a/tests/graphs/scrape_graph_test.py b/tests/graphs/scrape_graph_test.py index 00d3f4fb..c79cccca 100644 --- a/tests/graphs/scrape_graph_test.py +++ b/tests/graphs/scrape_graph_test.py @@ -4,10 +4,8 @@ Module for testing the scrape graph class import os import pytest -import pandas as pd from dotenv import load_dotenv from scrapegraphai.graphs import ScrapeGraph -from scrapegraphai.utils import prettify_exec_info load_dotenv() diff --git a/tests/graphs/smart_scraper_fireworks_test.py b/tests/graphs/smart_scraper_fireworks_test.py index 818f15b9..cfa5e908 100644 --- a/tests/graphs/smart_scraper_fireworks_test.py +++ b/tests/graphs/smart_scraper_fireworks_test.py @@ -4,10 +4,8 @@ Module for testing the smart scraper class import os import pytest -import pandas as pd from dotenv import load_dotenv from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info load_dotenv() diff --git a/tests/graphs/smart_scraper_multi_lite_graph_openai_test.py b/tests/graphs/smart_scraper_multi_lite_graph_openai_test.py index 0a0e0a69..7e5b88de 100644 --- a/tests/graphs/smart_scraper_multi_lite_graph_openai_test.py +++ b/tests/graphs/smart_scraper_multi_lite_graph_openai_test.py @@ -4,10 +4,8 @@ Module for testing the smart scraper class import os import pytest -import pandas as pd from dotenv import load_dotenv from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info load_dotenv() diff --git a/tests/graphs/smart_scraper_openai_test.py b/tests/graphs/smart_scraper_openai_test.py index 08a60118..0d25db91 100644 --- a/tests/graphs/smart_scraper_openai_test.py +++ b/tests/graphs/smart_scraper_openai_test.py @@ -4,10 +4,8 @@ Module for testing the smart scraper class import os import pytest -import pandas as pd from dotenv import load_dotenv from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info load_dotenv()