diff --git a/examples/extras/reasoning.py b/examples/extras/reasoning.py new file mode 100644 index 00000000..80e57faa --- /dev/null +++ b/examples/extras/reasoning.py @@ -0,0 +1,46 @@ +""" +Basic example of scraping pipeline using SmartScraper +""" + +import os +import json +from dotenv import load_dotenv +from scrapegraphai.graphs import SmartScraperGraph +from scrapegraphai.utils import prettify_exec_info + +load_dotenv() + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + + +graph_config = { + "llm": { + "api_key": os.getenv("OPENAI_API_KEY"), + "model": "openai/gpt-4o", + }, + "reasoning": True, + "verbose": True, + "headless": False, +} + +# ************************************************ +# Create the SmartScraperGraph instance and run it +# ************************************************ + +smart_scraper_graph = SmartScraperGraph( + prompt="List me what does the company do, the name and a contact email.", + source="https://scrapegraphai.com/", + config=graph_config +) + +result = smart_scraper_graph.run() +print(json.dumps(result, indent=4)) + +# ************************************************ +# Get graph execution info +# ************************************************ + +graph_exec_info = smart_scraper_graph.get_execution_info() +print(prettify_exec_info(graph_exec_info)) diff --git a/scrapegraphai/graphs/smart_scraper_graph.py b/scrapegraphai/graphs/smart_scraper_graph.py index 0c025c3a..95c2b460 100644 --- a/scrapegraphai/graphs/smart_scraper_graph.py +++ b/scrapegraphai/graphs/smart_scraper_graph.py @@ -9,6 +9,7 @@ from .abstract_graph import AbstractGraph from ..nodes import ( FetchNode, ParseNode, + ReasoningNode, GenerateAnswerNode ) @@ -88,6 +89,33 @@ class SmartScraperGraph(AbstractGraph): } ) + if self.config.get("reasoning"): + reasoning_node = ReasoningNode( + input="user_prompt & (relevant_chunks | parsed_doc | doc)", + output=["answer"], + node_config={ + "llm_model": self.llm_model, + "additional_info": self.config.get("additional_info"), + "schema": self.schema, + } + ) + + return BaseGraph( + nodes=[ + fetch_node, + parse_node, + reasoning_node, + generate_answer_node, + ], + edges=[ + (fetch_node, parse_node), + (parse_node, reasoning_node), + (reasoning_node, generate_answer_node) + ], + entry_point=fetch_node, + graph_name=self.__class__.__name__ + ) + return BaseGraph( nodes=[ fetch_node, diff --git a/scrapegraphai/nodes/__init__.py b/scrapegraphai/nodes/__init__.py index 2a0f261a..7ed99808 100644 --- a/scrapegraphai/nodes/__init__.py +++ b/scrapegraphai/nodes/__init__.py @@ -26,4 +26,4 @@ from .concat_answers_node import ConcatAnswersNode from .prompt_refiner_node import PromptRefinerNode from .html_analyzer_node import HtmlAnalyzerNode from .generate_code_node import GenerateCodeNode -from .reasoning_node import ReasoningNode \ No newline at end of file +from .reasoning_node import ReasoningNode diff --git a/scrapegraphai/nodes/reasoning_node.py b/scrapegraphai/nodes/reasoning_node.py index 431d8ab1..6b91155c 100644 --- a/scrapegraphai/nodes/reasoning_node.py +++ b/scrapegraphai/nodes/reasoning_node.py @@ -50,12 +50,13 @@ class ReasoningNode(BaseNode): ) self.additional_info = node_config.get("additional_info", None) - + self.output_schema = node_config.get("schema") def execute(self, state: dict) -> dict: """ - Generate a refined prompt for the reasoning task based on the user's input and the JSON schema. + Generate a refined prompt for the reasoning task based + on the user's input and the JSON schema. Args: state (dict): The current state of the graph. The input keys will be used @@ -70,11 +71,11 @@ class ReasoningNode(BaseNode): """ self.logger.info(f"--- Executing {self.node_name} Node ---") - + user_prompt = state['user_prompt'] self.simplefied_schema = transform_schema(self.output_schema.schema()) - + if self.additional_info is not None: prompt = PromptTemplate( template=TEMPLATE_REASONING_WITH_CONTEXT, diff --git a/scrapegraphai/prompts/reasoning_node_prompts.py b/scrapegraphai/prompts/reasoning_node_prompts.py index 47ceaa41..d9caf937 100644 --- a/scrapegraphai/prompts/reasoning_node_prompts.py +++ b/scrapegraphai/prompts/reasoning_node_prompts.py @@ -31,7 +31,7 @@ This analysis will be used to instruct an LLM that has the HTML content in its c **Reasoning Output**: [Your detailed analysis based on the above instructions] """ - + TEMPLATE_REASONING_WITH_CONTEXT = """ **Task**: Analyze the user's request and the provided JSON schema to guide an LLM in extracting information directly from a markdown file previously parsed froma a HTML file.