diff --git a/scrapegraphai/nodes/search_link_node.py b/scrapegraphai/nodes/search_link_node.py index 6fbe51dd..180a7ba1 100644 --- a/scrapegraphai/nodes/search_link_node.py +++ b/scrapegraphai/nodes/search_link_node.py @@ -10,6 +10,7 @@ from langchain_core.output_parsers import JsonOutputParser from langchain_core.runnables import RunnableParallel from ..utils.logging import get_logger from .base_node import BaseNode +from ..prompts import prompt_relevant_links class SearchLinkNode(BaseNode): @@ -83,29 +84,6 @@ class SearchLinkNode(BaseNode): except Exception as e: # Fallback approach: Using the LLM to extract links self.logger.error(f"Error extracting links: {e}. Falling back to LLM.") - prompt_relevant_links = """ - You are a website scraper and you have just scraped the following content from a website. - Content: {content} - - Assume relevance broadly, including any links that might be related or potentially useful - in relation to the task. - - Sort it in order of importance, the first one should be the most important one, the last one - the least important - - Please list only valid URLs and make sure to err on the side of inclusion if it's uncertain - whether the content at the link is directly relevant. - - Output only a list of relevant links in the format: - [ - "link1", - "link2", - "link3", - . - . - . - ] - """ merge_prompt = PromptTemplate( template=prompt_relevant_links, diff --git a/scrapegraphai/prompts/__init__.py b/scrapegraphai/prompts/__init__.py index 2b32431a..b4d22985 100644 --- a/scrapegraphai/prompts/__init__.py +++ b/scrapegraphai/prompts/__init__.py @@ -8,4 +8,5 @@ from .generate_answer_node_pdf_prompts import template_chunks_pdf, template_no_c from .generate_answer_node_omni_prompts import template_chunks_omni, template_no_chunk_omni, template_merge_omni from .merge_answer_node_prompts import template_combined from .robots_node_prompts import template_robot -from .search_internet_node_prompts import search_internet_template \ No newline at end of file +from .search_internet_node_prompts import search_internet_template +from .search_link_node_prompts import prompt_relevant_links \ No newline at end of file diff --git a/scrapegraphai/prompts/search_link_node_prompts.py b/scrapegraphai/prompts/search_link_node_prompts.py new file mode 100644 index 00000000..d3bc4553 --- /dev/null +++ b/scrapegraphai/prompts/search_link_node_prompts.py @@ -0,0 +1,27 @@ +""" +Search link node prompts helper +""" + +prompt_relevant_links = """ + You are a website scraper and you have just scraped the following content from a website. + Content: {content} + + Assume relevance broadly, including any links that might be related or potentially useful + in relation to the task. + + Sort it in order of importance, the first one should be the most important one, the last one + the least important + + Please list only valid URLs and make sure to err on the side of inclusion if it's uncertain + whether the content at the link is directly relevant. + + Output only a list of relevant links in the format: + [ + "link1", + "link2", + "link3", + . + . + . + ] + """ \ No newline at end of file