diff --git a/examples/integrations/indexify_node_example.py b/examples/integrations/indexify_node_example.py new file mode 100644 index 00000000..07a184ec --- /dev/null +++ b/examples/integrations/indexify_node_example.py @@ -0,0 +1,72 @@ +""" +Basic example of scraping pipeline using SmartScraper with schema +""" + +import os, json +from typing import List + +from dotenv import load_dotenv +load_dotenv() + +from pydantic import BaseModel, Field +from scrapegraphai.graphs import SmartScraperGraph +from scrapegraphai.integrations import IndexifyNode + + +# ************************************************ +# Define the output schema for the graph +# ************************************************ + +class Image(BaseModel): + url: str = Field(description="The url of the image") + +class Images(BaseModel): + images: List[Image] + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +openai_key = os.getenv("OPENAI_APIKEY") + +graph_config = { + "llm": { + "api_key":openai_key, + "model": "gpt-3.5-turbo", + }, + "verbose": True, + "headless": False, +} + +# ************************************************ +# Define the custom nodes for the graph +# ************************************************ + +indexify_node = IndexifyNode( + input="answer & img_urls", + output=["is_indexed"], + node_config={ + "verbose": True + } +) + +# ************************************************ +# Create the SmartScraperGraph instance +# ************************************************ + +smart_scraper_graph = SmartScraperGraph( + prompt="List me all the images with their url", + source="https://giphy.com/", + schema=Images, + config=graph_config +) + +# Add the custom node to the graph +smart_scraper_graph.append_node(indexify_node) + +# ************************************************ +# Run the SmartScraperGraph +# ************************************************ + +result = smart_scraper_graph.run() +print(json.dumps(result, indent=2)) diff --git a/scrapegraphai/integrations/__init__.py b/scrapegraphai/integrations/__init__.py index 556ccc2f..be6b4bf7 100644 --- a/scrapegraphai/integrations/__init__.py +++ b/scrapegraphai/integrations/__init__.py @@ -2,4 +2,5 @@ Init file for integrations module """ -from .burr_bridge import BurrBridge \ No newline at end of file +from .burr_bridge import BurrBridge +from .indexify_node import IndexifyNode \ No newline at end of file diff --git a/scrapegraphai/integrations/indexify_node.py b/scrapegraphai/integrations/indexify_node.py new file mode 100644 index 00000000..e12adc69 --- /dev/null +++ b/scrapegraphai/integrations/indexify_node.py @@ -0,0 +1,79 @@ +""" +IndexifyNode Module +""" + +from typing import List, Optional + +from ..utils.logging import get_logger +from ..nodes.base_node import BaseNode + +# try: +# import indexify +# except ImportError: +# raise ImportError("indexify package is not installed. Please install it with 'pip install scrapegraphai[indexify]'") + + +class IndexifyNode(BaseNode): + """ + A node responsible for indexing the content present in the state. + + Attributes: + verbose (bool): A flag indicating whether to show print statements during execution. + + Args: + input (str): Boolean expression defining the input keys needed from the state. + output (List[str]): List of output keys to be updated in the state. + node_config (dict): Additional configuration for the node. + node_name (str): The unique identifier name for the node, defaulting to "Parse". + """ + + def __init__( + self, + input: str, + output: List[str], + node_config: Optional[dict] = None, + node_name: str = "Indexify", + ): + super().__init__(node_name, "node", input, output, 2, node_config) + + self.verbose = ( + False if node_config is None else node_config.get("verbose", False) + ) + + def execute(self, state: dict) -> dict: + """ + Executes the node's logic to index the content present in the state. + + Args: + state (dict): The current state of the graph. The input keys will be used to fetch the + correct data from the state. + + Returns: + dict: The updated state with the output key containing the parsed content chunks. + + Raises: + KeyError: If the input keys are not found in the state, indicating that the + necessary information for parsing the content is missing. + """ + + self.logger.info(f"--- Executing {self.node_name} Node ---") + + # Interpret input keys based on the provided input expression + # input_keys length matches the min_input_len parameter in the __init__ method + # e.g. "answer & parsed_doc" or "answer | img_urls" + + input_keys = self.get_input_keys(state) + + # Fetching data from the state based on the input keys + input_data = [state[key] for key in input_keys] + + answer = input_data[0] + img_urls = input_data[1] + + # Indexify the content + # ... + + isIndexified = True + state.update({self.output[0]: isIndexified}) + + return state