From 209b4456fd668d9d124fd5586b32a4be677d4bf8 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Thu, 28 Nov 2024 13:23:57 +0100 Subject: [PATCH] feat: add sdk integration --- pyproject.toml | 2 +- scrapegraphai/graphs/smart_scraper_graph.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f99e484b..cfe73ead 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ "googlesearch-python>=1.2.5", "simpleeval>=1.0.0", "async_timeout>=4.0.3", - "scrapegraph-py>=0.0.3" + "scrapegraph-py>=0.0.4" ] license = "MIT" diff --git a/scrapegraphai/graphs/smart_scraper_graph.py b/scrapegraphai/graphs/smart_scraper_graph.py index f6316ec6..340f69bb 100644 --- a/scrapegraphai/graphs/smart_scraper_graph.py +++ b/scrapegraphai/graphs/smart_scraper_graph.py @@ -13,7 +13,7 @@ from ..nodes import ( ConditionalNode ) from ..prompts import REGEN_ADDITIONAL_INFO -from scrapegraph_py import ScrapeGraphClient, smart_scraper +from scrapegraph_py import SyncClient class SmartScraperGraph(AbstractGraph): """ @@ -61,10 +61,14 @@ class SmartScraperGraph(AbstractGraph): BaseGraph: A graph instance representing the web scraping workflow. """ if self.llm_model == "scrapegraphai/smart-scraper": - client = ScrapeGraphClient(self.config.get("api_key")) - result = smart_scraper(client, self.source, self.prompt) - return result + sgai_client = SyncClient(api_key=self.config.get("api_key")) + + response = sgai_client.smartscraper( + website_url=self.source, + user_prompt=self.prompt + ) + return response fetch_node = FetchNode( input="url| local_dir",