CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-01 21:00:48 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: add sdk integration
Some checks failed
/ build (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-11-28 13:23:57 +01:00
parent 8aa9103f02
commit 209b4456fd
2 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pyproject.toml
View File

@ -44,7 +44,7 @@ dependencies = [
"googlesearch-python>=1.2.5", "googlesearch-python>=1.2.5",
"simpleeval>=1.0.0", "simpleeval>=1.0.0",
"async_timeout>=4.0.3", "async_timeout>=4.0.3",
"scrapegraph-py>=0.0.3" "scrapegraph-py>=0.0.4"
] ]
license = "MIT" license = "MIT"

12
scrapegraphai/graphs/smart_scraper_graph.py
View File

@ -13,7 +13,7 @@ from ..nodes import (
ConditionalNode ConditionalNode
) )
from ..prompts import REGEN_ADDITIONAL_INFO from ..prompts import REGEN_ADDITIONAL_INFO
from scrapegraph_py import ScrapeGraphClient, smart_scraper from scrapegraph_py import SyncClient
class SmartScraperGraph(AbstractGraph): class SmartScraperGraph(AbstractGraph):
""" """
@ -61,10 +61,14 @@ class SmartScraperGraph(AbstractGraph):
BaseGraph: A graph instance representing the web scraping workflow. BaseGraph: A graph instance representing the web scraping workflow.
""" """
if self.llm_model == "scrapegraphai/smart-scraper": if self.llm_model == "scrapegraphai/smart-scraper":
client = ScrapeGraphClient(self.config.get("api_key"))
result = smart_scraper(client, self.source, self.prompt) sgai_client = SyncClient(api_key=self.config.get("api_key"))
return result
response = sgai_client.smartscraper(
website_url=self.source,
user_prompt=self.prompt
)
return response
fetch_node = FetchNode( fetch_node = FetchNode(
input="url| local_dir", input="url| local_dir",