CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-06-28 21:01:55 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: add sdk integration
Some checks failed
/ build (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-11-28 13:23:57 +01:00
parent 8aa9103f02
commit 209b4456fd
2 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pyproject.toml
View File

@ -44,7 +44,7 @@ dependencies = [
"googlesearch-python>=1.2.5",
"simpleeval>=1.0.0",
"async_timeout>=4.0.3",
"scrapegraph-py>=0.0.3"
"scrapegraph-py>=0.0.4"
]
license = "MIT"

12
scrapegraphai/graphs/smart_scraper_graph.py
View File

@ -13,7 +13,7 @@ from ..nodes import (
ConditionalNode
)
from ..prompts import REGEN_ADDITIONAL_INFO
from scrapegraph_py import ScrapeGraphClient, smart_scraper
from scrapegraph_py import SyncClient
class SmartScraperGraph(AbstractGraph):
"""
@ -61,10 +61,14 @@ class SmartScraperGraph(AbstractGraph):
BaseGraph: A graph instance representing the web scraping workflow.
"""
if self.llm_model == "scrapegraphai/smart-scraper":
client = ScrapeGraphClient(self.config.get("api_key"))
result = smart_scraper(client, self.source, self.prompt)
return result
sgai_client = SyncClient(api_key=self.config.get("api_key"))
response = sgai_client.smartscraper(
website_url=self.source,
user_prompt=self.prompt
)
return response
fetch_node = FetchNode(
input="url| local_dir",