From fbb131ce159bf69b9110edd217f21f099cd9fde5 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Tue, 10 Dec 2024 15:16:21 +0100 Subject: [PATCH 01/19] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a3f18146..e16bce6c 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ Remember to have [Ollama](https://ollama.com/) installed and download the models ## šŸ” Demo Official streamlit demo: -[![My Skills](https://skillicons.dev/icons?i=react)](https://scrapegraph-ai-web-dashboard.streamlit.app) +[![My Skills](https://skillicons.dev/icons?i=react)](https://scrapegraph-demo-demo.streamlit.app) Try it directly on the web using Google Colab: From bae92b0dcca353c137f8cdb12ef95b3bc2cd962a Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Tue, 10 Dec 2024 16:30:21 +0100 Subject: [PATCH 02/19] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e16bce6c..3da2d3ce 100644 --- a/README.md +++ b/README.md @@ -203,3 +203,5 @@ ScrapeGraphAI is licensed under the MIT License. See the [LICENSE](https://githu - We would like to thank all the contributors to the project and the open-source community for their support. - ScrapeGraphAI is meant to be used for data exploration and research purposes only. We are not responsible for any misuse of the library. + +Made with ā¤ļø by [ScrapeGraph AI](https://scrapegraphai.com) From d1b2104f28d84c5129edb29a5efdaf5bf7d22bfb Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Wed, 11 Dec 2024 17:18:05 +0100 Subject: [PATCH 03/19] fix: formatting --- scrapegraphai/graphs/base_graph.py | 33 ++++++++----------- scrapegraphai/graphs/code_generator_graph.py | 1 - scrapegraphai/graphs/csv_scraper_graph.py | 2 +- scrapegraphai/graphs/depth_search_graph.py | 1 - .../graphs/document_scraper_graph.py | 1 - scrapegraphai/graphs/omni_scraper_graph.py | 1 - scrapegraphai/graphs/script_creator_graph.py | 2 -- scrapegraphai/graphs/search_graph.py | 1 - scrapegraphai/graphs/search_link_graph.py | 2 -- scrapegraphai/graphs/smart_scraper_graph.py | 1 - .../graphs/smart_scraper_lite_graph.py | 2 -- scrapegraphai/graphs/speech_graph.py | 2 -- scrapegraphai/nodes/fetch_node.py | 2 -- scrapegraphai/nodes/fetch_node_level_k.py | 1 - 14 files changed, 14 insertions(+), 38 deletions(-) diff --git a/scrapegraphai/graphs/base_graph.py b/scrapegraphai/graphs/base_graph.py index 18a16ba3..0b11ffa4 100644 --- a/scrapegraphai/graphs/base_graph.py +++ b/scrapegraphai/graphs/base_graph.py @@ -56,13 +56,11 @@ class BaseGraph: self.callback_manager = CustomLLMCallbackManager() if nodes[0].node_name != entry_point.node_name: - # raise a warning if the entry point is not the first node in the list warnings.warn( "Careful! The entry point node is different from the first node in the graph.") self._set_conditional_node_edges() - # Burr configuration self.use_burr = use_burr self.burr_config = burr_config or {} @@ -91,7 +89,8 @@ class BaseGraph: if node.node_type == 'conditional_node': outgoing_edges = [(from_node, to_node) for from_node, to_node in self.raw_edges if from_node.node_name == node.node_name] if len(outgoing_edges) != 2: - raise ValueError(f"ConditionalNode '{node.node_name}' must have exactly two outgoing edges.") + raise ValueError(f"""ConditionalNode '{node.node_name}' + must have exactly two outgoing edges.""") node.true_node_name = outgoing_edges[0][1].node_name try: node.false_node_name = outgoing_edges[1][1].node_name @@ -151,14 +150,14 @@ class BaseGraph: """Extracts schema information from the node configuration.""" if not hasattr(current_node, "node_config"): return None - + if not isinstance(current_node.node_config, dict): return None - + schema_config = current_node.node_config.get("schema") if not schema_config or isinstance(schema_config, dict): return None - + try: return schema_config.schema() except Exception: @@ -167,7 +166,7 @@ class BaseGraph: def _execute_node(self, current_node, state, llm_model, llm_model_name): """Executes a single node and returns execution information.""" curr_time = time.time() - + with self.callback_manager.exclusive_get_callback(llm_model, llm_model_name) as cb: result = current_node.execute(state) node_exec_time = time.time() - curr_time @@ -197,17 +196,17 @@ class BaseGraph: raise ValueError( f"Conditional Node returned a node name '{result}' that does not exist in the graph" ) - + return self.edges.get(current_node.node_name) def _execute_standard(self, initial_state: dict) -> Tuple[dict, list]: """ - Executes the graph by traversing nodes starting from the entry point using the standard method. + Executes the graph by traversing nodes + starting from the entry point using the standard method. """ current_node_name = self.entry_point state = initial_state - - # Tracking variables + total_exec_time = 0.0 exec_info = [] cb_total = { @@ -230,16 +229,13 @@ class BaseGraph: while current_node_name: current_node = self._get_node_by_name(current_node_name) - - # Update source information if needed + if source_type is None: source_type, source, prompt = self._update_source_info(current_node, state) - - # Get model information if needed + if llm_model is None: llm_model, llm_model_name, embedder_model = self._get_model_info(current_node) - - # Get schema if needed + if schema is None: schema = self._get_schema(current_node) @@ -273,7 +269,6 @@ class BaseGraph: ) raise e - # Add total results to execution info exec_info.append({ "node_name": "TOTAL RESULT", "total_tokens": cb_total["total_tokens"], @@ -284,7 +279,6 @@ class BaseGraph: "exec_time": total_exec_time, }) - # Log final execution results graph_execution_time = time.time() - start_time response = state.get("answer", None) if source_type == "url" else None content = state.get("parsed_doc", None) if response is not None else None @@ -343,4 +337,3 @@ class BaseGraph: self.raw_edges.append((last_node, node)) self.nodes.append(node) self.edges = self._create_edges({e for e in self.raw_edges}) - diff --git a/scrapegraphai/graphs/code_generator_graph.py b/scrapegraphai/graphs/code_generator_graph.py index fe94e9d5..359b3b1a 100644 --- a/scrapegraphai/graphs/code_generator_graph.py +++ b/scrapegraphai/graphs/code_generator_graph.py @@ -17,7 +17,6 @@ from ..nodes import ( GenerateCodeNode, ) - class CodeGeneratorGraph(AbstractGraph): """ CodeGeneratorGraph is a script generator pipeline that generates diff --git a/scrapegraphai/graphs/csv_scraper_graph.py b/scrapegraphai/graphs/csv_scraper_graph.py index a4165a9d..071bc910 100644 --- a/scrapegraphai/graphs/csv_scraper_graph.py +++ b/scrapegraphai/graphs/csv_scraper_graph.py @@ -59,7 +59,7 @@ class CSVScraperGraph(AbstractGraph): """ Creates the graph of nodes representing the workflow for web scraping. """ - + fetch_node = FetchNode( input="csv | csv_dir", output=["doc"], diff --git a/scrapegraphai/graphs/depth_search_graph.py b/scrapegraphai/graphs/depth_search_graph.py index 0df9c061..92e54de0 100644 --- a/scrapegraphai/graphs/depth_search_graph.py +++ b/scrapegraphai/graphs/depth_search_graph.py @@ -15,7 +15,6 @@ from ..nodes import ( GenerateAnswerNodeKLevel, ) - class DepthSearchGraph(AbstractGraph): """ CodeGeneratorGraph is a script generator pipeline that generates diff --git a/scrapegraphai/graphs/document_scraper_graph.py b/scrapegraphai/graphs/document_scraper_graph.py index db3244c5..58c19ed3 100644 --- a/scrapegraphai/graphs/document_scraper_graph.py +++ b/scrapegraphai/graphs/document_scraper_graph.py @@ -9,7 +9,6 @@ from .base_graph import BaseGraph from .abstract_graph import AbstractGraph from ..nodes import FetchNode, ParseNode, GenerateAnswerNode - class DocumentScraperGraph(AbstractGraph): """ DocumentScraperGraph is a scraping pipeline that automates the process of diff --git a/scrapegraphai/graphs/omni_scraper_graph.py b/scrapegraphai/graphs/omni_scraper_graph.py index 035ad6a7..a7af6bf5 100644 --- a/scrapegraphai/graphs/omni_scraper_graph.py +++ b/scrapegraphai/graphs/omni_scraper_graph.py @@ -9,7 +9,6 @@ from .abstract_graph import AbstractGraph from ..nodes import FetchNode, ParseNode, ImageToTextNode, GenerateAnswerOmniNode from ..models import OpenAIImageToText - class OmniScraperGraph(AbstractGraph): """ OmniScraper is a scraping pipeline that automates the process of diff --git a/scrapegraphai/graphs/script_creator_graph.py b/scrapegraphai/graphs/script_creator_graph.py index 1e785c92..35c6d2ba 100644 --- a/scrapegraphai/graphs/script_creator_graph.py +++ b/scrapegraphai/graphs/script_creator_graph.py @@ -1,14 +1,12 @@ """ ScriptCreatorGraph Module """ - from typing import Optional from pydantic import BaseModel from .base_graph import BaseGraph from .abstract_graph import AbstractGraph from ..nodes import FetchNode, ParseNode, GenerateScraperNode - class ScriptCreatorGraph(AbstractGraph): """ ScriptCreatorGraph defines a scraping pipeline for generating web scraping scripts. diff --git a/scrapegraphai/graphs/search_graph.py b/scrapegraphai/graphs/search_graph.py index 313cb768..2fb4b949 100644 --- a/scrapegraphai/graphs/search_graph.py +++ b/scrapegraphai/graphs/search_graph.py @@ -1,7 +1,6 @@ """ SearchGraph Module """ - from copy import deepcopy from typing import Optional, List from pydantic import BaseModel diff --git a/scrapegraphai/graphs/search_link_graph.py b/scrapegraphai/graphs/search_link_graph.py index e8baf1d8..fa1b6f18 100644 --- a/scrapegraphai/graphs/search_link_graph.py +++ b/scrapegraphai/graphs/search_link_graph.py @@ -1,7 +1,6 @@ """ SearchLinkGraph Module """ - from typing import Optional import logging from pydantic import BaseModel @@ -9,7 +8,6 @@ from .base_graph import BaseGraph from .abstract_graph import AbstractGraph from ..nodes import FetchNode, SearchLinkNode, SearchLinksWithContext - class SearchLinkGraph(AbstractGraph): """ SearchLinkGraph is a scraping pipeline that automates the process of diff --git a/scrapegraphai/graphs/smart_scraper_graph.py b/scrapegraphai/graphs/smart_scraper_graph.py index cd9e75bf..404bdcd9 100644 --- a/scrapegraphai/graphs/smart_scraper_graph.py +++ b/scrapegraphai/graphs/smart_scraper_graph.py @@ -1,7 +1,6 @@ """ SmartScraperGraph Module """ - from typing import Optional from pydantic import BaseModel from scrapegraph_py import Client diff --git a/scrapegraphai/graphs/smart_scraper_lite_graph.py b/scrapegraphai/graphs/smart_scraper_lite_graph.py index b751a8c3..fbc8a087 100644 --- a/scrapegraphai/graphs/smart_scraper_lite_graph.py +++ b/scrapegraphai/graphs/smart_scraper_lite_graph.py @@ -1,7 +1,6 @@ """ SmartScraperGraph Module """ - from typing import Optional from pydantic import BaseModel from .base_graph import BaseGraph @@ -11,7 +10,6 @@ from ..nodes import ( ParseNode, ) - class SmartScraperLiteGraph(AbstractGraph): """ SmartScraperLiteGraph is a scraping pipeline that automates the process of diff --git a/scrapegraphai/graphs/speech_graph.py b/scrapegraphai/graphs/speech_graph.py index d9d107c0..8cec90d4 100644 --- a/scrapegraphai/graphs/speech_graph.py +++ b/scrapegraphai/graphs/speech_graph.py @@ -1,7 +1,6 @@ """ SpeechGraph Module """ - from typing import Optional from pydantic import BaseModel from .base_graph import BaseGraph @@ -15,7 +14,6 @@ from ..nodes import ( from ..utils.save_audio_from_bytes import save_audio_from_bytes from ..models import OpenAITextToSpeech - class SpeechGraph(AbstractGraph): """ SpeechyGraph is a scraping pipeline that scrapes the web, provide an answer diff --git a/scrapegraphai/nodes/fetch_node.py b/scrapegraphai/nodes/fetch_node.py index 88225a20..284868ff 100644 --- a/scrapegraphai/nodes/fetch_node.py +++ b/scrapegraphai/nodes/fetch_node.py @@ -1,7 +1,6 @@ """ FetchNode Module """ - import json from typing import List, Optional from langchain_openai import ChatOpenAI, AzureChatOpenAI @@ -15,7 +14,6 @@ from ..utils.convert_to_md import convert_to_md from ..utils.logging import get_logger from .base_node import BaseNode - class FetchNode(BaseNode): """ A node responsible for fetching the HTML content of a specified URL and updating diff --git a/scrapegraphai/nodes/fetch_node_level_k.py b/scrapegraphai/nodes/fetch_node_level_k.py index 3307f129..8be392aa 100644 --- a/scrapegraphai/nodes/fetch_node_level_k.py +++ b/scrapegraphai/nodes/fetch_node_level_k.py @@ -1,7 +1,6 @@ """ fetch_node_level_k module """ - from typing import List, Optional from urllib.parse import urljoin from langchain_core.documents import Document From 488093a63fcc1dc01eabdab301d752416a025139 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 11 Dec 2024 16:20:02 +0000 Subject: [PATCH 04/19] ci(release): 1.33.3 [skip ci] ## [1.33.3](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.33.2...v1.33.3) (2024-12-11) ### Bug Fixes * formatting ([d1b2104](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/d1b2104f28d84c5129edb29a5efdaf5bf7d22bfb)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 499722c9..5d9ea70f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.33.3](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.33.2...v1.33.3) (2024-12-11) + + +### Bug Fixes + +* formatting ([d1b2104](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/d1b2104f28d84c5129edb29a5efdaf5bf7d22bfb)) + ## [1.33.2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.33.1...v1.33.2) (2024-12-06) diff --git a/pyproject.toml b/pyproject.toml index 982ff701..7beba632 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "scrapegraphai" -version = "1.33.2" +version = "1.33.3" From 2c4d06aea2964cfccff98c2de3e32b08d21c1ee6 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Sat, 14 Dec 2024 12:02:53 +0100 Subject: [PATCH 05/19] Update README.md --- README.md | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3da2d3ce..63a6b41e 100644 --- a/README.md +++ b/README.md @@ -87,8 +87,8 @@ graph_config = { # Create the SmartScraperGraph instance smart_scraper_graph = SmartScraperGraph( - prompt="Find some information about what does the company do, the name and a contact email.", - source="https://scrapegraphai.com/", + prompt="Extract me all the news from the website", + source="https://www.wired.com", config=graph_config ) @@ -100,10 +100,20 @@ print(json.dumps(result, indent=4)) The output will be a dictionary like the following: ```python -{ - "company": "ScrapeGraphAI", - "name": "ScrapeGraphAI Extracting content from websites and local documents using LLM", - "contact_email": "contact@scrapegraphai.com" +"result": { + "news": [ + { + "title": "The New Jersey Drone Mystery May Not Actually Be That Mysterious", + "link": "https://www.wired.com/story/new-jersey-drone-mystery-maybe-not-drones/", + "author": "Lily Hay Newman" + }, + { + "title": "Former ByteDance Intern Accused of Sabotage Among Winners of Prestigious AI Award", + "link": "https://www.wired.com/story/bytedance-intern-best-paper-neurips/", + "author": "Louise Matsakis" + }, + ... + ] } ``` There are other pipelines that can be used to extract information from multiple pages, generate Python scripts, or even generate audio files. From ffdadaed6fe3f17da535e6eddb73851fce2f4bf2 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Mon, 16 Dec 2024 10:54:11 +0100 Subject: [PATCH 06/19] fix: context window --- scrapegraphai/helpers/models_tokens.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapegraphai/helpers/models_tokens.py b/scrapegraphai/helpers/models_tokens.py index 705e2969..0d4d4e33 100644 --- a/scrapegraphai/helpers/models_tokens.py +++ b/scrapegraphai/helpers/models_tokens.py @@ -79,7 +79,7 @@ models_tokens = { "llama3.2": 128000, "llama3.2:1b": 128000, "scrapegraph": 8192, - "mistral": 8192, + "mistral": 4096, "mistral-small": 128000, "mistral-openorca": 32000, "mistral-large": 128000, From a78917997060edbd61df5279546587e4ef123ea1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 16 Dec 2024 09:55:19 +0000 Subject: [PATCH 07/19] ci(release): 1.33.4 [skip ci] ## [1.33.4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.33.3...v1.33.4) (2024-12-16) ### Bug Fixes * context window ([ffdadae](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/ffdadaed6fe3f17da535e6eddb73851fce2f4bf2)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d9ea70f..24cd936d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.33.4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.33.3...v1.33.4) (2024-12-16) + + +### Bug Fixes + +* context window ([ffdadae](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/ffdadaed6fe3f17da535e6eddb73851fce2f4bf2)) + ## [1.33.3](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.33.2...v1.33.3) (2024-12-11) diff --git a/pyproject.toml b/pyproject.toml index 7beba632..5a5affd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "scrapegraphai" -version = "1.33.3" +version = "1.33.4" From 0a7fc392dea2b62122b977d62f4d85b117fc8351 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Mon, 16 Dec 2024 11:01:55 +0100 Subject: [PATCH 08/19] fix: uv.lock --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index 3840481c..57b7f611 100644 --- a/uv.lock +++ b/uv.lock @@ -4081,7 +4081,7 @@ wheels = [ [[package]] name = "scrapegraphai" -version = "1.33.0" +version = "1.33.3" source = { editable = "." } dependencies = [ { name = "async-timeout", version = "4.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, From 7a6164f1dc6dbb8ff0b4f7fc653f3910445f0754 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 16 Dec 2024 10:03:07 +0000 Subject: [PATCH 09/19] ci(release): 1.33.5 [skip ci] ## [1.33.5](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.33.4...v1.33.5) (2024-12-16) ### Bug Fixes * uv.lock ([0a7fc39](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/0a7fc392dea2b62122b977d62f4d85b117fc8351)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24cd936d..c00725cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.33.5](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.33.4...v1.33.5) (2024-12-16) + + +### Bug Fixes + +* uv.lock ([0a7fc39](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/0a7fc392dea2b62122b977d62f4d85b117fc8351)) + ## [1.33.4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.33.3...v1.33.4) (2024-12-16) diff --git a/pyproject.toml b/pyproject.toml index 5a5affd2..ca7531d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "scrapegraphai" -version = "1.33.4" +version = "1.33.5" From bf6cb0a582004617724e11ed04ba617eb39abc0c Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Mon, 16 Dec 2024 11:08:39 +0100 Subject: [PATCH 10/19] fix: pyproject --- pyproject.toml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index ca7531d7..e66d5d37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,6 +118,18 @@ screenshot_scraper = [ requires = ["hatchling"] build-backend = "hatchling.build" +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.build.targets.wheel] +packages = ["scrapegraphai"] + +[tool.hatch.build.targets.sdist] +include = [ + "/scrapegraphai", + "/tests", +] + [dependency-groups] dev = [ "burr[start]==0.22.1", From ca96c3d4309bd2b92c87a2b0095578dda302ad92 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 16 Dec 2024 10:09:48 +0000 Subject: [PATCH 11/19] ci(release): 1.33.6 [skip ci] ## [1.33.6](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.33.5...v1.33.6) (2024-12-16) ### Bug Fixes * pyproject ([bf6cb0a](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/bf6cb0a582004617724e11ed04ba617eb39abc0c)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c00725cf..d86aad7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.33.6](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.33.5...v1.33.6) (2024-12-16) + + +### Bug Fixes + +* pyproject ([bf6cb0a](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/bf6cb0a582004617724e11ed04ba617eb39abc0c)) + ## [1.33.5](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.33.4...v1.33.5) (2024-12-16) diff --git a/pyproject.toml b/pyproject.toml index e66d5d37..6094c05a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "scrapegraphai" -version = "1.33.5" +version = "1.33.6" From 3dcfcd492e71297031a7df1dba9dd135f1fae60e Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Mon, 16 Dec 2024 11:11:01 +0100 Subject: [PATCH 12/19] fix: pyproject --- pyproject.toml | 32 +++++++++++++++++++++----------- scrapegraphai/__init__.py | 1 + 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6094c05a..2ba1daa5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,6 @@ [project] name = "scrapegraphai" - - - -version = "1.33.6" - - - - +version = "1.33.5" description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines." authors = [ { name = "Marco Vinciguerra", email = "mvincig11@gmail.com" }, @@ -115,11 +108,22 @@ screenshot_scraper = [ ] [build-system] -requires = ["hatchling"] +requires = ["hatchling>=1.0.0"] build-backend = "hatchling.build" -[tool.hatch.metadata] -allow-direct-references = true +[tool.hatch.version] +path = "pyproject.toml" + +[tool.hatch.build] +include = [ + "scrapegraphai/**/*.py", + "scrapegraphai/**/*.typed", + "/tests", +] +exclude = [ + "tests/**", + "examples/**", +] [tool.hatch.build.targets.wheel] packages = ["scrapegraphai"] @@ -128,8 +132,14 @@ packages = ["scrapegraphai"] include = [ "/scrapegraphai", "/tests", + "pyproject.toml", + "README.md", + "LICENSE", ] +[tool.hatch.metadata] +allow-direct-references = true + [dependency-groups] dev = [ "burr[start]==0.22.1", diff --git a/scrapegraphai/__init__.py b/scrapegraphai/__init__.py index 52b4d951..09643198 100644 --- a/scrapegraphai/__init__.py +++ b/scrapegraphai/__init__.py @@ -1,3 +1,4 @@ """ __init__.py file for scrapegraphai folder """ +__version__ = "1.33.5" From 7a5764e3fdbfea12b04ea0686a28025a9d89cb2f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 16 Dec 2024 10:12:11 +0000 Subject: [PATCH 13/19] ci(release): 1.33.7 [skip ci] ## [1.33.7](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.33.6...v1.33.7) (2024-12-16) ### Bug Fixes * pyproject ([3dcfcd4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/3dcfcd492e71297031a7df1dba9dd135f1fae60e)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d86aad7c..d601fdfb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.33.7](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.33.6...v1.33.7) (2024-12-16) + + +### Bug Fixes + +* pyproject ([3dcfcd4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/3dcfcd492e71297031a7df1dba9dd135f1fae60e)) + ## [1.33.6](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.33.5...v1.33.6) (2024-12-16) diff --git a/pyproject.toml b/pyproject.toml index 2ba1daa5..353c4fda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "scrapegraphai" -version = "1.33.5" +version = "1.33.7" description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines." authors = [ { name = "Marco Vinciguerra", email = "mvincig11@gmail.com" }, From 76ac0a2141d9d53af023a405e2c61849921e4f0e Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Mon, 16 Dec 2024 11:13:16 +0100 Subject: [PATCH 14/19] fix: pyproject --- pyproject.toml | 18 ++++++++---------- scrapegraphai/__init__.py | 2 +- scrapegraphai/_version.py | 3 +++ 3 files changed, 12 insertions(+), 11 deletions(-) create mode 100644 scrapegraphai/_version.py diff --git a/pyproject.toml b/pyproject.toml index 353c4fda..6744ef50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,30 +108,28 @@ screenshot_scraper = [ ] [build-system] -requires = ["hatchling>=1.0.0"] +requires = ["hatchling>=1.0.0", "hatch-vcs"] build-backend = "hatchling.build" -[tool.hatch.version] -path = "pyproject.toml" - [tool.hatch.build] -include = [ - "scrapegraphai/**/*.py", - "scrapegraphai/**/*.typed", - "/tests", -] +packages = ["scrapegraphai"] exclude = [ "tests/**", "examples/**", ] +[tool.hatch.version] +source = "vcs" + +[tool.hatch.build.hooks.vcs] +version-file = "scrapegraphai/_version.py" + [tool.hatch.build.targets.wheel] packages = ["scrapegraphai"] [tool.hatch.build.targets.sdist] include = [ "/scrapegraphai", - "/tests", "pyproject.toml", "README.md", "LICENSE", diff --git a/scrapegraphai/__init__.py b/scrapegraphai/__init__.py index 09643198..7ab88b26 100644 --- a/scrapegraphai/__init__.py +++ b/scrapegraphai/__init__.py @@ -1,4 +1,4 @@ """ __init__.py file for scrapegraphai folder """ -__version__ = "1.33.5" +__version__ = "1.33.7" diff --git a/scrapegraphai/_version.py b/scrapegraphai/_version.py new file mode 100644 index 00000000..54ca2ed0 --- /dev/null +++ b/scrapegraphai/_version.py @@ -0,0 +1,3 @@ +"""Version information.""" +__version__ = "1.33.7" +version = __version__ \ No newline at end of file From bdd6a392e2c18de8c3e4e47e2f91a4a366365ff2 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 16 Dec 2024 10:14:31 +0000 Subject: [PATCH 15/19] ci(release): 1.33.8 [skip ci] ## [1.33.8](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.33.7...v1.33.8) (2024-12-16) ### Bug Fixes * pyproject ([76ac0a2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/76ac0a2141d9d53af023a405e2c61849921e4f0e)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d601fdfb..ee548274 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.33.8](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.33.7...v1.33.8) (2024-12-16) + + +### Bug Fixes + +* pyproject ([76ac0a2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/76ac0a2141d9d53af023a405e2c61849921e4f0e)) + ## [1.33.7](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.33.6...v1.33.7) (2024-12-16) diff --git a/pyproject.toml b/pyproject.toml index 6744ef50..07c3113e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "scrapegraphai" -version = "1.33.7" +version = "1.33.8" description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines." authors = [ { name = "Marco Vinciguerra", email = "mvincig11@gmail.com" }, From a4f0f5d16d7a2f218b51f198a1ad00a6edfcca7e Mon Sep 17 00:00:00 2001 From: SwapnilSonker Date: Mon, 16 Dec 2024 20:26:20 +0530 Subject: [PATCH 16/19] Add function to select backend (Selenium or other) for issue #171 --- scrapegraphai/docloaders/chromium.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scrapegraphai/docloaders/chromium.py b/scrapegraphai/docloaders/chromium.py index 942827ac..54493887 100644 --- a/scrapegraphai/docloaders/chromium.py +++ b/scrapegraphai/docloaders/chromium.py @@ -65,6 +65,15 @@ class ChromiumLoader(BaseLoader): self.load_state = load_state self.requires_js_support = requires_js_support self.storage_state = storage_state + + async def scrape(self, url:str) -> str: + if self.backend == "playwright": + return await self.ascrape_playwright(url) + elif self.backend == "selenium": + return await self.ascrape_undetected_chromedriver(url) + else: + raise ValueError(f"Unsupported backend: {self.backend}") + async def ascrape_undetected_chromedriver(self, url: str) -> str: """ From 753737e655b89bcb7cbf15aa69215fdc58a5c3ae Mon Sep 17 00:00:00 2001 From: SwapnilSonker Date: Tue, 17 Dec 2024 08:08:10 +0530 Subject: [PATCH 17/19] added the example in example/extras #171 --- examples/extras/chromium_selenium.py | 57 ++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 examples/extras/chromium_selenium.py diff --git a/examples/extras/chromium_selenium.py b/examples/extras/chromium_selenium.py new file mode 100644 index 00000000..e8ade7c0 --- /dev/null +++ b/examples/extras/chromium_selenium.py @@ -0,0 +1,57 @@ +import asyncio +from chromium import ChromiumLoader # Import the ChromiumLoader class from chromium.py +from aiohttp import ClientError + + +async def test_scraper(scraper: ChromiumLoader, urls: list): + """ + Test scraper for the given backend and URLs. + Args: + scraper (ChromiumLoader): The ChromiumLoader instance. + urls (list): A list of URLs to scrape. + """ + for url in urls: + try: + print(f"Scraping: {url} using {scraper.backend}...") + result = await scraper.scrape(url) + if "Error" in result or not result.strip(): + print(f"āŒ Failed to scrape {url}: {result}") + else: + print(f"āœ… Successfully scraped {url}. Content (first 200 chars): {result[:200]}") + except ClientError as ce: + print(f"āŒ Network error while scraping {url}: {ce}") + except Exception as e: + print(f"āŒ Unexpected error while scraping {url}: {e}") + + +async def main(): + urls_to_scrape = ["https://example.com", "https://www.python.org", "https://invalid-url.test"] + + # Test with Playwright backend + print("\n--- Testing Playwright Backend ---") + try: + scraper_playwright = ChromiumLoader(urls=urls_to_scrape, backend="playwright", headless=True) + await test_scraper(scraper_playwright, urls_to_scrape) + except ImportError as ie: + print(f"āŒ Playwright ImportError: {ie}") + except Exception as e: + print(f"āŒ Error initializing Playwright ChromiumLoader: {e}") + + # Test with Selenium backend + print("\n--- Testing Selenium Backend ---") + try: + scraper_selenium = ChromiumLoader(urls=urls_to_scrape, backend="selenium", headless=True) + await test_scraper(scraper_selenium, urls_to_scrape) + except ImportError as ie: + print(f"āŒ Selenium ImportError: {ie}") + except Exception as e: + print(f"āŒ Error initializing Selenium ChromiumLoader: {e}") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + print("Program interrupted by user.") + except Exception as e: + print(f"āŒ Program crashed: {e}") From fe66e3d28dcab063d79c2b21bd8850adb31f2f50 Mon Sep 17 00:00:00 2001 From: SwapnilSonker Date: Tue, 17 Dec 2024 15:58:19 +0530 Subject: [PATCH 18/19] #171 used ScrapegraphAI library and created a more consistent example. --- examples/extras/chromium_selenium.py | 76 ++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 10 deletions(-) diff --git a/examples/extras/chromium_selenium.py b/examples/extras/chromium_selenium.py index e8ade7c0..fb7686a1 100644 --- a/examples/extras/chromium_selenium.py +++ b/examples/extras/chromium_selenium.py @@ -1,37 +1,94 @@ import asyncio -from chromium import ChromiumLoader # Import the ChromiumLoader class from chromium.py +import os +import json +from dotenv import load_dotenv +from chromium import ChromiumLoader # Import your ChromiumLoader class +from scrapegraphai.graphs import SmartScraperGraph +from scrapegraphai.utils import prettify_exec_info from aiohttp import ClientError +# Load environment variables for API keys +load_dotenv() -async def test_scraper(scraper: ChromiumLoader, urls: list): +# ************************************************ +# Define function to analyze content with ScrapegraphAI +# ************************************************ +async def analyze_content_with_scrapegraph(content: str): """ - Test scraper for the given backend and URLs. + Analyze scraped content using ScrapegraphAI. + + Args: + content (str): The scraped HTML or text content. + + Returns: + dict: The result from ScrapegraphAI analysis. + """ + try: + # Initialize ScrapegraphAI SmartScraperGraph + smart_scraper = SmartScraperGraph( + prompt="Summarize the main content of this webpage and extract any contact information.", + source=content, # Pass the content directly + config={ + "llm": { + "api_key": os.getenv("OPENAI_API_KEY"), + "model": "openai/gpt-4o", + }, + "verbose": True + } + ) + result = smart_scraper.run() + return result + except Exception as e: + print(f"āŒ ScrapegraphAI analysis failed: {e}") + return {"error": str(e)} + +# ************************************************ +# Test scraper and ScrapegraphAI pipeline +# ************************************************ +async def test_scraper_with_analysis(scraper: ChromiumLoader, urls: list): + """ + Test scraper for the given backend and URLs, then analyze content with ScrapegraphAI. + Args: scraper (ChromiumLoader): The ChromiumLoader instance. urls (list): A list of URLs to scrape. """ for url in urls: try: - print(f"Scraping: {url} using {scraper.backend}...") + print(f"\nšŸ”Ž Scraping: {url} using {scraper.backend}...") result = await scraper.scrape(url) + if "Error" in result or not result.strip(): print(f"āŒ Failed to scrape {url}: {result}") else: print(f"āœ… Successfully scraped {url}. Content (first 200 chars): {result[:200]}") + + # Pass scraped content to ScrapegraphAI for analysis + print("šŸ¤– Analyzing content with ScrapegraphAI...") + analysis_result = await analyze_content_with_scrapegraph(result) + print("šŸ“ Analysis Result:") + print(json.dumps(analysis_result, indent=4)) + except ClientError as ce: print(f"āŒ Network error while scraping {url}: {ce}") except Exception as e: print(f"āŒ Unexpected error while scraping {url}: {e}") - +# ************************************************ +# Main Execution +# ************************************************ async def main(): - urls_to_scrape = ["https://example.com", "https://www.python.org", "https://invalid-url.test"] + urls_to_scrape = [ + "https://example.com", + "https://www.python.org", + "https://invalid-url.test" + ] # Test with Playwright backend print("\n--- Testing Playwright Backend ---") try: scraper_playwright = ChromiumLoader(urls=urls_to_scrape, backend="playwright", headless=True) - await test_scraper(scraper_playwright, urls_to_scrape) + await test_scraper_with_analysis(scraper_playwright, urls_to_scrape) except ImportError as ie: print(f"āŒ Playwright ImportError: {ie}") except Exception as e: @@ -41,17 +98,16 @@ async def main(): print("\n--- Testing Selenium Backend ---") try: scraper_selenium = ChromiumLoader(urls=urls_to_scrape, backend="selenium", headless=True) - await test_scraper(scraper_selenium, urls_to_scrape) + await test_scraper_with_analysis(scraper_selenium, urls_to_scrape) except ImportError as ie: print(f"āŒ Selenium ImportError: {ie}") except Exception as e: print(f"āŒ Error initializing Selenium ChromiumLoader: {e}") - if __name__ == "__main__": try: asyncio.run(main()) except KeyboardInterrupt: - print("Program interrupted by user.") + print("āŒ Program interrupted by user.") except Exception as e: print(f"āŒ Program crashed: {e}") From cbc75add0d92a64244c47165b9900354c2ab6221 Mon Sep 17 00:00:00 2001 From: SwapnilSonker Date: Tue, 17 Dec 2024 17:43:03 +0530 Subject: [PATCH 19/19] #171 imported ChromiumLoader from ScrapegraphAI --- examples/extras/chromium_selenium.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/extras/chromium_selenium.py b/examples/extras/chromium_selenium.py index fb7686a1..5e647bce 100644 --- a/examples/extras/chromium_selenium.py +++ b/examples/extras/chromium_selenium.py @@ -2,7 +2,7 @@ import asyncio import os import json from dotenv import load_dotenv -from chromium import ChromiumLoader # Import your ChromiumLoader class +from scrapegraphai.docloaders.chromium import ChromiumLoader # Import your ChromiumLoader class from scrapegraphai.graphs import SmartScraperGraph from scrapegraphai.utils import prettify_exec_info from aiohttp import ClientError