diff --git a/scrapegraphai/graphs/csv_scraper_graph.py b/scrapegraphai/graphs/csv_scraper_graph.py index 42153be5..48d84c18 100644 --- a/scrapegraphai/graphs/csv_scraper_graph.py +++ b/scrapegraphai/graphs/csv_scraper_graph.py @@ -13,8 +13,38 @@ from ..nodes import ( class CSVScraperGraph(AbstractGraph): """ - SmartScraper is a comprehensive web scraping tool that automates the process of extracting - information from web pages using a natural language model to interpret and answer prompts. + A class representing a graph for extracting information from CSV files. + + Attributes: + prompt (str): The prompt used to generate an answer. + source (str): The source of the data, which can be either a CSV + file or a directory containing multiple CSV files. + config (dict): Additional configuration parameters needed by some nodes in the graph. + + Methods: + __init__ (prompt: str, source: str, config: dict, schema: Optional[BaseModel] = None): + Initializes the CSVScraperGraph with a prompt, source, and configuration. + + __init__ initializes the CSVScraperGraph class. It requires the user's prompt as input, + along with the source of the data (which can be either a single CSV file or a directory + containing multiple CSV files), and any necessary configuration parameters. + + Methods: + _create_graph (): Creates the graph of nodes representing the workflow for web scraping. + + _create_graph generates the web scraping process workflow + represented by a directed acyclic graph. + This method is used internally to create the scraping pipeline + without having to execute it immediately. The result is a BaseGraph instance + containing nodes that fetch and process data from a source, and other helper functions. + + Methods: + run () -> str: Executes the web scraping process and returns + the answer to the prompt as a string. + run runs the CSVScraperGraph class to extract information from a CSV file based + on the user's prompt. It requires no additional arguments since all necessary data + is stored within the class instance. The method fetches the relevant chunks of text or speech, + generates an answer based on these chunks, and returns this answer as a string. """ def __init__(self, prompt: str, source: str, config: dict, schema: Optional[BaseModel] = None): diff --git a/scrapegraphai/graphs/pdf_scraper_multi_graph.py b/scrapegraphai/graphs/pdf_scraper_multi_graph.py index 6803e27a..a7386267 100644 --- a/scrapegraphai/graphs/pdf_scraper_multi_graph.py +++ b/scrapegraphai/graphs/pdf_scraper_multi_graph.py @@ -41,7 +41,8 @@ class PdfScraperMultiGraph(AbstractGraph): >>> result = search_graph.run() """ - def __init__(self, prompt: str, source: List[str], config: dict, schema: Optional[BaseModel] = None): + def __init__(self, prompt: str, source: List[str], + config: dict, schema: Optional[BaseModel] = None): if all(isinstance(value, str) for value in config.values()): self.copy_config = copy(config) diff --git a/scrapegraphai/graphs/xml_scraper_multi_graph.py b/scrapegraphai/graphs/xml_scraper_multi_graph.py index 36831580..8050d50c 100644 --- a/scrapegraphai/graphs/xml_scraper_multi_graph.py +++ b/scrapegraphai/graphs/xml_scraper_multi_graph.py @@ -43,7 +43,8 @@ class XMLScraperMultiGraph(AbstractGraph): >>> result = search_graph.run() """ - def __init__(self, prompt: str, source: List[str], config: dict, schema: Optional[BaseModel] = None): + def __init__(self, prompt: str, source: List[str], + config: dict, schema: Optional[BaseModel] = None): if all(isinstance(value, str) for value in config.values()): self.copy_config = copy(config) diff --git a/scrapegraphai/telemetry/telemetry.py b/scrapegraphai/telemetry/telemetry.py index 2289afd0..c68c0d08 100644 --- a/scrapegraphai/telemetry/telemetry.py +++ b/scrapegraphai/telemetry/telemetry.py @@ -188,4 +188,4 @@ def capture_function_usage(call_fn: Callable) -> Callable: log_event("function_usage", {"function_name": function_name}) except Exception as e: logger.debug(f"Failed to send telemetry for function usage. Encountered: {e}") - return wrapped_fn \ No newline at end of file + return wrapped_fn diff --git a/scrapegraphai/utils/cleanup_html.py b/scrapegraphai/utils/cleanup_html.py index 8a0fc269..23c9f803 100644 --- a/scrapegraphai/utils/cleanup_html.py +++ b/scrapegraphai/utils/cleanup_html.py @@ -1,9 +1,9 @@ """ Module for minimizing the code """ +from urllib.parse import urljoin from bs4 import BeautifulSoup from minify_html import minify -from urllib.parse import urljoin def cleanup_html(html_content: str, base_url: str) -> str: """ diff --git a/scrapegraphai/utils/convert_to_md.py b/scrapegraphai/utils/convert_to_md.py index 1db7f037..123f3457 100644 --- a/scrapegraphai/utils/convert_to_md.py +++ b/scrapegraphai/utils/convert_to_md.py @@ -1,5 +1,5 @@ """ -convert_to_md modul +convert_to_md module """ from urllib.parse import urlparse import html2text