From 8fba7e5490f916b325588443bba3fff5c0733c17 Mon Sep 17 00:00:00 2001 From: "EURAC\\marperini" Date: Tue, 30 Apr 2024 14:28:30 +0200 Subject: [PATCH] feat(refactor): changed variable names --- examples/gemini/json_scraper_gemini.py | 12 ++++++------ examples/gemini/xml_scraper_openai.py | 12 ++++++------ examples/local_models/Docker/json_scraper_docker.py | 12 ++++++------ examples/local_models/Docker/xml_scraper_docker.py | 12 ++++++------ examples/local_models/Ollama/json_scraper_ollama.py | 12 ++++++------ examples/local_models/Ollama/xml_scraper_ollama.py | 12 ++++++------ examples/local_models/result.json | 1 + examples/openai/json_scraper_openai.py | 12 ++++++------ examples/openai/result.json | 1 + examples/openai/xml_scraper_openai.py | 12 ++++++------ scrapegraphai/graphs/__init__.py | 4 ++-- scrapegraphai/graphs/json_scraper_graph.py | 2 +- scrapegraphai/graphs/xml_scraper_graph.py | 2 +- 13 files changed, 54 insertions(+), 52 deletions(-) create mode 100644 examples/local_models/result.json create mode 100644 examples/openai/result.json diff --git a/examples/gemini/json_scraper_gemini.py b/examples/gemini/json_scraper_gemini.py index 52592dd5..b038657c 100644 --- a/examples/gemini/json_scraper_gemini.py +++ b/examples/gemini/json_scraper_gemini.py @@ -1,10 +1,10 @@ """ -Basic example of scraping pipeline using SmartScraper from JSON documents +Basic example of scraping pipeline using JSONScraperGraph from JSON documents """ import os from dotenv import load_dotenv -from scrapegraphai.graphs import JsonScraperGraph +from scrapegraphai.graphs import JSONScraperGraph from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info load_dotenv() @@ -33,23 +33,23 @@ graph_config = { } # ************************************************ -# Create the JsonScraperGraph instance and run it +# Create the JSONScraperGraph instance and run it # ************************************************ -smart_scraper_graph = JsonScraperGraph( +json_scraper_graph = JSONScraperGraph( prompt="List me all the authors, title and genres of the books", source=text, # Pass the content of the file, not the file object config=graph_config ) -result = smart_scraper_graph.run() +result = json_scraper_graph.run() print(result) # ************************************************ # Get graph execution info # ************************************************ -graph_exec_info = smart_scraper_graph.get_execution_info() +graph_exec_info = json_scraper_graph.get_execution_info() print(prettify_exec_info(graph_exec_info)) # Save to json or csv diff --git a/examples/gemini/xml_scraper_openai.py b/examples/gemini/xml_scraper_openai.py index fde8246e..e82458ed 100644 --- a/examples/gemini/xml_scraper_openai.py +++ b/examples/gemini/xml_scraper_openai.py @@ -1,10 +1,10 @@ """ -Basic example of scraping pipeline using XmlScraperGraph from XML documents +Basic example of scraping pipeline using XMLScraperGraph from XML documents """ import os from dotenv import load_dotenv -from scrapegraphai.graphs import XmlScraperGraph +from scrapegraphai.graphs import XMLScraperGraph from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info load_dotenv() @@ -33,23 +33,23 @@ graph_config = { } # ************************************************ -# Create the XmlScraperGraph instance and run it +# Create the XMLScraperGraph instance and run it # ************************************************ -smart_scraper_graph = XmlScraperGraph( +xml_scraper_graph = XMLScraperGraph( prompt="List me all the authors, title and genres of the books", source=text, # Pass the content of the file, not the file object config=graph_config ) -result = smart_scraper_graph.run() +result = xml_scraper_graph.run() print(result) # ************************************************ # Get graph execution info # ************************************************ -graph_exec_info = smart_scraper_graph.get_execution_info() +graph_exec_info = xml_scraper_graph.get_execution_info() print(prettify_exec_info(graph_exec_info)) # Save to json or csv diff --git a/examples/local_models/Docker/json_scraper_docker.py b/examples/local_models/Docker/json_scraper_docker.py index f94cfa4d..758de09e 100644 --- a/examples/local_models/Docker/json_scraper_docker.py +++ b/examples/local_models/Docker/json_scraper_docker.py @@ -1,10 +1,10 @@ """ -Basic example of scraping pipeline using JsonScraperGraph from JSON documents +Basic example of scraping pipeline using JSONScraperGraph from JSON documents """ import os from dotenv import load_dotenv -from scrapegraphai.graphs import JsonScraperGraph +from scrapegraphai.graphs import JSONScraperGraph from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info load_dotenv() @@ -37,23 +37,23 @@ graph_config = { } # ************************************************ -# Create the JsonScraperGraph instance and run it +# Create the JSONScraperGraph instance and run it # ************************************************ -smart_scraper_graph = JsonScraperGraph( +json_scraper_graph = JSONScraperGraph( prompt="List me all the authors, title and genres of the books", source=text, # Pass the content of the file, not the file object config=graph_config ) -result = smart_scraper_graph.run() +result = json_scraper_graph.run() print(result) # ************************************************ # Get graph execution info # ************************************************ -graph_exec_info = smart_scraper_graph.get_execution_info() +graph_exec_info = json_scraper_graph.get_execution_info() print(prettify_exec_info(graph_exec_info)) # Save to json or csv diff --git a/examples/local_models/Docker/xml_scraper_docker.py b/examples/local_models/Docker/xml_scraper_docker.py index c52a7d0d..6a8c86cc 100644 --- a/examples/local_models/Docker/xml_scraper_docker.py +++ b/examples/local_models/Docker/xml_scraper_docker.py @@ -1,10 +1,10 @@ """ -Basic example of scraping pipeline using XmlScraperGraph from XML documents +Basic example of scraping pipeline using XMLScraperGraph from XML documents """ import os from dotenv import load_dotenv -from scrapegraphai.graphs import XmlScraperGraph +from scrapegraphai.graphs import XMLScraperGraph from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info load_dotenv() @@ -37,23 +37,23 @@ graph_config = { } # ************************************************ -# Create the XmlScraperGraph instance and run it +# Create the XMLScraperGraph instance and run it # ************************************************ -smart_scraper_graph = XmlScraperGraph( +xml_scraper_graph = XMLScraperGraph( prompt="List me all the authors, title and genres of the books", source=text, # Pass the content of the file, not the file object config=graph_config ) -result = smart_scraper_graph.run() +result = xml_scraper_graph.run() print(result) # ************************************************ # Get graph execution info # ************************************************ -graph_exec_info = smart_scraper_graph.get_execution_info() +graph_exec_info = xml_scraper_graph.get_execution_info() print(prettify_exec_info(graph_exec_info)) # Save to json or csv diff --git a/examples/local_models/Ollama/json_scraper_ollama.py b/examples/local_models/Ollama/json_scraper_ollama.py index 42d38753..90c4a151 100644 --- a/examples/local_models/Ollama/json_scraper_ollama.py +++ b/examples/local_models/Ollama/json_scraper_ollama.py @@ -1,10 +1,10 @@ """ -Basic example of scraping pipeline using JsonScraperGraph from JSON documents +Basic example of scraping pipeline using JSONScraperGraph from JSON documents """ import os from dotenv import load_dotenv -from scrapegraphai.graphs import JsonScraperGraph +from scrapegraphai.graphs import JSONScraperGraph from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info load_dotenv() @@ -39,23 +39,23 @@ graph_config = { } # ************************************************ -# Create the XmlScraperGraph instance and run it +# Create the JSONScraperGraph instance and run it # ************************************************ -smart_scraper_graph = JsonScraperGraph( +json_scraper_graph = JSONScraperGraph( prompt="List me all the authors, title and genres of the books", source=text, # Pass the content of the file, not the file object config=graph_config ) -result = smart_scraper_graph.run() +result = json_scraper_graph.run() print(result) # ************************************************ # Get graph execution info # ************************************************ -graph_exec_info = smart_scraper_graph.get_execution_info() +graph_exec_info = json_scraper_graph.get_execution_info() print(prettify_exec_info(graph_exec_info)) # Save to json or csv diff --git a/examples/local_models/Ollama/xml_scraper_ollama.py b/examples/local_models/Ollama/xml_scraper_ollama.py index 46f2564d..4c149a2b 100644 --- a/examples/local_models/Ollama/xml_scraper_ollama.py +++ b/examples/local_models/Ollama/xml_scraper_ollama.py @@ -1,10 +1,10 @@ """ -Basic example of scraping pipeline using XmlScraperGraph from XML documents +Basic example of scraping pipeline using XMLScraperGraph from XML documents """ import os from dotenv import load_dotenv -from scrapegraphai.graphs import XmlScraperGraph +from scrapegraphai.graphs import XMLScraperGraph from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info load_dotenv() @@ -39,23 +39,23 @@ graph_config = { } # ************************************************ -# Create the XmlScraperGraph instance and run it +# Create the XMLScraperGraph instance and run it # ************************************************ -smart_scraper_graph = XmlScraperGraph( +xml_scraper_graph = XMLScraperGraph( prompt="List me all the authors, title and genres of the books", source=text, # Pass the content of the file, not the file object config=graph_config ) -result = smart_scraper_graph.run() +result = xml_scraper_graph.run() print(result) # ************************************************ # Get graph execution info # ************************************************ -graph_exec_info = smart_scraper_graph.get_execution_info() +graph_exec_info = xml_scraper_graph.get_execution_info() print(prettify_exec_info(graph_exec_info)) # Save to json or csv diff --git a/examples/local_models/result.json b/examples/local_models/result.json new file mode 100644 index 00000000..8a4e7057 --- /dev/null +++ b/examples/local_models/result.json @@ -0,0 +1 @@ +{"projects": [{"title": "Rotary Pendulum RL", "description": "Open Source project aimed at controlling a real life rotary pendulum using RL algorithms"}, {"title": "DQN Implementation from scratch", "description": "Developed a Deep Q-Network algorithm to train a simple and double pendulum"}, {"title": "Multi Agents HAED", "description": "University project which focuses on simulating a multi-agent system to perform environment mapping. Agents, equipped with sensors, explore and record their surroundings, considering uncertainties in their readings."}, {"title": "Wireless ESC for Modular Drones", "description": "Modular drone architecture proposal and proof of concept. The project received maximum grade."}]} \ No newline at end of file diff --git a/examples/openai/json_scraper_openai.py b/examples/openai/json_scraper_openai.py index b52f9359..5e271006 100644 --- a/examples/openai/json_scraper_openai.py +++ b/examples/openai/json_scraper_openai.py @@ -1,10 +1,10 @@ """ -Basic example of scraping pipeline using JsonScraperGraph from JSON documents +Basic example of scraping pipeline using JSONScraperGraph from JSON documents """ import os from dotenv import load_dotenv -from scrapegraphai.graphs import JsonScraperGraph +from scrapegraphai.graphs import JSONScraperGraph from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info load_dotenv() @@ -33,23 +33,23 @@ graph_config = { } # ************************************************ -# Create the XmlScraperGraph instance and run it +# Create the JSONScraperGraph instance and run it # ************************************************ -smart_scraper_graph = JsonScraperGraph( +json_scraper_graph = JSONScraperGraph( prompt="List me all the authors, title and genres of the books", source=text, # Pass the content of the file, not the file object config=graph_config ) -result = smart_scraper_graph.run() +result = json_scraper_graph.run() print(result) # ************************************************ # Get graph execution info # ************************************************ -graph_exec_info = smart_scraper_graph.get_execution_info() +graph_exec_info = json_scraper_graph.get_execution_info() print(prettify_exec_info(graph_exec_info)) # Save to json or csv diff --git a/examples/openai/result.json b/examples/openai/result.json new file mode 100644 index 00000000..8867c8d6 --- /dev/null +++ b/examples/openai/result.json @@ -0,0 +1 @@ +{"top_5_eyeliner_products_for_gift": [{"product_name": "Tarte Double Take Eyeliner", "type": "Liquid, Gel", "price": "$26", "link": "https://www.sephora.com/product/double-take-eyeliner-P421701"}, {"product_name": "AppleDoll Velvet Liner", "type": "Liquid", "price": "$22", "link": "https://www.appledoll.com/products/velvet-liner"}, {"product_name": "Rare Beauty Perfect Strokes Gel Eyeliner", "type": "Gel", "price": "$19", "link": "https://www.sephora.com/product/perfect-strokes-gel-eyeliner-P468000"}, {"product_name": "Laura Mercier Caviar Tightline Eyeliner", "type": "Gel", "price": "$29", "link": "https://www.sephora.com/product/caviar-tightline-eyeliner-P448800"}, {"product_name": "Ilia Clean Line Liquid Eyeliner", "type": "Liquid", "price": "$28", "link": "https://www.amazon.com/ILIA-Clean-Line-Liquid-Eyeliner/dp/B08Z7JZQZP"}, {"brand": "Tom Ford", "product_name": "Eye Defining Pen", "price": "$62", "type": "Liquid", "colors": 1, "retailer": "Nordstrom"}, {"brand": "Fenty Beauty", "product_name": "Flyliner", "price": "$24", "type": "Liquid", "colors": 2, "retailer": "Sephora"}, {"brand": "Lanc\u00f4me", "product_name": "Le Crayon Kh\u00f4l Smoky Eyeliner", "price": "$28", "type": "Kohl", "colors": 2, "retailer": "Macy's"}, {"brand": "Jillian Dempsey", "product_name": "Kh\u00f4l Eyeliner", "price": "$20", "type": "Kohl", "colors": 6, "retailer": "Amazon"}, {"brand": "R\u00f3en", "product_name": "Eyeline Define Eyeliner Pencil", "price": "$26", "type": "Kohl", "colors": 4, "retailer": "Credo Beauty"}]} \ No newline at end of file diff --git a/examples/openai/xml_scraper_openai.py b/examples/openai/xml_scraper_openai.py index 7057de2b..32b79981 100644 --- a/examples/openai/xml_scraper_openai.py +++ b/examples/openai/xml_scraper_openai.py @@ -1,10 +1,10 @@ """ -Basic example of scraping pipeline using XmlScraperGraph from XML documents +Basic example of scraping pipeline using XMLScraperGraph from XML documents """ import os from dotenv import load_dotenv -from scrapegraphai.graphs import XmlScraperGraph +from scrapegraphai.graphs import XMLScraperGraph from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info load_dotenv() @@ -33,23 +33,23 @@ graph_config = { } # ************************************************ -# Create the XmlScraperGraph instance and run it +# Create the XMLScraperGraph instance and run it # ************************************************ -smart_scraper_graph = XmlScraperGraph( +xml_scraper_graph = XMLScraperGraph( prompt="List me all the authors, title and genres of the books", source=text, # Pass the content of the file, not the file object config=graph_config ) -result = smart_scraper_graph.run() +result = xml_scraper_graph.run() print(result) # ************************************************ # Get graph execution info # ************************************************ -graph_exec_info = smart_scraper_graph.get_execution_info() +graph_exec_info = xml_scraper_graph.get_execution_info() print(prettify_exec_info(graph_exec_info)) # Save to json or csv diff --git a/scrapegraphai/graphs/__init__.py b/scrapegraphai/graphs/__init__.py index b7fbcef7..d943a4dc 100644 --- a/scrapegraphai/graphs/__init__.py +++ b/scrapegraphai/graphs/__init__.py @@ -6,5 +6,5 @@ from .smart_scraper_graph import SmartScraperGraph from .speech_graph import SpeechGraph from .search_graph import SearchGraph from .script_creator_graph import ScriptCreatorGraph -from .xml_scraper_graph import XmlScraperGraph -from .json_scraper_graph import JsonScraperGraph +from .xml_scraper_graph import XMLScraperGraph +from .json_scraper_graph import JSONScraperGraph diff --git a/scrapegraphai/graphs/json_scraper_graph.py b/scrapegraphai/graphs/json_scraper_graph.py index b6a11179..02092544 100644 --- a/scrapegraphai/graphs/json_scraper_graph.py +++ b/scrapegraphai/graphs/json_scraper_graph.py @@ -11,7 +11,7 @@ from ..nodes import ( from .abstract_graph import AbstractGraph -class JsonScraperGraph(AbstractGraph): +class JSONScraperGraph(AbstractGraph): """ SmartScraper is a comprehensive web scraping tool that automates the process of extracting information from web pages using a natural language model to interpret and answer prompts. diff --git a/scrapegraphai/graphs/xml_scraper_graph.py b/scrapegraphai/graphs/xml_scraper_graph.py index f8a697bd..0dad83e3 100644 --- a/scrapegraphai/graphs/xml_scraper_graph.py +++ b/scrapegraphai/graphs/xml_scraper_graph.py @@ -11,7 +11,7 @@ from ..nodes import ( from .abstract_graph import AbstractGraph -class XmlScraperGraph(AbstractGraph): +class XMLScraperGraph(AbstractGraph): """ SmartScraper is a comprehensive web scraping tool that automates the process of extracting information from web pages using a natural language model to interpret and answer prompts.