From 14b4b19f60e33c855bee4eea0a1a6fcc01a98c1a Mon Sep 17 00:00:00 2001 From: PeriniM Date: Wed, 8 Jan 2025 04:16:38 +0100 Subject: [PATCH] docs: improved readme + fix csv scraper imports --- .../ollama/csv_scraper_graph_multi_ollama.py | 13 ++-- .../ollama/csv_scraper_ollama.py | 13 ++-- .../openai/csv_scraper_graph_multi_openai.py | 17 +++-- .../openai/csv_scraper_openai.py | 15 ++-- examples/readme.md | 68 +++++++++++++------ .../.env.example | 2 +- .../README.md | 2 +- .../ollama/smart_scraper_lite_ollama.py | 8 ++- .../smart_scraper_multi_concat_ollama.py | 16 ++--- .../ollama/smart_scraper_multi_lite_ollama.py | 14 ++-- .../ollama/smart_scraper_multi_ollama.py | 13 ++-- .../ollama/smart_scraper_ollama.py | 0 .../ollama/smart_scraper_schema_ollama.py | 19 +++++- .../openai/smart_scraper_lite_openai.py | 10 +-- .../smart_scraper_multi_concat_openai.py | 14 ++-- .../openai/smart_scraper_multi_lite_openai.py | 14 ++-- .../openai/smart_scraper_multi_openai.py | 14 ++-- .../openai/smart_scraper_openai.py | 0 .../openai/smart_scraper_schema_openai.py | 10 ++- 19 files changed, 150 insertions(+), 112 deletions(-) rename examples/{smart_scraper => smart_scraper_graph}/.env.example (89%) rename examples/{smart_scraper => smart_scraper_graph}/README.md (93%) rename examples/{smart_scraper => smart_scraper_graph}/ollama/smart_scraper_lite_ollama.py (92%) rename examples/{smart_scraper => smart_scraper_graph}/ollama/smart_scraper_multi_concat_ollama.py (78%) rename examples/{smart_scraper => smart_scraper_graph}/ollama/smart_scraper_multi_lite_ollama.py (88%) rename examples/{smart_scraper => smart_scraper_graph}/ollama/smart_scraper_multi_ollama.py (85%) rename examples/{smart_scraper => smart_scraper_graph}/ollama/smart_scraper_ollama.py (100%) rename examples/{smart_scraper => smart_scraper_graph}/ollama/smart_scraper_schema_ollama.py (81%) rename examples/{smart_scraper => smart_scraper_graph}/openai/smart_scraper_lite_openai.py (95%) rename examples/{smart_scraper => smart_scraper_graph}/openai/smart_scraper_multi_concat_openai.py (86%) rename examples/{smart_scraper => smart_scraper_graph}/openai/smart_scraper_multi_lite_openai.py (89%) rename examples/{smart_scraper => smart_scraper_graph}/openai/smart_scraper_multi_openai.py (86%) rename examples/{smart_scraper => smart_scraper_graph}/openai/smart_scraper_openai.py (100%) rename examples/{smart_scraper => smart_scraper_graph}/openai/smart_scraper_schema_openai.py (97%) diff --git a/examples/csv_scraper_graph/ollama/csv_scraper_graph_multi_ollama.py b/examples/csv_scraper_graph/ollama/csv_scraper_graph_multi_ollama.py index fb6bce51..558a876f 100644 --- a/examples/csv_scraper_graph/ollama/csv_scraper_graph_multi_ollama.py +++ b/examples/csv_scraper_graph/ollama/csv_scraper_graph_multi_ollama.py @@ -3,9 +3,9 @@ Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents """ import os -import pandas as pd + from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info +from scrapegraphai.utils import prettify_exec_info # ************************************************ # Read the CSV file @@ -15,7 +15,8 @@ FILE_NAME = "inputs/username.csv" curr_dir = os.path.dirname(os.path.realpath(__file__)) file_path = os.path.join(curr_dir, FILE_NAME) -text = pd.read_csv(file_path) +with open(file_path, "r") as file: + text = file.read() # ************************************************ # Define the configuration for the graph @@ -44,7 +45,7 @@ graph_config = { csv_scraper_graph = CSVScraperMultiGraph( prompt="List me all the last names", source=[str(text), str(text)], - config=graph_config + config=graph_config, ) result = csv_scraper_graph.run() @@ -56,7 +57,3 @@ print(result) graph_exec_info = csv_scraper_graph.get_execution_info() print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/examples/csv_scraper_graph/ollama/csv_scraper_ollama.py b/examples/csv_scraper_graph/ollama/csv_scraper_ollama.py index 8d1edbd7..d6e6eab2 100644 --- a/examples/csv_scraper_graph/ollama/csv_scraper_ollama.py +++ b/examples/csv_scraper_graph/ollama/csv_scraper_ollama.py @@ -3,9 +3,9 @@ Basic example of scraping pipeline using CSVScraperGraph from CSV documents """ import os -import pandas as pd + from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info +from scrapegraphai.utils import prettify_exec_info # ************************************************ # Read the CSV file @@ -15,7 +15,8 @@ FILE_NAME = "inputs/username.csv" curr_dir = os.path.dirname(os.path.realpath(__file__)) file_path = os.path.join(curr_dir, FILE_NAME) -text = pd.read_csv(file_path) +with open(file_path, "r") as file: + text = file.read() # ************************************************ # Define the configuration for the graph @@ -44,7 +45,7 @@ graph_config = { csv_scraper_graph = CSVScraperGraph( prompt="List me all the last names", source=str(text), # Pass the content of the file, not the file object - config=graph_config + config=graph_config, ) result = csv_scraper_graph.run() @@ -56,7 +57,3 @@ print(result) graph_exec_info = csv_scraper_graph.get_execution_info() print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/examples/csv_scraper_graph/openai/csv_scraper_graph_multi_openai.py b/examples/csv_scraper_graph/openai/csv_scraper_graph_multi_openai.py index 6ed33c90..b7bc83ae 100644 --- a/examples/csv_scraper_graph/openai/csv_scraper_graph_multi_openai.py +++ b/examples/csv_scraper_graph/openai/csv_scraper_graph_multi_openai.py @@ -1,11 +1,13 @@ """ Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents """ + import os + from dotenv import load_dotenv -import pandas as pd + from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info +from scrapegraphai.utils import prettify_exec_info load_dotenv() # ************************************************ @@ -16,7 +18,8 @@ FILE_NAME = "inputs/username.csv" curr_dir = os.path.dirname(os.path.realpath(__file__)) file_path = os.path.join(curr_dir, FILE_NAME) -text = pd.read_csv(file_path) +with open(file_path, "r") as file: + text = file.read() # ************************************************ # Define the configuration for the graph @@ -24,7 +27,7 @@ text = pd.read_csv(file_path) openai_key = os.getenv("OPENAI_APIKEY") graph_config = { - "llm": { + "llm": { "api_key": openai_key, "model": "openai/gpt-4o", }, @@ -37,7 +40,7 @@ graph_config = { csv_scraper_graph = CSVScraperMultiGraph( prompt="List me all the last names", source=[str(text), str(text)], - config=graph_config + config=graph_config, ) result = csv_scraper_graph.run() @@ -49,7 +52,3 @@ print(result) graph_exec_info = csv_scraper_graph.get_execution_info() print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/examples/csv_scraper_graph/openai/csv_scraper_openai.py b/examples/csv_scraper_graph/openai/csv_scraper_openai.py index d9527b86..a0abd714 100644 --- a/examples/csv_scraper_graph/openai/csv_scraper_openai.py +++ b/examples/csv_scraper_graph/openai/csv_scraper_openai.py @@ -1,11 +1,13 @@ """ Basic example of scraping pipeline using CSVScraperGraph from CSV documents """ + import os + from dotenv import load_dotenv -import pandas as pd + from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info +from scrapegraphai.utils import prettify_exec_info load_dotenv() @@ -17,7 +19,8 @@ FILE_NAME = "inputs/username.csv" curr_dir = os.path.dirname(os.path.realpath(__file__)) file_path = os.path.join(curr_dir, FILE_NAME) -text = pd.read_csv(file_path) +with open(file_path, "r") as file: + text = file.read() # ************************************************ # Define the configuration for the graph @@ -39,7 +42,7 @@ graph_config = { csv_scraper_graph = CSVScraperGraph( prompt="List me all the last names", source=str(text), # Pass the content of the file, not the file object - config=graph_config + config=graph_config, ) result = csv_scraper_graph.run() @@ -51,7 +54,3 @@ print(result) graph_exec_info = csv_scraper_graph.get_execution_info() print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/examples/readme.md b/examples/readme.md index 1d7b375f..1a5aae8b 100644 --- a/examples/readme.md +++ b/examples/readme.md @@ -1,32 +1,62 @@ -# Scrapegraph-ai Examples +# 🕷️ Scrapegraph-ai Examples -This directory contains various example implementations of Scrapegraph-ai for different use cases. +This directory contains various example implementations of Scrapegraph-ai for different use cases. Each example demonstrates how to leverage the power of Scrapegraph-ai for specific scenarios. -If you want more specific examples, visit [this](https://github.com/ScrapeGraphAI/ScrapegraphLib-Examples). +> **Note:** While these examples showcase implementations using OpenAI and Ollama, Scrapegraph-ai supports many other LLM providers! Check out our [documentation](https://docs-oss.scrapegraphai.com/examples) for the full list of supported providers. -## Available Examples +## 📚 Available Examples -- `smart_scraper/` - Advanced web scraping with intelligent content extraction -- `depth_search_graph/` - Deep web crawling and content exploration -- `csv_scraper_graph/` - Scraping and processing data into CSV format -- `xml_scraper_graph/` - XML data extraction and processing -- `speech_graph/` - Speech processing and analysis -- `omni_scraper_graph/` - Universal web scraping for multiple data types -- `omni_search_graph/` - Comprehensive search across multiple sources -- `document_scraper_graph/` - Document parsing and data extraction -- `script_generator_graph/` - Automated script generation -- `custom_graph/` - Custom graph implementation examples -- `code_generator_graph/` - Code generation utilities -- `json_scraper_graph/` - JSON data extraction and processing -- `search_graph/` - Web search and data retrieval +- 🧠 `smart_scraper/` - Advanced web scraping with intelligent content extraction +- 🔎 `search_graph/` - Web search and data retrieval +- ⚙️ `script_generator_graph/` - Automated script generation +- 🌐 `depth_search_graph/` - Deep web crawling and content exploration +- 📊 `csv_scraper_graph/` - Scraping and processing data into CSV format +- 📑 `xml_scraper_graph/` - XML data extraction and processing +- 🎤 `speech_graph/` - Speech processing and analysis +- 🔄 `omni_scraper_graph/` - Universal web scraping for multiple data types +- 🔍 `omni_search_graph/` - Comprehensive search across multiple sources +- 📄 `document_scraper_graph/` - Document parsing and data extraction +- 🛠️ `custom_graph/` - Custom graph implementation examples +- 💻 `code_generator_graph/` - Code generation utilities +- 📋 `json_scraper_graph/` - JSON data extraction and processing -## Getting Started +## 🚀 Getting Started 1. Choose the example that best fits your use case 2. Navigate to the corresponding directory 3. Follow the README instructions in each directory 4. Configure any required environment variables using the provided `.env.example` files -## Requirements +## ⚡ Quick Setup + +```bash +pip install scrapegraphai + +playwright install + +# choose an example +cd examples/smart_scraper_graph/openai + +# run the example +python smart_scraper_openai.py +``` + +## 📋 Requirements Each example may have its own specific requirements. Please refer to the individual README files in each directory for detailed setup instructions. + +## 📚 Additional Resources + +- 📖 [Full Documentation](https://docs-oss.scrapegraphai.com/examples) +- 💡 [Examples Repository](https://github.com/ScrapeGraphAI/ScrapegraphLib-Examples) +- 🤝 [Community Support](https://github.com/ScrapeGraphAI/scrapegraph-ai/discussions) + +## 🤔 Need Help? + +- Check out our [documentation](https://docs-oss.scrapegraphai.com) +- Join our [Discord community](https://discord.gg/scrapegraphai) +- Open an [issue](https://github.com/ScrapeGraphAI/scrapegraph-ai/issues) + +--- + +⭐ Don't forget to star our repository if you find these examples helpful! diff --git a/examples/smart_scraper/.env.example b/examples/smart_scraper_graph/.env.example similarity index 89% rename from examples/smart_scraper/.env.example rename to examples/smart_scraper_graph/.env.example index 7d2c49c0..0c8d0b86 100644 --- a/examples/smart_scraper/.env.example +++ b/examples/smart_scraper_graph/.env.example @@ -4,4 +4,4 @@ OPENAI_API_KEY=your-openai-api-key-here # Optional Configurations MAX_TOKENS=4000 MODEL_NAME=gpt-4-1106-preview -TEMPERATURE=0.7 \ No newline at end of file +TEMPERATURE=0.7 diff --git a/examples/smart_scraper/README.md b/examples/smart_scraper_graph/README.md similarity index 93% rename from examples/smart_scraper/README.md rename to examples/smart_scraper_graph/README.md index ba4f638c..d5f0d564 100644 --- a/examples/smart_scraper/README.md +++ b/examples/smart_scraper_graph/README.md @@ -27,4 +27,4 @@ results = graph.scrape("https://example.com") ## Environment Variables Required environment variables: -- `OPENAI_API_KEY`: Your OpenAI API key \ No newline at end of file +- `OPENAI_API_KEY`: Your OpenAI API key diff --git a/examples/smart_scraper/ollama/smart_scraper_lite_ollama.py b/examples/smart_scraper_graph/ollama/smart_scraper_lite_ollama.py similarity index 92% rename from examples/smart_scraper/ollama/smart_scraper_lite_ollama.py rename to examples/smart_scraper_graph/ollama/smart_scraper_lite_ollama.py index 2cf6c402..fd23f39c 100644 --- a/examples/smart_scraper/ollama/smart_scraper_lite_ollama.py +++ b/examples/smart_scraper_graph/ollama/smart_scraper_lite_ollama.py @@ -1,8 +1,10 @@ -""" +""" Basic example of scraping pipeline using SmartScraper """ + import json + from scrapegraphai.graphs import SmartScraperLiteGraph from scrapegraphai.utils import prettify_exec_info @@ -14,13 +16,13 @@ graph_config = { "base_url": "http://localhost:11434", }, "verbose": True, - "headless": False + "headless": False, } smart_scraper_lite_graph = SmartScraperLiteGraph( prompt="Who is Marco Perini?", source="https://perinim.github.io/", - config=graph_config + config=graph_config, ) result = smart_scraper_lite_graph.run() diff --git a/examples/smart_scraper/ollama/smart_scraper_multi_concat_ollama.py b/examples/smart_scraper_graph/ollama/smart_scraper_multi_concat_ollama.py similarity index 78% rename from examples/smart_scraper/ollama/smart_scraper_multi_concat_ollama.py rename to examples/smart_scraper_graph/ollama/smart_scraper_multi_concat_ollama.py index 665b5db4..566be922 100644 --- a/examples/smart_scraper/ollama/smart_scraper_multi_concat_ollama.py +++ b/examples/smart_scraper_graph/ollama/smart_scraper_multi_concat_ollama.py @@ -1,10 +1,11 @@ -""" +""" Basic example of scraping pipeline using SmartScraper """ -import os import json + from dotenv import load_dotenv + from scrapegraphai.graphs import SmartScraperMultiConcatGraph load_dotenv() @@ -18,10 +19,10 @@ graph_config = { "model": "ollama/llama3.1", "temperature": 0, "format": "json", # Ollama needs the format to be specified explicitly - "base_url": "http://localhost:11434", # set ollama URL arbitrarily + "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, - "headless": False + "headless": False, } # ******************************************************* @@ -30,12 +31,9 @@ graph_config = { multiple_search_graph = SmartScraperMultiConcatGraph( prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], + source=["https://perinim.github.io/", "https://perinim.github.io/cv/"], schema=None, - config=graph_config + config=graph_config, ) result = multiple_search_graph.run() diff --git a/examples/smart_scraper/ollama/smart_scraper_multi_lite_ollama.py b/examples/smart_scraper_graph/ollama/smart_scraper_multi_lite_ollama.py similarity index 88% rename from examples/smart_scraper/ollama/smart_scraper_multi_lite_ollama.py rename to examples/smart_scraper_graph/ollama/smart_scraper_multi_lite_ollama.py index f09c4cb4..90c58f9d 100644 --- a/examples/smart_scraper/ollama/smart_scraper_multi_lite_ollama.py +++ b/examples/smart_scraper_graph/ollama/smart_scraper_multi_lite_ollama.py @@ -1,7 +1,9 @@ -""" +""" Basic example of scraping pipeline using SmartScraper """ + import json + from scrapegraphai.graphs import SmartScraperMultiLiteGraph from scrapegraphai.utils import prettify_exec_info @@ -17,7 +19,7 @@ graph_config = { "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, - "headless": False + "headless": False, } # ************************************************ @@ -26,11 +28,8 @@ graph_config = { smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config + source=["https://perinim.github.io/", "https://perinim.github.io/cv/"], + config=graph_config, ) result = smart_scraper_multi_lite_graph.run() @@ -42,4 +41,3 @@ print(json.dumps(result, indent=4)) graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() print(prettify_exec_info(graph_exec_info)) - diff --git a/examples/smart_scraper/ollama/smart_scraper_multi_ollama.py b/examples/smart_scraper_graph/ollama/smart_scraper_multi_ollama.py similarity index 85% rename from examples/smart_scraper/ollama/smart_scraper_multi_ollama.py rename to examples/smart_scraper_graph/ollama/smart_scraper_multi_ollama.py index c9d49793..c42bb627 100644 --- a/examples/smart_scraper/ollama/smart_scraper_multi_ollama.py +++ b/examples/smart_scraper_graph/ollama/smart_scraper_multi_ollama.py @@ -1,8 +1,9 @@ -""" +""" Basic example of scraping pipeline using SmartScraper """ import json + from scrapegraphai.graphs import SmartScraperMultiGraph # ************************************************ @@ -15,9 +16,8 @@ graph_config = { "format": "json", # Ollama needs the format to be specified explicitly # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, - "verbose": True, - "headless": False + "headless": False, } @@ -27,12 +27,9 @@ graph_config = { multiple_search_graph = SmartScraperMultiGraph( prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], + source=["https://perinim.github.io/", "https://perinim.github.io/cv/"], schema=None, - config=graph_config + config=graph_config, ) result = multiple_search_graph.run() diff --git a/examples/smart_scraper/ollama/smart_scraper_ollama.py b/examples/smart_scraper_graph/ollama/smart_scraper_ollama.py similarity index 100% rename from examples/smart_scraper/ollama/smart_scraper_ollama.py rename to examples/smart_scraper_graph/ollama/smart_scraper_ollama.py diff --git a/examples/smart_scraper/ollama/smart_scraper_schema_ollama.py b/examples/smart_scraper_graph/ollama/smart_scraper_schema_ollama.py similarity index 81% rename from examples/smart_scraper/ollama/smart_scraper_schema_ollama.py rename to examples/smart_scraper_graph/ollama/smart_scraper_schema_ollama.py index 5a5b3cea..38eeaeb5 100644 --- a/examples/smart_scraper/ollama/smart_scraper_schema_ollama.py +++ b/examples/smart_scraper_graph/ollama/smart_scraper_schema_ollama.py @@ -1,12 +1,16 @@ -""" +""" Basic example of scraping pipeline using SmartScraper with schema """ + import json from typing import List + from pydantic import BaseModel, Field + from scrapegraphai.graphs import SmartScraperGraph from scrapegraphai.utils import prettify_exec_info + # ************************************************ # Define the configuration for the graph # ************************************************ @@ -14,9 +18,11 @@ class Project(BaseModel): title: str = Field(description="The title of the project") description: str = Field(description="The description of the project") + class Projects(BaseModel): projects: List[Project] + graph_config = { "llm": { "model": "ollama/llama3.1", @@ -25,7 +31,7 @@ graph_config = { # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, - "headless": False + "headless": False, } # ************************************************ @@ -36,8 +42,15 @@ smart_scraper_graph = SmartScraperGraph( prompt="List me all the projects with their description", source="https://perinim.github.io/projects/", schema=Projects, - config=graph_config + config=graph_config, ) result = smart_scraper_graph.run() print(json.dumps(result, indent=4)) + +# ************************************************ +# Get graph execution info +# ************************************************ + +graph_exec_info = smart_scraper_graph.get_execution_info() +print(prettify_exec_info(graph_exec_info)) diff --git a/examples/smart_scraper/openai/smart_scraper_lite_openai.py b/examples/smart_scraper_graph/openai/smart_scraper_lite_openai.py similarity index 95% rename from examples/smart_scraper/openai/smart_scraper_lite_openai.py rename to examples/smart_scraper_graph/openai/smart_scraper_lite_openai.py index 5de725bb..3d768548 100644 --- a/examples/smart_scraper/openai/smart_scraper_lite_openai.py +++ b/examples/smart_scraper_graph/openai/smart_scraper_lite_openai.py @@ -1,9 +1,12 @@ -""" +""" Basic example of scraping pipeline using SmartScraper """ -import os + import json +import os + from dotenv import load_dotenv + from scrapegraphai.graphs import SmartScraperLiteGraph from scrapegraphai.utils import prettify_exec_info @@ -21,7 +24,7 @@ graph_config = { smart_scraper_lite_graph = SmartScraperLiteGraph( prompt="Who is Marco Perini?", source="https://perinim.github.io/", - config=graph_config + config=graph_config, ) result = smart_scraper_lite_graph.run() @@ -29,4 +32,3 @@ print(json.dumps(result, indent=4)) graph_exec_info = smart_scraper_lite_graph.get_execution_info() print(prettify_exec_info(graph_exec_info)) - diff --git a/examples/smart_scraper/openai/smart_scraper_multi_concat_openai.py b/examples/smart_scraper_graph/openai/smart_scraper_multi_concat_openai.py similarity index 86% rename from examples/smart_scraper/openai/smart_scraper_multi_concat_openai.py rename to examples/smart_scraper_graph/openai/smart_scraper_multi_concat_openai.py index 650971f1..4774e620 100644 --- a/examples/smart_scraper/openai/smart_scraper_multi_concat_openai.py +++ b/examples/smart_scraper_graph/openai/smart_scraper_multi_concat_openai.py @@ -1,9 +1,12 @@ -""" +""" Basic example of scraping pipeline using SmartScraper """ -import os + import json +import os + from dotenv import load_dotenv + from scrapegraphai.graphs import SmartScraperMultiConcatGraph load_dotenv() @@ -28,12 +31,9 @@ graph_config = { multiple_search_graph = SmartScraperMultiConcatGraph( prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], + source=["https://perinim.github.io/", "https://perinim.github.io/cv/"], schema=None, - config=graph_config + config=graph_config, ) result = multiple_search_graph.run() diff --git a/examples/smart_scraper/openai/smart_scraper_multi_lite_openai.py b/examples/smart_scraper_graph/openai/smart_scraper_multi_lite_openai.py similarity index 89% rename from examples/smart_scraper/openai/smart_scraper_multi_lite_openai.py rename to examples/smart_scraper_graph/openai/smart_scraper_multi_lite_openai.py index 69eeafc7..acc970be 100644 --- a/examples/smart_scraper/openai/smart_scraper_multi_lite_openai.py +++ b/examples/smart_scraper_graph/openai/smart_scraper_multi_lite_openai.py @@ -1,9 +1,12 @@ -""" +""" Basic example of scraping pipeline using SmartScraper """ -import os + import json +import os + from dotenv import load_dotenv + from scrapegraphai.graphs import SmartScraperMultiLiteGraph from scrapegraphai.utils import prettify_exec_info @@ -29,11 +32,8 @@ graph_config = { smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config + source=["https://perinim.github.io/", "https://perinim.github.io/cv/"], + config=graph_config, ) result = smart_scraper_multi_lite_graph.run() diff --git a/examples/smart_scraper/openai/smart_scraper_multi_openai.py b/examples/smart_scraper_graph/openai/smart_scraper_multi_openai.py similarity index 86% rename from examples/smart_scraper/openai/smart_scraper_multi_openai.py rename to examples/smart_scraper_graph/openai/smart_scraper_multi_openai.py index ba889c96..ec510fc2 100644 --- a/examples/smart_scraper/openai/smart_scraper_multi_openai.py +++ b/examples/smart_scraper_graph/openai/smart_scraper_multi_openai.py @@ -1,9 +1,12 @@ -""" +""" Basic example of scraping pipeline using SmartScraper """ -import os + import json +import os + from dotenv import load_dotenv + from scrapegraphai.graphs import SmartScraperMultiGraph load_dotenv() @@ -29,12 +32,9 @@ graph_config = { multiple_search_graph = SmartScraperMultiGraph( prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], + source=["https://perinim.github.io/", "https://perinim.github.io/cv/"], schema=None, - config=graph_config + config=graph_config, ) result = multiple_search_graph.run() diff --git a/examples/smart_scraper/openai/smart_scraper_openai.py b/examples/smart_scraper_graph/openai/smart_scraper_openai.py similarity index 100% rename from examples/smart_scraper/openai/smart_scraper_openai.py rename to examples/smart_scraper_graph/openai/smart_scraper_openai.py diff --git a/examples/smart_scraper/openai/smart_scraper_schema_openai.py b/examples/smart_scraper_graph/openai/smart_scraper_schema_openai.py similarity index 97% rename from examples/smart_scraper/openai/smart_scraper_schema_openai.py rename to examples/smart_scraper_graph/openai/smart_scraper_schema_openai.py index 32e8891a..3a75bd5a 100644 --- a/examples/smart_scraper/openai/smart_scraper_schema_openai.py +++ b/examples/smart_scraper_graph/openai/smart_scraper_schema_openai.py @@ -1,10 +1,13 @@ -""" +""" Basic example of scraping pipeline using SmartScraper with schema """ + import os from typing import List + from dotenv import load_dotenv from pydantic import BaseModel, Field + from scrapegraphai.graphs import SmartScraperGraph load_dotenv() @@ -13,13 +16,16 @@ load_dotenv() # Define the output schema for the graph # ************************************************ + class Project(BaseModel): title: str = Field(description="The title of the project") description: str = Field(description="The description of the project") + class Projects(BaseModel): projects: List[Project] + # ************************************************ # Define the configuration for the graph # ************************************************ @@ -43,7 +49,7 @@ smart_scraper_graph = SmartScraperGraph( prompt="List me all the projects with their description", source="https://perinim.github.io/projects/", schema=Projects, - config=graph_config + config=graph_config, ) result = smart_scraper_graph.run()