diff --git a/README.md b/README.md index 3bed4ffc..87a8fdb5 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ graph_config = { "model": "ollama/mistral", "temperature": 0, "format": "json", # Ollama needs the format to be specified explicitly + # "model_tokens": 2000, # set context length arbitrarily }, } diff --git a/examples/gemini/smart_scraper_gemini.py b/examples/gemini/smart_scraper_gemini.py new file mode 100644 index 00000000..58f3788d --- /dev/null +++ b/examples/gemini/smart_scraper_gemini.py @@ -0,0 +1,36 @@ +""" +Basic example of scraping pipeline using SmartScraper +""" + +import os +from dotenv import load_dotenv +from scrapegraphai.graphs import SmartScraperGraph +load_dotenv() + + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +openai_key = os.getenv("OPENAI_APIKEY") + +graph_config = { + "llm": { + "api_key": openai_key, + "model": "gpt-3.5-turbo", + }, +} + +# ************************************************ +# Create the SmartScraperGraph instance and run it +# ************************************************ + +smart_scraper_graph = SmartScraperGraph( + prompt="List me all the news with their description.", + # also accepts a string with the already downloaded HTML code + source="https://www.wired.com", + config=graph_config +) + +result = smart_scraper_graph.run() +print(result) diff --git a/examples/local/scrape_plain_text_local.py b/examples/local/scrape_plain_text_local.py index 186c788c..0af0a167 100644 --- a/examples/local/scrape_plain_text_local.py +++ b/examples/local/scrape_plain_text_local.py @@ -27,6 +27,7 @@ graph_config = { "model": "ollama/mistral", "temperature": 0, "format": "json", # Ollama needs the format to be specified explicitly + # "model_tokens": 2000, # set context length arbitrarily }, } diff --git a/examples/local/scrape_xml_local.py b/examples/local/scrape_xml_local.py index 6bc0ac40..e647fa50 100644 --- a/examples/local/scrape_xml_local.py +++ b/examples/local/scrape_xml_local.py @@ -26,6 +26,7 @@ graph_config = { "model": "ollama/mistral", "temperature": 0, "format": "json", # Ollama needs the format to be specified explicitly + # "model_tokens": 2000, # set context length arbitrarily }, } diff --git a/examples/local/search_graph_local.py b/examples/local/search_graph_local.py index 15cc3aee..ed01128e 100644 --- a/examples/local/search_graph_local.py +++ b/examples/local/search_graph_local.py @@ -15,6 +15,7 @@ graph_config = { "model": "ollama/mistral", "temperature": 0, "format": "json", # Ollama needs the format to be specified explicitly + # "model_tokens": 2000, # set context length arbitrarily }, }