From 41ea3e0e2714c0b414c33a2aafdb1e648a71f610 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Wed, 24 Jul 2024 13:38:27 +0200 Subject: [PATCH 1/5] add llama3.1 model --- examples/local_models/smart_scraper_ollama.py | 2 +- scrapegraphai/helpers/models_tokens.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/local_models/smart_scraper_ollama.py b/examples/local_models/smart_scraper_ollama.py index c3d60559..8ac579e5 100644 --- a/examples/local_models/smart_scraper_ollama.py +++ b/examples/local_models/smart_scraper_ollama.py @@ -9,7 +9,7 @@ from scrapegraphai.utils import prettify_exec_info graph_config = { "llm": { - "model": "ollama/llama3", + "model": "ollama/llama3.1", "temperature": 0, "format": "json", # Ollama needs the format to be specified explicitly # "base_url": "http://localhost:11434", # set ollama URL arbitrarily diff --git a/scrapegraphai/helpers/models_tokens.py b/scrapegraphai/helpers/models_tokens.py index b3d61065..0724ee95 100644 --- a/scrapegraphai/helpers/models_tokens.py +++ b/scrapegraphai/helpers/models_tokens.py @@ -51,6 +51,10 @@ models_tokens = { "falcon": 2048, "llama2": 4096, "llama3": 8192, + "llama3:70b": 8192, + "llama3.1":128000, + "llama3.1:70b": 128000, + "lama3.1:405b": 128000, "scrapegraph": 8192, "llava": 4096, "mixtral:8x22b-instruct": 65536, From e08b30495d9254257498e682170284f8f8c354dd Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Wed, 24 Jul 2024 14:33:55 +0200 Subject: [PATCH 2/5] boh --- CHANGELOG.md | 31 +-- .../csv_scraper_graph_multi_nemotron.py | 55 ------ examples/nemotron/csv_scraper_nemotron.py | 57 ------ examples/nemotron/custom_graph_nemotron.py | 109 ----------- examples/nemotron/deep_scraper_nemotron.py | 47 ----- examples/nemotron/inputs/books.xml | 120 ------------ examples/nemotron/inputs/example.json | 182 ------------------ examples/nemotron/inputs/markdown_example.md | 35 ---- .../nemotron/inputs/plain_html_example.txt | 105 ---------- examples/nemotron/inputs/username.csv | 7 - .../nemotron/json_scraper_multi_nemotron.py | 37 ---- examples/nemotron/json_scraper_nemotron.py | 58 ------ examples/nemotron/md_scraper_nemotron.py | 57 ------ examples/nemotron/pdf_scraper_nemotron.py | 40 ---- .../nemotron/scrape_plain_text_nemotron.py | 55 ------ .../nemotron/script_generator_nemotron.py | 46 ----- .../script_generator_schema_nemotron.py | 62 ------ .../script_multi_generator_nemotron.py | 54 ------ examples/nemotron/search_graph_nemotron.py | 35 ---- .../nemotron/search_graph_schema_nemotron.py | 63 ------ .../nemotron/search_link_graph_nemotron.py | 40 ---- .../nemotron/smart_scraper_multi_nemotron.py | 42 ---- examples/nemotron/smart_scraper_nemotron.py | 43 ----- .../nemotron/smart_scraper_schema_nemotron.py | 51 ----- examples/nemotron/speech_graph_nemotron.py | 50 ----- .../nemotron/xml_scraper_graph_nemotron.py | 60 ------ examples/nemotron/xml_scraper_nemotron.py | 59 ------ .../openai/csv_scraper_graph_multi_openai.py | 10 +- examples/openai/custom_graph_openai.py | 10 +- examples/openai/json_scraper_multi_openai.py | 2 +- examples/openai/smart_scraper_openai.py | 4 +- pyproject.toml | 6 +- requirements-dev.lock | 143 ++++++-------- requirements.lock | 106 ++++------ requirements.txt | 6 +- scrapegraphai/graphs/abstract_graph.py | 21 +- scrapegraphai/graphs/base_graph.py | 3 - scrapegraphai/helpers/models_tokens.py | 18 -- scrapegraphai/models/__init__.py | 1 - scrapegraphai/models/nvidia.py | 25 --- scrapegraphai/nodes/fetch_node.py | 10 +- .../nodes/generate_answer_csv_node.py | 53 ++--- .../nodes/generate_answer_omni_node.py | 59 +++--- .../nodes/generate_answer_pdf_node.py | 55 +++--- scrapegraphai/telemetry/telemetry.py | 5 +- scrapegraphai/utils/convert_to_md.py | 3 +- 46 files changed, 221 insertions(+), 1919 deletions(-) delete mode 100644 examples/nemotron/csv_scraper_graph_multi_nemotron.py delete mode 100644 examples/nemotron/csv_scraper_nemotron.py delete mode 100644 examples/nemotron/custom_graph_nemotron.py delete mode 100644 examples/nemotron/deep_scraper_nemotron.py delete mode 100644 examples/nemotron/inputs/books.xml delete mode 100644 examples/nemotron/inputs/example.json delete mode 100644 examples/nemotron/inputs/markdown_example.md delete mode 100644 examples/nemotron/inputs/plain_html_example.txt delete mode 100644 examples/nemotron/inputs/username.csv delete mode 100644 examples/nemotron/json_scraper_multi_nemotron.py delete mode 100644 examples/nemotron/json_scraper_nemotron.py delete mode 100644 examples/nemotron/md_scraper_nemotron.py delete mode 100644 examples/nemotron/pdf_scraper_nemotron.py delete mode 100644 examples/nemotron/scrape_plain_text_nemotron.py delete mode 100644 examples/nemotron/script_generator_nemotron.py delete mode 100644 examples/nemotron/script_generator_schema_nemotron.py delete mode 100644 examples/nemotron/script_multi_generator_nemotron.py delete mode 100644 examples/nemotron/search_graph_nemotron.py delete mode 100644 examples/nemotron/search_graph_schema_nemotron.py delete mode 100644 examples/nemotron/search_link_graph_nemotron.py delete mode 100644 examples/nemotron/smart_scraper_multi_nemotron.py delete mode 100644 examples/nemotron/smart_scraper_nemotron.py delete mode 100644 examples/nemotron/smart_scraper_schema_nemotron.py delete mode 100644 examples/nemotron/speech_graph_nemotron.py delete mode 100644 examples/nemotron/xml_scraper_graph_nemotron.py delete mode 100644 examples/nemotron/xml_scraper_nemotron.py delete mode 100644 scrapegraphai/models/nvidia.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d5306d62..7d0803bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ +<<<<<<< HEAD ## [1.11.2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.11.1...v1.11.2) (2024-07-23) @@ -49,31 +50,13 @@ ## [1.10.0-beta.8](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.10.0-beta.7...v1.10.0-beta.8) (2024-07-23) +======= +>>>>>>> parent of 7708828 (Merge pull request #488 from ScrapeGraphAI/pre/beta) ## [1.10.4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.10.3...v1.10.4) (2024-07-22) - ### Bug Fixes - -* **md_conversion:** add absolute links md, added missing dependency ([12b5ead](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/12b5eada6ea783770afd630ede69b8cf867a7ded)) - -## [1.10.0-beta.7](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.10.0-beta.6...v1.10.0-beta.7) (2024-07-23) - - -### Features - -* add nvidia connection ([fc0dadb](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/fc0dadb8f812dfd636dec856921a971b58695ce3)) - - -### chore - -* **dependecies:** add script to auto-update requirements ([3289c7b](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/3289c7bf5ec58ac3d04e9e5e8e654af9abcee228)) -* **ci:** set up workflow for requirements auto-update ([295fc28](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/295fc28ceb02c78198f7fbe678352503b3259b6b)) -* update requirements.txt ([c7bac98](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/c7bac98d2e79e5ab98fa65d7efa858a2cdda1622)) - -## [1.10.0-beta.6](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.10.0-beta.5...v1.10.0-beta.6) (2024-07-22) - * parse node ([09256f7](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/09256f7b11a7a1c2aba01cf8de70401af1e86fe4)) ## [1.10.3](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.10.2...v1.10.3) (2024-07-22) @@ -100,12 +83,8 @@ ## [1.10.0](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.9.2...v1.10.0) (2024-07-20) - ### Features - -* add new toml ([fcb3220](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/fcb3220868e7ef1127a7a47f40d0379be282e6eb)) - * add gpt4o omni ([431edb7](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/431edb7bb2504f4c1335c3ae3ce2f91867fa7222)) * add searchngx integration ([5c92186](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/5c9218608140bf694fbfd96aa90276bc438bb475)) * refactoring_to_md function ([602dd00](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/602dd00209ee1d72a1223fc4793759450921fcf9)) @@ -118,11 +97,8 @@ * search link node ([cf3ab55](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/cf3ab5564ae5c415c63d1771b32ea68f5169ca82)) - ### chore - -* **pyproject:** upgrade dependencies ([0425124](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/0425124c570f765b98fcf67ba6649f4f9fe76b15)) * correct search engine name ([7ba2f6a](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/7ba2f6ae0b9d2e9336e973e1f57ab8355c739e27)) * remove unused import ([fd1b7cb](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/fd1b7cb24a7c252277607abde35826e3c58e34ef)) * **ci:** upgrade lockfiles ([c7b05a4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/c7b05a4993df14d6ed4848121a3cd209571232f7)) @@ -146,7 +122,6 @@ * **release:** 1.9.0-beta.5 [skip ci] ([bb62439](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/bb624399cfc3924825892dd48697fc298ad3b002)) * **release:** 1.9.0-beta.6 [skip ci] ([54a69de](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/54a69de69e8077e02fd5584783ca62cc2e0ec5bb)) - ## [1.10.0-beta.5](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.10.0-beta.4...v1.10.0-beta.5) (2024-07-20) diff --git a/examples/nemotron/csv_scraper_graph_multi_nemotron.py b/examples/nemotron/csv_scraper_graph_multi_nemotron.py deleted file mode 100644 index 1dc7f9ce..00000000 --- a/examples/nemotron/csv_scraper_graph_multi_nemotron.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents -""" - -import os -import pandas as pd -from dotenv import load_dotenv -from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("NEMOTRON_APIKEY"), - "model": "nvidia/meta/llama3-70b-instruct", - } -} - -# ************************************************ -# Create the CSVScraperMultiGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperMultiGraph( - prompt="List me all the last names", - source=[str(text), str(text)], - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/examples/nemotron/csv_scraper_nemotron.py b/examples/nemotron/csv_scraper_nemotron.py deleted file mode 100644 index 3fede206..00000000 --- a/examples/nemotron/csv_scraper_nemotron.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -Basic example of scraping pipeline using CSVScraperGraph from CSV documents -""" - -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, -} - -# ************************************************ -# Create the CSVScraperGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperGraph( - prompt="List me all the last names", - source=str(text), # Pass the content of the file, not the file object - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/examples/nemotron/custom_graph_nemotron.py b/examples/nemotron/custom_graph_nemotron.py deleted file mode 100644 index 14057446..00000000 --- a/examples/nemotron/custom_graph_nemotron.py +++ /dev/null @@ -1,109 +0,0 @@ -""" -Example of custom graph using existing nodes -""" - -import os -from dotenv import load_dotenv - -from langchain_openai import OpenAIEmbeddings -from scrapegraphai.models import OpenAI -from scrapegraphai.graphs import BaseGraph -from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode, RobotsNode -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("NEMOTRON_KEY"), - "model": "claude-3-haiku-20240307", - }, -} - -# ************************************************ -# Define the graph nodes -# ************************************************ - -llm_model = OpenAI(graph_config["llm"]) -embedder = OpenAIEmbeddings(api_key=llm_model.openai_api_key) - -# define the nodes for the graph -robot_node = RobotsNode( - input="url", - output=["is_scrapable"], - node_config={ - "llm_model": llm_model, - "force_scraping": True, - "verbose": True, - } -) - -fetch_node = FetchNode( - input="url | local_dir", - output=["doc", "link_urls", "img_urls"], - node_config={ - "verbose": True, - "headless": True, - } -) -parse_node = ParseNode( - input="doc", - output=["parsed_doc"], - node_config={ - "chunk_size": 4096, - "verbose": True, - } -) -rag_node = RAGNode( - input="user_prompt & (parsed_doc | doc)", - output=["relevant_chunks"], - node_config={ - "llm_model": llm_model, - "embedder_model": embedder, - "verbose": True, - } -) -generate_answer_node = GenerateAnswerNode( - input="user_prompt & (relevant_chunks | parsed_doc | doc)", - output=["answer"], - node_config={ - "llm_model": llm_model, - "verbose": True, - } -) - -# ************************************************ -# Create the graph by defining the connections -# ************************************************ - -graph = BaseGraph( - nodes=[ - robot_node, - fetch_node, - parse_node, - rag_node, - generate_answer_node, - ], - edges=[ - (robot_node, fetch_node), - (fetch_node, parse_node), - (parse_node, rag_node), - (rag_node, generate_answer_node) - ], - entry_point=robot_node -) - -# ************************************************ -# Execute the graph -# ************************************************ - -result, execution_info = graph.execute({ - "user_prompt": "Describe the content", - "url": "https://example.com/" -}) - -# get the answer from the result -result = result.get("answer", "No answer found.") -print(result) diff --git a/examples/nemotron/deep_scraper_nemotron.py b/examples/nemotron/deep_scraper_nemotron.py deleted file mode 100644 index 35f54b38..00000000 --- a/examples/nemotron/deep_scraper_nemotron.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -Basic example of scraping pipeline using SmartScraper -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import DeepScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "verbose": True, - "max_depth": 1 -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -deep_scraper_graph = DeepScraperGraph( - prompt="List me all the job titles and detailed job description.", - # also accepts a string with the already downloaded HTML code - source="https://www.google.com/about/careers/applications/jobs/results/?location=Bangalore%20India", - config=graph_config -) - -result = deep_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = deep_scraper_graph.get_execution_info() -print(deep_scraper_graph.get_state("relevant_links")) -print(prettify_exec_info(graph_exec_info)) \ No newline at end of file diff --git a/examples/nemotron/inputs/books.xml b/examples/nemotron/inputs/books.xml deleted file mode 100644 index e3d1fe87..00000000 --- a/examples/nemotron/inputs/books.xml +++ /dev/null @@ -1,120 +0,0 @@ - - - - Gambardella, Matthew - XML Developer's Guide - Computer - 44.95 - 2000-10-01 - An in-depth look at creating applications - with XML. - - - Ralls, Kim - Midnight Rain - Fantasy - 5.95 - 2000-12-16 - A former architect battles corporate zombies, - an evil sorceress, and her own childhood to become queen - of the world. - - - Corets, Eva - Maeve Ascendant - Fantasy - 5.95 - 2000-11-17 - After the collapse of a nanotechnology - society in England, the young survivors lay the - foundation for a new society. - - - Corets, Eva - Oberon's Legacy - Fantasy - 5.95 - 2001-03-10 - In post-apocalypse England, the mysterious - agent known only as Oberon helps to create a new life - for the inhabitants of London. Sequel to Maeve - Ascendant. - - - Corets, Eva - The Sundered Grail - Fantasy - 5.95 - 2001-09-10 - The two daughters of Maeve, half-sisters, - battle one another for control of England. Sequel to - Oberon's Legacy. - - - Randall, Cynthia - Lover Birds - Romance - 4.95 - 2000-09-02 - When Carla meets Paul at an ornithology - conference, tempers fly as feathers get ruffled. - - - Thurman, Paula - Splish Splash - Romance - 4.95 - 2000-11-02 - A deep sea diver finds true love twenty - thousand leagues beneath the sea. - - - Knorr, Stefan - Creepy Crawlies - Horror - 4.95 - 2000-12-06 - An anthology of horror stories about roaches, - centipedes, scorpions and other insects. - - - Kress, Peter - Paradox Lost - Science Fiction - 6.95 - 2000-11-02 - After an inadvertant trip through a Heisenberg - Uncertainty Device, James Salway discovers the problems - of being quantum. - - - O'Brien, Tim - Microsoft .NET: The Programming Bible - Computer - 36.95 - 2000-12-09 - Microsoft's .NET initiative is explored in - detail in this deep programmer's reference. - - - O'Brien, Tim - MSXML3: A Comprehensive Guide - Computer - 36.95 - 2000-12-01 - The Microsoft MSXML3 parser is covered in - detail, with attention to XML DOM interfaces, XSLT processing, - SAX and more. - - - Galos, Mike - Visual Studio 7: A Comprehensive Guide - Computer - 49.95 - 2001-04-16 - Microsoft Visual Studio 7 is explored in depth, - looking at how Visual Basic, Visual C++, C#, and ASP+ are - integrated into a comprehensive development - environment. - - \ No newline at end of file diff --git a/examples/nemotron/inputs/example.json b/examples/nemotron/inputs/example.json deleted file mode 100644 index 2263184c..00000000 --- a/examples/nemotron/inputs/example.json +++ /dev/null @@ -1,182 +0,0 @@ -{ - "kind":"youtube#searchListResponse", - "etag":"q4ibjmYp1KA3RqMF4jFLl6PBwOg", - "nextPageToken":"CAUQAA", - "regionCode":"NL", - "pageInfo":{ - "totalResults":1000000, - "resultsPerPage":5 - }, - "items":[ - { - "kind":"youtube#searchResult", - "etag":"QCsHBifbaernVCbLv8Cu6rAeaDQ", - "id":{ - "kind":"youtube#video", - "videoId":"TvWDY4Mm5GM" - }, - "snippet":{ - "publishedAt":"2023-07-24T14:15:01Z", - "channelId":"UCwozCpFp9g9x0wAzuFh0hwQ", - "title":"3 Football Clubs Kylian Mbappe Should Avoid Signing ✍️❌⚽️ #football #mbappe #shorts", - "description":"", - "thumbnails":{ - "default":{ - "url":"https://i.ytimg.com/vi/TvWDY4Mm5GM/default.jpg", - "width":120, - "height":90 - }, - "medium":{ - "url":"https://i.ytimg.com/vi/TvWDY4Mm5GM/mqdefault.jpg", - "width":320, - "height":180 - }, - "high":{ - "url":"https://i.ytimg.com/vi/TvWDY4Mm5GM/hqdefault.jpg", - "width":480, - "height":360 - } - }, - "channelTitle":"FC Motivate", - "liveBroadcastContent":"none", - "publishTime":"2023-07-24T14:15:01Z" - } - }, - { - "kind":"youtube#searchResult", - "etag":"0NG5QHdtIQM_V-DBJDEf-jK_Y9k", - "id":{ - "kind":"youtube#video", - "videoId":"aZM_42CcNZ4" - }, - "snippet":{ - "publishedAt":"2023-07-24T16:09:27Z", - "channelId":"UCM5gMM_HqfKHYIEJ3lstMUA", - "title":"Which Football Club Could Cristiano Ronaldo Afford To Buy? 💰", - "description":"Sign up to Sorare and get a FREE card: https://sorare.pxf.io/NellisShorts Give Soraredata a go for FREE: ...", - "thumbnails":{ - "default":{ - "url":"https://i.ytimg.com/vi/aZM_42CcNZ4/default.jpg", - "width":120, - "height":90 - }, - "medium":{ - "url":"https://i.ytimg.com/vi/aZM_42CcNZ4/mqdefault.jpg", - "width":320, - "height":180 - }, - "high":{ - "url":"https://i.ytimg.com/vi/aZM_42CcNZ4/hqdefault.jpg", - "width":480, - "height":360 - } - }, - "channelTitle":"John Nellis", - "liveBroadcastContent":"none", - "publishTime":"2023-07-24T16:09:27Z" - } - }, - { - "kind":"youtube#searchResult", - "etag":"WbBz4oh9I5VaYj91LjeJvffrBVY", - "id":{ - "kind":"youtube#video", - "videoId":"wkP3XS3aNAY" - }, - "snippet":{ - "publishedAt":"2023-07-24T16:00:50Z", - "channelId":"UC4EP1dxFDPup_aFLt0ElsDw", - "title":"PAULO DYBALA vs THE WORLD'S LONGEST FREEKICK WALL", - "description":"Can Paulo Dybala curl a football around the World's longest free kick wall? We met up with the World Cup winner and put him to ...", - "thumbnails":{ - "default":{ - "url":"https://i.ytimg.com/vi/wkP3XS3aNAY/default.jpg", - "width":120, - "height":90 - }, - "medium":{ - "url":"https://i.ytimg.com/vi/wkP3XS3aNAY/mqdefault.jpg", - "width":320, - "height":180 - }, - "high":{ - "url":"https://i.ytimg.com/vi/wkP3XS3aNAY/hqdefault.jpg", - "width":480, - "height":360 - } - }, - "channelTitle":"Shoot for Love", - "liveBroadcastContent":"none", - "publishTime":"2023-07-24T16:00:50Z" - } - }, - { - "kind":"youtube#searchResult", - "etag":"juxv_FhT_l4qrR05S1QTrb4CGh8", - "id":{ - "kind":"youtube#video", - "videoId":"rJkDZ0WvfT8" - }, - "snippet":{ - "publishedAt":"2023-07-24T10:00:39Z", - "channelId":"UCO8qj5u80Ga7N_tP3BZWWhQ", - "title":"TOP 10 DEFENDERS 2023", - "description":"SoccerKingz https://soccerkingz.nl Use code: 'ILOVEHOF' to get 10% off. TOP 10 DEFENDERS 2023 Follow us! • Instagram ...", - "thumbnails":{ - "default":{ - "url":"https://i.ytimg.com/vi/rJkDZ0WvfT8/default.jpg", - "width":120, - "height":90 - }, - "medium":{ - "url":"https://i.ytimg.com/vi/rJkDZ0WvfT8/mqdefault.jpg", - "width":320, - "height":180 - }, - "high":{ - "url":"https://i.ytimg.com/vi/rJkDZ0WvfT8/hqdefault.jpg", - "width":480, - "height":360 - } - }, - "channelTitle":"Home of Football", - "liveBroadcastContent":"none", - "publishTime":"2023-07-24T10:00:39Z" - } - }, - { - "kind":"youtube#searchResult", - "etag":"wtuknXTmI1txoULeH3aWaOuXOow", - "id":{ - "kind":"youtube#video", - "videoId":"XH0rtu4U6SE" - }, - "snippet":{ - "publishedAt":"2023-07-21T16:30:05Z", - "channelId":"UCwozCpFp9g9x0wAzuFh0hwQ", - "title":"3 Things You Didn't Know About Erling Haaland ⚽️🇳🇴 #football #haaland #shorts", - "description":"", - "thumbnails":{ - "default":{ - "url":"https://i.ytimg.com/vi/XH0rtu4U6SE/default.jpg", - "width":120, - "height":90 - }, - "medium":{ - "url":"https://i.ytimg.com/vi/XH0rtu4U6SE/mqdefault.jpg", - "width":320, - "height":180 - }, - "high":{ - "url":"https://i.ytimg.com/vi/XH0rtu4U6SE/hqdefault.jpg", - "width":480, - "height":360 - } - }, - "channelTitle":"FC Motivate", - "liveBroadcastContent":"none", - "publishTime":"2023-07-21T16:30:05Z" - } - } - ] -} \ No newline at end of file diff --git a/examples/nemotron/inputs/markdown_example.md b/examples/nemotron/inputs/markdown_example.md deleted file mode 100644 index 85088f29..00000000 --- a/examples/nemotron/inputs/markdown_example.md +++ /dev/null @@ -1,35 +0,0 @@ -Marco Perini Toggle navigation - - * About - * Projects(current) - -Projects - -Competitions - - * CV - * ____ - -# Projects - - ![project thumbnail Rotary Pendulum RL -Open Source project aimed at controlling a real life rotary pendulum using RL -algorithms ](/projects/rotary-pendulum-rl/) - - ![project thumbnail DQN -Implementation from scratch Developed a Deep Q-Network algorithm to train a -simple and double pendulum ](https://github.com/PeriniM/DQN-SwingUp) - - ![project thumbnail Multi Agents HAED -University project which focuses on simulating a multi-agent system to perform -environment mapping. Agents, equipped with sensors, explore and record their -surroundings, considering uncertainties in their readings. -](https://github.com/PeriniM/Multi-Agents-HAED) - - ![project thumbnail Wireless ESC for Modular -Drones Modular drone architecture proposal and proof of concept. The project -received maximum grade. ](/projects/wireless-esc-drone/) - -© Copyright 2023 Marco Perini. Powered by Jekyll with -al-folio theme. Hosted by [GitHub -Pages](https://pages.github.com/). \ No newline at end of file diff --git a/examples/nemotron/inputs/plain_html_example.txt b/examples/nemotron/inputs/plain_html_example.txt deleted file mode 100644 index 78f814ae..00000000 --- a/examples/nemotron/inputs/plain_html_example.txt +++ /dev/null @@ -1,105 +0,0 @@ - -
- - -
- -
-
-
-
-

Projects

-

-
-
-
-
-
-
- -
-
- project thumbnail -
-
-

Rotary Pendulum RL

-

Open Source project aimed at controlling a real life rotary pendulum using RL algorithms

-
-
-
- -
-
-
- -
-
- project thumbnail -
-
-

DQN Implementation from scratch

-

Developed a Deep Q-Network algorithm to train a simple and double pendulum

-
-
-
- -
-
-
- -
-
- project thumbnail -
-
-

Multi Agents HAED

-

University project which focuses on simulating a multi-agent system to perform environment mapping. Agents, equipped with sensors, explore and record their surroundings, considering uncertainties in their readings.

-
-
-
- -
-
-
- -
-
- project thumbnail -
-
-

Wireless ESC for Modular Drones

-

Modular drone architecture proposal and proof of concept. The project received maximum grade.

-
-
-
- -
-
-
-
-
-
- -
- \ No newline at end of file diff --git a/examples/nemotron/inputs/username.csv b/examples/nemotron/inputs/username.csv deleted file mode 100644 index 006ac8e6..00000000 --- a/examples/nemotron/inputs/username.csv +++ /dev/null @@ -1,7 +0,0 @@ -Username; Identifier;First name;Last name -booker12;9012;Rachel;Booker -grey07;2070;Laura;Grey -johnson81;4081;Craig;Johnson -jenkins46;9346;Mary;Jenkins -smith79;5079;Jamie;Smith - diff --git a/examples/nemotron/json_scraper_multi_nemotron.py b/examples/nemotron/json_scraper_multi_nemotron.py deleted file mode 100644 index 846c7f48..00000000 --- a/examples/nemotron/json_scraper_multi_nemotron.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -Module for showing how PDFScraper multi works -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperMultiGraph - -load_dotenv() - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, -} - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -sources = [text, text] - -multiple_search_graph = JSONScraperMultiGraph( - prompt= "List me all the authors, title and genres of the books", - source= sources, - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/examples/nemotron/json_scraper_nemotron.py b/examples/nemotron/json_scraper_nemotron.py deleted file mode 100644 index 7f19d15e..00000000 --- a/examples/nemotron/json_scraper_nemotron.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -Basic example of scraping pipeline using JSONScraperGraph from JSON documents -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the JSON file -# ************************************************ - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, -} - -# ************************************************ -# Create the JSONScraperGraph instance and run it -# ************************************************ - -json_scraper_graph = JSONScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = json_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = json_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") - diff --git a/examples/nemotron/md_scraper_nemotron.py b/examples/nemotron/md_scraper_nemotron.py deleted file mode 100644 index 71073bd4..00000000 --- a/examples/nemotron/md_scraper_nemotron.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -Basic example of scraping pipeline using MDScraperGraph from XML documents -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import MDScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/markdown_example.md" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, -} - -# ************************************************ -# Create the MDScraperGraph instance and run it -# ************************************************ - -md_scraper_graph = MDScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = md_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = md_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/examples/nemotron/pdf_scraper_nemotron.py b/examples/nemotron/pdf_scraper_nemotron.py deleted file mode 100644 index b8b4482e..00000000 --- a/examples/nemotron/pdf_scraper_nemotron.py +++ /dev/null @@ -1,40 +0,0 @@ -import os, json -from dotenv import load_dotenv -from scrapegraphai.graphs import PDFScraperGraph - -load_dotenv() - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "verbose": True, -} - -source = """ - The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian - circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. - Divided into three major sections—Inferno, Purgatorio, and Paradiso—the narrative traces the journey of Dante - from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. - Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood - through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided - by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, - the Beatrice of his earlier poetry, through the celestial spheres of Paradise. -""" - -pdf_scraper_graph = PDFScraperGraph( - prompt="Summarize the text and find the main topics", - source=source, - config=graph_config, -) -result = pdf_scraper_graph.run() - -print(json.dumps(result, indent=4)) diff --git a/examples/nemotron/scrape_plain_text_nemotron.py b/examples/nemotron/scrape_plain_text_nemotron.py deleted file mode 100644 index e5e7f764..00000000 --- a/examples/nemotron/scrape_plain_text_nemotron.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -Basic example of scraping pipeline using SmartScraper from text -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ - -FILE_NAME = "inputs/plain_html_example.txt" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -# It could be also a http request using the request model -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - source=text, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/examples/nemotron/script_generator_nemotron.py b/examples/nemotron/script_generator_nemotron.py deleted file mode 100644 index d8863214..00000000 --- a/examples/nemotron/script_generator_nemotron.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "library": "beautifulsoup" -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/examples/nemotron/script_generator_schema_nemotron.py b/examples/nemotron/script_generator_schema_nemotron.py deleted file mode 100644 index 3f0713a4..00000000 --- a/examples/nemotron/script_generator_schema_nemotron.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -from pydantic import BaseModel, Field -from typing import List - -load_dotenv() - -# ************************************************ -# Define the schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "library": "beautifulsoup", - "verbose": True, -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config, - schema=Projects -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/examples/nemotron/script_multi_generator_nemotron.py b/examples/nemotron/script_multi_generator_nemotron.py deleted file mode 100644 index ad0b4b48..00000000 --- a/examples/nemotron/script_multi_generator_nemotron.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorMultiGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "library": "beautifulsoup", - "verbose": True, -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -urls=[ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" -] - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorMultiGraph( - prompt="Who is Marco Perini?", - source=urls, - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/examples/nemotron/search_graph_nemotron.py b/examples/nemotron/search_graph_nemotron.py deleted file mode 100644 index 3e6a7050..00000000 --- a/examples/nemotron/search_graph_nemotron.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -Example of Search Graph -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "max_results": 2, - "verbose": True, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/examples/nemotron/search_graph_schema_nemotron.py b/examples/nemotron/search_graph_schema_nemotron.py deleted file mode 100644 index eec72daf..00000000 --- a/examples/nemotron/search_graph_schema_nemotron.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -Example of Search Graph -""" - -import os -from dotenv import load_dotenv -load_dotenv() - -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -from pydantic import BaseModel, Field -from typing import List - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Dish(BaseModel): - name: str = Field(description="The name of the dish") - description: str = Field(description="The description of the dish") - -class Dishes(BaseModel): - dishes: List[Dish] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "max_results": 2, - "verbose": True, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config, - schema=Dishes -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/examples/nemotron/search_link_graph_nemotron.py b/examples/nemotron/search_link_graph_nemotron.py deleted file mode 100644 index 6d1edbde..00000000 --- a/examples/nemotron/search_link_graph_nemotron.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchLinkGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("NEMOTRON_APIKEY"), - "model": "nvidia/meta/llama3-70b-instruct", - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SearchLinkGraph instance and run it -# ************************************************ - -smart_scraper_graph = SearchLinkGraph( - source="https://sport.sky.it/nba?gr=www", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/examples/nemotron/smart_scraper_multi_nemotron.py b/examples/nemotron/smart_scraper_multi_nemotron.py deleted file mode 100644 index c8e167ad..00000000 --- a/examples/nemotron/smart_scraper_multi_nemotron.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Basic example of scraping pipeline using SmartScraper -""" - -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "verbose": True, - "headless": False, -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/examples/nemotron/smart_scraper_nemotron.py b/examples/nemotron/smart_scraper_nemotron.py deleted file mode 100644 index 182a12d1..00000000 --- a/examples/nemotron/smart_scraper_nemotron.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -Basic example of scraping pipeline using SmartScraper -""" - -import os, json -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info -from dotenv import load_dotenv -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "api_key": os.getenv("NEMOTRON_KEY"), - "model": "nvidia/meta/llama3-70b-instruct", - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="Extract me the python code inside the page", - source="https://www.exploit-db.com/exploits/51447", - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/examples/nemotron/smart_scraper_schema_nemotron.py b/examples/nemotron/smart_scraper_schema_nemotron.py deleted file mode 100644 index e1462e85..00000000 --- a/examples/nemotron/smart_scraper_schema_nemotron.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -Basic example of scraping pipeline using SmartScraper with schema -""" - -import os, json -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import SmartScraperGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key":nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) diff --git a/examples/nemotron/speech_graph_nemotron.py b/examples/nemotron/speech_graph_nemotron.py deleted file mode 100644 index 3d46b9e2..00000000 --- a/examples/nemotron/speech_graph_nemotron.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -Basic example of scraping pipeline using SpeechSummaryGraph -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SpeechGraph -from scrapegraphai.utils import prettify_exec_info -load_dotenv() - -# ************************************************ -# Define audio output path -# ************************************************ - -FILE_NAME = "website_summary.mp3" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -output_path = os.path.join(curr_dir, FILE_NAME) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, -} - -# ************************************************ -# Create the SpeechGraph instance and run it -# ************************************************ - -speech_graph = SpeechGraph( - prompt="Make a detailed audio summary of the projects.", - source="https://perinim.github.io/projects/", - config=graph_config, -) - -result = speech_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = speech_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/examples/nemotron/xml_scraper_graph_nemotron.py b/examples/nemotron/xml_scraper_graph_nemotron.py deleted file mode 100644 index 4b53e082..00000000 --- a/examples/nemotron/xml_scraper_graph_nemotron.py +++ /dev/null @@ -1,60 +0,0 @@ -""" - -Basic example of scraping pipeline using XMLScraperMultiGraph from XML documents -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key":nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "verbose": True, - "headless": False, -} -# ************************************************ -# Create the XMLScraperMultiGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperMultiGraph( - prompt="List me all the authors, title and genres of the books", - source=[text, text], # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/examples/nemotron/xml_scraper_nemotron.py b/examples/nemotron/xml_scraper_nemotron.py deleted file mode 100644 index a3291cce..00000000 --- a/examples/nemotron/xml_scraper_nemotron.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -Basic example of scraping pipeline using XMLScraperGraph from XML documents -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "verbose":False, -} - -# ************************************************ -# Create the XMLScraperGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") - diff --git a/examples/openai/csv_scraper_graph_multi_openai.py b/examples/openai/csv_scraper_graph_multi_openai.py index 771ad679..890765df 100644 --- a/examples/openai/csv_scraper_graph_multi_openai.py +++ b/examples/openai/csv_scraper_graph_multi_openai.py @@ -22,13 +22,13 @@ text = pd.read_csv(file_path) # ************************************************ # Define the configuration for the graph # ************************************************ -openai_key = os.getenv("OPENAI_APIKEY") graph_config = { - "llm": { - "api_key": openai_key, - "model": "gpt-3.5-turbo", - }, + "llm": { + "api_key": "***************************", + "model": "oneapi/qwen-turbo", + "base_url": "http://127.0.0.1:3000/v1", # 设置 OneAPI URL + } } # ************************************************ diff --git a/examples/openai/custom_graph_openai.py b/examples/openai/custom_graph_openai.py index 905473e0..9580e88a 100644 --- a/examples/openai/custom_graph_openai.py +++ b/examples/openai/custom_graph_openai.py @@ -15,12 +15,12 @@ load_dotenv() # Define the configuration for the graph # ************************************************ -openai_key = os.getenv("OPENAI_APIKEY") graph_config = { - "llm": { - "api_key": openai_key, - "model": "gpt-3.5-turbo", - }, + "llm": { + "api_key": os.getenv("ANTHROPIC_API_KEY"), + "model": "claude-3-haiku-20240307", + "max_tokens": 4000 + }, } # ************************************************ diff --git a/examples/openai/json_scraper_multi_openai.py b/examples/openai/json_scraper_multi_openai.py index 021cd6e1..5f3d9fc2 100644 --- a/examples/openai/json_scraper_multi_openai.py +++ b/examples/openai/json_scraper_multi_openai.py @@ -14,7 +14,7 @@ graph_config = { "llm": { "api_key": openai_key, "model": "gpt-3.5-turbo", - } + }, } FILE_NAME = "inputs/example.json" diff --git a/examples/openai/smart_scraper_openai.py b/examples/openai/smart_scraper_openai.py index 6771b817..b0515770 100644 --- a/examples/openai/smart_scraper_openai.py +++ b/examples/openai/smart_scraper_openai.py @@ -27,8 +27,8 @@ graph_config = { # ************************************************ smart_scraper_graph = SmartScraperGraph( - prompt="List me what does the company do, the name and a contact email.", - source="https://scrapegraphai.com/", + prompt="Extract me the python code inside the page", + source="https://www.exploit-db.com/exploits/51447", config=graph_config ) diff --git a/pyproject.toml b/pyproject.toml index 8e74cca3..e6e992a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,16 +13,12 @@ authors = [ ] dependencies = [ "langchain>=0.2.10", - - "langchain-fireworks>=0.1.3", - "langchain_community>=0.2.9", "langchain-google-genai>=1.0.7", "langchain-google-vertexai", "langchain-openai>=0.1.17", "langchain-groq>=0.1.3", "langchain-aws>=0.1.3", "langchain-anthropic>=0.1.11", - "langchain-nvidia-ai-endpoints>=0.1.6", "html2text>=2024.2.26", "faiss-cpu>=1.8.0", "beautifulsoup4>=4.12.3", @@ -37,6 +33,8 @@ dependencies = [ "google>=3.0.0", "undetected-playwright>=0.3.0", "semchunk>=1.0.1", + "html2text>=2024.2.26", + "langchain-fireworks>=0.1.3", ] license = "MIT" diff --git a/requirements-dev.lock b/requirements-dev.lock index 405395c4..e9337e50 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -8,12 +8,11 @@ # with-sources: false -e file:. -aiofiles==24.1.0 +aiofiles==23.2.1 # via burr aiohttp==3.9.5 # via langchain # via langchain-fireworks - # via langchain-nvidia-ai-endpoints aiosignal==1.3.1 # via aiohttp alabaster==0.7.16 @@ -24,14 +23,14 @@ annotated-types==0.7.0 # via pydantic anthropic==0.31.2 # via langchain-anthropic -anyio==4.4.0 +anyio==4.3.0 # via anthropic # via groq # via httpx # via openai # via starlette # via watchfiles -astroid==3.2.4 +astroid==3.2.2 # via pylint async-timeout==4.0.3 # via aiohttp @@ -48,17 +47,17 @@ beautifulsoup4==4.12.3 # via scrapegraphai blinker==1.8.2 # via streamlit -boto3==1.34.146 +boto3==1.34.145 # via langchain-aws -botocore==1.34.146 +botocore==1.34.145 # via boto3 # via s3transfer burr==0.22.1 # via scrapegraphai -cachetools==5.4.0 +cachetools==5.3.3 # via google-auth # via streamlit -certifi==2024.7.4 +certifi==2024.2.2 # via httpcore # via httpx # via requests @@ -73,12 +72,9 @@ contourpy==1.2.1 # via matplotlib cycler==0.12.1 # via matplotlib -dataclasses-json==0.6.7 - # via langchain-community defusedxml==0.7.1 # via langchain-anthropic dill==0.3.8 - # via multiprocess # via pylint distro==1.9.0 # via anthropic @@ -90,31 +86,32 @@ docstring-parser==0.16 # via google-cloud-aiplatform docutils==0.19 # via sphinx -email-validator==2.2.0 +email-validator==2.1.1 # via fastapi -exceptiongroup==1.2.2 +exceptiongroup==1.2.1 # via anyio # via pytest -faiss-cpu==1.8.0.post1 +faiss-cpu==1.8.0 # via scrapegraphai -fastapi==0.111.1 +fastapi==0.111.0 # via burr + # via fastapi-pagination fastapi-cli==0.0.4 # via fastapi -fastapi-pagination==0.12.26 +fastapi-pagination==0.12.24 # via burr -filelock==3.15.4 +filelock==3.14.0 # via huggingface-hub fireworks-ai==0.14.0 # via langchain-fireworks -fonttools==4.53.1 +fonttools==4.52.1 # via matplotlib free-proxy==1.1.1 # via scrapegraphai frozenlist==1.4.1 # via aiohttp # via aiosignal -fsspec==2024.6.1 +fsspec==2024.5.0 # via huggingface-hub furo==2024.5.6 # via scrapegraphai @@ -126,7 +123,7 @@ google==3.0.0 # via scrapegraphai google-ai-generativelanguage==0.6.6 # via google-generativeai -google-api-core==2.19.1 +google-api-core==2.19.0 # via google-ai-generativelanguage # via google-api-python-client # via google-cloud-aiplatform @@ -135,9 +132,9 @@ google-api-core==2.19.1 # via google-cloud-resource-manager # via google-cloud-storage # via google-generativeai -google-api-python-client==2.137.0 +google-api-python-client==2.130.0 # via google-generativeai -google-auth==2.32.0 +google-auth==2.29.0 # via google-ai-generativelanguage # via google-api-core # via google-api-python-client @@ -150,16 +147,16 @@ google-auth==2.32.0 # via google-generativeai google-auth-httplib2==0.2.0 # via google-api-python-client -google-cloud-aiplatform==1.59.0 +google-cloud-aiplatform==1.58.0 # via langchain-google-vertexai google-cloud-bigquery==3.25.0 # via google-cloud-aiplatform google-cloud-core==2.4.1 # via google-cloud-bigquery # via google-cloud-storage -google-cloud-resource-manager==1.12.4 +google-cloud-resource-manager==1.12.3 # via google-cloud-aiplatform -google-cloud-storage==2.18.0 +google-cloud-storage==2.17.0 # via google-cloud-aiplatform # via langchain-google-vertexai google-crc32c==1.5.0 @@ -170,7 +167,7 @@ google-generativeai==0.7.2 google-resumable-media==2.7.1 # via google-cloud-bigquery # via google-cloud-storage -googleapis-common-protos==1.63.2 +googleapis-common-protos==1.63.0 # via google-api-core # via grpc-google-iam-v1 # via grpcio-status @@ -179,11 +176,12 @@ graphviz==0.20.3 # via scrapegraphai greenlet==3.0.3 # via playwright -groq==0.9.0 + # via sqlalchemy +groq==0.8.0 # via langchain-groq grpc-google-iam-v1==0.13.1 # via google-cloud-resource-manager -grpcio==1.65.1 +grpcio==1.64.0 # via google-api-core # via googleapis-common-protos # via grpc-google-iam-v1 @@ -210,7 +208,7 @@ httpx==0.27.0 # via openai httpx-sse==0.4.0 # via fireworks-ai -huggingface-hub==0.24.0 +huggingface-hub==0.23.1 # via tokenizers idna==3.7 # via anyio @@ -234,29 +232,26 @@ jinja2==3.1.4 # via fastapi # via pydeck # via sphinx -jiter==0.5.0 +jiter==0.4.0 # via anthropic jmespath==1.0.1 # via boto3 # via botocore jsonpatch==1.33 # via langchain-core -jsonpointer==3.0.0 +jsonpointer==2.4 # via jsonpatch -jsonschema==4.23.0 +jsonschema==4.22.0 # via altair jsonschema-specifications==2023.12.1 # via jsonschema kiwisolver==1.4.5 # via matplotlib langchain==0.2.10 - # via langchain-community # via scrapegraphai langchain-anthropic==0.1.20 # via scrapegraphai -langchain-aws==0.1.12 - # via scrapegraphai -langchain-community==0.2.9 +langchain-aws==0.1.11 # via scrapegraphai langchain-core==0.2.22 # via langchain @@ -266,19 +261,16 @@ langchain-core==0.2.22 # via langchain-google-genai # via langchain-google-vertexai # via langchain-groq - # via langchain-nvidia-ai-endpoints # via langchain-openai # via langchain-text-splitters -langchain-fireworks==0.1.5 +langchain-fireworks==0.1.3 # via scrapegraphai -langchain-google-genai==1.0.8 +langchain-google-genai==1.0.7 # via scrapegraphai -langchain-google-vertexai==1.0.7 +langchain-google-vertexai==1.0.4 # via scrapegraphai langchain-groq==0.1.6 # via scrapegraphai -langchain-nvidia-ai-endpoints==0.1.6 - # via scrapegraphai langchain-openai==0.1.17 # via scrapegraphai langchain-text-splitters==0.2.2 @@ -294,9 +286,7 @@ markdown-it-py==3.0.0 # via rich markupsafe==2.1.5 # via jinja2 -marshmallow==3.21.3 - # via dataclasses-json -matplotlib==3.9.1 +matplotlib==3.9.0 # via burr mccabe==0.7.0 # via pylint @@ -304,13 +294,9 @@ mdurl==0.1.2 # via markdown-it-py minify-html==0.15.0 # via scrapegraphai -mpire==2.10.2 - # via semchunk multidict==6.0.5 # via aiohttp # via yarl -multiprocess==0.70.16 - # via mpire mypy-extensions==1.0.0 # via typing-inspect numpy==1.26.4 @@ -330,11 +316,11 @@ openai==1.37.0 # via burr # via langchain-fireworks # via langchain-openai -orjson==3.10.6 +orjson==3.10.3 + # via fastapi # via langsmith -packaging==24.1 +packaging==23.2 # via altair - # via faiss-cpu # via google-cloud-aiplatform # via google-cloud-bigquery # via huggingface-hub @@ -348,19 +334,18 @@ pandas==2.2.2 # via scrapegraphai # via sf-hamilton # via streamlit -pillow==10.4.0 +pillow==10.3.0 # via fireworks-ai - # via langchain-nvidia-ai-endpoints # via matplotlib # via streamlit platformdirs==4.2.2 # via pylint -playwright==1.45.0 +playwright==1.43.0 # via scrapegraphai # via undetected-playwright pluggy==1.5.0 # via pytest -proto-plus==1.24.0 +proto-plus==1.23.0 # via google-ai-generativelanguage # via google-api-core # via google-cloud-aiplatform @@ -376,14 +361,14 @@ protobuf==4.25.3 # via grpcio-status # via proto-plus # via streamlit -pyarrow==17.0.0 +pyarrow==16.1.0 # via streamlit pyasn1==0.6.0 # via pyasn1-modules # via rsa pyasn1-modules==0.4.0 # via google-auth -pydantic==2.8.2 +pydantic==2.7.1 # via anthropic # via burr # via fastapi @@ -396,7 +381,7 @@ pydantic==2.8.2 # via langchain-core # via langsmith # via openai -pydantic-core==2.20.1 +pydantic-core==2.18.2 # via pydantic pydeck==0.9.1 # via streamlit @@ -404,10 +389,9 @@ pyee==11.1.0 # via playwright pygments==2.18.0 # via furo - # via mpire # via rich # via sphinx -pylint==3.2.6 +pylint==3.2.5 pyparsing==3.1.2 # via httplib2 # via matplotlib @@ -436,7 +420,7 @@ referencing==0.35.1 # via jsonschema-specifications regex==2024.5.15 # via tiktoken -requests==2.32.3 +requests==2.32.2 # via burr # via free-proxy # via google-api-core @@ -452,18 +436,18 @@ requests==2.32.3 rich==13.7.1 # via streamlit # via typer -rpds-py==0.19.0 +rpds-py==0.18.1 # via jsonschema # via referencing rsa==4.9 # via google-auth -s3transfer==0.10.2 +s3transfer==0.10.1 # via boto3 -semchunk==2.2.0 +semchunk==1.0.1 # via scrapegraphai -sf-hamilton==1.72.1 +sf-hamilton==1.63.0 # via burr -shapely==2.0.5 +shapely==2.0.4 # via google-cloud-aiplatform shellingham==1.5.4 # via typer @@ -491,21 +475,21 @@ sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 # via sphinx -sphinxcontrib-htmlhelp==2.0.6 +sphinxcontrib-htmlhelp==2.0.5 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.8 +sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy==2.0.31 +sqlalchemy==2.0.30 # via langchain starlette==0.37.2 # via fastapi -streamlit==1.36.0 +streamlit==1.35.0 # via burr -tenacity==8.5.0 +tenacity==8.3.0 # via langchain # via langchain-core # via streamlit @@ -519,22 +503,21 @@ toml==0.10.2 tomli==2.0.1 # via pylint # via pytest -tomlkit==0.13.0 +tomlkit==0.12.5 # via pylint toolz==0.12.1 # via altair -tornado==6.4.1 +tornado==6.4 # via streamlit tqdm==4.66.4 # via google-generativeai # via huggingface-hub - # via mpire # via openai # via scrapegraphai # via semchunk typer==0.12.3 # via fastapi-cli -typing-extensions==4.12.2 +typing-extensions==4.12.0 # via altair # via anthropic # via anyio @@ -560,19 +543,21 @@ typing-inspect==0.9.0 # via sf-hamilton tzdata==2024.1 # via pandas +ujson==5.10.0 + # via fastapi undetected-playwright==0.3.0 # via scrapegraphai uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.19 +urllib3==1.26.18 # via botocore # via requests -uvicorn==0.30.3 +uvicorn==0.29.0 # via burr # via fastapi uvloop==0.19.0 # via uvicorn -watchfiles==0.22.0 +watchfiles==0.21.0 # via uvicorn websockets==12.0 # via uvicorn diff --git a/requirements.lock b/requirements.lock index 9d0602db..79828b70 100644 --- a/requirements.lock +++ b/requirements.lock @@ -11,14 +11,13 @@ aiohttp==3.9.5 # via langchain # via langchain-fireworks - # via langchain-nvidia-ai-endpoints aiosignal==1.3.1 # via aiohttp annotated-types==0.7.0 # via pydantic anthropic==0.31.2 # via langchain-anthropic -anyio==4.4.0 +anyio==4.3.0 # via anthropic # via groq # via httpx @@ -31,36 +30,32 @@ attrs==23.2.0 beautifulsoup4==4.12.3 # via google # via scrapegraphai -boto3==1.34.146 +boto3==1.34.145 # via langchain-aws -botocore==1.34.146 +botocore==1.34.145 # via boto3 # via s3transfer -cachetools==5.4.0 +cachetools==5.3.3 # via google-auth -certifi==2024.7.4 +certifi==2024.2.2 # via httpcore # via httpx # via requests charset-normalizer==3.3.2 # via requests -dataclasses-json==0.6.7 - # via langchain-community defusedxml==0.7.1 # via langchain-anthropic -dill==0.3.8 - # via multiprocess distro==1.9.0 # via anthropic # via groq # via openai docstring-parser==0.16 # via google-cloud-aiplatform -exceptiongroup==1.2.2 +exceptiongroup==1.2.1 # via anyio -faiss-cpu==1.8.0.post1 +faiss-cpu==1.8.0 # via scrapegraphai -filelock==3.15.4 +filelock==3.14.0 # via huggingface-hub fireworks-ai==0.14.0 # via langchain-fireworks @@ -69,13 +64,13 @@ free-proxy==1.1.1 frozenlist==1.4.1 # via aiohttp # via aiosignal -fsspec==2024.6.1 +fsspec==2024.5.0 # via huggingface-hub google==3.0.0 # via scrapegraphai google-ai-generativelanguage==0.6.6 # via google-generativeai -google-api-core==2.19.1 +google-api-core==2.19.0 # via google-ai-generativelanguage # via google-api-python-client # via google-cloud-aiplatform @@ -84,9 +79,9 @@ google-api-core==2.19.1 # via google-cloud-resource-manager # via google-cloud-storage # via google-generativeai -google-api-python-client==2.137.0 +google-api-python-client==2.130.0 # via google-generativeai -google-auth==2.32.0 +google-auth==2.29.0 # via google-ai-generativelanguage # via google-api-core # via google-api-python-client @@ -99,16 +94,16 @@ google-auth==2.32.0 # via google-generativeai google-auth-httplib2==0.2.0 # via google-api-python-client -google-cloud-aiplatform==1.59.0 +google-cloud-aiplatform==1.58.0 # via langchain-google-vertexai google-cloud-bigquery==3.25.0 # via google-cloud-aiplatform google-cloud-core==2.4.1 # via google-cloud-bigquery # via google-cloud-storage -google-cloud-resource-manager==1.12.4 +google-cloud-resource-manager==1.12.3 # via google-cloud-aiplatform -google-cloud-storage==2.18.0 +google-cloud-storage==2.17.0 # via google-cloud-aiplatform # via langchain-google-vertexai google-crc32c==1.5.0 @@ -119,7 +114,7 @@ google-generativeai==0.7.2 google-resumable-media==2.7.1 # via google-cloud-bigquery # via google-cloud-storage -googleapis-common-protos==1.63.2 +googleapis-common-protos==1.63.0 # via google-api-core # via grpc-google-iam-v1 # via grpcio-status @@ -127,11 +122,12 @@ graphviz==0.20.3 # via scrapegraphai greenlet==3.0.3 # via playwright -groq==0.9.0 + # via sqlalchemy +groq==0.8.0 # via langchain-groq grpc-google-iam-v1==0.13.1 # via google-cloud-resource-manager -grpcio==1.65.1 +grpcio==1.64.0 # via google-api-core # via googleapis-common-protos # via grpc-google-iam-v1 @@ -154,30 +150,27 @@ httpx==0.27.0 # via openai httpx-sse==0.4.0 # via fireworks-ai -huggingface-hub==0.24.0 +huggingface-hub==0.23.1 # via tokenizers idna==3.7 # via anyio # via httpx # via requests # via yarl -jiter==0.5.0 +jiter==0.4.0 # via anthropic jmespath==1.0.1 # via boto3 # via botocore jsonpatch==1.33 # via langchain-core -jsonpointer==3.0.0 +jsonpointer==2.4 # via jsonpatch langchain==0.2.10 - # via langchain-community # via scrapegraphai langchain-anthropic==0.1.20 # via scrapegraphai -langchain-aws==0.1.12 - # via scrapegraphai -langchain-community==0.2.9 +langchain-aws==0.1.11 # via scrapegraphai langchain-core==0.2.22 # via langchain @@ -187,19 +180,16 @@ langchain-core==0.2.22 # via langchain-google-genai # via langchain-google-vertexai # via langchain-groq - # via langchain-nvidia-ai-endpoints # via langchain-openai # via langchain-text-splitters -langchain-fireworks==0.1.5 +langchain-fireworks==0.1.3 # via scrapegraphai -langchain-google-genai==1.0.8 +langchain-google-genai==1.0.7 # via scrapegraphai -langchain-google-vertexai==1.0.7 +langchain-google-vertexai==1.0.4 # via scrapegraphai langchain-groq==0.1.6 # via scrapegraphai -langchain-nvidia-ai-endpoints==0.1.6 - # via scrapegraphai langchain-openai==0.1.17 # via scrapegraphai langchain-text-splitters==0.2.2 @@ -209,19 +199,11 @@ langsmith==0.1.93 # via langchain-core lxml==5.2.2 # via free-proxy -marshmallow==3.21.3 - # via dataclasses-json minify-html==0.15.0 # via scrapegraphai -mpire==2.10.2 - # via semchunk multidict==6.0.5 # via aiohttp # via yarl -multiprocess==0.70.16 - # via mpire -mypy-extensions==1.0.0 - # via typing-inspect numpy==1.26.4 # via faiss-cpu # via langchain @@ -231,23 +213,21 @@ numpy==1.26.4 openai==1.37.0 # via langchain-fireworks # via langchain-openai -orjson==3.10.6 +orjson==3.10.3 # via langsmith -packaging==24.1 - # via faiss-cpu +packaging==23.2 # via google-cloud-aiplatform # via google-cloud-bigquery # via huggingface-hub # via langchain-core pandas==2.2.2 # via scrapegraphai -pillow==10.4.0 +pillow==10.3.0 # via fireworks-ai - # via langchain-nvidia-ai-endpoints -playwright==1.45.0 +playwright==1.43.0 # via scrapegraphai # via undetected-playwright -proto-plus==1.24.0 +proto-plus==1.23.0 # via google-ai-generativelanguage # via google-api-core # via google-cloud-aiplatform @@ -267,7 +247,7 @@ pyasn1==0.6.0 # via rsa pyasn1-modules==0.4.0 # via google-auth -pydantic==2.8.2 +pydantic==2.7.1 # via anthropic # via fireworks-ai # via google-cloud-aiplatform @@ -277,12 +257,10 @@ pydantic==2.8.2 # via langchain-core # via langsmith # via openai -pydantic-core==2.20.1 +pydantic-core==2.18.2 # via pydantic pyee==11.1.0 # via playwright -pygments==2.18.0 - # via mpire pyparsing==3.1.2 # via httplib2 python-dateutil==2.9.0.post0 @@ -299,7 +277,7 @@ pyyaml==6.0.1 # via langchain-core regex==2024.5.15 # via tiktoken -requests==2.32.3 +requests==2.32.2 # via free-proxy # via google-api-core # via google-cloud-bigquery @@ -311,11 +289,11 @@ requests==2.32.3 # via tiktoken rsa==4.9 # via google-auth -s3transfer==0.10.2 +s3transfer==0.10.1 # via boto3 -semchunk==2.2.0 +semchunk==1.0.1 # via scrapegraphai -shapely==2.0.5 +shapely==2.0.4 # via google-cloud-aiplatform six==1.16.0 # via python-dateutil @@ -327,10 +305,9 @@ sniffio==1.3.1 # via openai soupsieve==2.5 # via beautifulsoup4 -sqlalchemy==2.0.31 +sqlalchemy==2.0.30 # via langchain - # via langchain-community -tenacity==8.5.0 +tenacity==8.3.0 # via langchain # via langchain-core tiktoken==0.7.0 @@ -341,11 +318,10 @@ tokenizers==0.19.1 tqdm==4.66.4 # via google-generativeai # via huggingface-hub - # via mpire # via openai # via scrapegraphai # via semchunk -typing-extensions==4.12.2 +typing-extensions==4.12.0 # via anthropic # via anyio # via google-generativeai @@ -362,7 +338,7 @@ undetected-playwright==0.3.0 # via scrapegraphai uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.19 +urllib3==1.26.18 # via botocore # via requests yarl==1.9.4 diff --git a/requirements.txt b/requirements.txt index 124840e5..4a3d525c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,10 @@ langchain>=0.2.10 -langchain_community>=0.2.9 langchain-google-genai>=1.0.7 -langchain-fireworks>=0.1.3 langchain-google-vertexai langchain-openai>=0.1.17 langchain-groq>=0.1.3 langchain-aws>=0.1.3 langchain-anthropic>=0.1.11 -langchain-nvidia-ai-endpoints>=0.1.6 html2text>=2024.2.26 faiss-cpu>=1.8.0 beautifulsoup4>=4.12.3 @@ -22,4 +19,5 @@ playwright>=1.43.0 google>=3.0.0 undetected-playwright>=0.3.0 semchunk>=1.0.1 - +html2text>=2024.2.26 +langchain-fireworks>=0.1.3 diff --git a/scrapegraphai/graphs/abstract_graph.py b/scrapegraphai/graphs/abstract_graph.py index 91396ae0..ca2b2e81 100644 --- a/scrapegraphai/graphs/abstract_graph.py +++ b/scrapegraphai/graphs/abstract_graph.py @@ -14,7 +14,6 @@ from langchain_google_vertexai import VertexAIEmbeddings from langchain_google_genai.embeddings import GoogleGenerativeAIEmbeddings from langchain_fireworks import FireworksEmbeddings from langchain_openai import AzureOpenAIEmbeddings, OpenAIEmbeddings -from langchain_nvidia_ai_endpoints import NVIDIAEmbeddings from ..helpers import models_tokens from ..models import ( Anthropic, @@ -27,8 +26,7 @@ from ..models import ( OpenAI, OneApi, Fireworks, - VertexAI, - Nvidia + VertexAI ) from ..models.ernie import Ernie from ..utils.logging import set_verbosity_debug, set_verbosity_warning, set_verbosity_info @@ -186,13 +184,6 @@ class AbstractGraph(ABC): except KeyError as exc: raise KeyError("Model not supported") from exc return AzureOpenAI(llm_params) - elif "nvidia" in llm_params["model"]: - try: - self.model_token = models_tokens["nvidia"][llm_params["model"].split("/")[-1]] - llm_params["model"] = "/".join(llm_params["model"].split("/")[1:]) - except KeyError as exc: - raise KeyError("Model not supported") from exc - return Nvidia(llm_params) elif "gemini" in llm_params["model"]: llm_params["model"] = llm_params["model"].split("/")[-1] try: @@ -318,8 +309,6 @@ class AbstractGraph(ABC): return AzureOpenAIEmbeddings() elif isinstance(self.llm_model, Fireworks): return FireworksEmbeddings(model=self.llm_model.model_name) - elif isinstance(self.llm_model, Nvidia): - return NVIDIAEmbeddings(model=self.llm_model.model_name) elif isinstance(self.llm_model, Ollama): # unwrap the kwargs from the model whihc is a dict params = self.llm_model._lc_kwargs @@ -356,14 +345,6 @@ class AbstractGraph(ABC): return OpenAIEmbeddings(api_key=embedder_params["api_key"]) elif "azure" in embedder_params["model"]: return AzureOpenAIEmbeddings() - if "nvidia" in embedder_params["model"]: - embedder_params["model"] = "/".join(embedder_params["model"].split("/")[1:]) - try: - models_tokens["nvidia"][embedder_params["model"]] - except KeyError as exc: - raise KeyError("Model not supported") from exc - return NVIDIAEmbeddings(model=embedder_params["model"], - nvidia_api_key=embedder_params["api_key"]) elif "ollama" in embedder_params["model"]: embedder_params["model"] = "/".join(embedder_params["model"].split("/")[1:]) try: diff --git a/scrapegraphai/graphs/base_graph.py b/scrapegraphai/graphs/base_graph.py index 21f564d7..e6df2dec 100644 --- a/scrapegraphai/graphs/base_graph.py +++ b/scrapegraphai/graphs/base_graph.py @@ -220,8 +220,6 @@ class BaseGraph: # Log the graph execution telemetry graph_execution_time = time.time() - start_time response = state.get("answer", None) if source_type == "url" else None - content = state.get("parsed_doc", None) if response is not None else None - log_graph_execution( graph_name=self.graph_name, source=source, @@ -230,7 +228,6 @@ class BaseGraph: llm_model=llm_model, embedder_model=embedder_model, source_type=source_type, - content=content, response=response, execution_time=graph_execution_time, total_tokens=cb_total["total_tokens"] if cb_total["total_tokens"] > 0 else None, diff --git a/scrapegraphai/helpers/models_tokens.py b/scrapegraphai/helpers/models_tokens.py index 0724ee95..7ae061f1 100644 --- a/scrapegraphai/helpers/models_tokens.py +++ b/scrapegraphai/helpers/models_tokens.py @@ -83,24 +83,6 @@ models_tokens = { "oneapi": { "qwen-turbo": 6000 }, - "nvidia": { - "meta/llama3-70b-instruct": 419, - "meta/llama3-8b-instruct": 419, - "nemotron-4-340b-instruct": 1024, - "databricks/dbrx-instruct": 4096, - "google/codegemma-7b": 8192, - "google/gemma-2b": 2048, - "google/gemma-7b": 8192, - "google/recurrentgemma-2b": 2048, - "meta/codellama-70b": 16384, - "meta/llama2-70b": 4096, - "microsoft/phi-3-mini-128k-instruct": 122880, - "mistralai/mistral-7b-instruct-v0.2": 4096, - "mistralai/mistral-large": 8192, - "mistralai/mixtral-8x22b-instruct-v0.1": 32768, - "mistralai/mixtral-8x7b-instruct-v0.1": 8192, - "snowflake/arctic": 16384, - }, "groq": { "llama3-8b-8192": 8192, "llama3-70b-8192": 8192, diff --git a/scrapegraphai/models/__init__.py b/scrapegraphai/models/__init__.py index bfcb84d6..a408d9ac 100644 --- a/scrapegraphai/models/__init__.py +++ b/scrapegraphai/models/__init__.py @@ -16,4 +16,3 @@ from .deepseek import DeepSeek from .oneapi import OneApi from .fireworks import Fireworks from .vertex import VertexAI -from .nvidia import Nvidia diff --git a/scrapegraphai/models/nvidia.py b/scrapegraphai/models/nvidia.py deleted file mode 100644 index 48ce3c0f..00000000 --- a/scrapegraphai/models/nvidia.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -This is a Python wrapper class for ChatNVIDIA. -It provides default configuration and could be extended with additional methods if needed. -The purpose of this wrapper is to simplify the creation of instances of ChatNVIDIA by providing -default configurations for certain parameters, -allowing users to focus on specifying other important parameters without having -to understand all the details of the underlying class's constructor. -It inherits from the base class ChatNVIDIA and overrides -its init method to provide a more user-friendly interface. -The constructor takes one argument: llm_config, which is used to initialize the superclass -with default configuration. -""" - -from langchain_nvidia_ai_endpoints import ChatNVIDIA - -class Nvidia(ChatNVIDIA): - """ A wrapper for the Nvidia class that provides default configuration - and could be extended with additional methods if needed. - - Args: - llm_config (dict): Configuration parameters for the language model. - """ - - def __init__(self, llm_config: dict): - super().__init__(**llm_config) diff --git a/scrapegraphai/nodes/fetch_node.py b/scrapegraphai/nodes/fetch_node.py index 5f55b6e6..0fcfed66 100644 --- a/scrapegraphai/nodes/fetch_node.py +++ b/scrapegraphai/nodes/fetch_node.py @@ -184,8 +184,8 @@ class FetchNode(BaseNode): if not self.cut: parsed_content = cleanup_html(response, source) - if (not self.script_creator) or (self.force and not self.script_creator): - parsed_content = convert_to_md(parsed_content, source) + if (isinstance(self.llm_model, OpenAI) and not self.script_creator) or (self.force and not self.script_creator): + parsed_content = convert_to_md(source) compressed_document = [Document(page_content=parsed_content)] else: self.logger.warning( @@ -206,10 +206,8 @@ class FetchNode(BaseNode): raise ValueError("No HTML body content found in the document fetched by ChromiumLoader.") parsed_content = document[0].page_content - if (not self.script_creator) or (self.force and not self.script_creator and not self.openai_md_enabled): - - parsed_content = convert_to_md(document[0].page_content, source) - + if isinstance(self.llm_model, OpenAI) and not self.script_creator or self.force and not self.script_creator and not self.openai_md_enabled: + parsed_content = convert_to_md(document[0].page_content) compressed_document = [ Document(page_content=parsed_content, metadata={"source": "html file"}) diff --git a/scrapegraphai/nodes/generate_answer_csv_node.py b/scrapegraphai/nodes/generate_answer_csv_node.py index 6ce19ef2..43657b50 100644 --- a/scrapegraphai/nodes/generate_answer_csv_node.py +++ b/scrapegraphai/nodes/generate_answer_csv_node.py @@ -116,24 +116,24 @@ class GenerateAnswerCSVNode(BaseNode): chains_dict = {} - if len(doc) == 1: - prompt = PromptTemplate( - template=template_no_chunks_csv_prompt, - input_variables=["question"], - partial_variables={ - "context": doc, - "format_instructions": format_instructions, - }, - ) - - chain = prompt | self.llm_model | output_parser - answer = chain.invoke({"question": user_prompt}) - state.update({self.output[0]: answer}) - return state - + # Use tqdm to add progress bar for i, chunk in enumerate( tqdm(doc, desc="Processing chunks", disable=not self.verbose) ): + if len(doc) == 1: + prompt = PromptTemplate( + template=template_no_chunks_csv_prompt, + input_variables=["question"], + partial_variables={ + "context": chunk, + "format_instructions": format_instructions, + }, + ) + + chain = prompt | self.llm_model | output_parser + answer = chain.invoke({"question": user_prompt}) + break + prompt = PromptTemplate( template=template_chunks_csv_prompt, input_variables=["question"], @@ -144,21 +144,24 @@ class GenerateAnswerCSVNode(BaseNode): }, ) + # Dynamically name the chains based on their index chain_name = f"chunk{i+1}" chains_dict[chain_name] = prompt | self.llm_model | output_parser - async_runner = RunnableParallel(**chains_dict) - - batch_results = async_runner.invoke({"question": user_prompt}) - - merge_prompt = PromptTemplate( - template = template_merge_csv_prompt, + if len(chains_dict) > 1: + # Use dictionary unpacking to pass the dynamically named chains to RunnableParallel + map_chain = RunnableParallel(**chains_dict) + # Chain + answer = map_chain.invoke({"question": user_prompt}) + # Merge the answers from the chunks + merge_prompt = PromptTemplate( + template=template_merge_csv_prompt, input_variables=["context", "question"], partial_variables={"format_instructions": format_instructions}, ) + merge_chain = merge_prompt | self.llm_model | output_parser + answer = merge_chain.invoke({"context": answer, "question": user_prompt}) - merge_chain = merge_prompt | self.llm_model | output_parser - answer = merge_chain.invoke({"context": batch_results, "question": user_prompt}) - + # Update the state with the generated answer state.update({self.output[0]: answer}) - return state \ No newline at end of file + return state diff --git a/scrapegraphai/nodes/generate_answer_omni_node.py b/scrapegraphai/nodes/generate_answer_omni_node.py index c2f2b65d..7a030c6f 100644 --- a/scrapegraphai/nodes/generate_answer_omni_node.py +++ b/scrapegraphai/nodes/generate_answer_omni_node.py @@ -100,26 +100,26 @@ class GenerateAnswerOmniNode(BaseNode): chains_dict = {} - if len(doc) == 1: - prompt = PromptTemplate( - template=template_no_chunk_omni_prompt, - input_variables=["question"], - partial_variables={ - "context": chunk, - "format_instructions": format_instructions, - "img_desc": imag_desc, - }, - ) - - chain = prompt | self.llm_model | output_parser - answer = chain.invoke({"question": user_prompt}) - - state.update({self.output[0]: answer}) - return state + # Use tqdm to add progress bar for i, chunk in enumerate( tqdm(doc, desc="Processing chunks", disable=not self.verbose) ): + if len(doc) == 1: + prompt = PromptTemplate( + template=template_no_chunk_omni_prompt, + input_variables=["question"], + partial_variables={ + "context": chunk, + "format_instructions": format_instructions, + "img_desc": imag_desc, + }, + ) + + chain = prompt | self.llm_model | output_parser + answer = chain.invoke({"question": user_prompt}) + break + prompt = PromptTemplate( template=template_chunks_omni_prompt, input_variables=["question"], @@ -134,18 +134,23 @@ class GenerateAnswerOmniNode(BaseNode): chain_name = f"chunk{i+1}" chains_dict[chain_name] = prompt | self.llm_model | output_parser - async_runner = RunnableParallel(**chains_dict) - - batch_results = async_runner.invoke({"question": user_prompt}) - - merge_prompt = PromptTemplate( - template = template_merge_omni_prompt, + if len(chains_dict) > 1: + # Use dictionary unpacking to pass the dynamically named chains to RunnableParallel + map_chain = RunnableParallel(**chains_dict) + # Chain + answer = map_chain.invoke({"question": user_prompt}) + # Merge the answers from the chunks + merge_prompt = PromptTemplate( + template=template_merge_omni_prompt, input_variables=["context", "question"], - partial_variables={"format_instructions": format_instructions}, + partial_variables={ + "format_instructions": format_instructions, + "img_desc": imag_desc, + }, ) + merge_chain = merge_prompt | self.llm_model | output_parser + answer = merge_chain.invoke({"context": answer, "question": user_prompt}) - merge_chain = merge_prompt | self.llm_model | output_parser - answer = merge_chain.invoke({"context": batch_results, "question": user_prompt}) - + # Update the state with the generated answer state.update({self.output[0]: answer}) - return state \ No newline at end of file + return state diff --git a/scrapegraphai/nodes/generate_answer_pdf_node.py b/scrapegraphai/nodes/generate_answer_pdf_node.py index 7add7948..db6152bc 100644 --- a/scrapegraphai/nodes/generate_answer_pdf_node.py +++ b/scrapegraphai/nodes/generate_answer_pdf_node.py @@ -114,26 +114,24 @@ class GenerateAnswerPDFNode(BaseNode): format_instructions = output_parser.get_format_instructions() - if len(doc) == 1: - prompt = PromptTemplate( - template=template_no_chunks_pdf_prompt, - input_variables=["question"], - partial_variables={ - "context":chunk, - "format_instructions": format_instructions, - }, - ) - chain = prompt | self.llm_model | output_parser - answer = chain.invoke({"question": user_prompt}) - - - state.update({self.output[0]: answer}) - return state - chains_dict = {} - + # Use tqdm to add progress bar for i, chunk in enumerate( - tqdm(doc, desc="Processing chunks", disable=not self.verbose)): + tqdm(doc, desc="Processing chunks", disable=not self.verbose) + ): + if len(doc) == 1: + prompt = PromptTemplate( + template=template_no_chunks_pdf_prompt, + input_variables=["question"], + partial_variables={ + "context":chunk, + "format_instructions": format_instructions, + }, + ) + chain = prompt | self.llm_model | output_parser + answer = chain.invoke({"question": user_prompt}) + + break prompt = PromptTemplate( template=template_chunks_pdf_prompt, input_variables=["question"], @@ -144,21 +142,24 @@ class GenerateAnswerPDFNode(BaseNode): }, ) + # Dynamically name the chains based on their index chain_name = f"chunk{i+1}" chains_dict[chain_name] = prompt | self.llm_model | output_parser - async_runner = RunnableParallel(**chains_dict) - - batch_results = async_runner.invoke({"question": user_prompt}) - - merge_prompt = PromptTemplate( - template = template_merge_pdf_prompt, + if len(chains_dict) > 1: + # Use dictionary unpacking to pass the dynamically named chains to RunnableParallel + map_chain = RunnableParallel(**chains_dict) + # Chain + answer = map_chain.invoke({"question": user_prompt}) + # Merge the answers from the chunks + merge_prompt = PromptTemplate( + template=template_merge_pdf_prompt, input_variables=["context", "question"], partial_variables={"format_instructions": format_instructions}, ) + merge_chain = merge_prompt | self.llm_model | output_parser + answer = merge_chain.invoke({"context": answer, "question": user_prompt}) - merge_chain = merge_prompt | self.llm_model | output_parser - answer = merge_chain.invoke({"context": batch_results, "question": user_prompt}) - + # Update the state with the generated answer state.update({self.output[0]: answer}) return state diff --git a/scrapegraphai/telemetry/telemetry.py b/scrapegraphai/telemetry/telemetry.py index 2289afd0..e44a0c28 100644 --- a/scrapegraphai/telemetry/telemetry.py +++ b/scrapegraphai/telemetry/telemetry.py @@ -156,7 +156,7 @@ def log_event(event: str, properties: Dict[str, any]): send_event_json(event_json) -def log_graph_execution(graph_name: str, source: str, prompt:str, schema:dict, llm_model: str, embedder_model: str, source_type: str, execution_time: float, content: str = None, response: dict = None, error_node: str = None, exception: str = None, total_tokens: int = None): +def log_graph_execution(graph_name: str, source: str, prompt:str, schema:dict, llm_model: str, embedder_model: str, source_type: str, execution_time: float, response: dict = None, error_node: str = None, exception: str = None, total_tokens: int = None, is_library=True): properties = { "graph_name": graph_name, "source": source, @@ -165,13 +165,12 @@ def log_graph_execution(graph_name: str, source: str, prompt:str, schema:dict, l "llm_model": llm_model, "embedder_model": embedder_model, "source_type": source_type, - "content": content, "response": response, "execution_time": execution_time, "error_node": error_node, "exception": exception, "total_tokens": total_tokens, - "type": "community-library" + "is_library": is_library } log_event("graph_execution", properties) diff --git a/scrapegraphai/utils/convert_to_md.py b/scrapegraphai/utils/convert_to_md.py index ece41c60..9257a95e 100644 --- a/scrapegraphai/utils/convert_to_md.py +++ b/scrapegraphai/utils/convert_to_md.py @@ -2,9 +2,8 @@ convert_to_md modul """ import html2text -from urllib.parse import urlparse -def convert_to_md(html: str, url: str = None) -> str: +def convert_to_md(html): """ Convert HTML to Markdown. This function uses the html2text library to convert the provided HTML content to Markdown format. From 5c08eea189d7ede6f29399a67d897aa3b3f6a7b0 Mon Sep 17 00:00:00 2001 From: Marco Perini Date: Wed, 24 Jul 2024 23:37:57 +0200 Subject: [PATCH 3/5] docs: prev version --- CHANGELOG.md | 31 ++- .../csv_scraper_graph_multi_nemotron.py | 55 ++++++ examples/nemotron/csv_scraper_nemotron.py | 57 ++++++ examples/nemotron/custom_graph_nemotron.py | 109 +++++++++++ examples/nemotron/deep_scraper_nemotron.py | 47 +++++ examples/nemotron/inputs/books.xml | 120 ++++++++++++ examples/nemotron/inputs/example.json | 182 ++++++++++++++++++ examples/nemotron/inputs/markdown_example.md | 35 ++++ .../nemotron/inputs/plain_html_example.txt | 105 ++++++++++ examples/nemotron/inputs/username.csv | 7 + .../nemotron/json_scraper_multi_nemotron.py | 37 ++++ examples/nemotron/json_scraper_nemotron.py | 58 ++++++ examples/nemotron/md_scraper_nemotron.py | 57 ++++++ examples/nemotron/pdf_scraper_nemotron.py | 40 ++++ .../nemotron/scrape_plain_text_nemotron.py | 55 ++++++ .../nemotron/script_generator_nemotron.py | 46 +++++ .../script_generator_schema_nemotron.py | 62 ++++++ .../script_multi_generator_nemotron.py | 54 ++++++ examples/nemotron/search_graph_nemotron.py | 35 ++++ .../nemotron/search_graph_schema_nemotron.py | 63 ++++++ .../nemotron/search_link_graph_nemotron.py | 40 ++++ .../nemotron/smart_scraper_multi_nemotron.py | 42 ++++ examples/nemotron/smart_scraper_nemotron.py | 43 +++++ .../nemotron/smart_scraper_schema_nemotron.py | 51 +++++ examples/nemotron/speech_graph_nemotron.py | 50 +++++ .../nemotron/xml_scraper_graph_nemotron.py | 60 ++++++ examples/nemotron/xml_scraper_nemotron.py | 59 ++++++ .../openai/csv_scraper_graph_multi_openai.py | 10 +- examples/openai/custom_graph_openai.py | 10 +- examples/openai/json_scraper_multi_openai.py | 2 +- examples/openai/smart_scraper_openai.py | 4 +- pyproject.toml | 6 +- requirements-dev.lock | 143 ++++++++------ requirements.lock | 106 ++++++---- requirements.txt | 6 +- scrapegraphai/graphs/abstract_graph.py | 21 +- scrapegraphai/graphs/base_graph.py | 3 + scrapegraphai/helpers/models_tokens.py | 18 ++ scrapegraphai/models/__init__.py | 1 + scrapegraphai/models/nvidia.py | 25 +++ scrapegraphai/nodes/fetch_node.py | 10 +- .../nodes/generate_answer_csv_node.py | 53 +++-- .../nodes/generate_answer_omni_node.py | 63 +++--- .../nodes/generate_answer_pdf_node.py | 55 +++--- scrapegraphai/telemetry/telemetry.py | 5 +- scrapegraphai/utils/convert_to_md.py | 3 +- 46 files changed, 1921 insertions(+), 223 deletions(-) create mode 100644 examples/nemotron/csv_scraper_graph_multi_nemotron.py create mode 100644 examples/nemotron/csv_scraper_nemotron.py create mode 100644 examples/nemotron/custom_graph_nemotron.py create mode 100644 examples/nemotron/deep_scraper_nemotron.py create mode 100644 examples/nemotron/inputs/books.xml create mode 100644 examples/nemotron/inputs/example.json create mode 100644 examples/nemotron/inputs/markdown_example.md create mode 100644 examples/nemotron/inputs/plain_html_example.txt create mode 100644 examples/nemotron/inputs/username.csv create mode 100644 examples/nemotron/json_scraper_multi_nemotron.py create mode 100644 examples/nemotron/json_scraper_nemotron.py create mode 100644 examples/nemotron/md_scraper_nemotron.py create mode 100644 examples/nemotron/pdf_scraper_nemotron.py create mode 100644 examples/nemotron/scrape_plain_text_nemotron.py create mode 100644 examples/nemotron/script_generator_nemotron.py create mode 100644 examples/nemotron/script_generator_schema_nemotron.py create mode 100644 examples/nemotron/script_multi_generator_nemotron.py create mode 100644 examples/nemotron/search_graph_nemotron.py create mode 100644 examples/nemotron/search_graph_schema_nemotron.py create mode 100644 examples/nemotron/search_link_graph_nemotron.py create mode 100644 examples/nemotron/smart_scraper_multi_nemotron.py create mode 100644 examples/nemotron/smart_scraper_nemotron.py create mode 100644 examples/nemotron/smart_scraper_schema_nemotron.py create mode 100644 examples/nemotron/speech_graph_nemotron.py create mode 100644 examples/nemotron/xml_scraper_graph_nemotron.py create mode 100644 examples/nemotron/xml_scraper_nemotron.py create mode 100644 scrapegraphai/models/nvidia.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d0803bf..d5306d62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,3 @@ -<<<<<<< HEAD ## [1.11.2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.11.1...v1.11.2) (2024-07-23) @@ -50,13 +49,31 @@ ## [1.10.0-beta.8](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.10.0-beta.7...v1.10.0-beta.8) (2024-07-23) -======= ->>>>>>> parent of 7708828 (Merge pull request #488 from ScrapeGraphAI/pre/beta) ## [1.10.4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.10.3...v1.10.4) (2024-07-22) + ### Bug Fixes + +* **md_conversion:** add absolute links md, added missing dependency ([12b5ead](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/12b5eada6ea783770afd630ede69b8cf867a7ded)) + +## [1.10.0-beta.7](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.10.0-beta.6...v1.10.0-beta.7) (2024-07-23) + + +### Features + +* add nvidia connection ([fc0dadb](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/fc0dadb8f812dfd636dec856921a971b58695ce3)) + + +### chore + +* **dependecies:** add script to auto-update requirements ([3289c7b](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/3289c7bf5ec58ac3d04e9e5e8e654af9abcee228)) +* **ci:** set up workflow for requirements auto-update ([295fc28](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/295fc28ceb02c78198f7fbe678352503b3259b6b)) +* update requirements.txt ([c7bac98](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/c7bac98d2e79e5ab98fa65d7efa858a2cdda1622)) + +## [1.10.0-beta.6](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.10.0-beta.5...v1.10.0-beta.6) (2024-07-22) + * parse node ([09256f7](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/09256f7b11a7a1c2aba01cf8de70401af1e86fe4)) ## [1.10.3](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.10.2...v1.10.3) (2024-07-22) @@ -83,8 +100,12 @@ ## [1.10.0](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.9.2...v1.10.0) (2024-07-20) + ### Features + +* add new toml ([fcb3220](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/fcb3220868e7ef1127a7a47f40d0379be282e6eb)) + * add gpt4o omni ([431edb7](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/431edb7bb2504f4c1335c3ae3ce2f91867fa7222)) * add searchngx integration ([5c92186](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/5c9218608140bf694fbfd96aa90276bc438bb475)) * refactoring_to_md function ([602dd00](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/602dd00209ee1d72a1223fc4793759450921fcf9)) @@ -97,8 +118,11 @@ * search link node ([cf3ab55](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/cf3ab5564ae5c415c63d1771b32ea68f5169ca82)) + ### chore + +* **pyproject:** upgrade dependencies ([0425124](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/0425124c570f765b98fcf67ba6649f4f9fe76b15)) * correct search engine name ([7ba2f6a](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/7ba2f6ae0b9d2e9336e973e1f57ab8355c739e27)) * remove unused import ([fd1b7cb](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/fd1b7cb24a7c252277607abde35826e3c58e34ef)) * **ci:** upgrade lockfiles ([c7b05a4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/c7b05a4993df14d6ed4848121a3cd209571232f7)) @@ -122,6 +146,7 @@ * **release:** 1.9.0-beta.5 [skip ci] ([bb62439](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/bb624399cfc3924825892dd48697fc298ad3b002)) * **release:** 1.9.0-beta.6 [skip ci] ([54a69de](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/54a69de69e8077e02fd5584783ca62cc2e0ec5bb)) + ## [1.10.0-beta.5](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.10.0-beta.4...v1.10.0-beta.5) (2024-07-20) diff --git a/examples/nemotron/csv_scraper_graph_multi_nemotron.py b/examples/nemotron/csv_scraper_graph_multi_nemotron.py new file mode 100644 index 00000000..1dc7f9ce --- /dev/null +++ b/examples/nemotron/csv_scraper_graph_multi_nemotron.py @@ -0,0 +1,55 @@ +""" +Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents +""" + +import os +import pandas as pd +from dotenv import load_dotenv +from scrapegraphai.graphs import CSVScraperMultiGraph +from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info + +load_dotenv() +# ************************************************ +# Read the CSV file +# ************************************************ + +FILE_NAME = "inputs/username.csv" +curr_dir = os.path.dirname(os.path.realpath(__file__)) +file_path = os.path.join(curr_dir, FILE_NAME) + +text = pd.read_csv(file_path) + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +graph_config = { + "llm": { + "api_key": os.getenv("NEMOTRON_APIKEY"), + "model": "nvidia/meta/llama3-70b-instruct", + } +} + +# ************************************************ +# Create the CSVScraperMultiGraph instance and run it +# ************************************************ + +csv_scraper_graph = CSVScraperMultiGraph( + prompt="List me all the last names", + source=[str(text), str(text)], + config=graph_config +) + +result = csv_scraper_graph.run() +print(result) + +# ************************************************ +# Get graph execution info +# ************************************************ + +graph_exec_info = csv_scraper_graph.get_execution_info() +print(prettify_exec_info(graph_exec_info)) + +# Save to json or csv +convert_to_csv(result, "result") +convert_to_json(result, "result") diff --git a/examples/nemotron/csv_scraper_nemotron.py b/examples/nemotron/csv_scraper_nemotron.py new file mode 100644 index 00000000..3fede206 --- /dev/null +++ b/examples/nemotron/csv_scraper_nemotron.py @@ -0,0 +1,57 @@ +""" +Basic example of scraping pipeline using CSVScraperGraph from CSV documents +""" + +import os +from dotenv import load_dotenv +import pandas as pd +from scrapegraphai.graphs import CSVScraperGraph +from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info +load_dotenv() + +# ************************************************ +# Read the CSV file +# ************************************************ + +FILE_NAME = "inputs/username.csv" +curr_dir = os.path.dirname(os.path.realpath(__file__)) +file_path = os.path.join(curr_dir, FILE_NAME) + +text = pd.read_csv(file_path) + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +nemotron_key = os.getenv("NEMOTRON_APIKEY") + +graph_config = { + "llm": { + "api_key": nemotron_key, + "model": "nvidia/meta/llama3-70b-instruct", + }, +} + +# ************************************************ +# Create the CSVScraperGraph instance and run it +# ************************************************ + +csv_scraper_graph = CSVScraperGraph( + prompt="List me all the last names", + source=str(text), # Pass the content of the file, not the file object + config=graph_config +) + +result = csv_scraper_graph.run() +print(result) + +# ************************************************ +# Get graph execution info +# ************************************************ + +graph_exec_info = csv_scraper_graph.get_execution_info() +print(prettify_exec_info(graph_exec_info)) + +# Save to json or csv +convert_to_csv(result, "result") +convert_to_json(result, "result") diff --git a/examples/nemotron/custom_graph_nemotron.py b/examples/nemotron/custom_graph_nemotron.py new file mode 100644 index 00000000..14057446 --- /dev/null +++ b/examples/nemotron/custom_graph_nemotron.py @@ -0,0 +1,109 @@ +""" +Example of custom graph using existing nodes +""" + +import os +from dotenv import load_dotenv + +from langchain_openai import OpenAIEmbeddings +from scrapegraphai.models import OpenAI +from scrapegraphai.graphs import BaseGraph +from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode, RobotsNode +load_dotenv() + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +graph_config = { + "llm": { + "api_key": os.getenv("NEMOTRON_KEY"), + "model": "claude-3-haiku-20240307", + }, +} + +# ************************************************ +# Define the graph nodes +# ************************************************ + +llm_model = OpenAI(graph_config["llm"]) +embedder = OpenAIEmbeddings(api_key=llm_model.openai_api_key) + +# define the nodes for the graph +robot_node = RobotsNode( + input="url", + output=["is_scrapable"], + node_config={ + "llm_model": llm_model, + "force_scraping": True, + "verbose": True, + } +) + +fetch_node = FetchNode( + input="url | local_dir", + output=["doc", "link_urls", "img_urls"], + node_config={ + "verbose": True, + "headless": True, + } +) +parse_node = ParseNode( + input="doc", + output=["parsed_doc"], + node_config={ + "chunk_size": 4096, + "verbose": True, + } +) +rag_node = RAGNode( + input="user_prompt & (parsed_doc | doc)", + output=["relevant_chunks"], + node_config={ + "llm_model": llm_model, + "embedder_model": embedder, + "verbose": True, + } +) +generate_answer_node = GenerateAnswerNode( + input="user_prompt & (relevant_chunks | parsed_doc | doc)", + output=["answer"], + node_config={ + "llm_model": llm_model, + "verbose": True, + } +) + +# ************************************************ +# Create the graph by defining the connections +# ************************************************ + +graph = BaseGraph( + nodes=[ + robot_node, + fetch_node, + parse_node, + rag_node, + generate_answer_node, + ], + edges=[ + (robot_node, fetch_node), + (fetch_node, parse_node), + (parse_node, rag_node), + (rag_node, generate_answer_node) + ], + entry_point=robot_node +) + +# ************************************************ +# Execute the graph +# ************************************************ + +result, execution_info = graph.execute({ + "user_prompt": "Describe the content", + "url": "https://example.com/" +}) + +# get the answer from the result +result = result.get("answer", "No answer found.") +print(result) diff --git a/examples/nemotron/deep_scraper_nemotron.py b/examples/nemotron/deep_scraper_nemotron.py new file mode 100644 index 00000000..35f54b38 --- /dev/null +++ b/examples/nemotron/deep_scraper_nemotron.py @@ -0,0 +1,47 @@ +""" +Basic example of scraping pipeline using SmartScraper +""" + +import os +from dotenv import load_dotenv +from scrapegraphai.graphs import DeepScraperGraph +from scrapegraphai.utils import prettify_exec_info + +load_dotenv() + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +nemotron_key = os.getenv("NEMOTRON_APIKEY") + +graph_config = { + "llm": { + "api_key": nemotron_key, + "model": "nvidia/meta/llama3-70b-instruct", + }, + "verbose": True, + "max_depth": 1 +} + +# ************************************************ +# Create the SmartScraperGraph instance and run it +# ************************************************ + +deep_scraper_graph = DeepScraperGraph( + prompt="List me all the job titles and detailed job description.", + # also accepts a string with the already downloaded HTML code + source="https://www.google.com/about/careers/applications/jobs/results/?location=Bangalore%20India", + config=graph_config +) + +result = deep_scraper_graph.run() +print(result) + +# ************************************************ +# Get graph execution info +# ************************************************ + +graph_exec_info = deep_scraper_graph.get_execution_info() +print(deep_scraper_graph.get_state("relevant_links")) +print(prettify_exec_info(graph_exec_info)) \ No newline at end of file diff --git a/examples/nemotron/inputs/books.xml b/examples/nemotron/inputs/books.xml new file mode 100644 index 00000000..e3d1fe87 --- /dev/null +++ b/examples/nemotron/inputs/books.xml @@ -0,0 +1,120 @@ + + + + Gambardella, Matthew + XML Developer's Guide + Computer + 44.95 + 2000-10-01 + An in-depth look at creating applications + with XML. + + + Ralls, Kim + Midnight Rain + Fantasy + 5.95 + 2000-12-16 + A former architect battles corporate zombies, + an evil sorceress, and her own childhood to become queen + of the world. + + + Corets, Eva + Maeve Ascendant + Fantasy + 5.95 + 2000-11-17 + After the collapse of a nanotechnology + society in England, the young survivors lay the + foundation for a new society. + + + Corets, Eva + Oberon's Legacy + Fantasy + 5.95 + 2001-03-10 + In post-apocalypse England, the mysterious + agent known only as Oberon helps to create a new life + for the inhabitants of London. Sequel to Maeve + Ascendant. + + + Corets, Eva + The Sundered Grail + Fantasy + 5.95 + 2001-09-10 + The two daughters of Maeve, half-sisters, + battle one another for control of England. Sequel to + Oberon's Legacy. + + + Randall, Cynthia + Lover Birds + Romance + 4.95 + 2000-09-02 + When Carla meets Paul at an ornithology + conference, tempers fly as feathers get ruffled. + + + Thurman, Paula + Splish Splash + Romance + 4.95 + 2000-11-02 + A deep sea diver finds true love twenty + thousand leagues beneath the sea. + + + Knorr, Stefan + Creepy Crawlies + Horror + 4.95 + 2000-12-06 + An anthology of horror stories about roaches, + centipedes, scorpions and other insects. + + + Kress, Peter + Paradox Lost + Science Fiction + 6.95 + 2000-11-02 + After an inadvertant trip through a Heisenberg + Uncertainty Device, James Salway discovers the problems + of being quantum. + + + O'Brien, Tim + Microsoft .NET: The Programming Bible + Computer + 36.95 + 2000-12-09 + Microsoft's .NET initiative is explored in + detail in this deep programmer's reference. + + + O'Brien, Tim + MSXML3: A Comprehensive Guide + Computer + 36.95 + 2000-12-01 + The Microsoft MSXML3 parser is covered in + detail, with attention to XML DOM interfaces, XSLT processing, + SAX and more. + + + Galos, Mike + Visual Studio 7: A Comprehensive Guide + Computer + 49.95 + 2001-04-16 + Microsoft Visual Studio 7 is explored in depth, + looking at how Visual Basic, Visual C++, C#, and ASP+ are + integrated into a comprehensive development + environment. + + \ No newline at end of file diff --git a/examples/nemotron/inputs/example.json b/examples/nemotron/inputs/example.json new file mode 100644 index 00000000..2263184c --- /dev/null +++ b/examples/nemotron/inputs/example.json @@ -0,0 +1,182 @@ +{ + "kind":"youtube#searchListResponse", + "etag":"q4ibjmYp1KA3RqMF4jFLl6PBwOg", + "nextPageToken":"CAUQAA", + "regionCode":"NL", + "pageInfo":{ + "totalResults":1000000, + "resultsPerPage":5 + }, + "items":[ + { + "kind":"youtube#searchResult", + "etag":"QCsHBifbaernVCbLv8Cu6rAeaDQ", + "id":{ + "kind":"youtube#video", + "videoId":"TvWDY4Mm5GM" + }, + "snippet":{ + "publishedAt":"2023-07-24T14:15:01Z", + "channelId":"UCwozCpFp9g9x0wAzuFh0hwQ", + "title":"3 Football Clubs Kylian Mbappe Should Avoid Signing ✍️❌⚽️ #football #mbappe #shorts", + "description":"", + "thumbnails":{ + "default":{ + "url":"https://i.ytimg.com/vi/TvWDY4Mm5GM/default.jpg", + "width":120, + "height":90 + }, + "medium":{ + "url":"https://i.ytimg.com/vi/TvWDY4Mm5GM/mqdefault.jpg", + "width":320, + "height":180 + }, + "high":{ + "url":"https://i.ytimg.com/vi/TvWDY4Mm5GM/hqdefault.jpg", + "width":480, + "height":360 + } + }, + "channelTitle":"FC Motivate", + "liveBroadcastContent":"none", + "publishTime":"2023-07-24T14:15:01Z" + } + }, + { + "kind":"youtube#searchResult", + "etag":"0NG5QHdtIQM_V-DBJDEf-jK_Y9k", + "id":{ + "kind":"youtube#video", + "videoId":"aZM_42CcNZ4" + }, + "snippet":{ + "publishedAt":"2023-07-24T16:09:27Z", + "channelId":"UCM5gMM_HqfKHYIEJ3lstMUA", + "title":"Which Football Club Could Cristiano Ronaldo Afford To Buy? 💰", + "description":"Sign up to Sorare and get a FREE card: https://sorare.pxf.io/NellisShorts Give Soraredata a go for FREE: ...", + "thumbnails":{ + "default":{ + "url":"https://i.ytimg.com/vi/aZM_42CcNZ4/default.jpg", + "width":120, + "height":90 + }, + "medium":{ + "url":"https://i.ytimg.com/vi/aZM_42CcNZ4/mqdefault.jpg", + "width":320, + "height":180 + }, + "high":{ + "url":"https://i.ytimg.com/vi/aZM_42CcNZ4/hqdefault.jpg", + "width":480, + "height":360 + } + }, + "channelTitle":"John Nellis", + "liveBroadcastContent":"none", + "publishTime":"2023-07-24T16:09:27Z" + } + }, + { + "kind":"youtube#searchResult", + "etag":"WbBz4oh9I5VaYj91LjeJvffrBVY", + "id":{ + "kind":"youtube#video", + "videoId":"wkP3XS3aNAY" + }, + "snippet":{ + "publishedAt":"2023-07-24T16:00:50Z", + "channelId":"UC4EP1dxFDPup_aFLt0ElsDw", + "title":"PAULO DYBALA vs THE WORLD'S LONGEST FREEKICK WALL", + "description":"Can Paulo Dybala curl a football around the World's longest free kick wall? We met up with the World Cup winner and put him to ...", + "thumbnails":{ + "default":{ + "url":"https://i.ytimg.com/vi/wkP3XS3aNAY/default.jpg", + "width":120, + "height":90 + }, + "medium":{ + "url":"https://i.ytimg.com/vi/wkP3XS3aNAY/mqdefault.jpg", + "width":320, + "height":180 + }, + "high":{ + "url":"https://i.ytimg.com/vi/wkP3XS3aNAY/hqdefault.jpg", + "width":480, + "height":360 + } + }, + "channelTitle":"Shoot for Love", + "liveBroadcastContent":"none", + "publishTime":"2023-07-24T16:00:50Z" + } + }, + { + "kind":"youtube#searchResult", + "etag":"juxv_FhT_l4qrR05S1QTrb4CGh8", + "id":{ + "kind":"youtube#video", + "videoId":"rJkDZ0WvfT8" + }, + "snippet":{ + "publishedAt":"2023-07-24T10:00:39Z", + "channelId":"UCO8qj5u80Ga7N_tP3BZWWhQ", + "title":"TOP 10 DEFENDERS 2023", + "description":"SoccerKingz https://soccerkingz.nl Use code: 'ILOVEHOF' to get 10% off. TOP 10 DEFENDERS 2023 Follow us! • Instagram ...", + "thumbnails":{ + "default":{ + "url":"https://i.ytimg.com/vi/rJkDZ0WvfT8/default.jpg", + "width":120, + "height":90 + }, + "medium":{ + "url":"https://i.ytimg.com/vi/rJkDZ0WvfT8/mqdefault.jpg", + "width":320, + "height":180 + }, + "high":{ + "url":"https://i.ytimg.com/vi/rJkDZ0WvfT8/hqdefault.jpg", + "width":480, + "height":360 + } + }, + "channelTitle":"Home of Football", + "liveBroadcastContent":"none", + "publishTime":"2023-07-24T10:00:39Z" + } + }, + { + "kind":"youtube#searchResult", + "etag":"wtuknXTmI1txoULeH3aWaOuXOow", + "id":{ + "kind":"youtube#video", + "videoId":"XH0rtu4U6SE" + }, + "snippet":{ + "publishedAt":"2023-07-21T16:30:05Z", + "channelId":"UCwozCpFp9g9x0wAzuFh0hwQ", + "title":"3 Things You Didn't Know About Erling Haaland ⚽️🇳🇴 #football #haaland #shorts", + "description":"", + "thumbnails":{ + "default":{ + "url":"https://i.ytimg.com/vi/XH0rtu4U6SE/default.jpg", + "width":120, + "height":90 + }, + "medium":{ + "url":"https://i.ytimg.com/vi/XH0rtu4U6SE/mqdefault.jpg", + "width":320, + "height":180 + }, + "high":{ + "url":"https://i.ytimg.com/vi/XH0rtu4U6SE/hqdefault.jpg", + "width":480, + "height":360 + } + }, + "channelTitle":"FC Motivate", + "liveBroadcastContent":"none", + "publishTime":"2023-07-21T16:30:05Z" + } + } + ] +} \ No newline at end of file diff --git a/examples/nemotron/inputs/markdown_example.md b/examples/nemotron/inputs/markdown_example.md new file mode 100644 index 00000000..85088f29 --- /dev/null +++ b/examples/nemotron/inputs/markdown_example.md @@ -0,0 +1,35 @@ +Marco Perini Toggle navigation + + * About + * Projects(current) + +Projects + +Competitions + + * CV + * ____ + +# Projects + + ![project thumbnail Rotary Pendulum RL +Open Source project aimed at controlling a real life rotary pendulum using RL +algorithms ](/projects/rotary-pendulum-rl/) + + ![project thumbnail DQN +Implementation from scratch Developed a Deep Q-Network algorithm to train a +simple and double pendulum ](https://github.com/PeriniM/DQN-SwingUp) + + ![project thumbnail Multi Agents HAED +University project which focuses on simulating a multi-agent system to perform +environment mapping. Agents, equipped with sensors, explore and record their +surroundings, considering uncertainties in their readings. +](https://github.com/PeriniM/Multi-Agents-HAED) + + ![project thumbnail Wireless ESC for Modular +Drones Modular drone architecture proposal and proof of concept. The project +received maximum grade. ](/projects/wireless-esc-drone/) + +© Copyright 2023 Marco Perini. Powered by Jekyll with +al-folio theme. Hosted by [GitHub +Pages](https://pages.github.com/). \ No newline at end of file diff --git a/examples/nemotron/inputs/plain_html_example.txt b/examples/nemotron/inputs/plain_html_example.txt new file mode 100644 index 00000000..78f814ae --- /dev/null +++ b/examples/nemotron/inputs/plain_html_example.txt @@ -0,0 +1,105 @@ + +
+ + +
+ +
+
+
+
+

Projects

+

+
+
+
+
+
+
+ +
+
+ project thumbnail +
+
+

Rotary Pendulum RL

+

Open Source project aimed at controlling a real life rotary pendulum using RL algorithms

+
+
+
+ +
+
+
+ +
+
+ project thumbnail +
+
+

DQN Implementation from scratch

+

Developed a Deep Q-Network algorithm to train a simple and double pendulum

+
+
+
+ +
+
+
+ +
+
+ project thumbnail +
+
+

Multi Agents HAED

+

University project which focuses on simulating a multi-agent system to perform environment mapping. Agents, equipped with sensors, explore and record their surroundings, considering uncertainties in their readings.

+
+
+
+ +
+
+
+ +
+
+ project thumbnail +
+
+

Wireless ESC for Modular Drones

+

Modular drone architecture proposal and proof of concept. The project received maximum grade.

+
+
+
+ +
+
+
+
+
+
+ +
+ \ No newline at end of file diff --git a/examples/nemotron/inputs/username.csv b/examples/nemotron/inputs/username.csv new file mode 100644 index 00000000..006ac8e6 --- /dev/null +++ b/examples/nemotron/inputs/username.csv @@ -0,0 +1,7 @@ +Username; Identifier;First name;Last name +booker12;9012;Rachel;Booker +grey07;2070;Laura;Grey +johnson81;4081;Craig;Johnson +jenkins46;9346;Mary;Jenkins +smith79;5079;Jamie;Smith + diff --git a/examples/nemotron/json_scraper_multi_nemotron.py b/examples/nemotron/json_scraper_multi_nemotron.py new file mode 100644 index 00000000..846c7f48 --- /dev/null +++ b/examples/nemotron/json_scraper_multi_nemotron.py @@ -0,0 +1,37 @@ +""" +Module for showing how PDFScraper multi works +""" +import os +import json +from dotenv import load_dotenv +from scrapegraphai.graphs import JSONScraperMultiGraph + +load_dotenv() + +nemotron_key = os.getenv("NEMOTRON_APIKEY") + +graph_config = { + "llm": { + "api_key": nemotron_key, + "model": "nvidia/meta/llama3-70b-instruct", + }, +} + +FILE_NAME = "inputs/example.json" +curr_dir = os.path.dirname(os.path.realpath(__file__)) +file_path = os.path.join(curr_dir, FILE_NAME) + +with open(file_path, 'r', encoding="utf-8") as file: + text = file.read() + +sources = [text, text] + +multiple_search_graph = JSONScraperMultiGraph( + prompt= "List me all the authors, title and genres of the books", + source= sources, + schema=None, + config=graph_config +) + +result = multiple_search_graph.run() +print(json.dumps(result, indent=4)) diff --git a/examples/nemotron/json_scraper_nemotron.py b/examples/nemotron/json_scraper_nemotron.py new file mode 100644 index 00000000..7f19d15e --- /dev/null +++ b/examples/nemotron/json_scraper_nemotron.py @@ -0,0 +1,58 @@ +""" +Basic example of scraping pipeline using JSONScraperGraph from JSON documents +""" + +import os +from dotenv import load_dotenv +from scrapegraphai.graphs import JSONScraperGraph +from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info +load_dotenv() + +# ************************************************ +# Read the JSON file +# ************************************************ + +FILE_NAME = "inputs/example.json" +curr_dir = os.path.dirname(os.path.realpath(__file__)) +file_path = os.path.join(curr_dir, FILE_NAME) + +with open(file_path, 'r', encoding="utf-8") as file: + text = file.read() + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +nemotron_key = os.getenv("NEMOTRON_APIKEY") + +graph_config = { + "llm": { + "api_key": nemotron_key, + "model": "nvidia/meta/llama3-70b-instruct", + }, +} + +# ************************************************ +# Create the JSONScraperGraph instance and run it +# ************************************************ + +json_scraper_graph = JSONScraperGraph( + prompt="List me all the authors, title and genres of the books", + source=text, # Pass the content of the file, not the file object + config=graph_config +) + +result = json_scraper_graph.run() +print(result) + +# ************************************************ +# Get graph execution info +# ************************************************ + +graph_exec_info = json_scraper_graph.get_execution_info() +print(prettify_exec_info(graph_exec_info)) + +# Save to json or csv +convert_to_csv(result, "result") +convert_to_json(result, "result") + diff --git a/examples/nemotron/md_scraper_nemotron.py b/examples/nemotron/md_scraper_nemotron.py new file mode 100644 index 00000000..71073bd4 --- /dev/null +++ b/examples/nemotron/md_scraper_nemotron.py @@ -0,0 +1,57 @@ +""" +Basic example of scraping pipeline using MDScraperGraph from XML documents +""" + +import os +from dotenv import load_dotenv +from scrapegraphai.graphs import MDScraperGraph +from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info +load_dotenv() + +# ************************************************ +# Read the XML file +# ************************************************ + +FILE_NAME = "inputs/markdown_example.md" +curr_dir = os.path.dirname(os.path.realpath(__file__)) +file_path = os.path.join(curr_dir, FILE_NAME) + +with open(file_path, 'r', encoding="utf-8") as file: + text = file.read() + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +nemotron_key = os.getenv("NEMOTRON_APIKEY") + +graph_config = { + "llm": { + "api_key": nemotron_key, + "model": "nvidia/meta/llama3-70b-instruct", + }, +} + +# ************************************************ +# Create the MDScraperGraph instance and run it +# ************************************************ + +md_scraper_graph = MDScraperGraph( + prompt="List me all the authors, title and genres of the books", + source=text, # Pass the content of the file, not the file object + config=graph_config +) + +result = md_scraper_graph.run() +print(result) + +# ************************************************ +# Get graph execution info +# ************************************************ + +graph_exec_info = md_scraper_graph.get_execution_info() +print(prettify_exec_info(graph_exec_info)) + +# Save to json or csv +convert_to_csv(result, "result") +convert_to_json(result, "result") diff --git a/examples/nemotron/pdf_scraper_nemotron.py b/examples/nemotron/pdf_scraper_nemotron.py new file mode 100644 index 00000000..b8b4482e --- /dev/null +++ b/examples/nemotron/pdf_scraper_nemotron.py @@ -0,0 +1,40 @@ +import os, json +from dotenv import load_dotenv +from scrapegraphai.graphs import PDFScraperGraph + +load_dotenv() + + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +nemotron_key = os.getenv("NEMOTRON_APIKEY") + +graph_config = { + "llm": { + "api_key": nemotron_key, + "model": "nvidia/meta/llama3-70b-instruct", + }, + "verbose": True, +} + +source = """ + The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian + circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. + Divided into three major sections—Inferno, Purgatorio, and Paradiso—the narrative traces the journey of Dante + from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. + Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood + through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided + by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, + the Beatrice of his earlier poetry, through the celestial spheres of Paradise. +""" + +pdf_scraper_graph = PDFScraperGraph( + prompt="Summarize the text and find the main topics", + source=source, + config=graph_config, +) +result = pdf_scraper_graph.run() + +print(json.dumps(result, indent=4)) diff --git a/examples/nemotron/scrape_plain_text_nemotron.py b/examples/nemotron/scrape_plain_text_nemotron.py new file mode 100644 index 00000000..e5e7f764 --- /dev/null +++ b/examples/nemotron/scrape_plain_text_nemotron.py @@ -0,0 +1,55 @@ +""" +Basic example of scraping pipeline using SmartScraper from text +""" + +import os +from dotenv import load_dotenv +from scrapegraphai.graphs import SmartScraperGraph +from scrapegraphai.utils import prettify_exec_info + +load_dotenv() + +# ************************************************ +# Read the text file +# ************************************************ + +FILE_NAME = "inputs/plain_html_example.txt" +curr_dir = os.path.dirname(os.path.realpath(__file__)) +file_path = os.path.join(curr_dir, FILE_NAME) + +# It could be also a http request using the request model +with open(file_path, 'r', encoding="utf-8") as file: + text = file.read() + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +nemotron_key = os.getenv("NEMOTRON_APIKEY") + +graph_config = { + "llm": { + "api_key": nemotron_key, + "model": "nvidia/meta/llama3-70b-instruct", + }, +} + +# ************************************************ +# Create the SmartScraperGraph instance and run it +# ************************************************ + +smart_scraper_graph = SmartScraperGraph( + prompt="List me all the projects with their description.", + source=text, + config=graph_config +) + +result = smart_scraper_graph.run() +print(result) + +# ************************************************ +# Get graph execution info +# ************************************************ + +graph_exec_info = smart_scraper_graph.get_execution_info() +print(prettify_exec_info(graph_exec_info)) diff --git a/examples/nemotron/script_generator_nemotron.py b/examples/nemotron/script_generator_nemotron.py new file mode 100644 index 00000000..d8863214 --- /dev/null +++ b/examples/nemotron/script_generator_nemotron.py @@ -0,0 +1,46 @@ +""" +Basic example of scraping pipeline using ScriptCreatorGraph +""" + +import os +from dotenv import load_dotenv +from scrapegraphai.graphs import ScriptCreatorGraph +from scrapegraphai.utils import prettify_exec_info + +load_dotenv() + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +nemotron_key = os.getenv("NEMOTRON_APIKEY") + +graph_config = { + "llm": { + "api_key": nemotron_key, + "model": "nvidia/meta/llama3-70b-instruct", + }, + "library": "beautifulsoup" +} + +# ************************************************ +# Create the ScriptCreatorGraph instance and run it +# ************************************************ + +script_creator_graph = ScriptCreatorGraph( + prompt="List me all the projects with their description.", + # also accepts a string with the already downloaded HTML code + source="https://perinim.github.io/projects", + config=graph_config +) + +result = script_creator_graph.run() +print(result) + +# ************************************************ +# Get graph execution info +# ************************************************ + +graph_exec_info = script_creator_graph.get_execution_info() +print(prettify_exec_info(graph_exec_info)) + diff --git a/examples/nemotron/script_generator_schema_nemotron.py b/examples/nemotron/script_generator_schema_nemotron.py new file mode 100644 index 00000000..3f0713a4 --- /dev/null +++ b/examples/nemotron/script_generator_schema_nemotron.py @@ -0,0 +1,62 @@ +""" +Basic example of scraping pipeline using ScriptCreatorGraph +""" + +import os +from dotenv import load_dotenv +from scrapegraphai.graphs import ScriptCreatorGraph +from scrapegraphai.utils import prettify_exec_info + +from pydantic import BaseModel, Field +from typing import List + +load_dotenv() + +# ************************************************ +# Define the schema for the graph +# ************************************************ + +class Project(BaseModel): + title: str = Field(description="The title of the project") + description: str = Field(description="The description of the project") + +class Projects(BaseModel): + projects: List[Project] + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +nemotron_key = os.getenv("NEMOTRON_APIKEY") + +graph_config = { + "llm": { + "api_key": nemotron_key, + "model": "nvidia/meta/llama3-70b-instruct", + }, + "library": "beautifulsoup", + "verbose": True, +} + +# ************************************************ +# Create the ScriptCreatorGraph instance and run it +# ************************************************ + +script_creator_graph = ScriptCreatorGraph( + prompt="List me all the projects with their description.", + # also accepts a string with the already downloaded HTML code + source="https://perinim.github.io/projects", + config=graph_config, + schema=Projects +) + +result = script_creator_graph.run() +print(result) + +# ************************************************ +# Get graph execution info +# ************************************************ + +graph_exec_info = script_creator_graph.get_execution_info() +print(prettify_exec_info(graph_exec_info)) + diff --git a/examples/nemotron/script_multi_generator_nemotron.py b/examples/nemotron/script_multi_generator_nemotron.py new file mode 100644 index 00000000..ad0b4b48 --- /dev/null +++ b/examples/nemotron/script_multi_generator_nemotron.py @@ -0,0 +1,54 @@ +""" +Basic example of scraping pipeline using ScriptCreatorGraph +""" + +import os +from dotenv import load_dotenv +from scrapegraphai.graphs import ScriptCreatorMultiGraph +from scrapegraphai.utils import prettify_exec_info + +load_dotenv() + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +nemotron_key = os.getenv("NEMOTRON_APIKEY") + +graph_config = { + "llm": { + "api_key": nemotron_key, + "model": "nvidia/meta/llama3-70b-instruct", + }, + "library": "beautifulsoup", + "verbose": True, +} + +# ************************************************ +# Create the ScriptCreatorGraph instance and run it +# ************************************************ + +urls=[ + "https://perinim.github.io/", + "https://perinim.github.io/cv/" +] + +# ************************************************ +# Create the ScriptCreatorGraph instance and run it +# ************************************************ + +script_creator_graph = ScriptCreatorMultiGraph( + prompt="Who is Marco Perini?", + source=urls, + config=graph_config +) + +result = script_creator_graph.run() +print(result) + +# ************************************************ +# Get graph execution info +# ************************************************ + +graph_exec_info = script_creator_graph.get_execution_info() +print(prettify_exec_info(graph_exec_info)) diff --git a/examples/nemotron/search_graph_nemotron.py b/examples/nemotron/search_graph_nemotron.py new file mode 100644 index 00000000..3e6a7050 --- /dev/null +++ b/examples/nemotron/search_graph_nemotron.py @@ -0,0 +1,35 @@ +""" +Example of Search Graph +""" + +import os +from dotenv import load_dotenv +from scrapegraphai.graphs import SearchGraph +load_dotenv() + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +nemotron_key = os.getenv("NEMOTRON_APIKEY") + +graph_config = { + "llm": { + "api_key": nemotron_key, + "model": "nvidia/meta/llama3-70b-instruct", + }, + "max_results": 2, + "verbose": True, +} + +# ************************************************ +# Create the SearchGraph instance and run it +# ************************************************ + +search_graph = SearchGraph( + prompt="List me Chioggia's famous dishes", + config=graph_config +) + +result = search_graph.run() +print(result) diff --git a/examples/nemotron/search_graph_schema_nemotron.py b/examples/nemotron/search_graph_schema_nemotron.py new file mode 100644 index 00000000..eec72daf --- /dev/null +++ b/examples/nemotron/search_graph_schema_nemotron.py @@ -0,0 +1,63 @@ +""" +Example of Search Graph +""" + +import os +from dotenv import load_dotenv +load_dotenv() + +from scrapegraphai.graphs import SearchGraph +from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info + +from pydantic import BaseModel, Field +from typing import List + +# ************************************************ +# Define the output schema for the graph +# ************************************************ + +class Dish(BaseModel): + name: str = Field(description="The name of the dish") + description: str = Field(description="The description of the dish") + +class Dishes(BaseModel): + dishes: List[Dish] + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +nemotron_key = os.getenv("NEMOTRON_APIKEY") + +graph_config = { + "llm": { + "api_key": nemotron_key, + "model": "nvidia/meta/llama3-70b-instruct", + }, + "max_results": 2, + "verbose": True, +} + +# ************************************************ +# Create the SearchGraph instance and run it +# ************************************************ + +search_graph = SearchGraph( + prompt="List me Chioggia's famous dishes", + config=graph_config, + schema=Dishes +) + +result = search_graph.run() +print(result) + +# ************************************************ +# Get graph execution info +# ************************************************ + +graph_exec_info = search_graph.get_execution_info() +print(prettify_exec_info(graph_exec_info)) + +# Save to json and csv +convert_to_csv(result, "result") +convert_to_json(result, "result") diff --git a/examples/nemotron/search_link_graph_nemotron.py b/examples/nemotron/search_link_graph_nemotron.py new file mode 100644 index 00000000..6d1edbde --- /dev/null +++ b/examples/nemotron/search_link_graph_nemotron.py @@ -0,0 +1,40 @@ +""" +Basic example of scraping pipeline using SmartScraper +""" +import os +from dotenv import load_dotenv +from scrapegraphai.graphs import SearchLinkGraph +from scrapegraphai.utils import prettify_exec_info + +load_dotenv() +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +graph_config = { + "llm": { + "api_key": os.getenv("NEMOTRON_APIKEY"), + "model": "nvidia/meta/llama3-70b-instruct", + }, + "verbose": True, + "headless": False, +} + +# ************************************************ +# Create the SearchLinkGraph instance and run it +# ************************************************ + +smart_scraper_graph = SearchLinkGraph( + source="https://sport.sky.it/nba?gr=www", + config=graph_config +) + +result = smart_scraper_graph.run() +print(result) + +# ************************************************ +# Get graph execution info +# ************************************************ + +graph_exec_info = smart_scraper_graph.get_execution_info() +print(prettify_exec_info(graph_exec_info)) diff --git a/examples/nemotron/smart_scraper_multi_nemotron.py b/examples/nemotron/smart_scraper_multi_nemotron.py new file mode 100644 index 00000000..c8e167ad --- /dev/null +++ b/examples/nemotron/smart_scraper_multi_nemotron.py @@ -0,0 +1,42 @@ +""" +Basic example of scraping pipeline using SmartScraper +""" + +import os +import json +from dotenv import load_dotenv +from scrapegraphai.graphs import SmartScraperMultiGraph + +load_dotenv() + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +nemotron_key = os.getenv("NEMOTRON_APIKEY") + +graph_config = { + "llm": { + "api_key": nemotron_key, + "model": "nvidia/meta/llama3-70b-instruct", + }, + "verbose": True, + "headless": False, +} + +# ******************************************************* +# Create the SmartScraperMultiGraph instance and run it +# ******************************************************* + +multiple_search_graph = SmartScraperMultiGraph( + prompt="Who is Marco Perini?", + source= [ + "https://perinim.github.io/", + "https://perinim.github.io/cv/" + ], + schema=None, + config=graph_config +) + +result = multiple_search_graph.run() +print(json.dumps(result, indent=4)) diff --git a/examples/nemotron/smart_scraper_nemotron.py b/examples/nemotron/smart_scraper_nemotron.py new file mode 100644 index 00000000..182a12d1 --- /dev/null +++ b/examples/nemotron/smart_scraper_nemotron.py @@ -0,0 +1,43 @@ +""" +Basic example of scraping pipeline using SmartScraper +""" + +import os, json +from scrapegraphai.graphs import SmartScraperGraph +from scrapegraphai.utils import prettify_exec_info +from dotenv import load_dotenv +load_dotenv() + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + + +graph_config = { + "llm": { + "api_key": os.getenv("NEMOTRON_KEY"), + "model": "nvidia/meta/llama3-70b-instruct", + }, + "verbose": True, + "headless": False, +} + +# ************************************************ +# Create the SmartScraperGraph instance and run it +# ************************************************ + +smart_scraper_graph = SmartScraperGraph( + prompt="Extract me the python code inside the page", + source="https://www.exploit-db.com/exploits/51447", + config=graph_config +) + +result = smart_scraper_graph.run() +print(json.dumps(result, indent=4)) + +# ************************************************ +# Get graph execution info +# ************************************************ + +graph_exec_info = smart_scraper_graph.get_execution_info() +print(prettify_exec_info(graph_exec_info)) diff --git a/examples/nemotron/smart_scraper_schema_nemotron.py b/examples/nemotron/smart_scraper_schema_nemotron.py new file mode 100644 index 00000000..e1462e85 --- /dev/null +++ b/examples/nemotron/smart_scraper_schema_nemotron.py @@ -0,0 +1,51 @@ +""" +Basic example of scraping pipeline using SmartScraper with schema +""" + +import os, json +from typing import List +from dotenv import load_dotenv +from pydantic import BaseModel, Field +from scrapegraphai.graphs import SmartScraperGraph + +load_dotenv() + +# ************************************************ +# Define the output schema for the graph +# ************************************************ + +class Project(BaseModel): + title: str = Field(description="The title of the project") + description: str = Field(description="The description of the project") + +class Projects(BaseModel): + projects: List[Project] + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +nemotron_key = os.getenv("NEMOTRON_APIKEY") + +graph_config = { + "llm": { + "api_key":nemotron_key, + "model": "nvidia/meta/llama3-70b-instruct", + }, + "verbose": True, + "headless": False, +} + +# ************************************************ +# Create the SmartScraperGraph instance and run it +# ************************************************ + +smart_scraper_graph = SmartScraperGraph( + prompt="List me all the projects with their description", + source="https://perinim.github.io/projects/", + schema=Projects, + config=graph_config +) + +result = smart_scraper_graph.run() +print(result) diff --git a/examples/nemotron/speech_graph_nemotron.py b/examples/nemotron/speech_graph_nemotron.py new file mode 100644 index 00000000..3d46b9e2 --- /dev/null +++ b/examples/nemotron/speech_graph_nemotron.py @@ -0,0 +1,50 @@ +""" +Basic example of scraping pipeline using SpeechSummaryGraph +""" + +import os +from dotenv import load_dotenv +from scrapegraphai.graphs import SpeechGraph +from scrapegraphai.utils import prettify_exec_info +load_dotenv() + +# ************************************************ +# Define audio output path +# ************************************************ + +FILE_NAME = "website_summary.mp3" +curr_dir = os.path.dirname(os.path.realpath(__file__)) +output_path = os.path.join(curr_dir, FILE_NAME) + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +nemotron_key = os.getenv("NEMOTRON_APIKEY") + +graph_config = { + "llm": { + "api_key": nemotron_key, + "model": "nvidia/meta/llama3-70b-instruct", + }, +} + +# ************************************************ +# Create the SpeechGraph instance and run it +# ************************************************ + +speech_graph = SpeechGraph( + prompt="Make a detailed audio summary of the projects.", + source="https://perinim.github.io/projects/", + config=graph_config, +) + +result = speech_graph.run() +print(result) + +# ************************************************ +# Get graph execution info +# ************************************************ + +graph_exec_info = speech_graph.get_execution_info() +print(prettify_exec_info(graph_exec_info)) diff --git a/examples/nemotron/xml_scraper_graph_nemotron.py b/examples/nemotron/xml_scraper_graph_nemotron.py new file mode 100644 index 00000000..4b53e082 --- /dev/null +++ b/examples/nemotron/xml_scraper_graph_nemotron.py @@ -0,0 +1,60 @@ +""" + +Basic example of scraping pipeline using XMLScraperMultiGraph from XML documents +""" + +import os +from dotenv import load_dotenv +from scrapegraphai.graphs import XMLScraperMultiGraph +from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info +load_dotenv() + +# ************************************************ +# Read the XML file +# ************************************************ + +FILE_NAME = "inputs/books.xml" +curr_dir = os.path.dirname(os.path.realpath(__file__)) +file_path = os.path.join(curr_dir, FILE_NAME) + +with open(file_path, 'r', encoding="utf-8") as file: + text = file.read() + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + + +nemotron_key = os.getenv("NEMOTRON_APIKEY") + +graph_config = { + "llm": { + "api_key":nemotron_key, + "model": "nvidia/meta/llama3-70b-instruct", + }, + "verbose": True, + "headless": False, +} +# ************************************************ +# Create the XMLScraperMultiGraph instance and run it +# ************************************************ + +xml_scraper_graph = XMLScraperMultiGraph( + prompt="List me all the authors, title and genres of the books", + source=[text, text], # Pass the content of the file, not the file object + config=graph_config +) + +result = xml_scraper_graph.run() +print(result) + +# ************************************************ +# Get graph execution info +# ************************************************ + +graph_exec_info = xml_scraper_graph.get_execution_info() +print(prettify_exec_info(graph_exec_info)) + +# Save to json or csv +convert_to_csv(result, "result") +convert_to_json(result, "result") diff --git a/examples/nemotron/xml_scraper_nemotron.py b/examples/nemotron/xml_scraper_nemotron.py new file mode 100644 index 00000000..a3291cce --- /dev/null +++ b/examples/nemotron/xml_scraper_nemotron.py @@ -0,0 +1,59 @@ +""" +Basic example of scraping pipeline using XMLScraperGraph from XML documents +""" + +import os +from dotenv import load_dotenv +from scrapegraphai.graphs import XMLScraperGraph +from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info +load_dotenv() + +# ************************************************ +# Read the XML file +# ************************************************ + +FILE_NAME = "inputs/books.xml" +curr_dir = os.path.dirname(os.path.realpath(__file__)) +file_path = os.path.join(curr_dir, FILE_NAME) + +with open(file_path, 'r', encoding="utf-8") as file: + text = file.read() + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + +nemotron_key = os.getenv("NEMOTRON_APIKEY") + +graph_config = { + "llm": { + "api_key": nemotron_key, + "model": "nvidia/meta/llama3-70b-instruct", + }, + "verbose":False, +} + +# ************************************************ +# Create the XMLScraperGraph instance and run it +# ************************************************ + +xml_scraper_graph = XMLScraperGraph( + prompt="List me all the authors, title and genres of the books", + source=text, # Pass the content of the file, not the file object + config=graph_config +) + +result = xml_scraper_graph.run() +print(result) + +# ************************************************ +# Get graph execution info +# ************************************************ + +graph_exec_info = xml_scraper_graph.get_execution_info() +print(prettify_exec_info(graph_exec_info)) + +# Save to json or csv +convert_to_csv(result, "result") +convert_to_json(result, "result") + diff --git a/examples/openai/csv_scraper_graph_multi_openai.py b/examples/openai/csv_scraper_graph_multi_openai.py index 890765df..771ad679 100644 --- a/examples/openai/csv_scraper_graph_multi_openai.py +++ b/examples/openai/csv_scraper_graph_multi_openai.py @@ -22,13 +22,13 @@ text = pd.read_csv(file_path) # ************************************************ # Define the configuration for the graph # ************************************************ +openai_key = os.getenv("OPENAI_APIKEY") graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # 设置 OneAPI URL - } + "llm": { + "api_key": openai_key, + "model": "gpt-3.5-turbo", + }, } # ************************************************ diff --git a/examples/openai/custom_graph_openai.py b/examples/openai/custom_graph_openai.py index 9580e88a..905473e0 100644 --- a/examples/openai/custom_graph_openai.py +++ b/examples/openai/custom_graph_openai.py @@ -15,12 +15,12 @@ load_dotenv() # Define the configuration for the graph # ************************************************ +openai_key = os.getenv("OPENAI_APIKEY") graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "claude-3-haiku-20240307", - "max_tokens": 4000 - }, + "llm": { + "api_key": openai_key, + "model": "gpt-3.5-turbo", + }, } # ************************************************ diff --git a/examples/openai/json_scraper_multi_openai.py b/examples/openai/json_scraper_multi_openai.py index 5f3d9fc2..021cd6e1 100644 --- a/examples/openai/json_scraper_multi_openai.py +++ b/examples/openai/json_scraper_multi_openai.py @@ -14,7 +14,7 @@ graph_config = { "llm": { "api_key": openai_key, "model": "gpt-3.5-turbo", - }, + } } FILE_NAME = "inputs/example.json" diff --git a/examples/openai/smart_scraper_openai.py b/examples/openai/smart_scraper_openai.py index b0515770..6771b817 100644 --- a/examples/openai/smart_scraper_openai.py +++ b/examples/openai/smart_scraper_openai.py @@ -27,8 +27,8 @@ graph_config = { # ************************************************ smart_scraper_graph = SmartScraperGraph( - prompt="Extract me the python code inside the page", - source="https://www.exploit-db.com/exploits/51447", + prompt="List me what does the company do, the name and a contact email.", + source="https://scrapegraphai.com/", config=graph_config ) diff --git a/pyproject.toml b/pyproject.toml index e6e992a4..8e74cca3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,12 +13,16 @@ authors = [ ] dependencies = [ "langchain>=0.2.10", + + "langchain-fireworks>=0.1.3", + "langchain_community>=0.2.9", "langchain-google-genai>=1.0.7", "langchain-google-vertexai", "langchain-openai>=0.1.17", "langchain-groq>=0.1.3", "langchain-aws>=0.1.3", "langchain-anthropic>=0.1.11", + "langchain-nvidia-ai-endpoints>=0.1.6", "html2text>=2024.2.26", "faiss-cpu>=1.8.0", "beautifulsoup4>=4.12.3", @@ -33,8 +37,6 @@ dependencies = [ "google>=3.0.0", "undetected-playwright>=0.3.0", "semchunk>=1.0.1", - "html2text>=2024.2.26", - "langchain-fireworks>=0.1.3", ] license = "MIT" diff --git a/requirements-dev.lock b/requirements-dev.lock index e9337e50..405395c4 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -8,11 +8,12 @@ # with-sources: false -e file:. -aiofiles==23.2.1 +aiofiles==24.1.0 # via burr aiohttp==3.9.5 # via langchain # via langchain-fireworks + # via langchain-nvidia-ai-endpoints aiosignal==1.3.1 # via aiohttp alabaster==0.7.16 @@ -23,14 +24,14 @@ annotated-types==0.7.0 # via pydantic anthropic==0.31.2 # via langchain-anthropic -anyio==4.3.0 +anyio==4.4.0 # via anthropic # via groq # via httpx # via openai # via starlette # via watchfiles -astroid==3.2.2 +astroid==3.2.4 # via pylint async-timeout==4.0.3 # via aiohttp @@ -47,17 +48,17 @@ beautifulsoup4==4.12.3 # via scrapegraphai blinker==1.8.2 # via streamlit -boto3==1.34.145 +boto3==1.34.146 # via langchain-aws -botocore==1.34.145 +botocore==1.34.146 # via boto3 # via s3transfer burr==0.22.1 # via scrapegraphai -cachetools==5.3.3 +cachetools==5.4.0 # via google-auth # via streamlit -certifi==2024.2.2 +certifi==2024.7.4 # via httpcore # via httpx # via requests @@ -72,9 +73,12 @@ contourpy==1.2.1 # via matplotlib cycler==0.12.1 # via matplotlib +dataclasses-json==0.6.7 + # via langchain-community defusedxml==0.7.1 # via langchain-anthropic dill==0.3.8 + # via multiprocess # via pylint distro==1.9.0 # via anthropic @@ -86,32 +90,31 @@ docstring-parser==0.16 # via google-cloud-aiplatform docutils==0.19 # via sphinx -email-validator==2.1.1 +email-validator==2.2.0 # via fastapi -exceptiongroup==1.2.1 +exceptiongroup==1.2.2 # via anyio # via pytest -faiss-cpu==1.8.0 +faiss-cpu==1.8.0.post1 # via scrapegraphai -fastapi==0.111.0 +fastapi==0.111.1 # via burr - # via fastapi-pagination fastapi-cli==0.0.4 # via fastapi -fastapi-pagination==0.12.24 +fastapi-pagination==0.12.26 # via burr -filelock==3.14.0 +filelock==3.15.4 # via huggingface-hub fireworks-ai==0.14.0 # via langchain-fireworks -fonttools==4.52.1 +fonttools==4.53.1 # via matplotlib free-proxy==1.1.1 # via scrapegraphai frozenlist==1.4.1 # via aiohttp # via aiosignal -fsspec==2024.5.0 +fsspec==2024.6.1 # via huggingface-hub furo==2024.5.6 # via scrapegraphai @@ -123,7 +126,7 @@ google==3.0.0 # via scrapegraphai google-ai-generativelanguage==0.6.6 # via google-generativeai -google-api-core==2.19.0 +google-api-core==2.19.1 # via google-ai-generativelanguage # via google-api-python-client # via google-cloud-aiplatform @@ -132,9 +135,9 @@ google-api-core==2.19.0 # via google-cloud-resource-manager # via google-cloud-storage # via google-generativeai -google-api-python-client==2.130.0 +google-api-python-client==2.137.0 # via google-generativeai -google-auth==2.29.0 +google-auth==2.32.0 # via google-ai-generativelanguage # via google-api-core # via google-api-python-client @@ -147,16 +150,16 @@ google-auth==2.29.0 # via google-generativeai google-auth-httplib2==0.2.0 # via google-api-python-client -google-cloud-aiplatform==1.58.0 +google-cloud-aiplatform==1.59.0 # via langchain-google-vertexai google-cloud-bigquery==3.25.0 # via google-cloud-aiplatform google-cloud-core==2.4.1 # via google-cloud-bigquery # via google-cloud-storage -google-cloud-resource-manager==1.12.3 +google-cloud-resource-manager==1.12.4 # via google-cloud-aiplatform -google-cloud-storage==2.17.0 +google-cloud-storage==2.18.0 # via google-cloud-aiplatform # via langchain-google-vertexai google-crc32c==1.5.0 @@ -167,7 +170,7 @@ google-generativeai==0.7.2 google-resumable-media==2.7.1 # via google-cloud-bigquery # via google-cloud-storage -googleapis-common-protos==1.63.0 +googleapis-common-protos==1.63.2 # via google-api-core # via grpc-google-iam-v1 # via grpcio-status @@ -176,12 +179,11 @@ graphviz==0.20.3 # via scrapegraphai greenlet==3.0.3 # via playwright - # via sqlalchemy -groq==0.8.0 +groq==0.9.0 # via langchain-groq grpc-google-iam-v1==0.13.1 # via google-cloud-resource-manager -grpcio==1.64.0 +grpcio==1.65.1 # via google-api-core # via googleapis-common-protos # via grpc-google-iam-v1 @@ -208,7 +210,7 @@ httpx==0.27.0 # via openai httpx-sse==0.4.0 # via fireworks-ai -huggingface-hub==0.23.1 +huggingface-hub==0.24.0 # via tokenizers idna==3.7 # via anyio @@ -232,26 +234,29 @@ jinja2==3.1.4 # via fastapi # via pydeck # via sphinx -jiter==0.4.0 +jiter==0.5.0 # via anthropic jmespath==1.0.1 # via boto3 # via botocore jsonpatch==1.33 # via langchain-core -jsonpointer==2.4 +jsonpointer==3.0.0 # via jsonpatch -jsonschema==4.22.0 +jsonschema==4.23.0 # via altair jsonschema-specifications==2023.12.1 # via jsonschema kiwisolver==1.4.5 # via matplotlib langchain==0.2.10 + # via langchain-community # via scrapegraphai langchain-anthropic==0.1.20 # via scrapegraphai -langchain-aws==0.1.11 +langchain-aws==0.1.12 + # via scrapegraphai +langchain-community==0.2.9 # via scrapegraphai langchain-core==0.2.22 # via langchain @@ -261,16 +266,19 @@ langchain-core==0.2.22 # via langchain-google-genai # via langchain-google-vertexai # via langchain-groq + # via langchain-nvidia-ai-endpoints # via langchain-openai # via langchain-text-splitters -langchain-fireworks==0.1.3 +langchain-fireworks==0.1.5 # via scrapegraphai -langchain-google-genai==1.0.7 +langchain-google-genai==1.0.8 # via scrapegraphai -langchain-google-vertexai==1.0.4 +langchain-google-vertexai==1.0.7 # via scrapegraphai langchain-groq==0.1.6 # via scrapegraphai +langchain-nvidia-ai-endpoints==0.1.6 + # via scrapegraphai langchain-openai==0.1.17 # via scrapegraphai langchain-text-splitters==0.2.2 @@ -286,7 +294,9 @@ markdown-it-py==3.0.0 # via rich markupsafe==2.1.5 # via jinja2 -matplotlib==3.9.0 +marshmallow==3.21.3 + # via dataclasses-json +matplotlib==3.9.1 # via burr mccabe==0.7.0 # via pylint @@ -294,9 +304,13 @@ mdurl==0.1.2 # via markdown-it-py minify-html==0.15.0 # via scrapegraphai +mpire==2.10.2 + # via semchunk multidict==6.0.5 # via aiohttp # via yarl +multiprocess==0.70.16 + # via mpire mypy-extensions==1.0.0 # via typing-inspect numpy==1.26.4 @@ -316,11 +330,11 @@ openai==1.37.0 # via burr # via langchain-fireworks # via langchain-openai -orjson==3.10.3 - # via fastapi +orjson==3.10.6 # via langsmith -packaging==23.2 +packaging==24.1 # via altair + # via faiss-cpu # via google-cloud-aiplatform # via google-cloud-bigquery # via huggingface-hub @@ -334,18 +348,19 @@ pandas==2.2.2 # via scrapegraphai # via sf-hamilton # via streamlit -pillow==10.3.0 +pillow==10.4.0 # via fireworks-ai + # via langchain-nvidia-ai-endpoints # via matplotlib # via streamlit platformdirs==4.2.2 # via pylint -playwright==1.43.0 +playwright==1.45.0 # via scrapegraphai # via undetected-playwright pluggy==1.5.0 # via pytest -proto-plus==1.23.0 +proto-plus==1.24.0 # via google-ai-generativelanguage # via google-api-core # via google-cloud-aiplatform @@ -361,14 +376,14 @@ protobuf==4.25.3 # via grpcio-status # via proto-plus # via streamlit -pyarrow==16.1.0 +pyarrow==17.0.0 # via streamlit pyasn1==0.6.0 # via pyasn1-modules # via rsa pyasn1-modules==0.4.0 # via google-auth -pydantic==2.7.1 +pydantic==2.8.2 # via anthropic # via burr # via fastapi @@ -381,7 +396,7 @@ pydantic==2.7.1 # via langchain-core # via langsmith # via openai -pydantic-core==2.18.2 +pydantic-core==2.20.1 # via pydantic pydeck==0.9.1 # via streamlit @@ -389,9 +404,10 @@ pyee==11.1.0 # via playwright pygments==2.18.0 # via furo + # via mpire # via rich # via sphinx -pylint==3.2.5 +pylint==3.2.6 pyparsing==3.1.2 # via httplib2 # via matplotlib @@ -420,7 +436,7 @@ referencing==0.35.1 # via jsonschema-specifications regex==2024.5.15 # via tiktoken -requests==2.32.2 +requests==2.32.3 # via burr # via free-proxy # via google-api-core @@ -436,18 +452,18 @@ requests==2.32.2 rich==13.7.1 # via streamlit # via typer -rpds-py==0.18.1 +rpds-py==0.19.0 # via jsonschema # via referencing rsa==4.9 # via google-auth -s3transfer==0.10.1 +s3transfer==0.10.2 # via boto3 -semchunk==1.0.1 +semchunk==2.2.0 # via scrapegraphai -sf-hamilton==1.63.0 +sf-hamilton==1.72.1 # via burr -shapely==2.0.4 +shapely==2.0.5 # via google-cloud-aiplatform shellingham==1.5.4 # via typer @@ -475,21 +491,21 @@ sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 # via sphinx -sphinxcontrib-htmlhelp==2.0.5 +sphinxcontrib-htmlhelp==2.0.6 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.7 +sphinxcontrib-qthelp==1.0.8 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy==2.0.30 +sqlalchemy==2.0.31 # via langchain starlette==0.37.2 # via fastapi -streamlit==1.35.0 +streamlit==1.36.0 # via burr -tenacity==8.3.0 +tenacity==8.5.0 # via langchain # via langchain-core # via streamlit @@ -503,21 +519,22 @@ toml==0.10.2 tomli==2.0.1 # via pylint # via pytest -tomlkit==0.12.5 +tomlkit==0.13.0 # via pylint toolz==0.12.1 # via altair -tornado==6.4 +tornado==6.4.1 # via streamlit tqdm==4.66.4 # via google-generativeai # via huggingface-hub + # via mpire # via openai # via scrapegraphai # via semchunk typer==0.12.3 # via fastapi-cli -typing-extensions==4.12.0 +typing-extensions==4.12.2 # via altair # via anthropic # via anyio @@ -543,21 +560,19 @@ typing-inspect==0.9.0 # via sf-hamilton tzdata==2024.1 # via pandas -ujson==5.10.0 - # via fastapi undetected-playwright==0.3.0 # via scrapegraphai uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.18 +urllib3==1.26.19 # via botocore # via requests -uvicorn==0.29.0 +uvicorn==0.30.3 # via burr # via fastapi uvloop==0.19.0 # via uvicorn -watchfiles==0.21.0 +watchfiles==0.22.0 # via uvicorn websockets==12.0 # via uvicorn diff --git a/requirements.lock b/requirements.lock index 79828b70..9d0602db 100644 --- a/requirements.lock +++ b/requirements.lock @@ -11,13 +11,14 @@ aiohttp==3.9.5 # via langchain # via langchain-fireworks + # via langchain-nvidia-ai-endpoints aiosignal==1.3.1 # via aiohttp annotated-types==0.7.0 # via pydantic anthropic==0.31.2 # via langchain-anthropic -anyio==4.3.0 +anyio==4.4.0 # via anthropic # via groq # via httpx @@ -30,32 +31,36 @@ attrs==23.2.0 beautifulsoup4==4.12.3 # via google # via scrapegraphai -boto3==1.34.145 +boto3==1.34.146 # via langchain-aws -botocore==1.34.145 +botocore==1.34.146 # via boto3 # via s3transfer -cachetools==5.3.3 +cachetools==5.4.0 # via google-auth -certifi==2024.2.2 +certifi==2024.7.4 # via httpcore # via httpx # via requests charset-normalizer==3.3.2 # via requests +dataclasses-json==0.6.7 + # via langchain-community defusedxml==0.7.1 # via langchain-anthropic +dill==0.3.8 + # via multiprocess distro==1.9.0 # via anthropic # via groq # via openai docstring-parser==0.16 # via google-cloud-aiplatform -exceptiongroup==1.2.1 +exceptiongroup==1.2.2 # via anyio -faiss-cpu==1.8.0 +faiss-cpu==1.8.0.post1 # via scrapegraphai -filelock==3.14.0 +filelock==3.15.4 # via huggingface-hub fireworks-ai==0.14.0 # via langchain-fireworks @@ -64,13 +69,13 @@ free-proxy==1.1.1 frozenlist==1.4.1 # via aiohttp # via aiosignal -fsspec==2024.5.0 +fsspec==2024.6.1 # via huggingface-hub google==3.0.0 # via scrapegraphai google-ai-generativelanguage==0.6.6 # via google-generativeai -google-api-core==2.19.0 +google-api-core==2.19.1 # via google-ai-generativelanguage # via google-api-python-client # via google-cloud-aiplatform @@ -79,9 +84,9 @@ google-api-core==2.19.0 # via google-cloud-resource-manager # via google-cloud-storage # via google-generativeai -google-api-python-client==2.130.0 +google-api-python-client==2.137.0 # via google-generativeai -google-auth==2.29.0 +google-auth==2.32.0 # via google-ai-generativelanguage # via google-api-core # via google-api-python-client @@ -94,16 +99,16 @@ google-auth==2.29.0 # via google-generativeai google-auth-httplib2==0.2.0 # via google-api-python-client -google-cloud-aiplatform==1.58.0 +google-cloud-aiplatform==1.59.0 # via langchain-google-vertexai google-cloud-bigquery==3.25.0 # via google-cloud-aiplatform google-cloud-core==2.4.1 # via google-cloud-bigquery # via google-cloud-storage -google-cloud-resource-manager==1.12.3 +google-cloud-resource-manager==1.12.4 # via google-cloud-aiplatform -google-cloud-storage==2.17.0 +google-cloud-storage==2.18.0 # via google-cloud-aiplatform # via langchain-google-vertexai google-crc32c==1.5.0 @@ -114,7 +119,7 @@ google-generativeai==0.7.2 google-resumable-media==2.7.1 # via google-cloud-bigquery # via google-cloud-storage -googleapis-common-protos==1.63.0 +googleapis-common-protos==1.63.2 # via google-api-core # via grpc-google-iam-v1 # via grpcio-status @@ -122,12 +127,11 @@ graphviz==0.20.3 # via scrapegraphai greenlet==3.0.3 # via playwright - # via sqlalchemy -groq==0.8.0 +groq==0.9.0 # via langchain-groq grpc-google-iam-v1==0.13.1 # via google-cloud-resource-manager -grpcio==1.64.0 +grpcio==1.65.1 # via google-api-core # via googleapis-common-protos # via grpc-google-iam-v1 @@ -150,27 +154,30 @@ httpx==0.27.0 # via openai httpx-sse==0.4.0 # via fireworks-ai -huggingface-hub==0.23.1 +huggingface-hub==0.24.0 # via tokenizers idna==3.7 # via anyio # via httpx # via requests # via yarl -jiter==0.4.0 +jiter==0.5.0 # via anthropic jmespath==1.0.1 # via boto3 # via botocore jsonpatch==1.33 # via langchain-core -jsonpointer==2.4 +jsonpointer==3.0.0 # via jsonpatch langchain==0.2.10 + # via langchain-community # via scrapegraphai langchain-anthropic==0.1.20 # via scrapegraphai -langchain-aws==0.1.11 +langchain-aws==0.1.12 + # via scrapegraphai +langchain-community==0.2.9 # via scrapegraphai langchain-core==0.2.22 # via langchain @@ -180,16 +187,19 @@ langchain-core==0.2.22 # via langchain-google-genai # via langchain-google-vertexai # via langchain-groq + # via langchain-nvidia-ai-endpoints # via langchain-openai # via langchain-text-splitters -langchain-fireworks==0.1.3 +langchain-fireworks==0.1.5 # via scrapegraphai -langchain-google-genai==1.0.7 +langchain-google-genai==1.0.8 # via scrapegraphai -langchain-google-vertexai==1.0.4 +langchain-google-vertexai==1.0.7 # via scrapegraphai langchain-groq==0.1.6 # via scrapegraphai +langchain-nvidia-ai-endpoints==0.1.6 + # via scrapegraphai langchain-openai==0.1.17 # via scrapegraphai langchain-text-splitters==0.2.2 @@ -199,11 +209,19 @@ langsmith==0.1.93 # via langchain-core lxml==5.2.2 # via free-proxy +marshmallow==3.21.3 + # via dataclasses-json minify-html==0.15.0 # via scrapegraphai +mpire==2.10.2 + # via semchunk multidict==6.0.5 # via aiohttp # via yarl +multiprocess==0.70.16 + # via mpire +mypy-extensions==1.0.0 + # via typing-inspect numpy==1.26.4 # via faiss-cpu # via langchain @@ -213,21 +231,23 @@ numpy==1.26.4 openai==1.37.0 # via langchain-fireworks # via langchain-openai -orjson==3.10.3 +orjson==3.10.6 # via langsmith -packaging==23.2 +packaging==24.1 + # via faiss-cpu # via google-cloud-aiplatform # via google-cloud-bigquery # via huggingface-hub # via langchain-core pandas==2.2.2 # via scrapegraphai -pillow==10.3.0 +pillow==10.4.0 # via fireworks-ai -playwright==1.43.0 + # via langchain-nvidia-ai-endpoints +playwright==1.45.0 # via scrapegraphai # via undetected-playwright -proto-plus==1.23.0 +proto-plus==1.24.0 # via google-ai-generativelanguage # via google-api-core # via google-cloud-aiplatform @@ -247,7 +267,7 @@ pyasn1==0.6.0 # via rsa pyasn1-modules==0.4.0 # via google-auth -pydantic==2.7.1 +pydantic==2.8.2 # via anthropic # via fireworks-ai # via google-cloud-aiplatform @@ -257,10 +277,12 @@ pydantic==2.7.1 # via langchain-core # via langsmith # via openai -pydantic-core==2.18.2 +pydantic-core==2.20.1 # via pydantic pyee==11.1.0 # via playwright +pygments==2.18.0 + # via mpire pyparsing==3.1.2 # via httplib2 python-dateutil==2.9.0.post0 @@ -277,7 +299,7 @@ pyyaml==6.0.1 # via langchain-core regex==2024.5.15 # via tiktoken -requests==2.32.2 +requests==2.32.3 # via free-proxy # via google-api-core # via google-cloud-bigquery @@ -289,11 +311,11 @@ requests==2.32.2 # via tiktoken rsa==4.9 # via google-auth -s3transfer==0.10.1 +s3transfer==0.10.2 # via boto3 -semchunk==1.0.1 +semchunk==2.2.0 # via scrapegraphai -shapely==2.0.4 +shapely==2.0.5 # via google-cloud-aiplatform six==1.16.0 # via python-dateutil @@ -305,9 +327,10 @@ sniffio==1.3.1 # via openai soupsieve==2.5 # via beautifulsoup4 -sqlalchemy==2.0.30 +sqlalchemy==2.0.31 # via langchain -tenacity==8.3.0 + # via langchain-community +tenacity==8.5.0 # via langchain # via langchain-core tiktoken==0.7.0 @@ -318,10 +341,11 @@ tokenizers==0.19.1 tqdm==4.66.4 # via google-generativeai # via huggingface-hub + # via mpire # via openai # via scrapegraphai # via semchunk -typing-extensions==4.12.0 +typing-extensions==4.12.2 # via anthropic # via anyio # via google-generativeai @@ -338,7 +362,7 @@ undetected-playwright==0.3.0 # via scrapegraphai uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.18 +urllib3==1.26.19 # via botocore # via requests yarl==1.9.4 diff --git a/requirements.txt b/requirements.txt index 4a3d525c..124840e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,13 @@ langchain>=0.2.10 +langchain_community>=0.2.9 langchain-google-genai>=1.0.7 +langchain-fireworks>=0.1.3 langchain-google-vertexai langchain-openai>=0.1.17 langchain-groq>=0.1.3 langchain-aws>=0.1.3 langchain-anthropic>=0.1.11 +langchain-nvidia-ai-endpoints>=0.1.6 html2text>=2024.2.26 faiss-cpu>=1.8.0 beautifulsoup4>=4.12.3 @@ -19,5 +22,4 @@ playwright>=1.43.0 google>=3.0.0 undetected-playwright>=0.3.0 semchunk>=1.0.1 -html2text>=2024.2.26 -langchain-fireworks>=0.1.3 + diff --git a/scrapegraphai/graphs/abstract_graph.py b/scrapegraphai/graphs/abstract_graph.py index ca2b2e81..91396ae0 100644 --- a/scrapegraphai/graphs/abstract_graph.py +++ b/scrapegraphai/graphs/abstract_graph.py @@ -14,6 +14,7 @@ from langchain_google_vertexai import VertexAIEmbeddings from langchain_google_genai.embeddings import GoogleGenerativeAIEmbeddings from langchain_fireworks import FireworksEmbeddings from langchain_openai import AzureOpenAIEmbeddings, OpenAIEmbeddings +from langchain_nvidia_ai_endpoints import NVIDIAEmbeddings from ..helpers import models_tokens from ..models import ( Anthropic, @@ -26,7 +27,8 @@ from ..models import ( OpenAI, OneApi, Fireworks, - VertexAI + VertexAI, + Nvidia ) from ..models.ernie import Ernie from ..utils.logging import set_verbosity_debug, set_verbosity_warning, set_verbosity_info @@ -184,6 +186,13 @@ class AbstractGraph(ABC): except KeyError as exc: raise KeyError("Model not supported") from exc return AzureOpenAI(llm_params) + elif "nvidia" in llm_params["model"]: + try: + self.model_token = models_tokens["nvidia"][llm_params["model"].split("/")[-1]] + llm_params["model"] = "/".join(llm_params["model"].split("/")[1:]) + except KeyError as exc: + raise KeyError("Model not supported") from exc + return Nvidia(llm_params) elif "gemini" in llm_params["model"]: llm_params["model"] = llm_params["model"].split("/")[-1] try: @@ -309,6 +318,8 @@ class AbstractGraph(ABC): return AzureOpenAIEmbeddings() elif isinstance(self.llm_model, Fireworks): return FireworksEmbeddings(model=self.llm_model.model_name) + elif isinstance(self.llm_model, Nvidia): + return NVIDIAEmbeddings(model=self.llm_model.model_name) elif isinstance(self.llm_model, Ollama): # unwrap the kwargs from the model whihc is a dict params = self.llm_model._lc_kwargs @@ -345,6 +356,14 @@ class AbstractGraph(ABC): return OpenAIEmbeddings(api_key=embedder_params["api_key"]) elif "azure" in embedder_params["model"]: return AzureOpenAIEmbeddings() + if "nvidia" in embedder_params["model"]: + embedder_params["model"] = "/".join(embedder_params["model"].split("/")[1:]) + try: + models_tokens["nvidia"][embedder_params["model"]] + except KeyError as exc: + raise KeyError("Model not supported") from exc + return NVIDIAEmbeddings(model=embedder_params["model"], + nvidia_api_key=embedder_params["api_key"]) elif "ollama" in embedder_params["model"]: embedder_params["model"] = "/".join(embedder_params["model"].split("/")[1:]) try: diff --git a/scrapegraphai/graphs/base_graph.py b/scrapegraphai/graphs/base_graph.py index e6df2dec..21f564d7 100644 --- a/scrapegraphai/graphs/base_graph.py +++ b/scrapegraphai/graphs/base_graph.py @@ -220,6 +220,8 @@ class BaseGraph: # Log the graph execution telemetry graph_execution_time = time.time() - start_time response = state.get("answer", None) if source_type == "url" else None + content = state.get("parsed_doc", None) if response is not None else None + log_graph_execution( graph_name=self.graph_name, source=source, @@ -228,6 +230,7 @@ class BaseGraph: llm_model=llm_model, embedder_model=embedder_model, source_type=source_type, + content=content, response=response, execution_time=graph_execution_time, total_tokens=cb_total["total_tokens"] if cb_total["total_tokens"] > 0 else None, diff --git a/scrapegraphai/helpers/models_tokens.py b/scrapegraphai/helpers/models_tokens.py index 7ae061f1..0724ee95 100644 --- a/scrapegraphai/helpers/models_tokens.py +++ b/scrapegraphai/helpers/models_tokens.py @@ -83,6 +83,24 @@ models_tokens = { "oneapi": { "qwen-turbo": 6000 }, + "nvidia": { + "meta/llama3-70b-instruct": 419, + "meta/llama3-8b-instruct": 419, + "nemotron-4-340b-instruct": 1024, + "databricks/dbrx-instruct": 4096, + "google/codegemma-7b": 8192, + "google/gemma-2b": 2048, + "google/gemma-7b": 8192, + "google/recurrentgemma-2b": 2048, + "meta/codellama-70b": 16384, + "meta/llama2-70b": 4096, + "microsoft/phi-3-mini-128k-instruct": 122880, + "mistralai/mistral-7b-instruct-v0.2": 4096, + "mistralai/mistral-large": 8192, + "mistralai/mixtral-8x22b-instruct-v0.1": 32768, + "mistralai/mixtral-8x7b-instruct-v0.1": 8192, + "snowflake/arctic": 16384, + }, "groq": { "llama3-8b-8192": 8192, "llama3-70b-8192": 8192, diff --git a/scrapegraphai/models/__init__.py b/scrapegraphai/models/__init__.py index a408d9ac..bfcb84d6 100644 --- a/scrapegraphai/models/__init__.py +++ b/scrapegraphai/models/__init__.py @@ -16,3 +16,4 @@ from .deepseek import DeepSeek from .oneapi import OneApi from .fireworks import Fireworks from .vertex import VertexAI +from .nvidia import Nvidia diff --git a/scrapegraphai/models/nvidia.py b/scrapegraphai/models/nvidia.py new file mode 100644 index 00000000..48ce3c0f --- /dev/null +++ b/scrapegraphai/models/nvidia.py @@ -0,0 +1,25 @@ +""" +This is a Python wrapper class for ChatNVIDIA. +It provides default configuration and could be extended with additional methods if needed. +The purpose of this wrapper is to simplify the creation of instances of ChatNVIDIA by providing +default configurations for certain parameters, +allowing users to focus on specifying other important parameters without having +to understand all the details of the underlying class's constructor. +It inherits from the base class ChatNVIDIA and overrides +its init method to provide a more user-friendly interface. +The constructor takes one argument: llm_config, which is used to initialize the superclass +with default configuration. +""" + +from langchain_nvidia_ai_endpoints import ChatNVIDIA + +class Nvidia(ChatNVIDIA): + """ A wrapper for the Nvidia class that provides default configuration + and could be extended with additional methods if needed. + + Args: + llm_config (dict): Configuration parameters for the language model. + """ + + def __init__(self, llm_config: dict): + super().__init__(**llm_config) diff --git a/scrapegraphai/nodes/fetch_node.py b/scrapegraphai/nodes/fetch_node.py index 0fcfed66..5f55b6e6 100644 --- a/scrapegraphai/nodes/fetch_node.py +++ b/scrapegraphai/nodes/fetch_node.py @@ -184,8 +184,8 @@ class FetchNode(BaseNode): if not self.cut: parsed_content = cleanup_html(response, source) - if (isinstance(self.llm_model, OpenAI) and not self.script_creator) or (self.force and not self.script_creator): - parsed_content = convert_to_md(source) + if (not self.script_creator) or (self.force and not self.script_creator): + parsed_content = convert_to_md(parsed_content, source) compressed_document = [Document(page_content=parsed_content)] else: self.logger.warning( @@ -206,8 +206,10 @@ class FetchNode(BaseNode): raise ValueError("No HTML body content found in the document fetched by ChromiumLoader.") parsed_content = document[0].page_content - if isinstance(self.llm_model, OpenAI) and not self.script_creator or self.force and not self.script_creator and not self.openai_md_enabled: - parsed_content = convert_to_md(document[0].page_content) + if (not self.script_creator) or (self.force and not self.script_creator and not self.openai_md_enabled): + + parsed_content = convert_to_md(document[0].page_content, source) + compressed_document = [ Document(page_content=parsed_content, metadata={"source": "html file"}) diff --git a/scrapegraphai/nodes/generate_answer_csv_node.py b/scrapegraphai/nodes/generate_answer_csv_node.py index 43657b50..6ce19ef2 100644 --- a/scrapegraphai/nodes/generate_answer_csv_node.py +++ b/scrapegraphai/nodes/generate_answer_csv_node.py @@ -116,24 +116,24 @@ class GenerateAnswerCSVNode(BaseNode): chains_dict = {} - # Use tqdm to add progress bar + if len(doc) == 1: + prompt = PromptTemplate( + template=template_no_chunks_csv_prompt, + input_variables=["question"], + partial_variables={ + "context": doc, + "format_instructions": format_instructions, + }, + ) + + chain = prompt | self.llm_model | output_parser + answer = chain.invoke({"question": user_prompt}) + state.update({self.output[0]: answer}) + return state + for i, chunk in enumerate( tqdm(doc, desc="Processing chunks", disable=not self.verbose) ): - if len(doc) == 1: - prompt = PromptTemplate( - template=template_no_chunks_csv_prompt, - input_variables=["question"], - partial_variables={ - "context": chunk, - "format_instructions": format_instructions, - }, - ) - - chain = prompt | self.llm_model | output_parser - answer = chain.invoke({"question": user_prompt}) - break - prompt = PromptTemplate( template=template_chunks_csv_prompt, input_variables=["question"], @@ -144,24 +144,21 @@ class GenerateAnswerCSVNode(BaseNode): }, ) - # Dynamically name the chains based on their index chain_name = f"chunk{i+1}" chains_dict[chain_name] = prompt | self.llm_model | output_parser - if len(chains_dict) > 1: - # Use dictionary unpacking to pass the dynamically named chains to RunnableParallel - map_chain = RunnableParallel(**chains_dict) - # Chain - answer = map_chain.invoke({"question": user_prompt}) - # Merge the answers from the chunks - merge_prompt = PromptTemplate( - template=template_merge_csv_prompt, + async_runner = RunnableParallel(**chains_dict) + + batch_results = async_runner.invoke({"question": user_prompt}) + + merge_prompt = PromptTemplate( + template = template_merge_csv_prompt, input_variables=["context", "question"], partial_variables={"format_instructions": format_instructions}, ) - merge_chain = merge_prompt | self.llm_model | output_parser - answer = merge_chain.invoke({"context": answer, "question": user_prompt}) - # Update the state with the generated answer + merge_chain = merge_prompt | self.llm_model | output_parser + answer = merge_chain.invoke({"context": batch_results, "question": user_prompt}) + state.update({self.output[0]: answer}) - return state + return state \ No newline at end of file diff --git a/scrapegraphai/nodes/generate_answer_omni_node.py b/scrapegraphai/nodes/generate_answer_omni_node.py index 7a030c6f..c2f2b65d 100644 --- a/scrapegraphai/nodes/generate_answer_omni_node.py +++ b/scrapegraphai/nodes/generate_answer_omni_node.py @@ -100,26 +100,26 @@ class GenerateAnswerOmniNode(BaseNode): chains_dict = {} + if len(doc) == 1: + prompt = PromptTemplate( + template=template_no_chunk_omni_prompt, + input_variables=["question"], + partial_variables={ + "context": chunk, + "format_instructions": format_instructions, + "img_desc": imag_desc, + }, + ) + + chain = prompt | self.llm_model | output_parser + answer = chain.invoke({"question": user_prompt}) + + state.update({self.output[0]: answer}) + return state - # Use tqdm to add progress bar for i, chunk in enumerate( tqdm(doc, desc="Processing chunks", disable=not self.verbose) ): - if len(doc) == 1: - prompt = PromptTemplate( - template=template_no_chunk_omni_prompt, - input_variables=["question"], - partial_variables={ - "context": chunk, - "format_instructions": format_instructions, - "img_desc": imag_desc, - }, - ) - - chain = prompt | self.llm_model | output_parser - answer = chain.invoke({"question": user_prompt}) - break - prompt = PromptTemplate( template=template_chunks_omni_prompt, input_variables=["question"], @@ -134,23 +134,18 @@ class GenerateAnswerOmniNode(BaseNode): chain_name = f"chunk{i+1}" chains_dict[chain_name] = prompt | self.llm_model | output_parser - if len(chains_dict) > 1: - # Use dictionary unpacking to pass the dynamically named chains to RunnableParallel - map_chain = RunnableParallel(**chains_dict) - # Chain - answer = map_chain.invoke({"question": user_prompt}) - # Merge the answers from the chunks - merge_prompt = PromptTemplate( - template=template_merge_omni_prompt, - input_variables=["context", "question"], - partial_variables={ - "format_instructions": format_instructions, - "img_desc": imag_desc, - }, - ) - merge_chain = merge_prompt | self.llm_model | output_parser - answer = merge_chain.invoke({"context": answer, "question": user_prompt}) + async_runner = RunnableParallel(**chains_dict) + + batch_results = async_runner.invoke({"question": user_prompt}) + + merge_prompt = PromptTemplate( + template = template_merge_omni_prompt, + input_variables=["context", "question"], + partial_variables={"format_instructions": format_instructions}, + ) + + merge_chain = merge_prompt | self.llm_model | output_parser + answer = merge_chain.invoke({"context": batch_results, "question": user_prompt}) - # Update the state with the generated answer state.update({self.output[0]: answer}) - return state + return state \ No newline at end of file diff --git a/scrapegraphai/nodes/generate_answer_pdf_node.py b/scrapegraphai/nodes/generate_answer_pdf_node.py index db6152bc..7add7948 100644 --- a/scrapegraphai/nodes/generate_answer_pdf_node.py +++ b/scrapegraphai/nodes/generate_answer_pdf_node.py @@ -114,24 +114,26 @@ class GenerateAnswerPDFNode(BaseNode): format_instructions = output_parser.get_format_instructions() + if len(doc) == 1: + prompt = PromptTemplate( + template=template_no_chunks_pdf_prompt, + input_variables=["question"], + partial_variables={ + "context":chunk, + "format_instructions": format_instructions, + }, + ) + chain = prompt | self.llm_model | output_parser + answer = chain.invoke({"question": user_prompt}) + + + state.update({self.output[0]: answer}) + return state + chains_dict = {} - # Use tqdm to add progress bar + for i, chunk in enumerate( - tqdm(doc, desc="Processing chunks", disable=not self.verbose) - ): - if len(doc) == 1: - prompt = PromptTemplate( - template=template_no_chunks_pdf_prompt, - input_variables=["question"], - partial_variables={ - "context":chunk, - "format_instructions": format_instructions, - }, - ) - chain = prompt | self.llm_model | output_parser - answer = chain.invoke({"question": user_prompt}) - - break + tqdm(doc, desc="Processing chunks", disable=not self.verbose)): prompt = PromptTemplate( template=template_chunks_pdf_prompt, input_variables=["question"], @@ -142,24 +144,21 @@ class GenerateAnswerPDFNode(BaseNode): }, ) - # Dynamically name the chains based on their index chain_name = f"chunk{i+1}" chains_dict[chain_name] = prompt | self.llm_model | output_parser - if len(chains_dict) > 1: - # Use dictionary unpacking to pass the dynamically named chains to RunnableParallel - map_chain = RunnableParallel(**chains_dict) - # Chain - answer = map_chain.invoke({"question": user_prompt}) - # Merge the answers from the chunks - merge_prompt = PromptTemplate( - template=template_merge_pdf_prompt, + async_runner = RunnableParallel(**chains_dict) + + batch_results = async_runner.invoke({"question": user_prompt}) + + merge_prompt = PromptTemplate( + template = template_merge_pdf_prompt, input_variables=["context", "question"], partial_variables={"format_instructions": format_instructions}, ) - merge_chain = merge_prompt | self.llm_model | output_parser - answer = merge_chain.invoke({"context": answer, "question": user_prompt}) - # Update the state with the generated answer + merge_chain = merge_prompt | self.llm_model | output_parser + answer = merge_chain.invoke({"context": batch_results, "question": user_prompt}) + state.update({self.output[0]: answer}) return state diff --git a/scrapegraphai/telemetry/telemetry.py b/scrapegraphai/telemetry/telemetry.py index e44a0c28..2289afd0 100644 --- a/scrapegraphai/telemetry/telemetry.py +++ b/scrapegraphai/telemetry/telemetry.py @@ -156,7 +156,7 @@ def log_event(event: str, properties: Dict[str, any]): send_event_json(event_json) -def log_graph_execution(graph_name: str, source: str, prompt:str, schema:dict, llm_model: str, embedder_model: str, source_type: str, execution_time: float, response: dict = None, error_node: str = None, exception: str = None, total_tokens: int = None, is_library=True): +def log_graph_execution(graph_name: str, source: str, prompt:str, schema:dict, llm_model: str, embedder_model: str, source_type: str, execution_time: float, content: str = None, response: dict = None, error_node: str = None, exception: str = None, total_tokens: int = None): properties = { "graph_name": graph_name, "source": source, @@ -165,12 +165,13 @@ def log_graph_execution(graph_name: str, source: str, prompt:str, schema:dict, l "llm_model": llm_model, "embedder_model": embedder_model, "source_type": source_type, + "content": content, "response": response, "execution_time": execution_time, "error_node": error_node, "exception": exception, "total_tokens": total_tokens, - "is_library": is_library + "type": "community-library" } log_event("graph_execution", properties) diff --git a/scrapegraphai/utils/convert_to_md.py b/scrapegraphai/utils/convert_to_md.py index 9257a95e..ece41c60 100644 --- a/scrapegraphai/utils/convert_to_md.py +++ b/scrapegraphai/utils/convert_to_md.py @@ -2,8 +2,9 @@ convert_to_md modul """ import html2text +from urllib.parse import urlparse -def convert_to_md(html): +def convert_to_md(html: str, url: str = None) -> str: """ Convert HTML to Markdown. This function uses the html2text library to convert the provided HTML content to Markdown format. From f872bdd24f9874660eea04f9ade570c96b6e7e93 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Thu, 25 Jul 2024 09:17:28 +0200 Subject: [PATCH 4/5] fix: add llama 3.1 From a7359753d3c9fbdd6397433c724b9dcc4b665c39 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 25 Jul 2024 07:19:01 +0000 Subject: [PATCH 5/5] ci(release): 1.11.3 [skip ci] ## [1.11.3](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.11.2...v1.11.3) (2024-07-25) ### Bug Fixes * add llama 3.1 ([f872bdd](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/f872bdd24f9874660eea04f9ade570c96b6e7e93)) ### Docs * prev version ([5c08eea](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/5c08eea189d7ede6f29399a67d897aa3b3f6a7b0)) --- CHANGELOG.md | 12 ++++++++++++ pyproject.toml | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5306d62..2f8b4a8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +## [1.11.3](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.11.2...v1.11.3) (2024-07-25) + + +### Bug Fixes + +* add llama 3.1 ([f872bdd](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/f872bdd24f9874660eea04f9ade570c96b6e7e93)) + + +### Docs + +* prev version ([5c08eea](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/5c08eea189d7ede6f29399a67d897aa3b3f6a7b0)) + ## [1.11.2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.11.1...v1.11.2) (2024-07-23) diff --git a/pyproject.toml b/pyproject.toml index 8e74cca3..61d4782f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "scrapegraphai" -version = "1.11.2" +version = "1.11.3"