mirror of
https://github.com/VinciGit00/Scrapegraph-ai.git
synced 2026-06-23 21:00:30 +08:00
Merge branch 'main' into research_branch
This commit is contained in:
VinciGit00
commit
6dc4c6dec4
18
README.md
18
README.md
@ -6,9 +6,10 @@
|
||||
[](https://github.com/pylint-dev/pylint)
|
||||
[](https://github.com/VinciGit00/Scrapegraph-ai/actions/workflows/pylint.yml)
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://discord.gg/gkxQDAjfeX)
|
||||
|
||||
|
||||
ScrapeGraphAI is a *web scraping* python library based on LangChain which uses LLM and direct graph logic to create scraping pipelines for websites and documents.
|
||||
ScrapeGraphAI is a *web scraping* python library which uses LLM and direct graph logic to create scraping pipelines for websites, documents and XML files.
|
||||
Just say which information you want to extract and the library will do it for you!
|
||||
|
||||
<p align="center">
|
||||
@ -49,31 +50,26 @@ You can use the `SmartScraper` class to extract information from a website using
|
||||
The `SmartScraper` class is a direct graph implementation that uses the most common nodes present in a web scraping pipeline. For more information, please see the [documentation](https://scrapegraph-ai.readthedocs.io/en/latest/).
|
||||
|
||||
```python
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
|
||||
load_dotenv()
|
||||
openai_key = os.getenv("OPENAI_APIKEY")
|
||||
OPENAI_API_KEY = "YOUR_API_KEY"
|
||||
|
||||
# Define the configuration for the graph
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"api_key": openai_key,
|
||||
"api_key": OPENAI_API_KEY,
|
||||
"model": "gpt-3.5-turbo",
|
||||
},
|
||||
}
|
||||
|
||||
# Create the SmartScraperGraph instance
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="List me all the titles and project descriptions"
|
||||
file_source="https://perinim.github.io/projects/", # also accepts a local file path
|
||||
prompt="List me all the news with their description.",
|
||||
file_source="https://perinim.github.io/projects/", # also accepts a string with the already downloaded HTML code as string format
|
||||
config=graph_config
|
||||
)
|
||||
|
||||
result = smart_scraper_graph.run()
|
||||
print(result)
|
||||
|
||||
```
|
||||
|
||||
The output will be a dictionary with the extracted information, for example:
|
||||
@ -95,7 +91,7 @@ Fell free to contribute and join our Discord server to discuss with us improveme
|
||||
|
||||
For more information, please see the [contributing guidelines](https://github.com/VinciGit00/Scrapegraph-ai/blob/main/CONTRIBUTING.md).
|
||||
|
||||
[](https://discord.gg/bSgWTVXz)
|
||||
[](https://discord.gg/gkxQDAjfeX)
|
||||
[](https://www.linkedin.com/company/scrapegraphai/)
|
||||
[](https://twitter.com/scrapegraph)
|
||||
|
||||
|
||||
@ -1,31 +0,0 @@
|
||||
"""
|
||||
Example of graph builder
|
||||
"""
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from scrapegraphai.builders import GraphBuilder
|
||||
|
||||
load_dotenv()
|
||||
openai_key = os.getenv("OPENAI_APIKEY")
|
||||
|
||||
# Define the configuration for the graph
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"api_key": openai_key,
|
||||
"model": "gpt-3.5-turbo",
|
||||
},
|
||||
}
|
||||
|
||||
# Example usage of GraphBuilder
|
||||
graph_builder = GraphBuilder(
|
||||
user_prompt="Extract the news and generate a text summary with a voiceover.",
|
||||
config=graph_config
|
||||
)
|
||||
|
||||
graph_json = graph_builder.build_graph()
|
||||
|
||||
# Convert the resulting JSON to Graphviz format
|
||||
graphviz_graph = graph_builder.convert_json_to_graphviz(graph_json)
|
||||
|
||||
# Save the graph to a file and open it in the default viewer
|
||||
graphviz_graph.render('ScrapeGraphAI_generated_graph', view=True)
|
||||
120
examples/inputs/books.xml
Normal file
120
examples/inputs/books.xml
Normal file
@ -0,0 +1,120 @@
|
||||
<?xml version="1.0"?>
|
||||
<catalog>
|
||||
<book id="bk101">
|
||||
<author>Gambardella, Matthew</author>
|
||||
<title>XML Developer's Guide</title>
|
||||
<genre>Computer</genre>
|
||||
<price>44.95</price>
|
||||
<publish_date>2000-10-01</publish_date>
|
||||
<description>An in-depth look at creating applications
|
||||
with XML.</description>
|
||||
</book>
|
||||
<book id="bk102">
|
||||
<author>Ralls, Kim</author>
|
||||
<title>Midnight Rain</title>
|
||||
<genre>Fantasy</genre>
|
||||
<price>5.95</price>
|
||||
<publish_date>2000-12-16</publish_date>
|
||||
<description>A former architect battles corporate zombies,
|
||||
an evil sorceress, and her own childhood to become queen
|
||||
of the world.</description>
|
||||
</book>
|
||||
<book id="bk103">
|
||||
<author>Corets, Eva</author>
|
||||
<title>Maeve Ascendant</title>
|
||||
<genre>Fantasy</genre>
|
||||
<price>5.95</price>
|
||||
<publish_date>2000-11-17</publish_date>
|
||||
<description>After the collapse of a nanotechnology
|
||||
society in England, the young survivors lay the
|
||||
foundation for a new society.</description>
|
||||
</book>
|
||||
<book id="bk104">
|
||||
<author>Corets, Eva</author>
|
||||
<title>Oberon's Legacy</title>
|
||||
<genre>Fantasy</genre>
|
||||
<price>5.95</price>
|
||||
<publish_date>2001-03-10</publish_date>
|
||||
<description>In post-apocalypse England, the mysterious
|
||||
agent known only as Oberon helps to create a new life
|
||||
for the inhabitants of London. Sequel to Maeve
|
||||
Ascendant.</description>
|
||||
</book>
|
||||
<book id="bk105">
|
||||
<author>Corets, Eva</author>
|
||||
<title>The Sundered Grail</title>
|
||||
<genre>Fantasy</genre>
|
||||
<price>5.95</price>
|
||||
<publish_date>2001-09-10</publish_date>
|
||||
<description>The two daughters of Maeve, half-sisters,
|
||||
battle one another for control of England. Sequel to
|
||||
Oberon's Legacy.</description>
|
||||
</book>
|
||||
<book id="bk106">
|
||||
<author>Randall, Cynthia</author>
|
||||
<title>Lover Birds</title>
|
||||
<genre>Romance</genre>
|
||||
<price>4.95</price>
|
||||
<publish_date>2000-09-02</publish_date>
|
||||
<description>When Carla meets Paul at an ornithology
|
||||
conference, tempers fly as feathers get ruffled.</description>
|
||||
</book>
|
||||
<book id="bk107">
|
||||
<author>Thurman, Paula</author>
|
||||
<title>Splish Splash</title>
|
||||
<genre>Romance</genre>
|
||||
<price>4.95</price>
|
||||
<publish_date>2000-11-02</publish_date>
|
||||
<description>A deep sea diver finds true love twenty
|
||||
thousand leagues beneath the sea.</description>
|
||||
</book>
|
||||
<book id="bk108">
|
||||
<author>Knorr, Stefan</author>
|
||||
<title>Creepy Crawlies</title>
|
||||
<genre>Horror</genre>
|
||||
<price>4.95</price>
|
||||
<publish_date>2000-12-06</publish_date>
|
||||
<description>An anthology of horror stories about roaches,
|
||||
centipedes, scorpions and other insects.</description>
|
||||
</book>
|
||||
<book id="bk109">
|
||||
<author>Kress, Peter</author>
|
||||
<title>Paradox Lost</title>
|
||||
<genre>Science Fiction</genre>
|
||||
<price>6.95</price>
|
||||
<publish_date>2000-11-02</publish_date>
|
||||
<description>After an inadvertant trip through a Heisenberg
|
||||
Uncertainty Device, James Salway discovers the problems
|
||||
of being quantum.</description>
|
||||
</book>
|
||||
<book id="bk110">
|
||||
<author>O'Brien, Tim</author>
|
||||
<title>Microsoft .NET: The Programming Bible</title>
|
||||
<genre>Computer</genre>
|
||||
<price>36.95</price>
|
||||
<publish_date>2000-12-09</publish_date>
|
||||
<description>Microsoft's .NET initiative is explored in
|
||||
detail in this deep programmer's reference.</description>
|
||||
</book>
|
||||
<book id="bk111">
|
||||
<author>O'Brien, Tim</author>
|
||||
<title>MSXML3: A Comprehensive Guide</title>
|
||||
<genre>Computer</genre>
|
||||
<price>36.95</price>
|
||||
<publish_date>2000-12-01</publish_date>
|
||||
<description>The Microsoft MSXML3 parser is covered in
|
||||
detail, with attention to XML DOM interfaces, XSLT processing,
|
||||
SAX and more.</description>
|
||||
</book>
|
||||
<book id="bk112">
|
||||
<author>Galos, Mike</author>
|
||||
<title>Visual Studio 7: A Comprehensive Guide</title>
|
||||
<genre>Computer</genre>
|
||||
<price>49.95</price>
|
||||
<publish_date>2001-04-16</publish_date>
|
||||
<description>Microsoft Visual Studio 7 is explored in depth,
|
||||
looking at how Visual Basic, Visual C++, C#, and ASP+ are
|
||||
integrated into a comprehensive development
|
||||
environment.</description>
|
||||
</book>
|
||||
</catalog>
|
||||
2
examples/results/result.csv
Normal file
2
examples/results/result.csv
Normal file
@ -0,0 +1,2 @@
|
||||
0,1,2,3
|
||||
"{'title': 'Rotary Pendulum RL', 'description': 'Open Source project aimed at controlling a real life rotary pendulum using RL algorithms'}","{'title': 'DQN Implementation from scratch', 'description': 'Developed a Deep Q-Network algorithm to train a simple and double pendulum'}","{'title': 'Multi Agents HAED', 'description': 'University project which focuses on simulating a multi-agent system to perform environment mapping. Agents, equipped with sensors, explore and record their surroundings, considering uncertainties in their readings.'}","{'title': 'Wireless ESC for Modular Drones', 'description': 'Modular drone architecture proposal and proof of concept. The project received maximum grade.'}"
|
||||
|
1
examples/results/result.json
Normal file
1
examples/results/result.json
Normal file
@ -0,0 +1 @@
|
||||
{"projects": [{"title": "Rotary Pendulum RL", "description": "Open Source project aimed at controlling a real life rotary pendulum using RL algorithms"}, {"title": "DQN Implementation from scratch", "description": "Developed a Deep Q-Network algorithm to train a simple and double pendulum"}, {"title": "Multi Agents HAED", "description": "University project which focuses on simulating a multi-agent system to perform environment mapping. Agents, equipped with sensors, explore and record their surroundings, considering uncertainties in their readings."}, {"title": "Wireless ESC for Modular Drones", "description": "Modular drone architecture proposal and proof of concept. The project received maximum grade."}]}
|
||||
@ -1,10 +1,11 @@
|
||||
"""
|
||||
Basic example of scraping pipeline using SmartScraper
|
||||
Basic example of scraping pipeline using SmartScraper from text
|
||||
"""
|
||||
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
from scrapegraphai.utils import convert_to_csv, convert_to_json
|
||||
|
||||
load_dotenv()
|
||||
openai_key = os.getenv("OPENAI_APIKEY")
|
||||
@ -19,7 +20,7 @@ graph_config = {
|
||||
|
||||
|
||||
# It could be also a http request using the request model
|
||||
text = open('plain_html_example.txt', 'r', encoding="utf-8")
|
||||
text = open('inputs/plain_html_example.txt', 'r', encoding="utf-8")
|
||||
|
||||
# Create the SmartScraperGraph instance
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
@ -32,3 +33,5 @@ result = smart_scraper_graph.run()
|
||||
print(result)
|
||||
|
||||
# Save to json or csv
|
||||
convert_to_csv(result, "result")
|
||||
convert_to_json(result, "result")
|
||||
37
examples/scrape_xml_document.py
Normal file
37
examples/scrape_xml_document.py
Normal file
@ -0,0 +1,37 @@
|
||||
"""
|
||||
Basic example of scraping pipeline using SmartScraper from XML documents
|
||||
"""
|
||||
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
from scrapegraphai.utils import convert_to_csv, convert_to_json
|
||||
|
||||
load_dotenv()
|
||||
openai_key = os.getenv("OPENAI_APIKEY")
|
||||
|
||||
# Define the configuration for the graph
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"api_key": openai_key,
|
||||
"model": "gpt-3.5-turbo",
|
||||
},
|
||||
}
|
||||
|
||||
# Read the XML file
|
||||
with open('inputs/books.xml', 'r', encoding="utf-8") as file:
|
||||
text = file.read()
|
||||
|
||||
# Create the SmartScraperGraph instance
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="List me all the authors, title and genres of the books",
|
||||
file_source=text, # Pass the content of the file, not the file object
|
||||
config=graph_config
|
||||
)
|
||||
|
||||
result = smart_scraper_graph.run()
|
||||
print(result)
|
||||
|
||||
# Save to json or csv
|
||||
convert_to_csv(result, "result")
|
||||
convert_to_json(result, "result")
|
||||
@ -1,39 +0,0 @@
|
||||
"""
|
||||
Teest for convert_to_csv
|
||||
"""
|
||||
import os
|
||||
from scrapegraphai.utils.convert_to_csv import convert_to_csv
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
Example usage of the convert_to_csv function.
|
||||
"""
|
||||
# Example data
|
||||
data = {
|
||||
'Name': ['John', 'Alice', 'Bob'],
|
||||
'Age': [30, 25, 35],
|
||||
'City': ['New York', 'San Francisco', 'Seattle']
|
||||
}
|
||||
|
||||
# Example filename and position
|
||||
filename = "example_data"
|
||||
position = "./output"
|
||||
|
||||
try:
|
||||
# Convert data to CSV and save
|
||||
convert_to_csv(data, filename, position)
|
||||
print(
|
||||
f"Data saved successfully to {os.path.join(position, filename)}.csv")
|
||||
except ValueError as ve:
|
||||
print(f"ValueError: {ve}")
|
||||
except FileNotFoundError as fnfe:
|
||||
print(f"FileNotFoundError: {fnfe}")
|
||||
except PermissionError as pe:
|
||||
print(f"PermissionError: {pe}")
|
||||
except Exception as e:
|
||||
print(f"An unexpected error occurred: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,28 +0,0 @@
|
||||
"""
|
||||
Example of using convert_to_json function to save data in JSON format.
|
||||
"""
|
||||
import os
|
||||
from scrapegraphai.utils.convert_to_json import convert_to_json
|
||||
|
||||
# Data to save in JSON format
|
||||
data_to_save = {
|
||||
"name": "John Doe",
|
||||
"age": 30,
|
||||
"city": "New York"
|
||||
}
|
||||
|
||||
FILENAME = "example_data"
|
||||
DIRECTORY = "data_output"
|
||||
|
||||
try:
|
||||
convert_to_json(data_to_save, FILENAME, DIRECTORY)
|
||||
print(
|
||||
f"Data has been successfully saved to {os.path.join(DIRECTORY, FILENAME)}.json")
|
||||
except ValueError as value_error:
|
||||
print(value_error)
|
||||
except FileNotFoundError as file_not_found_error:
|
||||
print(file_not_found_error)
|
||||
except PermissionError as permission_error:
|
||||
print(permission_error)
|
||||
except Exception as exception:
|
||||
print(f"An error occurred: {exception}")
|
||||
@ -1,21 +0,0 @@
|
||||
"""
|
||||
Example of the remover method
|
||||
"""
|
||||
from scrapegraphai.utils.remover import remover
|
||||
|
||||
HTML_CONTENT = """
|
||||
<html>
|
||||
<head>
|
||||
<title>Test Page</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>This is a Test</h1>
|
||||
<p>Hello, World!</p>
|
||||
<script>alert("This is a script");</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
parsed_content = remover(HTML_CONTENT)
|
||||
|
||||
print(parsed_content)
|
||||
@ -1,10 +0,0 @@
|
||||
"""
|
||||
Example for th e file save_audio_from_bytes
|
||||
"""
|
||||
from scrapegraphai.utils.save_audio_from_bytes import save_audio_from_bytes
|
||||
|
||||
BYTE_RESPONSE = b'\x12\x34\x56\x78\x90'
|
||||
|
||||
OUTPUT_PATH = "generated_speech.wav"
|
||||
|
||||
save_audio_from_bytes(BYTE_RESPONSE, OUTPUT_PATH)
|
||||
@ -1,14 +0,0 @@
|
||||
"""
|
||||
Example for calclating the tokenizer
|
||||
"""
|
||||
from scrapegraphai.utils.token_calculator import truncate_text_tokens
|
||||
|
||||
INPUT_TEXT = "http://nba.com"
|
||||
|
||||
MODEL_NAME = "gpt-3.5-turbo"
|
||||
ENCODING_NAME = "EMBEDDING_ENCODING"
|
||||
|
||||
tokenized_chunks = truncate_text_tokens(INPUT_TEXT, MODEL_NAME, ENCODING_NAME)
|
||||
|
||||
for i, chunk in enumerate(tokenized_chunks):
|
||||
print(f"Chunk {i+1}: {chunk}")
|
||||
@ -1,46 +0,0 @@
|
||||
"""
|
||||
Basic example of scraping pipeline using SmartScraper
|
||||
"""
|
||||
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from scrapegraphai.models import OpenAIImageToText, OpenAITextToSpeech
|
||||
from scrapegraphai.utils import save_audio_from_bytes
|
||||
|
||||
load_dotenv()
|
||||
openai_key = os.getenv("OPENAI_APIKEY")
|
||||
|
||||
# Define the configuration for the graph
|
||||
config = {
|
||||
"itt_model": {
|
||||
"api_key": openai_key,
|
||||
"model": "gpt-4-vision-preview",
|
||||
},
|
||||
"tts_model": {
|
||||
"api_key": openai_key,
|
||||
"model": "tts-1",
|
||||
"voice": "alloy"
|
||||
},
|
||||
}
|
||||
|
||||
itt_model = OpenAIImageToText(config["itt_model"])
|
||||
img_to_text_result = itt_model.run(
|
||||
"https://raw.githubusercontent.com/VinciGit00/Scrapegraph-ai/main/docs/assets/scrapegraphai_logo.png"
|
||||
)
|
||||
|
||||
print(f"Image description: {img_to_text_result}")
|
||||
|
||||
tts_model = OpenAITextToSpeech(config["tts_model"])
|
||||
|
||||
audio = tts_model.run(
|
||||
img_to_text_result
|
||||
)
|
||||
|
||||
# Save the audio to a file
|
||||
file_name = "image_description.mp3"
|
||||
curr_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
output_path = os.path.join(curr_dir, file_name)
|
||||
|
||||
save_audio_from_bytes(audio, output_path)
|
||||
|
||||
print(f"Audio file saved to: {output_path}")
|
||||
213
poetry.lock
generated
213
poetry.lock
generated
@ -1,4 +1,4 @@
|
||||
# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
|
||||
# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
|
||||
|
||||
[[package]]
|
||||
name = "aiohttp"
|
||||
@ -611,13 +611,13 @@ files = [
|
||||
|
||||
[[package]]
|
||||
name = "faker"
|
||||
version = "24.7.1"
|
||||
version = "24.4.0"
|
||||
description = "Faker is a Python package that generates fake data for you."
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "Faker-24.7.1-py3-none-any.whl", hash = "sha256:73f2bd886e8ce751e660c7d37a6c0a128aab5e1551359335bb79cfea0f4fabfc"},
|
||||
{file = "Faker-24.7.1.tar.gz", hash = "sha256:39d34c63f0d62ed574161e23fe32008917b923d18098ce94c2650fe16463b7d5"},
|
||||
{file = "Faker-24.4.0-py3-none-any.whl", hash = "sha256:998c29ee7d64429bd59204abffa9ba11f784fb26c7b9df4def78d1a70feb36a7"},
|
||||
{file = "Faker-24.4.0.tar.gz", hash = "sha256:a5ddccbe97ab691fad6bd8036c31f5697cfaa550e62e000078d1935fa8a7ec2e"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
@ -640,53 +640,53 @@ requests = ">=2.21.0"
|
||||
|
||||
[[package]]
|
||||
name = "fonttools"
|
||||
version = "4.51.0"
|
||||
version = "4.50.0"
|
||||
description = "Tools to manipulate font files"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "fonttools-4.51.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:84d7751f4468dd8cdd03ddada18b8b0857a5beec80bce9f435742abc9a851a74"},
|
||||
{file = "fonttools-4.51.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b4850fa2ef2cfbc1d1f689bc159ef0f45d8d83298c1425838095bf53ef46308"},
|
||||
{file = "fonttools-4.51.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5b48a1121117047d82695d276c2af2ee3a24ffe0f502ed581acc2673ecf1037"},
|
||||
{file = "fonttools-4.51.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:180194c7fe60c989bb627d7ed5011f2bef1c4d36ecf3ec64daec8302f1ae0716"},
|
||||
{file = "fonttools-4.51.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:96a48e137c36be55e68845fc4284533bda2980f8d6f835e26bca79d7e2006438"},
|
||||
{file = "fonttools-4.51.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:806e7912c32a657fa39d2d6eb1d3012d35f841387c8fc6cf349ed70b7c340039"},
|
||||
{file = "fonttools-4.51.0-cp310-cp310-win32.whl", hash = "sha256:32b17504696f605e9e960647c5f64b35704782a502cc26a37b800b4d69ff3c77"},
|
||||
{file = "fonttools-4.51.0-cp310-cp310-win_amd64.whl", hash = "sha256:c7e91abdfae1b5c9e3a543f48ce96013f9a08c6c9668f1e6be0beabf0a569c1b"},
|
||||
{file = "fonttools-4.51.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a8feca65bab31479d795b0d16c9a9852902e3a3c0630678efb0b2b7941ea9c74"},
|
||||
{file = "fonttools-4.51.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8ac27f436e8af7779f0bb4d5425aa3535270494d3bc5459ed27de3f03151e4c2"},
|
||||
{file = "fonttools-4.51.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e19bd9e9964a09cd2433a4b100ca7f34e34731e0758e13ba9a1ed6e5468cc0f"},
|
||||
{file = "fonttools-4.51.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2b92381f37b39ba2fc98c3a45a9d6383bfc9916a87d66ccb6553f7bdd129097"},
|
||||
{file = "fonttools-4.51.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5f6bc991d1610f5c3bbe997b0233cbc234b8e82fa99fc0b2932dc1ca5e5afec0"},
|
||||
{file = "fonttools-4.51.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9696fe9f3f0c32e9a321d5268208a7cc9205a52f99b89479d1b035ed54c923f1"},
|
||||
{file = "fonttools-4.51.0-cp311-cp311-win32.whl", hash = "sha256:3bee3f3bd9fa1d5ee616ccfd13b27ca605c2b4270e45715bd2883e9504735034"},
|
||||
{file = "fonttools-4.51.0-cp311-cp311-win_amd64.whl", hash = "sha256:0f08c901d3866a8905363619e3741c33f0a83a680d92a9f0e575985c2634fcc1"},
|
||||
{file = "fonttools-4.51.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:4060acc2bfa2d8e98117828a238889f13b6f69d59f4f2d5857eece5277b829ba"},
|
||||
{file = "fonttools-4.51.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1250e818b5f8a679ad79660855528120a8f0288f8f30ec88b83db51515411fcc"},
|
||||
{file = "fonttools-4.51.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76f1777d8b3386479ffb4a282e74318e730014d86ce60f016908d9801af9ca2a"},
|
||||
{file = "fonttools-4.51.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b5ad456813d93b9c4b7ee55302208db2b45324315129d85275c01f5cb7e61a2"},
|
||||
{file = "fonttools-4.51.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:68b3fb7775a923be73e739f92f7e8a72725fd333eab24834041365d2278c3671"},
|
||||
{file = "fonttools-4.51.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8e2f1a4499e3b5ee82c19b5ee57f0294673125c65b0a1ff3764ea1f9db2f9ef5"},
|
||||
{file = "fonttools-4.51.0-cp312-cp312-win32.whl", hash = "sha256:278e50f6b003c6aed19bae2242b364e575bcb16304b53f2b64f6551b9c000e15"},
|
||||
{file = "fonttools-4.51.0-cp312-cp312-win_amd64.whl", hash = "sha256:b3c61423f22165541b9403ee39874dcae84cd57a9078b82e1dce8cb06b07fa2e"},
|
||||
{file = "fonttools-4.51.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:1621ee57da887c17312acc4b0e7ac30d3a4fb0fec6174b2e3754a74c26bbed1e"},
|
||||
{file = "fonttools-4.51.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e9d9298be7a05bb4801f558522adbe2feea1b0b103d5294ebf24a92dd49b78e5"},
|
||||
{file = "fonttools-4.51.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee1af4be1c5afe4c96ca23badd368d8dc75f611887fb0c0dac9f71ee5d6f110e"},
|
||||
{file = "fonttools-4.51.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c18b49adc721a7d0b8dfe7c3130c89b8704baf599fb396396d07d4aa69b824a1"},
|
||||
{file = "fonttools-4.51.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:de7c29bdbdd35811f14493ffd2534b88f0ce1b9065316433b22d63ca1cd21f14"},
|
||||
{file = "fonttools-4.51.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cadf4e12a608ef1d13e039864f484c8a968840afa0258b0b843a0556497ea9ed"},
|
||||
{file = "fonttools-4.51.0-cp38-cp38-win32.whl", hash = "sha256:aefa011207ed36cd280babfaa8510b8176f1a77261833e895a9d96e57e44802f"},
|
||||
{file = "fonttools-4.51.0-cp38-cp38-win_amd64.whl", hash = "sha256:865a58b6e60b0938874af0968cd0553bcd88e0b2cb6e588727117bd099eef836"},
|
||||
{file = "fonttools-4.51.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:60a3409c9112aec02d5fb546f557bca6efa773dcb32ac147c6baf5f742e6258b"},
|
||||
{file = "fonttools-4.51.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f7e89853d8bea103c8e3514b9f9dc86b5b4120afb4583b57eb10dfa5afbe0936"},
|
||||
{file = "fonttools-4.51.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56fc244f2585d6c00b9bcc59e6593e646cf095a96fe68d62cd4da53dd1287b55"},
|
||||
{file = "fonttools-4.51.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d145976194a5242fdd22df18a1b451481a88071feadf251221af110ca8f00ce"},
|
||||
{file = "fonttools-4.51.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c5b8cab0c137ca229433570151b5c1fc6af212680b58b15abd797dcdd9dd5051"},
|
||||
{file = "fonttools-4.51.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:54dcf21a2f2d06ded676e3c3f9f74b2bafded3a8ff12f0983160b13e9f2fb4a7"},
|
||||
{file = "fonttools-4.51.0-cp39-cp39-win32.whl", hash = "sha256:0118ef998a0699a96c7b28457f15546815015a2710a1b23a7bf6c1be60c01636"},
|
||||
{file = "fonttools-4.51.0-cp39-cp39-win_amd64.whl", hash = "sha256:599bdb75e220241cedc6faebfafedd7670335d2e29620d207dd0378a4e9ccc5a"},
|
||||
{file = "fonttools-4.51.0-py3-none-any.whl", hash = "sha256:15c94eeef6b095831067f72c825eb0e2d48bb4cea0647c1b05c981ecba2bf39f"},
|
||||
{file = "fonttools-4.51.0.tar.gz", hash = "sha256:dc0673361331566d7a663d7ce0f6fdcbfbdc1f59c6e3ed1165ad7202ca183c68"},
|
||||
{file = "fonttools-4.50.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:effd303fb422f8ce06543a36ca69148471144c534cc25f30e5be752bc4f46736"},
|
||||
{file = "fonttools-4.50.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7913992ab836f621d06aabac118fc258b9947a775a607e1a737eb3a91c360335"},
|
||||
{file = "fonttools-4.50.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e0a1c5bd2f63da4043b63888534b52c5a1fd7ae187c8ffc64cbb7ae475b9dab"},
|
||||
{file = "fonttools-4.50.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d40fc98540fa5360e7ecf2c56ddf3c6e7dd04929543618fd7b5cc76e66390562"},
|
||||
{file = "fonttools-4.50.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9fff65fbb7afe137bac3113827855e0204482727bddd00a806034ab0d3951d0d"},
|
||||
{file = "fonttools-4.50.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b1aeae3dd2ee719074a9372c89ad94f7c581903306d76befdaca2a559f802472"},
|
||||
{file = "fonttools-4.50.0-cp310-cp310-win32.whl", hash = "sha256:e9623afa319405da33b43c85cceb0585a6f5d3a1d7c604daf4f7e1dd55c03d1f"},
|
||||
{file = "fonttools-4.50.0-cp310-cp310-win_amd64.whl", hash = "sha256:778c5f43e7e654ef7fe0605e80894930bc3a7772e2f496238e57218610140f54"},
|
||||
{file = "fonttools-4.50.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3dfb102e7f63b78c832e4539969167ffcc0375b013080e6472350965a5fe8048"},
|
||||
{file = "fonttools-4.50.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9e58fe34cb379ba3d01d5d319d67dd3ce7ca9a47ad044ea2b22635cd2d1247fc"},
|
||||
{file = "fonttools-4.50.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c673ab40d15a442a4e6eb09bf007c1dda47c84ac1e2eecbdf359adacb799c24"},
|
||||
{file = "fonttools-4.50.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b3ac35cdcd1a4c90c23a5200212c1bb74fa05833cc7c14291d7043a52ca2aaa"},
|
||||
{file = "fonttools-4.50.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8844e7a2c5f7ecf977e82eb6b3014f025c8b454e046d941ece05b768be5847ae"},
|
||||
{file = "fonttools-4.50.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f849bd3c5c2249b49c98eca5aaebb920d2bfd92b3c69e84ca9bddf133e9f83f0"},
|
||||
{file = "fonttools-4.50.0-cp311-cp311-win32.whl", hash = "sha256:39293ff231b36b035575e81c14626dfc14407a20de5262f9596c2cbb199c3625"},
|
||||
{file = "fonttools-4.50.0-cp311-cp311-win_amd64.whl", hash = "sha256:c33d5023523b44d3481624f840c8646656a1def7630ca562f222eb3ead16c438"},
|
||||
{file = "fonttools-4.50.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b4a886a6dbe60100ba1cd24de962f8cd18139bd32808da80de1fa9f9f27bf1dc"},
|
||||
{file = "fonttools-4.50.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b2ca1837bfbe5eafa11313dbc7edada79052709a1fffa10cea691210af4aa1fa"},
|
||||
{file = "fonttools-4.50.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0493dd97ac8977e48ffc1476b932b37c847cbb87fd68673dee5182004906828"},
|
||||
{file = "fonttools-4.50.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77844e2f1b0889120b6c222fc49b2b75c3d88b930615e98893b899b9352a27ea"},
|
||||
{file = "fonttools-4.50.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3566bfb8c55ed9100afe1ba6f0f12265cd63a1387b9661eb6031a1578a28bad1"},
|
||||
{file = "fonttools-4.50.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:35e10ddbc129cf61775d58a14f2d44121178d89874d32cae1eac722e687d9019"},
|
||||
{file = "fonttools-4.50.0-cp312-cp312-win32.whl", hash = "sha256:cc8140baf9fa8f9b903f2b393a6c413a220fa990264b215bf48484f3d0bf8710"},
|
||||
{file = "fonttools-4.50.0-cp312-cp312-win_amd64.whl", hash = "sha256:0ccc85fd96373ab73c59833b824d7a73846670a0cb1f3afbaee2b2c426a8f931"},
|
||||
{file = "fonttools-4.50.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e270a406219af37581d96c810172001ec536e29e5593aa40d4c01cca3e145aa6"},
|
||||
{file = "fonttools-4.50.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ac2463de667233372e9e1c7e9de3d914b708437ef52a3199fdbf5a60184f190c"},
|
||||
{file = "fonttools-4.50.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47abd6669195abe87c22750dbcd366dc3a0648f1b7c93c2baa97429c4dc1506e"},
|
||||
{file = "fonttools-4.50.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:074841375e2e3d559aecc86e1224caf78e8b8417bb391e7d2506412538f21adc"},
|
||||
{file = "fonttools-4.50.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0743fd2191ad7ab43d78cd747215b12033ddee24fa1e088605a3efe80d6984de"},
|
||||
{file = "fonttools-4.50.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3d7080cce7be5ed65bee3496f09f79a82865a514863197ff4d4d177389e981b0"},
|
||||
{file = "fonttools-4.50.0-cp38-cp38-win32.whl", hash = "sha256:a467ba4e2eadc1d5cc1a11d355abb945f680473fbe30d15617e104c81f483045"},
|
||||
{file = "fonttools-4.50.0-cp38-cp38-win_amd64.whl", hash = "sha256:f77e048f805e00870659d6318fd89ef28ca4ee16a22b4c5e1905b735495fc422"},
|
||||
{file = "fonttools-4.50.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b6245eafd553c4e9a0708e93be51392bd2288c773523892fbd616d33fd2fda59"},
|
||||
{file = "fonttools-4.50.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a4062cc7e8de26f1603323ef3ae2171c9d29c8a9f5e067d555a2813cd5c7a7e0"},
|
||||
{file = "fonttools-4.50.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34692850dfd64ba06af61e5791a441f664cb7d21e7b544e8f385718430e8f8e4"},
|
||||
{file = "fonttools-4.50.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:678dd95f26a67e02c50dcb5bf250f95231d455642afbc65a3b0bcdacd4e4dd38"},
|
||||
{file = "fonttools-4.50.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4f2ce7b0b295fe64ac0a85aef46a0f2614995774bd7bc643b85679c0283287f9"},
|
||||
{file = "fonttools-4.50.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d346f4dc2221bfb7ab652d1e37d327578434ce559baf7113b0f55768437fe6a0"},
|
||||
{file = "fonttools-4.50.0-cp39-cp39-win32.whl", hash = "sha256:a51eeaf52ba3afd70bf489be20e52fdfafe6c03d652b02477c6ce23c995222f4"},
|
||||
{file = "fonttools-4.50.0-cp39-cp39-win_amd64.whl", hash = "sha256:8639be40d583e5d9da67795aa3eeeda0488fb577a1d42ae11a5036f18fb16d93"},
|
||||
{file = "fonttools-4.50.0-py3-none-any.whl", hash = "sha256:48fa36da06247aa8282766cfd63efff1bb24e55f020f29a335939ed3844d20d3"},
|
||||
{file = "fonttools-4.50.0.tar.gz", hash = "sha256:fa5cf61058c7dbb104c2ac4e782bf1b2016a8cf2f69de6e4dd6a865d2c969bb5"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
@ -866,20 +866,6 @@ gitdb = ">=4.0.1,<5"
|
||||
doc = ["sphinx (==4.3.2)", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "sphinxcontrib-applehelp (>=1.0.2,<=1.0.4)", "sphinxcontrib-devhelp (==1.0.2)", "sphinxcontrib-htmlhelp (>=2.0.0,<=2.0.1)", "sphinxcontrib-qthelp (==1.0.3)", "sphinxcontrib-serializinghtml (==1.1.5)"]
|
||||
test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"]
|
||||
|
||||
[[package]]
|
||||
name = "google"
|
||||
version = "3.0.0"
|
||||
description = "Python bindings to the Google search engine."
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
files = [
|
||||
{file = "google-3.0.0-py2.py3-none-any.whl", hash = "sha256:889cf695f84e4ae2c55fbc0cfdaf4c1e729417fa52ab1db0485202ba173e4935"},
|
||||
{file = "google-3.0.0.tar.gz", hash = "sha256:143530122ee5130509ad5e989f0512f7cb218b2d4eddbafbad40fd10e8d8ccbe"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
beautifulsoup4 = "*"
|
||||
|
||||
[[package]]
|
||||
name = "google-ai-generativelanguage"
|
||||
version = "0.4.0"
|
||||
@ -1727,7 +1713,6 @@ files = [
|
||||
{file = "lxml-5.2.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c38d7b9a690b090de999835f0443d8aa93ce5f2064035dfc48f27f02b4afc3d0"},
|
||||
{file = "lxml-5.2.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5670fb70a828663cc37552a2a85bf2ac38475572b0e9b91283dc09efb52c41d1"},
|
||||
{file = "lxml-5.2.1-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:958244ad566c3ffc385f47dddde4145088a0ab893504b54b52c041987a8c1863"},
|
||||
{file = "lxml-5.2.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b6241d4eee5f89453307c2f2bfa03b50362052ca0af1efecf9fef9a41a22bb4f"},
|
||||
{file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:2a66bf12fbd4666dd023b6f51223aed3d9f3b40fef06ce404cb75bafd3d89536"},
|
||||
{file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:9123716666e25b7b71c4e1789ec829ed18663152008b58544d95b008ed9e21e9"},
|
||||
{file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:0c3f67e2aeda739d1cc0b1102c9a9129f7dc83901226cc24dd72ba275ced4218"},
|
||||
@ -1995,39 +1980,39 @@ tests = ["pytest", "pytz", "simplejson"]
|
||||
|
||||
[[package]]
|
||||
name = "matplotlib"
|
||||
version = "3.8.4"
|
||||
version = "3.8.3"
|
||||
description = "Python plotting package"
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
files = [
|
||||
{file = "matplotlib-3.8.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:abc9d838f93583650c35eca41cfcec65b2e7cb50fd486da6f0c49b5e1ed23014"},
|
||||
{file = "matplotlib-3.8.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f65c9f002d281a6e904976007b2d46a1ee2bcea3a68a8c12dda24709ddc9106"},
|
||||
{file = "matplotlib-3.8.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce1edd9f5383b504dbc26eeea404ed0a00656c526638129028b758fd43fc5f10"},
|
||||
{file = "matplotlib-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecd79298550cba13a43c340581a3ec9c707bd895a6a061a78fa2524660482fc0"},
|
||||
{file = "matplotlib-3.8.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:90df07db7b599fe7035d2f74ab7e438b656528c68ba6bb59b7dc46af39ee48ef"},
|
||||
{file = "matplotlib-3.8.4-cp310-cp310-win_amd64.whl", hash = "sha256:ac24233e8f2939ac4fd2919eed1e9c0871eac8057666070e94cbf0b33dd9c338"},
|
||||
{file = "matplotlib-3.8.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:72f9322712e4562e792b2961971891b9fbbb0e525011e09ea0d1f416c4645661"},
|
||||
{file = "matplotlib-3.8.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:232ce322bfd020a434caaffbd9a95333f7c2491e59cfc014041d95e38ab90d1c"},
|
||||
{file = "matplotlib-3.8.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6addbd5b488aedb7f9bc19f91cd87ea476206f45d7116fcfe3d31416702a82fa"},
|
||||
{file = "matplotlib-3.8.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc4ccdc64e3039fc303defd119658148f2349239871db72cd74e2eeaa9b80b71"},
|
||||
{file = "matplotlib-3.8.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b7a2a253d3b36d90c8993b4620183b55665a429da8357a4f621e78cd48b2b30b"},
|
||||
{file = "matplotlib-3.8.4-cp311-cp311-win_amd64.whl", hash = "sha256:8080d5081a86e690d7688ffa542532e87f224c38a6ed71f8fbed34dd1d9fedae"},
|
||||
{file = "matplotlib-3.8.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6485ac1f2e84676cff22e693eaa4fbed50ef5dc37173ce1f023daef4687df616"},
|
||||
{file = "matplotlib-3.8.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c89ee9314ef48c72fe92ce55c4e95f2f39d70208f9f1d9db4e64079420d8d732"},
|
||||
{file = "matplotlib-3.8.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50bac6e4d77e4262c4340d7a985c30912054745ec99756ce213bfbc3cb3808eb"},
|
||||
{file = "matplotlib-3.8.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f51c4c869d4b60d769f7b4406eec39596648d9d70246428745a681c327a8ad30"},
|
||||
{file = "matplotlib-3.8.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b12ba985837e4899b762b81f5b2845bd1a28f4fdd1a126d9ace64e9c4eb2fb25"},
|
||||
{file = "matplotlib-3.8.4-cp312-cp312-win_amd64.whl", hash = "sha256:7a6769f58ce51791b4cb8b4d7642489df347697cd3e23d88266aaaee93b41d9a"},
|
||||
{file = "matplotlib-3.8.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:843cbde2f0946dadd8c5c11c6d91847abd18ec76859dc319362a0964493f0ba6"},
|
||||
{file = "matplotlib-3.8.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1c13f041a7178f9780fb61cc3a2b10423d5e125480e4be51beaf62b172413b67"},
|
||||
{file = "matplotlib-3.8.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb44f53af0a62dc80bba4443d9b27f2fde6acfdac281d95bc872dc148a6509cc"},
|
||||
{file = "matplotlib-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:606e3b90897554c989b1e38a258c626d46c873523de432b1462f295db13de6f9"},
|
||||
{file = "matplotlib-3.8.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9bb0189011785ea794ee827b68777db3ca3f93f3e339ea4d920315a0e5a78d54"},
|
||||
{file = "matplotlib-3.8.4-cp39-cp39-win_amd64.whl", hash = "sha256:6209e5c9aaccc056e63b547a8152661324404dd92340a6e479b3a7f24b42a5d0"},
|
||||
{file = "matplotlib-3.8.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c7064120a59ce6f64103c9cefba8ffe6fba87f2c61d67c401186423c9a20fd35"},
|
||||
{file = "matplotlib-3.8.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0e47eda4eb2614300fc7bb4657fced3e83d6334d03da2173b09e447418d499f"},
|
||||
{file = "matplotlib-3.8.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:493e9f6aa5819156b58fce42b296ea31969f2aab71c5b680b4ea7a3cb5c07d94"},
|
||||
{file = "matplotlib-3.8.4.tar.gz", hash = "sha256:8aac397d5e9ec158960e31c381c5ffc52ddd52bd9a47717e2a694038167dffea"},
|
||||
{file = "matplotlib-3.8.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:cf60138ccc8004f117ab2a2bad513cc4d122e55864b4fe7adf4db20ca68a078f"},
|
||||
{file = "matplotlib-3.8.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5f557156f7116be3340cdeef7f128fa99b0d5d287d5f41a16e169819dcf22357"},
|
||||
{file = "matplotlib-3.8.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f386cf162b059809ecfac3bcc491a9ea17da69fa35c8ded8ad154cd4b933d5ec"},
|
||||
{file = "matplotlib-3.8.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3c5f96f57b0369c288bf6f9b5274ba45787f7e0589a34d24bdbaf6d3344632f"},
|
||||
{file = "matplotlib-3.8.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:83e0f72e2c116ca7e571c57aa29b0fe697d4c6425c4e87c6e994159e0c008635"},
|
||||
{file = "matplotlib-3.8.3-cp310-cp310-win_amd64.whl", hash = "sha256:1c5c8290074ba31a41db1dc332dc2b62def469ff33766cbe325d32a3ee291aea"},
|
||||
{file = "matplotlib-3.8.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:5184e07c7e1d6d1481862ee361905b7059f7fe065fc837f7c3dc11eeb3f2f900"},
|
||||
{file = "matplotlib-3.8.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d7e7e0993d0758933b1a241a432b42c2db22dfa37d4108342ab4afb9557cbe3e"},
|
||||
{file = "matplotlib-3.8.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04b36ad07eac9740fc76c2aa16edf94e50b297d6eb4c081e3add863de4bb19a7"},
|
||||
{file = "matplotlib-3.8.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c42dae72a62f14982f1474f7e5c9959fc4bc70c9de11cc5244c6e766200ba65"},
|
||||
{file = "matplotlib-3.8.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bf5932eee0d428192c40b7eac1399d608f5d995f975cdb9d1e6b48539a5ad8d0"},
|
||||
{file = "matplotlib-3.8.3-cp311-cp311-win_amd64.whl", hash = "sha256:40321634e3a05ed02abf7c7b47a50be50b53ef3eaa3a573847431a545585b407"},
|
||||
{file = "matplotlib-3.8.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:09074f8057917d17ab52c242fdf4916f30e99959c1908958b1fc6032e2d0f6d4"},
|
||||
{file = "matplotlib-3.8.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5745f6d0fb5acfabbb2790318db03809a253096e98c91b9a31969df28ee604aa"},
|
||||
{file = "matplotlib-3.8.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b97653d869a71721b639714b42d87cda4cfee0ee74b47c569e4874c7590c55c5"},
|
||||
{file = "matplotlib-3.8.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:242489efdb75b690c9c2e70bb5c6550727058c8a614e4c7716f363c27e10bba1"},
|
||||
{file = "matplotlib-3.8.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:83c0653c64b73926730bd9ea14aa0f50f202ba187c307a881673bad4985967b7"},
|
||||
{file = "matplotlib-3.8.3-cp312-cp312-win_amd64.whl", hash = "sha256:ef6c1025a570354297d6c15f7d0f296d95f88bd3850066b7f1e7b4f2f4c13a39"},
|
||||
{file = "matplotlib-3.8.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:c4af3f7317f8a1009bbb2d0bf23dfaba859eb7dd4ccbd604eba146dccaaaf0a4"},
|
||||
{file = "matplotlib-3.8.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4c6e00a65d017d26009bac6808f637b75ceade3e1ff91a138576f6b3065eeeba"},
|
||||
{file = "matplotlib-3.8.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7b49ab49a3bea17802df6872f8d44f664ba8f9be0632a60c99b20b6db2165b7"},
|
||||
{file = "matplotlib-3.8.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6728dde0a3997396b053602dbd907a9bd64ec7d5cf99e728b404083698d3ca01"},
|
||||
{file = "matplotlib-3.8.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:813925d08fb86aba139f2d31864928d67511f64e5945ca909ad5bc09a96189bb"},
|
||||
{file = "matplotlib-3.8.3-cp39-cp39-win_amd64.whl", hash = "sha256:cd3a0c2be76f4e7be03d34a14d49ded6acf22ef61f88da600a18a5cd8b3c5f3c"},
|
||||
{file = "matplotlib-3.8.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:fa93695d5c08544f4a0dfd0965f378e7afc410d8672816aff1e81be1f45dbf2e"},
|
||||
{file = "matplotlib-3.8.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9764df0e8778f06414b9d281a75235c1e85071f64bb5d71564b97c1306a2afc"},
|
||||
{file = "matplotlib-3.8.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:5e431a09e6fab4012b01fc155db0ce6dccacdbabe8198197f523a4ef4805eb26"},
|
||||
{file = "matplotlib-3.8.3.tar.gz", hash = "sha256:7b416239e9ae38be54b028abbf9048aff5054a9aba5416bef0bd17f9162ce161"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
@ -2036,7 +2021,7 @@ cycler = ">=0.10"
|
||||
fonttools = ">=4.22.0"
|
||||
importlib-resources = {version = ">=3.2.0", markers = "python_version < \"3.10\""}
|
||||
kiwisolver = ">=1.3.1"
|
||||
numpy = ">=1.21"
|
||||
numpy = ">=1.21,<2"
|
||||
packaging = ">=20.0"
|
||||
pillow = ">=8"
|
||||
pyparsing = ">=2.3.1"
|
||||
@ -2268,13 +2253,13 @@ files = [
|
||||
|
||||
[[package]]
|
||||
name = "openai"
|
||||
version = "1.16.2"
|
||||
version = "1.16.1"
|
||||
description = "The official Python library for the openai API"
|
||||
optional = false
|
||||
python-versions = ">=3.7.1"
|
||||
files = [
|
||||
{file = "openai-1.16.2-py3-none-any.whl", hash = "sha256:46a435380921e42dae218d04d6dd0e89a30d7f3b9d8a778d5887f78003cf9354"},
|
||||
{file = "openai-1.16.2.tar.gz", hash = "sha256:c93d5efe5b73b6cb72c4cd31823852d2e7c84a138c0af3cbe4a8eb32b1164ab2"},
|
||||
{file = "openai-1.16.1-py3-none-any.whl", hash = "sha256:77ef3db6110071f7154859e234250fb945a36554207a30a4491092eadb73fcb5"},
|
||||
{file = "openai-1.16.1.tar.gz", hash = "sha256:58922c785d167458b46e3c76e7b1bc2306f313ee9b71791e84cbf590abe160f2"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
@ -3498,13 +3483,13 @@ htbuilder = "*"
|
||||
|
||||
[[package]]
|
||||
name = "streamlit"
|
||||
version = "1.33.0"
|
||||
version = "1.32.2"
|
||||
description = "A faster way to build and share data apps"
|
||||
optional = false
|
||||
python-versions = "!=3.9.7,>=3.8"
|
||||
python-versions = ">=3.8, !=3.9.7"
|
||||
files = [
|
||||
{file = "streamlit-1.33.0-py2.py3-none-any.whl", hash = "sha256:bfacb5d1edefcf803c2040b051a21b4c81317a9865448e6767d0a0c6aae7edae"},
|
||||
{file = "streamlit-1.33.0.tar.gz", hash = "sha256:a8da8ff46f5b948c56d2dc7aca7a61cf8d995f4f21744cf82258ae75e63004ba"},
|
||||
{file = "streamlit-1.32.2-py2.py3-none-any.whl", hash = "sha256:a0b8044e76fec364b07be145f8b40dbd8d083e20ebbb189ceb1fa9423f3dedea"},
|
||||
{file = "streamlit-1.32.2.tar.gz", hash = "sha256:1258b9cbc3ff957bf7d09b1bfc85cedc308f1065b30748545295a9af8d5577ab"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
@ -3514,7 +3499,7 @@ cachetools = ">=4.0,<6"
|
||||
click = ">=7.0,<9"
|
||||
gitpython = ">=3.0.7,<3.1.19 || >3.1.19,<4"
|
||||
numpy = ">=1.19.3,<2"
|
||||
packaging = ">=16.8,<25"
|
||||
packaging = ">=16.8,<24"
|
||||
pandas = ">=1.3.0,<3"
|
||||
pillow = ">=7.1.0,<11"
|
||||
protobuf = ">=3.20,<5"
|
||||
@ -3592,13 +3577,13 @@ streamlit = ">=0.63"
|
||||
|
||||
[[package]]
|
||||
name = "streamlit-extras"
|
||||
version = "0.4.2"
|
||||
version = "0.4.0"
|
||||
description = "A library to discover, try, install and share Streamlit extras"
|
||||
optional = false
|
||||
python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8"
|
||||
python-versions = ">=3.8, !=2.7.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.*"
|
||||
files = [
|
||||
{file = "streamlit_extras-0.4.2-py3-none-any.whl", hash = "sha256:a7d28c13f167f7c9bccd7344052584575fbbf2452e119c433bc4e5c8a98022a0"},
|
||||
{file = "streamlit_extras-0.4.2.tar.gz", hash = "sha256:483016e54cc25f7217e109dc4c63bc66325d2e0db75ad481df327bed32f3f64f"},
|
||||
{file = "streamlit_extras-0.4.0-py3-none-any.whl", hash = "sha256:7371963e9472065c38cb51e79b340f1bc8995d07e0837f1ccf0930df443c2439"},
|
||||
{file = "streamlit_extras-0.4.0.tar.gz", hash = "sha256:ac67645ab84accb5ae4de8ef7ca5dd5fc69965d934b9373f813770720814204d"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
@ -3617,7 +3602,6 @@ streamlit-image-coordinates = ">=0.1.1,<0.2.0"
|
||||
streamlit-keyup = ">=0.1.9"
|
||||
streamlit-toggle-switch = ">=1.0.2"
|
||||
streamlit-vertical-slider = ">=2.5.5"
|
||||
validators = ">=0.20.0"
|
||||
|
||||
[[package]]
|
||||
name = "streamlit-faker"
|
||||
@ -3868,13 +3852,13 @@ typing-inspect = ">=0.8.0"
|
||||
|
||||
[[package]]
|
||||
name = "typing-extensions"
|
||||
version = "4.11.0"
|
||||
version = "4.10.0"
|
||||
description = "Backported and Experimental Type Hints for Python 3.8+"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"},
|
||||
{file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"},
|
||||
{file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"},
|
||||
{file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3920,17 +3904,6 @@ h2 = ["h2 (>=4,<5)"]
|
||||
socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"]
|
||||
zstd = ["zstandard (>=0.18.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "validators"
|
||||
version = "0.28.0"
|
||||
description = "Python Data Validation for Humans™"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "validators-0.28.0-py3-none-any.whl", hash = "sha256:e0184691dea3ba82b52c161ba81d3ec1d8be8da9609f0137d1430b395b366521"},
|
||||
{file = "validators-0.28.0.tar.gz", hash = "sha256:85bc82511f6ccd0800f4c15d8c0dc546c15e369640c5ea1f24349ba0b3b17815"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "watchdog"
|
||||
version = "4.0.0"
|
||||
@ -4093,4 +4066,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
|
||||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = ">3.9,<3.9.7 || >3.9.7,<3.12"
|
||||
content-hash = "8e8c1c00e39f87a357da392c4e544a6f36f1b390ab558f9a9de7340998065724"
|
||||
content-hash = "e3b11e19ab2febb71da1dd3746bae4310fce98af6241cf2b8d2c996963237481"
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "scrapegraphai"
|
||||
version = "0.0.12"
|
||||
version = "0.0.15"
|
||||
description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines."
|
||||
authors = [
|
||||
"Marco Vinciguerra <mvincig11@gmail.com>",
|
||||
|
||||
@ -33,8 +33,7 @@ class SmartScraperGraph(AbstractGraph):
|
||||
return OpenAI(llm_params)
|
||||
elif "gemini" in llm_params["model"]:
|
||||
return Gemini(llm_params)
|
||||
else:
|
||||
raise ValueError("Model not supported")
|
||||
raise ValueError("Model not supported")
|
||||
|
||||
def _create_graph(self):
|
||||
"""
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
"""
|
||||
Module for configuration of Gemini
|
||||
"""
|
||||
Gemini module configuration
|
||||
"""
|
||||
from langchain_google_genai import ChatGoogleGenerativeAI
|
||||
|
||||
|
||||
@ -74,9 +74,8 @@ class BaseNode(ABC):
|
||||
pass
|
||||
|
||||
def get_input_keys(self, state: dict) -> List[str]:
|
||||
"""
|
||||
Use the _parse_input_keys method to identify which state keys are needed
|
||||
based on the input attribute
|
||||
"""Use the _parse_input_keys method to identify which state keys are
|
||||
needed based on the input attribute
|
||||
"""
|
||||
try:
|
||||
input_keys = self._parse_input_keys(state, self.input)
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
"""
|
||||
"""
|
||||
Module for the ImageToTextNode class.
|
||||
"""
|
||||
|
||||
from typing import List
|
||||
from .base_node import BaseNode
|
||||
|
||||
|
||||
@ -10,34 +10,43 @@ class ImageToTextNode(BaseNode):
|
||||
A class representing a node that processes an image and returns the text description.
|
||||
|
||||
Attributes:
|
||||
llm (OpenAIImageToText): An instance of the OpenAIImageToText class.
|
||||
llm_model (OpenAIImageToText): An instance of the OpenAIImageToText class.
|
||||
|
||||
Methods:
|
||||
execute(state, url): Execute the node's logic and return the updated state.
|
||||
"""
|
||||
|
||||
def __init__(self, llm, node_name: str):
|
||||
def __init__(self, input: str, output: List[str], model_config: dict,
|
||||
node_name: str = "ImageToText"):
|
||||
"""
|
||||
Initializes an instance of the ImageToTextNode class.
|
||||
|
||||
Args:
|
||||
llm (OpenAIImageToText): An instance of the OpenAIImageToText class.
|
||||
node_name (str): name of the node
|
||||
input (str): The input for the node.
|
||||
output (List[str]): The output of the node.
|
||||
model_config (dict): Configuration for the model.
|
||||
node_name (str): Name of the node.
|
||||
"""
|
||||
super().__init__(node_name, "node")
|
||||
self.llm = llm
|
||||
super().__init__(node_name, "node", input, output, 1, model_config)
|
||||
self.llm_model = model_config["llm_model"]
|
||||
|
||||
def execute(self, state: dict, url: str) -> dict:
|
||||
def execute(self, state: dict) -> dict:
|
||||
"""
|
||||
Execute the node's logic and return the updated state.
|
||||
|
||||
Args:
|
||||
state (dict): The current state of the graph.
|
||||
url (str): url of the image where to
|
||||
:return: The updated state after executing this node.
|
||||
"""
|
||||
|
||||
Returns:
|
||||
dict: The updated state after executing this node.
|
||||
"""
|
||||
print("---GENERATING TEXT FROM IMAGE---")
|
||||
text_answer = self.llm.run(url)
|
||||
input_keys = self.get_input_keys(state)
|
||||
|
||||
input_data = [state[key] for key in input_keys]
|
||||
url = input_data[0]
|
||||
|
||||
text_answer = self.llm_model.run(url)
|
||||
|
||||
state.update({"image_text": text_answer})
|
||||
return state
|
||||
|
||||
@ -7,6 +7,8 @@ import re
|
||||
def parse_expression(expression, state: dict):
|
||||
"""
|
||||
Function for parsing the expressions
|
||||
Args:
|
||||
state (dict): state to elaborate
|
||||
"""
|
||||
# Check for empty expression
|
||||
if not expression:
|
||||
@ -69,14 +71,14 @@ def parse_expression(expression, state: dict):
|
||||
'|'.join(sub_result) + expression[end+1:]
|
||||
return evaluate_simple_expression(expression)
|
||||
|
||||
result = evaluate_expression(expression)
|
||||
temp_result = evaluate_expression(expression)
|
||||
|
||||
if not result:
|
||||
if not temp_result:
|
||||
raise ValueError("No state keys matched the expression.")
|
||||
|
||||
# Remove redundant state keys from the result, without changing their order
|
||||
final_result = []
|
||||
for key in result:
|
||||
for key in temp_result:
|
||||
if key not in final_result:
|
||||
final_result.append(key)
|
||||
|
||||
|
||||
@ -18,14 +18,11 @@ def remover(html_content: str) -> str:
|
||||
|
||||
soup = BeautifulSoup(html_content, 'html.parser')
|
||||
|
||||
# Estrai il titolo
|
||||
title_tag = soup.find('title')
|
||||
title = title_tag.get_text() if title_tag else ""
|
||||
|
||||
# Rimuovi i tag <script> in tutto il documento
|
||||
[script.extract() for script in soup.find_all('script')]
|
||||
|
||||
# Estrai il corpo del documento
|
||||
body_content = soup.find('body')
|
||||
body = str(body_content) if body_content else ""
|
||||
|
||||
|
||||
@ -1,11 +1,11 @@
|
||||
"""
|
||||
This utility function saves the byte response as an audio file.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Union
|
||||
|
||||
|
||||
def save_audio_from_bytes(byte_response: bytes, output_path):
|
||||
def save_audio_from_bytes(byte_response: bytes, output_path: Union[str, Path]) -> None:
|
||||
"""
|
||||
Saves the byte response as an audio file.
|
||||
|
||||
|
||||
@ -1,58 +0,0 @@
|
||||
"""
|
||||
Module for making the tests
|
||||
"""
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from scrapegraphai.models import OpenAI
|
||||
from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# Define the configuration for the language model
|
||||
openai_key = os.getenv("OPENAI_APIKEY")
|
||||
|
||||
llm_config = {
|
||||
"api_key": openai_key,
|
||||
"model_name": "gpt-3.5-turbo",
|
||||
"temperature": 0,
|
||||
"streaming": True
|
||||
}
|
||||
llm_model = OpenAI(llm_config)
|
||||
|
||||
state = {
|
||||
"user_prompt": "List me all the projects",
|
||||
"url": "https://perinim.github.io/projects/",
|
||||
}
|
||||
|
||||
fetch_node = FetchNode(
|
||||
input="url | local_dir",
|
||||
output=["doc"],
|
||||
node_name="fetch_html"
|
||||
)
|
||||
|
||||
updated_state = fetch_node.execute(state)
|
||||
parse_node = ParseNode(
|
||||
input="doc",
|
||||
output=["parsed_doc"],
|
||||
node_name="parse_document"
|
||||
)
|
||||
|
||||
updated_state = parse_node.execute(updated_state)
|
||||
|
||||
rag_node = RAGNode(
|
||||
input="user_prompt & (parsed_doc | doc)",
|
||||
output=["relevant_chunks"],
|
||||
model_config={"llm_model": llm_model},
|
||||
node_name="rag_node"
|
||||
)
|
||||
|
||||
updated_state = rag_node.execute(updated_state)
|
||||
|
||||
generate_answer_node = GenerateAnswerNode(
|
||||
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
|
||||
output=["answer"],
|
||||
model_config={"llm_model": llm_model},
|
||||
node_name="generate_answer"
|
||||
)
|
||||
|
||||
print(generate_answer_node.execute(updated_state))
|
||||
Loading…
Reference in New Issue
Block a user