From 3289c7bf5ec58ac3d04e9e5e8e654af9abcee228 Mon Sep 17 00:00:00 2001
From: Federico Aguzzi <62149513+f-aguzzi@users.noreply.github.com>
Date: Mon, 22 Jul 2024 13:18:17 +0200
Subject: [PATCH] chore(dependecies): add script to auto-update requirements

---
 manual deployment/autorequirements.py | 30 +++++++++++++++++++++++++++
 pyproject.toml                        |  1 +
 requirements-dev.txt                  |  6 ++++++
 requirements.txt                      | 23 ++++++++++++++++++++
 4 files changed, 60 insertions(+)
 create mode 100644 manual deployment/autorequirements.py
 create mode 100644 requirements-dev.txt
 create mode 100644 requirements.txt

diff --git a/manual deployment/autorequirements.py b/manual deployment/autorequirements.py
new file mode 100644
index 00000000..2bb7e1e8
--- /dev/null
+++ b/manual deployment/autorequirements.py	
@@ -0,0 +1,30 @@
+import toml
+
+# Load the TOML file
+data = toml.load('pyproject.toml')
+
+# Get the dependencies
+dependencies = data['project']['dependencies']
+
+# Write the dependencies to a requirements.txt file
+with open('requirements.txt', 'w') as f:
+    for dependency in dependencies:
+        f.write(dependency + '\n')
+
+# Get the dev dependencies
+dev_dependencies = data['tool']['rye']['dev-dependencies']
+
+# Expand the optional dependencies
+optional_dependencies = data['project']['optional-dependencies']
+expanded_dev_dependencies = []
+for dependency in dev_dependencies:
+    if dependency.startswith('-e file:.'):
+        optional_dependency_name = dependency.split('.')[1][1:-1]
+        expanded_dev_dependencies.extend(optional_dependencies[optional_dependency_name])
+    else:
+        expanded_dev_dependencies.append(dependency)
+
+# Write the expanded dev dependencies to a requirements-dev.txt file
+with open('requirements-dev.txt', 'w') as f:
+    for dependency in expanded_dev_dependencies:
+        f.write(dependency + '\n')
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index 92b9a9af..b06411a0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -91,3 +91,4 @@ dev-dependencies = [
 [tool.rye.scripts]
 pylint-local = "pylint scrapegraphai/**/*.py"
 pylint-ci = "pylint --disable=C0114,C0115,C0116 --exit-zero scrapegraphai/**/*.py"
+update-requirements = "python 'manual deployment/autorequirements.py'"
\ No newline at end of file
diff --git a/requirements-dev.txt b/requirements-dev.txt
new file mode 100644
index 00000000..e04399e9
--- /dev/null
+++ b/requirements-dev.txt
@@ -0,0 +1,6 @@
+pytest==8.0.0
+pytest-mock==3.14.0
+burr[start]==0.22.1
+sphinx==6.0
+furo==2024.5.6
+pylint>=3.2.5
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 00000000..4a3d525c
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,23 @@
+langchain>=0.2.10
+langchain-google-genai>=1.0.7
+langchain-google-vertexai
+langchain-openai>=0.1.17
+langchain-groq>=0.1.3
+langchain-aws>=0.1.3
+langchain-anthropic>=0.1.11
+html2text>=2024.2.26
+faiss-cpu>=1.8.0
+beautifulsoup4>=4.12.3
+pandas>=2.2.2
+python-dotenv>=1.0.1
+tiktoken>=0.7
+tqdm>=4.66.4
+graphviz>=0.20.3
+minify-html>=0.15.0
+free-proxy>=1.1.1
+playwright>=1.43.0
+google>=3.0.0
+undetected-playwright>=0.3.0
+semchunk>=1.0.1
+html2text>=2024.2.26
+langchain-fireworks>=0.1.3