chore(dependecies): add script to auto-update requirements

manual deployment/autorequirements.py

@@ -0,0 +1,30 @@
+import toml
+
+# Load the TOML file
+data = toml.load('pyproject.toml')
+
+# Get the dependencies
+dependencies = data['project']['dependencies']
+
+# Write the dependencies to a requirements.txt file
+with open('requirements.txt', 'w') as f:
+    for dependency in dependencies:
+        f.write(dependency + '\n')
+
+# Get the dev dependencies
+dev_dependencies = data['tool']['rye']['dev-dependencies']
+
+# Expand the optional dependencies
+optional_dependencies = data['project']['optional-dependencies']
+expanded_dev_dependencies = []
+for dependency in dev_dependencies:
+    if dependency.startswith('-e file:.'):
+        optional_dependency_name = dependency.split('.')[1][1:-1]
+        expanded_dev_dependencies.extend(optional_dependencies[optional_dependency_name])
+    else:
+        expanded_dev_dependencies.append(dependency)
+
+# Write the expanded dev dependencies to a requirements-dev.txt file
+with open('requirements-dev.txt', 'w') as f:
+    for dependency in expanded_dev_dependencies:
+        f.write(dependency + '\n')

pyproject.toml

@@ -91,3 +91,4 @@ dev-dependencies = [
 [tool.rye.scripts]
 pylint-local = "pylint scrapegraphai/**/*.py"
 pylint-ci = "pylint --disable=C0114,C0115,C0116 --exit-zero scrapegraphai/**/*.py"
+update-requirements = "python 'manual deployment/autorequirements.py'"

requirements-dev.txt

@@ -0,0 +1,6 @@
+pytest==8.0.0
+pytest-mock==3.14.0
+burr[start]==0.22.1
+sphinx==6.0
+furo==2024.5.6
+pylint>=3.2.5

requirements.txt

@@ -0,0 +1,23 @@
+langchain>=0.2.10
+langchain-google-genai>=1.0.7
+langchain-google-vertexai
+langchain-openai>=0.1.17
+langchain-groq>=0.1.3
+langchain-aws>=0.1.3
+langchain-anthropic>=0.1.11
+html2text>=2024.2.26
+faiss-cpu>=1.8.0
+beautifulsoup4>=4.12.3
+pandas>=2.2.2
+python-dotenv>=1.0.1
+tiktoken>=0.7
+tqdm>=4.66.4
+graphviz>=0.20.3
+minify-html>=0.15.0
+free-proxy>=1.1.1
+playwright>=1.43.0
+google>=3.0.0
+undetected-playwright>=0.3.0
+semchunk>=1.0.1
+html2text>=2024.2.26
+langchain-fireworks>=0.1.3