diff --git a/docs/source/conf.py b/docs/source/conf.py index a64cfb33..43c849c4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -23,7 +23,7 @@ html_last_updated_fmt = "%b %d, %Y" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon','sphinx_wagtail_theme'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon'] templates_path = ['_templates'] exclude_patterns = [] @@ -31,19 +31,9 @@ exclude_patterns = [] # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -# html_theme = 'sphinx_rtd_theme' -html_theme = 'sphinx_wagtail_theme' - -html_theme_options = dict( - project_name = "ScrapeGraphAI", - logo = "scrapegraphai_logo.png", - logo_alt = "ScrapeGraphAI", - logo_height = 59, - logo_url = "https://scrapegraph-ai.readthedocs.io/en/latest/", - logo_width = 45, - github_url = "https://github.com/VinciGit00/Scrapegraph-ai/tree/main/docs/source/", - footer_links = ",".join( - ["Landing Page|https://scrapegraphai.com/", - "Docusaurus|https://scrapegraph-doc.onrender.com/docs/intro"] - ), -) +html_theme = 'furo' +html_theme_options = { + "source_repository": "https://github.com/VinciGit00/Scrapegraph-ai/", + "source_branch": "main", + "source_directory": "docs/source/", +} \ No newline at end of file diff --git a/docs/source/modules/modules.rst b/docs/source/modules/modules.rst index f22d1cea..eaa8b0f6 100644 --- a/docs/source/modules/modules.rst +++ b/docs/source/modules/modules.rst @@ -1,3 +1,6 @@ +scrapegraphai +============= + .. toctree:: :maxdepth: 4 diff --git a/docs/source/modules/scrapegraphai.builders.rst b/docs/source/modules/scrapegraphai.builders.rst new file mode 100644 index 00000000..668ea5bc --- /dev/null +++ b/docs/source/modules/scrapegraphai.builders.rst @@ -0,0 +1,21 @@ +scrapegraphai.builders package +============================== + +Submodules +---------- + +scrapegraphai.builders.graph\_builder module +-------------------------------------------- + +.. automodule:: scrapegraphai.builders.graph_builder + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: scrapegraphai.builders + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/modules/scrapegraphai.docloaders.rst b/docs/source/modules/scrapegraphai.docloaders.rst new file mode 100644 index 00000000..be66f042 --- /dev/null +++ b/docs/source/modules/scrapegraphai.docloaders.rst @@ -0,0 +1,21 @@ +scrapegraphai.docloaders package +================================ + +Submodules +---------- + +scrapegraphai.docloaders.chromium module +---------------------------------------- + +.. automodule:: scrapegraphai.docloaders.chromium + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: scrapegraphai.docloaders + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/modules/scrapegraphai.graphs.rst b/docs/source/modules/scrapegraphai.graphs.rst index 7201f2d4..7eca6683 100644 --- a/docs/source/modules/scrapegraphai.graphs.rst +++ b/docs/source/modules/scrapegraphai.graphs.rst @@ -4,6 +4,14 @@ scrapegraphai.graphs package Submodules ---------- +scrapegraphai.graphs.abstract\_graph module +------------------------------------------- + +.. automodule:: scrapegraphai.graphs.abstract_graph + :members: + :undoc-members: + :show-inheritance: + scrapegraphai.graphs.base\_graph module --------------------------------------- @@ -12,6 +20,70 @@ scrapegraphai.graphs.base\_graph module :undoc-members: :show-inheritance: +scrapegraphai.graphs.csv\_scraper\_graph module +----------------------------------------------- + +.. automodule:: scrapegraphai.graphs.csv_scraper_graph + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.graphs.deep\_scraper\_graph module +------------------------------------------------ + +.. automodule:: scrapegraphai.graphs.deep_scraper_graph + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.graphs.json\_scraper\_graph module +------------------------------------------------ + +.. automodule:: scrapegraphai.graphs.json_scraper_graph + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.graphs.omni\_scraper\_graph module +------------------------------------------------ + +.. automodule:: scrapegraphai.graphs.omni_scraper_graph + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.graphs.omni\_search\_graph module +----------------------------------------------- + +.. automodule:: scrapegraphai.graphs.omni_search_graph + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.graphs.pdf\_scraper\_graph module +----------------------------------------------- + +.. automodule:: scrapegraphai.graphs.pdf_scraper_graph + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.graphs.script\_creator\_graph module +-------------------------------------------------- + +.. automodule:: scrapegraphai.graphs.script_creator_graph + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.graphs.search\_graph module +----------------------------------------- + +.. automodule:: scrapegraphai.graphs.search_graph + :members: + :undoc-members: + :show-inheritance: + scrapegraphai.graphs.smart\_scraper\_graph module ------------------------------------------------- @@ -20,6 +92,38 @@ scrapegraphai.graphs.smart\_scraper\_graph module :undoc-members: :show-inheritance: +scrapegraphai.graphs.smart\_scraper\_graph\_burr module +------------------------------------------------------- + +.. automodule:: scrapegraphai.graphs.smart_scraper_graph_burr + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.graphs.smart\_scraper\_graph\_hamilton module +----------------------------------------------------------- + +.. automodule:: scrapegraphai.graphs.smart_scraper_graph_hamilton + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.graphs.speech\_graph module +----------------------------------------- + +.. automodule:: scrapegraphai.graphs.speech_graph + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.graphs.xml\_scraper\_graph module +----------------------------------------------- + +.. automodule:: scrapegraphai.graphs.xml_scraper_graph + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/source/modules/scrapegraphai.helpers.rst b/docs/source/modules/scrapegraphai.helpers.rst new file mode 100644 index 00000000..5bcdf457 --- /dev/null +++ b/docs/source/modules/scrapegraphai.helpers.rst @@ -0,0 +1,45 @@ +scrapegraphai.helpers package +============================= + +Submodules +---------- + +scrapegraphai.helpers.models\_tokens module +------------------------------------------- + +.. automodule:: scrapegraphai.helpers.models_tokens + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.helpers.nodes\_metadata module +-------------------------------------------- + +.. automodule:: scrapegraphai.helpers.nodes_metadata + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.helpers.robots module +----------------------------------- + +.. automodule:: scrapegraphai.helpers.robots + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.helpers.schemas module +------------------------------------ + +.. automodule:: scrapegraphai.helpers.schemas + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: scrapegraphai.helpers + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/modules/scrapegraphai.integrations.rst b/docs/source/modules/scrapegraphai.integrations.rst new file mode 100644 index 00000000..a90c8b7a --- /dev/null +++ b/docs/source/modules/scrapegraphai.integrations.rst @@ -0,0 +1,21 @@ +scrapegraphai.integrations package +================================== + +Submodules +---------- + +scrapegraphai.integrations.burr\_bridge module +---------------------------------------------- + +.. automodule:: scrapegraphai.integrations.burr_bridge + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: scrapegraphai.integrations + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/modules/scrapegraphai.models.rst b/docs/source/modules/scrapegraphai.models.rst new file mode 100644 index 00000000..f16ad476 --- /dev/null +++ b/docs/source/modules/scrapegraphai.models.rst @@ -0,0 +1,101 @@ +scrapegraphai.models package +============================ + +Submodules +---------- + +scrapegraphai.models.anthropic module +------------------------------------- + +.. automodule:: scrapegraphai.models.anthropic + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.models.azure\_openai module +----------------------------------------- + +.. automodule:: scrapegraphai.models.azure_openai + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.models.bedrock module +----------------------------------- + +.. automodule:: scrapegraphai.models.bedrock + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.models.deepseek module +------------------------------------ + +.. automodule:: scrapegraphai.models.deepseek + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.models.gemini module +---------------------------------- + +.. automodule:: scrapegraphai.models.gemini + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.models.groq module +-------------------------------- + +.. automodule:: scrapegraphai.models.groq + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.models.hugging\_face module +----------------------------------------- + +.. automodule:: scrapegraphai.models.hugging_face + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.models.ollama module +---------------------------------- + +.. automodule:: scrapegraphai.models.ollama + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.models.openai module +---------------------------------- + +.. automodule:: scrapegraphai.models.openai + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.models.openai\_itt module +--------------------------------------- + +.. automodule:: scrapegraphai.models.openai_itt + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.models.openai\_tts module +--------------------------------------- + +.. automodule:: scrapegraphai.models.openai_tts + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: scrapegraphai.models + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/modules/scrapegraphai.nodes.rst b/docs/source/modules/scrapegraphai.nodes.rst index fef036a1..c89eecfc 100644 --- a/docs/source/modules/scrapegraphai.nodes.rst +++ b/docs/source/modules/scrapegraphai.nodes.rst @@ -20,10 +20,18 @@ scrapegraphai.nodes.conditional\_node module :undoc-members: :show-inheritance: -scrapegraphai.nodes.fetch\_html\_node module --------------------------------------------- +scrapegraphai.nodes.fetch\_node module +-------------------------------------- -.. automodule:: scrapegraphai.nodes.fetch_html_node +.. automodule:: scrapegraphai.nodes.fetch_node + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.nodes.generate\_answer\_csv\_node module +------------------------------------------------------ + +.. automodule:: scrapegraphai.nodes.generate_answer_csv_node :members: :undoc-members: :show-inheritance: @@ -36,6 +44,30 @@ scrapegraphai.nodes.generate\_answer\_node module :undoc-members: :show-inheritance: +scrapegraphai.nodes.generate\_answer\_omni\_node module +------------------------------------------------------- + +.. automodule:: scrapegraphai.nodes.generate_answer_omni_node + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.nodes.generate\_answer\_pdf\_node module +------------------------------------------------------ + +.. automodule:: scrapegraphai.nodes.generate_answer_pdf_node + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.nodes.generate\_scraper\_node module +-------------------------------------------------- + +.. automodule:: scrapegraphai.nodes.generate_scraper_node + :members: + :undoc-members: + :show-inheritance: + scrapegraphai.nodes.get\_probable\_tags\_node module ---------------------------------------------------- @@ -44,10 +76,82 @@ scrapegraphai.nodes.get\_probable\_tags\_node module :undoc-members: :show-inheritance: -scrapegraphai.nodes.parse\_html\_node module --------------------------------------------- +scrapegraphai.nodes.graph\_iterator\_node module +------------------------------------------------ -.. automodule:: scrapegraphai.nodes.parse_html_node +.. automodule:: scrapegraphai.nodes.graph_iterator_node + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.nodes.image\_to\_text\_node module +------------------------------------------------ + +.. automodule:: scrapegraphai.nodes.image_to_text_node + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.nodes.merge\_answers\_node module +----------------------------------------------- + +.. automodule:: scrapegraphai.nodes.merge_answers_node + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.nodes.parse\_node module +-------------------------------------- + +.. automodule:: scrapegraphai.nodes.parse_node + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.nodes.rag\_node module +------------------------------------ + +.. automodule:: scrapegraphai.nodes.rag_node + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.nodes.robots\_node module +--------------------------------------- + +.. automodule:: scrapegraphai.nodes.robots_node + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.nodes.search\_internet\_node module +------------------------------------------------- + +.. automodule:: scrapegraphai.nodes.search_internet_node + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.nodes.search\_link\_node module +--------------------------------------------- + +.. automodule:: scrapegraphai.nodes.search_link_node + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.nodes.search\_node\_with\_context module +------------------------------------------------------ + +.. automodule:: scrapegraphai.nodes.search_node_with_context + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.nodes.text\_to\_speech\_node module +------------------------------------------------- + +.. automodule:: scrapegraphai.nodes.text_to_speech_node :members: :undoc-members: :show-inheritance: diff --git a/docs/source/modules/scrapegraphai.rst b/docs/source/modules/scrapegraphai.rst index 7ea1ab69..df0fb1a9 100644 --- a/docs/source/modules/scrapegraphai.rst +++ b/docs/source/modules/scrapegraphai.rst @@ -7,99 +7,14 @@ Subpackages .. toctree:: :maxdepth: 4 + scrapegraphai.builders + scrapegraphai.docloaders scrapegraphai.graphs + scrapegraphai.helpers + scrapegraphai.integrations + scrapegraphai.models scrapegraphai.nodes - -Submodules ----------- - -scrapegraphai.class\_creator module ------------------------------------ - -.. automodule:: scrapegraphai.class_creator - :members: - :undoc-members: - :show-inheritance: - -scrapegraphai.class\_generator module -------------------------------------- - -.. automodule:: scrapegraphai.class_generator - :members: - :undoc-members: - :show-inheritance: - -scrapegraphai.convert\_to\_csv module -------------------------------------- - -.. automodule:: scrapegraphai.convert_to_csv - :members: - :undoc-members: - :show-inheritance: - -scrapegraphai.convert\_to\_json module --------------------------------------- - -.. automodule:: scrapegraphai.convert_to_json - :members: - :undoc-members: - :show-inheritance: - -scrapegraphai.dictionaries module ---------------------------------- - -.. automodule:: scrapegraphai.dictionaries - :members: - :undoc-members: - :show-inheritance: - -scrapegraphai.getter module ---------------------------- - -.. automodule:: scrapegraphai.getter - :members: - :undoc-members: - :show-inheritance: - -scrapegraphai.json\_getter module ---------------------------------- - -.. automodule:: scrapegraphai.json_getter - :members: - :undoc-members: - :show-inheritance: - -scrapegraphai.pydantic\_class module ------------------------------------- - -.. automodule:: scrapegraphai.pydantic_class - :members: - :undoc-members: - :show-inheritance: - -scrapegraphai.remover module ----------------------------- - -.. automodule:: scrapegraphai.remover - :members: - :undoc-members: - :show-inheritance: - -scrapegraphai.request module ----------------------------- - -.. automodule:: scrapegraphai.request - :members: - :undoc-members: - :show-inheritance: - -scrapegraphai.token\_calculator module --------------------------------------- - -.. automodule:: scrapegraphai.token_calculator - :members: - :undoc-members: - :show-inheritance: + scrapegraphai.utils Module contents --------------- diff --git a/docs/source/modules/scrapegraphai.utils.rst b/docs/source/modules/scrapegraphai.utils.rst new file mode 100644 index 00000000..d9100f1e --- /dev/null +++ b/docs/source/modules/scrapegraphai.utils.rst @@ -0,0 +1,93 @@ +scrapegraphai.utils package +=========================== + +Submodules +---------- + +scrapegraphai.utils.cleanup\_html module +---------------------------------------- + +.. automodule:: scrapegraphai.utils.cleanup_html + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.utils.convert\_to\_csv module +------------------------------------------- + +.. automodule:: scrapegraphai.utils.convert_to_csv + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.utils.convert\_to\_json module +-------------------------------------------- + +.. automodule:: scrapegraphai.utils.convert_to_json + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.utils.parse\_state\_keys module +--------------------------------------------- + +.. automodule:: scrapegraphai.utils.parse_state_keys + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.utils.prettify\_exec\_info module +----------------------------------------------- + +.. automodule:: scrapegraphai.utils.prettify_exec_info + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.utils.proxy\_rotation module +------------------------------------------ + +.. automodule:: scrapegraphai.utils.proxy_rotation + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.utils.research\_web module +---------------------------------------- + +.. automodule:: scrapegraphai.utils.research_web + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.utils.save\_audio\_from\_bytes module +--------------------------------------------------- + +.. automodule:: scrapegraphai.utils.save_audio_from_bytes + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.utils.sys\_dynamic\_import module +----------------------------------------------- + +.. automodule:: scrapegraphai.utils.sys_dynamic_import + :members: + :undoc-members: + :show-inheritance: + +scrapegraphai.utils.token\_calculator module +-------------------------------------------- + +.. automodule:: scrapegraphai.utils.token_calculator + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: scrapegraphai.utils + :members: + :undoc-members: + :show-inheritance: diff --git a/pyproject.toml b/pyproject.toml index 46471433..81376cb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,8 +64,8 @@ classifiers = [ requires-python = ">= 3.9, < 3.12" [project.optional-dependencies] -burr = ["burr[start]==0.18.0"] -docs = ["sphinx==4.3.0", "sphinx-rtd-theme==1.0.0"] +burr = ["burr[start]==0.19.1"] +docs = ["sphinx==6.0", "furo==2024.5.6"] [build-system] requires = ["hatchling"] diff --git a/requirements-dev.lock b/requirements-dev.lock index 25be91f4..375debaf 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -40,6 +40,7 @@ attrs==23.2.0 babel==2.15.0 # via sphinx beautifulsoup4==4.12.3 + # via furo # via google # via scrapegraphai blinker==1.8.2 @@ -49,7 +50,7 @@ boto3==1.34.110 botocore==1.34.110 # via boto3 # via s3transfer -burr==0.18.0 +burr==0.19.1 # via burr # via scrapegraphai cachetools==5.3.3 @@ -88,9 +89,8 @@ distro==1.9.0 # via openai dnspython==2.6.1 # via email-validator -docutils==0.17.1 +docutils==0.19 # via sphinx - # via sphinx-rtd-theme email-validator==2.1.1 # via fastapi exceptiongroup==1.2.1 @@ -116,6 +116,8 @@ frozenlist==1.4.1 # via aiosignal fsspec==2024.5.0 # via huggingface-hub +furo==2024.5.6 + # via scrapegraphai gitdb==4.0.11 # via gitpython gitpython==3.1.43 @@ -335,6 +337,7 @@ pydeck==0.9.1 pyee==11.1.0 # via playwright pygments==2.18.0 + # via furo # via rich # via sphinx pyparsing==3.1.2 @@ -406,11 +409,12 @@ snowballstemmer==2.2.0 # via sphinx soupsieve==2.5 # via beautifulsoup4 -sphinx==4.3.0 - # via scrapegraphai - # via sphinx-rtd-theme -sphinx-rtd-theme==1.0.0 +sphinx==6.0.0 + # via furo # via scrapegraphai + # via sphinx-basic-ng +sphinx-basic-ng==1.0.0b2 + # via furo sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 @@ -502,5 +506,3 @@ yahoo-search-py==0.3 # via scrapegraphai yarl==1.9.4 # via aiohttp -setuptools==70.0.0 - # via sphinx diff --git a/requirements.txt b/requirements.txt index 00259542..1e6224b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,4 +19,3 @@ langchain-aws==0.1.2 langchain-anthropic==0.1.11 yahoo-search-py==0.3 pypdf==4.2.0 -burr[start]