diff --git a/pyproject.toml b/pyproject.toml index cf278260..8b474f38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,10 +27,10 @@ dependencies = [ "playwright>=1.43.0", "undetected-playwright>=0.3.0", "semchunk>=2.2.0", - "googlesearch-python>=1.2.5", "async-timeout>=4.0.3", "simpleeval>=1.0.0", - "jsonschema>=4.23.0" + "jsonschema>=4.23.0", + "duckduckgo-search>=7.2.1" ] readme = "README.md" diff --git a/scrapegraphai/nodes/search_internet_node.py b/scrapegraphai/nodes/search_internet_node.py index 482f5b09..d65bc89a 100644 --- a/scrapegraphai/nodes/search_internet_node.py +++ b/scrapegraphai/nodes/search_internet_node.py @@ -48,7 +48,7 @@ class SearchInternetNode(BaseNode): self.search_engine = ( node_config["search_engine"] if node_config.get("search_engine") - else "google" + else "duckduckgo" ) self.serper_api_key = ( diff --git a/scrapegraphai/utils/research_web.py b/scrapegraphai/utils/research_web.py index 0a30b76c..b9721306 100644 --- a/scrapegraphai/utils/research_web.py +++ b/scrapegraphai/utils/research_web.py @@ -7,13 +7,12 @@ from typing import List import requests from bs4 import BeautifulSoup -from googlesearch import search as google_search from langchain_community.tools import DuckDuckGoSearchResults def search_on_web( query: str, - search_engine: str = "Google", + search_engine: str = "duckduckgo", max_results: int = 10, port: int = 8080, timeout: int = 10, @@ -41,7 +40,7 @@ def search_on_web( raise ValueError("Query must be a non-empty string") search_engine = search_engine.lower() - valid_engines = {"google", "duckduckgo", "bing", "searxng", "serper"} + valid_engines = {"duckduckgo", "bing", "searxng", "serper"} if search_engine not in valid_engines: raise ValueError(f"Search engine must be one of: {', '.join(valid_engines)}") @@ -52,20 +51,12 @@ def search_on_web( try: results = [] - if search_engine == "google": - kwargs = { - "num_results": max_results, - "proxy": formatted_proxy, - "lang": language, - } - if region: - kwargs["region"] = region - - results = list(google_search(query, **kwargs)) - - elif search_engine == "duckduckgo": + if search_engine == "duckduckgo": + # Create a DuckDuckGo search object with max_results research = DuckDuckGoSearchResults(max_results=max_results) + # Run the search res = research.run(query) + # Extract URLs using regex results = re.findall(r"https?://[^\s,\]]+", res) elif search_engine == "bing": @@ -74,7 +65,7 @@ def search_on_web( elif search_engine == "searxng": results = _search_searxng(query, max_results, port, timeout) - elif search_engine.lower() == "serper": + elif search_engine == "serper": results = _search_serper(query, max_results, serper_api_key, timeout) return filter_pdf_links(results) diff --git a/uv.lock b/uv.lock index b5d953f1..000a667c 100644 --- a/uv.lock +++ b/uv.lock @@ -760,6 +760,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/93/69/e391bd51bc08ed9141ecd899a0ddb61ab6465309f1eb470905c0c8868081/docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc", size = 570472 }, ] +[[package]] +name = "duckduckgo-search" +version = "7.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "lxml" }, + { name = "primp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/e5/8ac183cadbefa444183f4aca22140b44ed399e80a93caf0b338a043a3c7f/duckduckgo_search-7.2.1.tar.gz", hash = "sha256:cb214b6cd9505a41c228445a9c254620b93519c59292662d62ef19d0220618a0", size = 23897 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/8f/ee72af555cd58feb928ff0fd3977913f4ecd0ce8ad92cf4031c36de91776/duckduckgo_search-7.2.1-py3-none-any.whl", hash = "sha256:72ebbf6ad8759e3c3c79521cd66256e7a4ac741c522fd9342db94de91745ef87", size = 19720 }, +] + [[package]] name = "exceptiongroup" version = "1.2.2" @@ -1012,19 +1026,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/bd/cc3a402a6439c15c3d4294333e13042b915bbeab54edc457c723931fed3f/GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff", size = 207337 }, ] -[[package]] -name = "googlesearch-python" -version = "1.2.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "beautifulsoup4" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d4/69/b285059d1e3ab119eb9fd072aa3fd9da8da8867296e3cee4349bd74c2d63/googlesearch_python-1.2.5.tar.gz", hash = "sha256:3c23488374f477fb36e8e2975abc5e4e18794d898d8a2c9c9bf21a1440213e87", size = 4510 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2e/5c/387b879de452c6c4b5400a52f1c0216aca3aa5740acfd93346f543a20133/googlesearch_python-1.2.5-py3-none-any.whl", hash = "sha256:28ef8ad737951b96ebe8963fbeb9bd2b45cae5c2533df1cc7cb8e72129a785f9", size = 4845 }, -] - [[package]] name = "graphviz" version = "0.20.3" @@ -2621,6 +2622,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/8f/496e10d51edd6671ebe0432e33ff800aa86775d2d147ce7d43389324a525/pre_commit-4.0.1-py2.py3-none-any.whl", hash = "sha256:efde913840816312445dc98787724647c65473daefe420785f885e8ed9a06878", size = 218713 }, ] +[[package]] +name = "primp" +version = "0.10.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/3a/6f26826abf7225678fc5e8eb27d1df99bf14ba1563840d1424c2ffcc5934/primp-0.10.1.tar.gz", hash = "sha256:1fab598cb7d9c1e509747c0ac4352b75268849c6c67262cdb5a603d373ddb2bb", size = 85190 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/fa/cc4c434111a98bf8e4909e8d9a1bd54c5b8080bcb9cbf4839230819637ab/primp-0.10.1-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:b90305c5fdaa63a049a62842d2a5357ad53eed04665bc6bb22c75d253cbe9a2e", size = 3251987 }, + { url = "https://files.pythonhosted.org/packages/da/ae/2290ed2a023f84b419a95247bb8bca3a9bda4d62099c774711df1a44b4f0/primp-0.10.1-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:97b7c216b3382a7cee55ab98622cd1ad364de9684be7a0607335705456ae24e1", size = 3005142 }, + { url = "https://files.pythonhosted.org/packages/6d/05/d3d9fe8d0e74448d1c5ba1909989683507cb94b895cfb63a3e200a83eff7/primp-0.10.1-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1192bfeb6e6ddd9ce52e327138a1489b8fe3828250483da41728c0c96316f40", size = 3345879 }, + { url = "https://files.pythonhosted.org/packages/0a/a6/a693893fb3a130194eb01c5b911f2e20f28c4b269200c76509494bcc7496/primp-0.10.1-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:6fe2a03c8d140b1077aabf4840b65978a3dbe1dbfbad240435c640e55e14d297", size = 3315795 }, + { url = "https://files.pythonhosted.org/packages/c7/1e/1176eab646b31ba5c085c764346b1ba73f87099d9f7ff308c4385bab673b/primp-0.10.1-cp38-abi3-manylinux_2_34_armv7l.whl", hash = "sha256:3d2aa3a82ca4a72d13817bbd5d148f308f431d27207882ab4c3453cdd063ad9d", size = 3096656 }, + { url = "https://files.pythonhosted.org/packages/85/43/a2fdfff83c69a6a22f28f6673fc246df59dee49c285dccdf248cc97befd8/primp-0.10.1-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a7cf7fe0652538d83519feb302e23d36108149369a6a2d59bbcb8bcdc1768fb1", size = 3476038 }, + { url = "https://files.pythonhosted.org/packages/63/59/da47d1d1507a8a86757bca2733a9e554039b345f1017d9ec3a82dd984078/primp-0.10.1-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:161d400d4786734377b64c3bc799a9055bf7537521647ca3ba80b8341e487bfe", size = 3675762 }, + { url = "https://files.pythonhosted.org/packages/f0/19/74fb024f85c71b3a45f2b88e859b12beea2db4a687ea2a93f2dedb17273d/primp-0.10.1-cp38-abi3-win_amd64.whl", hash = "sha256:b4b11310f7723d858ff810e7c056c87bdec8b9867f804972ae59153bc387ff2c", size = 3207609 }, +] + [[package]] name = "prompt-toolkit" version = "3.0.48" @@ -3435,8 +3452,8 @@ dependencies = [ { name = "async-timeout", version = "4.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "async-timeout", version = "5.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "beautifulsoup4" }, + { name = "duckduckgo-search" }, { name = "free-proxy" }, - { name = "googlesearch-python" }, { name = "html2text" }, { name = "jsonschema" }, { name = "langchain" }, @@ -3492,9 +3509,9 @@ requires-dist = [ { name = "async-timeout", specifier = ">=4.0.3" }, { name = "beautifulsoup4", specifier = ">=4.12.3" }, { name = "burr", extras = ["start"], marker = "extra == 'burr'", specifier = "==0.22.1" }, + { name = "duckduckgo-search", specifier = ">=7.2.1" }, { name = "free-proxy", specifier = ">=1.1.1" }, { name = "furo", marker = "extra == 'docs'", specifier = "==2024.5.6" }, - { name = "googlesearch-python", specifier = ">=1.2.5" }, { name = "html2text", specifier = ">=2024.2.26" }, { name = "ipywidgets", marker = "extra == 'ocr'", specifier = ">=8.1.0" }, { name = "jsonschema", specifier = ">=4.23.0" },