diff --git a/scrapegraphai/nodes/fetch_node.py b/scrapegraphai/nodes/fetch_node.py index bdbabfb1..2564d44d 100644 --- a/scrapegraphai/nodes/fetch_node.py +++ b/scrapegraphai/nodes/fetch_node.py @@ -7,7 +7,7 @@ from langchain_community.document_loaders import AsyncHtmlLoader from langchain_core.documents import Document from .base_node import BaseNode from ..utils.remover import remover -from ..utils.proxy_rotation import proxy_rotation +from ..utils.proxy_generator import proxy_generator class FetchNode(BaseNode): @@ -84,7 +84,7 @@ class FetchNode(BaseNode): else: if self.num_prox > 1: loader = AsyncHtmlLoader( - source, proxies=proxy_rotation(self.num_prox)) + source, proxies=proxy_generator(self.num_prox)) else: loader = AsyncHtmlLoader(source) document = loader.load() diff --git a/scrapegraphai/utils/__init__.py b/scrapegraphai/utils/__init__.py index 4d339ed4..3fd1d884 100644 --- a/scrapegraphai/utils/__init__.py +++ b/scrapegraphai/utils/__init__.py @@ -5,4 +5,4 @@ from .save_audio_from_bytes import save_audio_from_bytes from .convert_to_csv import convert_to_csv from .convert_to_json import convert_to_json from .prettify_exec_info import prettify_exec_info -from .proxy_rotation import proxy_rotation +from .proxy_generator import proxy_generator diff --git a/scrapegraphai/utils/proxy_rotation.py b/scrapegraphai/utils/proxy_rotation.py index 62f9f1de..0019b421 100644 --- a/scrapegraphai/utils/proxy_rotation.py +++ b/scrapegraphai/utils/proxy_rotation.py @@ -4,7 +4,7 @@ Module for rotating proxies from fp.fp import FreeProxy -def proxy_rotation(num_ips: int): +def proxy_generator(num_ips: int): """ Rotates through a specified number of proxy IPs using the FreeProxy library. @@ -15,7 +15,7 @@ def proxy_rotation(num_ips: int): dict: A dictionary containing the rotated proxy IPs, indexed by their position in rotation. Example: - >>> proxy_rotation(5) + >>> proxy_generator(5) { 0: '192.168.1.1:8080', 1: '103.10.63.135:8080', @@ -24,9 +24,8 @@ def proxy_rotation(num_ips: int): 4: '113.20.31.250:8080' } """ - res = {} + res = [] for i in range(0, num_ips): - res[i] = FreeProxy().get() - + res.append(FreeProxy().get()) return res