fix: changed proxy function

scrapegraphai/nodes/fetch_node.py

@@ -7,7 +7,7 @@ from langchain_community.document_loaders import AsyncHtmlLoader
 from langchain_core.documents import Document
 from .base_node import BaseNode
 from ..utils.remover import remover
-from ..utils.proxy_rotation import proxy_rotation
+from ..utils.proxy_generator import proxy_generator
 
 
 class FetchNode(BaseNode):
@@ -84,7 +84,7 @@ class FetchNode(BaseNode):
         else:
             if self.num_prox > 1:
                 loader = AsyncHtmlLoader(
-                    source, proxies=proxy_rotation(self.num_prox))
+                    source, proxies=proxy_generator(self.num_prox))
             else:
                 loader = AsyncHtmlLoader(source)
             document = loader.load()

scrapegraphai/utils/__init__.py

@@ -5,4 +5,4 @@ from .save_audio_from_bytes import save_audio_from_bytes
 from .convert_to_csv import convert_to_csv
 from .convert_to_json import convert_to_json
 from .prettify_exec_info import prettify_exec_info
-from .proxy_rotation import proxy_rotation
+from .proxy_generator import proxy_generator

scrapegraphai/utils/proxy_rotation.py

@@ -4,7 +4,7 @@ Module for rotating proxies
 from fp.fp import FreeProxy
 
 
-def proxy_rotation(num_ips: int):
+def proxy_generator(num_ips: int):
     """
     Rotates through a specified number of proxy IPs using the FreeProxy library.
 
@@ -15,7 +15,7 @@ def proxy_rotation(num_ips: int):
         dict: A dictionary containing the rotated proxy IPs, indexed by their position in rotation.
 
     Example:
-        >>> proxy_rotation(5)
+        >>> proxy_generator(5)
         {
             0: '192.168.1.1:8080',
             1: '103.10.63.135:8080',
@@ -24,9 +24,8 @@ def proxy_rotation(num_ips: int):
             4: '113.20.31.250:8080'
         }
     """
-    res = {}
+    res = []
 
     for i in range(0, num_ips):
-        res[i] = FreeProxy().get()
+        res.append(FreeProxy().get())
-
     return res