CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-06-28 21:01:55 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: refactoring of export functions

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-10-21 10:30:21 +02:00
parent 3d6bbcdaa3
commit 0ea00c078f
3 changed files with 1 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
scrapegraphai/utils/__init__.py
View File

@ -1,8 +1,6 @@
""" """
__init__.py file for utils folder __init__.py file for utils folder
""" """
from .convert_to_csv import convert_to_csv
from .convert_to_json import convert_to_json
from .prettify_exec_info import prettify_exec_info from .prettify_exec_info import prettify_exec_info
from .proxy_rotation import Proxy, parse_or_search_proxy, search_proxy_servers from .proxy_rotation import Proxy, parse_or_search_proxy, search_proxy_servers
from .save_audio_from_bytes import save_audio_from_bytes from .save_audio_from_bytes import save_audio_from_bytes
@ -28,3 +26,4 @@ from .code_error_correction import (syntax_focused_code_generation,
validation_focused_code_generation, validation_focused_code_generation,
semantic_focused_code_generation) semantic_focused_code_generation)
from .save_code_to_file import save_code_to_file from .save_code_to_file import save_code_to_file
from .data_export import export_to_json, export_to_csv, export_to_xml

55
scrapegraphai/utils/convert_to_csv.py
View File

@ -1,55 +0,0 @@
"""
Module that given a filename and a position saves the file in the csv format
"""
import os
import sys
import pandas as pd
def convert_to_csv(data: dict, filename: str, position: str = None) -> None:
"""
Converts a dictionary to a CSV file and saves it at a specified location.
Args:
data (dict): The data to be converted into CSV format.
filename (str): The name of the output CSV file, without the '.csv' extension.
position (str, optional): The file path where the CSV should be saved.
Defaults to the directory of the caller script if not provided.
Returns:
None: The function does not return anything.
Raises:
FileNotFoundError: If the specified directory does not exist.
PermissionError: If write permissions are lacking for the directory.
TypeError: If `data` is not a dictionary.
Exception: For other issues that may arise during the creation or saving of the CSV file.
Example:
>>> convert_to_csv({'id': [1, 2], 'value': [10, 20]}, 'output', '/path/to/save')
Saves a CSV file named 'output.csv' at '/path/to/save'.
"""
if ".csv" in filename:
filename = filename.replace(".csv", "")
if position is None:
caller_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
position = caller_dir
try:
if not isinstance(data, dict):
raise TypeError("Input data must be a dictionary")
os.makedirs(position, exist_ok=True)
df = pd.DataFrame.from_dict(data, orient='index')
df.to_csv(os.path.join(position, f"{filename}.csv"), index=False)
except FileNotFoundError as fnfe:
raise FileNotFoundError(
f"The specified directory '{position}' does not exist.") from fnfe
except PermissionError as pe:
raise PermissionError(
f"You don't have permission to write to '{position}'.") from pe
except Exception as e:
raise e

52
scrapegraphai/utils/convert_to_json.py
View File

@ -1,52 +0,0 @@
"""
Convert to json module
"""
import json
import os
import sys
def convert_to_json(data: dict, filename: str, position: str = None) -> None:
"""
Converts a dictionary to a JSON file and saves it at a specified location.
Args:
data (dict): The data to be converted into JSON format.
filename (str): The name of the output JSON file, without the '.json' extension.
position (str, optional): The file path where the JSON file should be saved.
Defaults to the directory of the caller script if not provided.
Returns:
None: The function does not return anything.
Raises:
ValueError: If 'filename' contains '.json'.
FileNotFoundError: If the specified directory does not exist.
PermissionError: If write permissions are lacking for the directory.
Example:
>>> convert_to_json({'id': [1, 2], 'value': [10, 20]}, 'output', '/path/to/save')
Saves a JSON file named 'output.json' at '/path/to/save'.
Notes:
This function automatically ensures the directory exists before
attempting to write the file.
If the directory does not exist, it will attempt to create it.
"""
if ".json" in filename:
filename = filename.replace(".json", "") # Remove .json extension
if position is None:
caller_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
position = caller_dir
try:
os.makedirs(position, exist_ok=True)
with open(os.path.join(position, f"{filename}.json"), "w", encoding="utf-8") as f:
f.write(json.dumps(data))
except FileNotFoundError as fnfe:
raise FileNotFoundError(
f"The specified directory '{position}' does not exist.") from fnfe
except PermissionError as pe:
raise PermissionError(
f"You don't have permission to write to '{position}'.") from pe