FormatManager.py

Go to the documentation of this file.

# from pandas import ExcelFile, DataFrame
from Validate import validate_data
from ErrorHandler import ErrorHandler
from TemplateManager import TemplateManager 
 
class csvManager:
    def df_csv(*args):
        """Executes full procedure
 
        Args:
            event (dict): is a dictionary with all client and sales information
            type_format (str): format name 
            file (dict): features required by file
            session (boto3 obj): AWS client connection
            write_DF (class): mapper columns method
        Returns: data_otto (pandas dataframe)
                 status (str)
        """
        event = args[0]
        type_format = args[1]
        file = args[2]
        session = args[3]
        write_DF = args[4]
        try:
            filename = file["file"]
            s3_path_file = "s3://{bucket}/{key}/{filename}".format(bucket=event["bucket"][0], key=event["path"][0], filename=filename)
            columns_csv = event["format"][type_format]["columns"]
            data = TemplateManager.preprocessing(dir_file=s3_path_file, type_format=type_format, file_features=file, session=session, release_col=columns_csv[0]["value"])
            print("ORIGINAL DATA \n {}".format(data.head()))
            low = [x.lower() for x in data.columns.to_list()]
            data.columns = low
            # data = validate_data().validate_columns(data, columns_csv)
            data_otto = write_DF.ftypes(data, columns_csv, type_format, filename)
            data_otto["file"] = filename
            status = "Ok"
            return data_otto, status
        except Exception as status:
            return "File could not be loaded", status
        
# class xlsManager:
#     def df_excel(event, s3_object, formats, file, write_DF): #s3_object=s3_object, formats=str, file=dict, write_DF=class
#         filename = file["file"]
#         sheet = file["sheet"]
#         delimiter = event["format"][formats]["delimiter"]
#         columns_xls = event["format"][formats]["columns"]
#         skip_rows = file["skip_rows"]
#         xls = ExcelFile(s3_object['Body'].read())
#         name = filename.split(".xls")
#         sheet_empty = 0
#         data = TemplateManager.preprocessing(xls, formats, sheet, delimiter, skip_rows)
#         print("ORIGINAL DATA \n {}".format(data.head()))
#         low = [x.lower() for x in data.columns.to_list()]
#         data.columns = low
#         data = validate_data().validate_columns(data, columns_xls)
#         # data = read_excel(xls, sheet_name=sheet, keep_default_na=False)
#         if len(data) == 0:
#             sheet_empty += 1
#             # print("sheet {} is empty".format(sheet))
#             return 0
#         else:
#             # write_DF.format_type(data, formats, filename)
#             data_otto = write_DF.ftypes(data, columns_xls, formats, filename)
#             filename = filename+"-"+sheet.replace(" ", "_")
#             data_otto["filename"] = filename
#             xls.close()
#             return data_otto, filename