UploadFilesS3.py

Go to the documentation of this file.

import os, sys, traceback
import gzip
import shutil
import zipfile
import boto3
import uuid
import subprocess
# from tkinter import messagebox
from datetime import date, datetime
from dotenv import load_dotenv
 
file_path = os.path.dirname(__file__)
dotenv_path = os.path.join(file_path, ".env")
load_dotenv(dotenv_path)
 
 
ACCESS_ID, ACCESS_KEY = os.environ.get("AWS_KEY_ID"), os.environ.get("AWS_KEY_SECRET")
 
 
s3_client = boto3.client("s3", aws_access_key_id=ACCESS_ID, aws_secret_access_key=ACCESS_KEY)
 
 
date_for_path = date.today()
date_for_path = datetime.strftime(date_for_path, "%Y-%m-%d")
# final_files = None
mode = None
global bucket, s3_path
 
def folder_or_file(local_path, files_list):
    """
    Moves all files (in fub folders) to root path
 
    Args:
        local_path (str): Root path where is saved files
        files_list (list): list where each file is allocated to be moved posteriorly
    Returns : Nothing.
    """
    for path, subdirs, files in os.walk(local_path):
        for name in files:
            files_list.append(os.path.join(path, name))
    for file in files_list:
        try:
            shutil.move(file, local_path)
        except:
            continue
 
def upload_files_s3(bucket, s3_path, local_path, file, mode):
    """
    Uploads all files to s3 path defined previously
 
    Args:
        local_path (str): Root path where is saved files
        files (str): File to upload
    Returns : Nothing.
    """
    if "manual" in mode:
        full_path = local_path+file
        s3_client.upload_file(full_path, bucket, s3_path+file)
        return file
    else:
        random_uuid = uuid.uuid4()
        random_text = str(random_uuid)[:8]
        file_id = random_text+"_"+file
        full_path = local_path+file
        s3_client.upload_file(full_path, bucket, s3_path+file_id)
        return file_id
 
def transform_xls_file(local_path, file):
    """
    Converts xls file (it contains fx rates for apple) to csv file
 
    Args:
        local_path (str): Root path where is saved files
        files (str): File to convert
    Returns : Nothing.
    """
    output_file = file.replace(" ", "_")
    output_file = os.path.join(local_path, output_file)
    ext_file = os.path.splitext(output_file)[1]
    output_file = output_file.replace(ext_file, ".csv")
    input_file = os.path.join(local_path, file)
    if os.name == 'nt':
        #ssc_path = file_path+r"\gnumeric\1.12.17\bin\ssconvert.exe" # testing
        ssc_path = file_path+r"\gnumeric\bin\ssconvert.exe" # app
        command = [ssc_path, input_file, output_file]
        subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
    elif os.name == 'posix':
        ssc_path = "gnumeric/bin/ssconvert" #for local testing
        # ssc_path = os.path.join(file_path, "gnumeric/bin/ssconvert") #for app
        # subprocess.call([ssc_path, input_file, output_file])
        subprocess.call(["ssconvert", input_file, output_file])
    # file_o = file.replace(" ", "_")
    # file_o = file_o.replace(".xlsx", ".csv")
    # subprocess.call(["ssconvert",file,local_path+file_o],cwd=local_path)
 
def extract_file(bucket, s3_path, local_path, file, compression, final_files):
    # global final_files
    """
    Extract files from a compressed file, delete blank spaces  
 
    Args:
        local_path (str): Root path where is saved files
        files (str): File to decompress
        compression (str): Type of compression (gz, zip)
    Returns : Nothing.
    """
    if compression == ".zip":
        file_replaced = file.replace(" ", "_")
        z = zipfile.ZipFile(local_path+file)
        for f in z.filelist:
            f.filename = f.filename.replace(" ", "_")
            z.extract(f, local_path)
        file_upload = file_replaced.replace(".zip", "")
        print(file_upload)
        file_upload_id = upload_files_s3(bucket, s3_path, local_path, file_upload, mode)
        final_files.append(file_upload_id)
    elif compression == ".gz":
        file_o = file.replace(".gz", "")
        file_o = file_o.replace(" ", "_")
        print(file_o)
        with gzip.open(local_path+file, 'rb') as f_in:
            with open(local_path+file_o, 'wb') as f_out:
                shutil.copyfileobj(f_in, f_out)
        file_upload_id = upload_files_s3(bucket, s3_path, local_path, file_o, mode)
        final_files.append(file_upload_id)
 
def clean_files(event, local_path):
    global mode
    """
    Makes blank spaces deletion, extraction, conversion files and upload procedure for each file into the defined path
 
    Args:
        event (dict): Contains the client info to save files in s3
        local_path (str): Root path where is saved files
    Returns : Nothing.
    """
    mode = event["module"]
    # Bucket to upload files (it's fixed)
    bucket = event["bucket"] 
    # path to upload files, each moth it's a different path
    s3_path = event["path"]
    client_id = event["client_id"]
    if s3_path.endswith("/"):
        pass
    else:
        s3_path = s3_path+"/"
    # s3_path = "{}/{}/".format(client_id, date_for_path)
    #Create path if it doesn't exist
    response_in = s3_client.list_objects(Bucket = bucket, Prefix=s3_path)
    if "Contents" not in response_in:
            s3_client.put_object(Bucket=bucket, Key=(s3_path))
    csv_files = []
    folder_or_file(local_path, csv_files)
    files_to_upload = os.listdir(local_path)
    files_to_upload.sort()
    for file in files_to_upload:
        try:
            file_replaced = file.replace(" ", "_")
            os.rename(local_path+file, local_path+file_replaced)
            extensions = [".csv", ".xls", ".xlsx", ".txt", ".tsv"]
            ext = os.path.splitext(file)[1]
            if ext == ".zip" or ext == ".gz":
                extract_file(bucket, s3_path, local_path, file_replaced, ext, event["final_files"])
                pass
            if ext in extensions:
                if "Exchange" in file_replaced:
                    transform_xls_file(local_path,file_replaced)
                    continue
                if (".csv" in file_replaced) and ("ZQGRO_" in file_replaced) and ("_Summary_Statement" in file_replaced):
                    transform_xls_file(local_path, file_replaced)
                    # subprocess.call(["ssconvert",file_replaced,local_path+file_replaced],cwd=local_path)
                elif ("Napster" in file_replaced) or ("revshare" in file_replaced) or ("Audiomack" in file_replaced) or ("WW_MN" in file_replaced) or ("KepachMusictas" in file_replaced):
                    transform_xls_file(local_path, file_replaced)
                    # subprocess.call(["ssconvert",file_replaced,local_path+file_replaced],cwd=local_path)
                elif ext == ".xls" or ext == ".xlsx":
                    file_replaced_1 = file_replaced.replace(".xlsx", ".csv")
                    transform_xls_file(local_path, file_replaced)
                    # subprocess.call(["ssconvert",file_replaced,local_path+file_replaced_1],cwd=local_path) # path to your file must contain the xlsx and csv files
                    file_replaced = file_replaced_1
                file_upload_id = upload_files_s3(bucket, s3_path, local_path, file_replaced, mode)
                print(file_upload_id)
                event["final_files"].append(file_upload_id)
        except:
            print("{} could not be uploaded".format(file_replaced))
            print(sys.exc_info()[2])
            print(traceback.format_exc())
            continue
    return bucket+"/"+s3_path, event["final_files"]