GenUnmatched.py

Go to the documentation of this file.

import ssl
import os
import csv
import re
import sys, traceback
from datetime import datetime
from pymongo import MongoClient
from dotenv import load_dotenv
from botocore.exceptions import ClientError
from .SendEmail import send_unmatched_report
from .UploadReports import upload_files_s3
f_path = __file__
index = f_path.find("report_generation/")
f_path = f_path[:index]
sys.path.insert(1, f_path+'importer')
from MongoConnection import mongo_connection
load_dotenv()
 
 
mongo_conn = mongo_connection()
 
collection = mongo_conn.mongo_conn_sales()
 
 
ROOT_DIR = os.path.expanduser('~')
 
path = ROOT_DIR+"/Reports/"
if not os.path.exists(path):
    os.mkdir(path)
 
def get_month(date):
    """
    Generate the filename according to date in 'MonthYear' format.
 
    Args:
        date (str): accounting date, i.e processing files date
    Returns: full_date (str) 
    """
    m = re.findall(r'-\d{2}-', date)[0]
    m = m.replace("-", "")
    month_dic = {"01": "Jan", "02": "Feb", "03": "Mar", "04": "Apr", "05": "May", "06": "Jun", "07": "Jul", "08": "Aug", "09": "Sep", "10": "Oct", "11": "Nov", "12": "Dec"}
    month = month_dic[m]
    year = re.findall(r'20\d{2}-', date)[0]
    year = year.replace("-", "")
    return (month+year).lower()
 
def get_num_files(csv_file):
    """
    Counts the total files which contains not matched lines
 
    Args:
        csv_file (str): path to csv report
    Returns: num_files (int) 
    """
    files = set()
    with open(csv_file, newline='') as f:
        reader = csv.reader(f, delimiter=",")
        for row in reader:
            files.add(row[0])
            # print(row)
    num_files = len(files)-1
    return num_files
 
def generate_unmatched_csv(event):
    """
    Generates the unmatched lines in compressed csv format according to sales which matches with 'accounting_date' field
 
    Args:
        date (str): accounting_date, i.e processing files date
    Returns (str): Nothing.
    """
    date = event["date"]
    bucket = event["bucket"] 
    # path to upload files, each month it's a different path
    s3_path = "{}/unmatched_lines/{}/".format(event["path"], date)
    file = get_month(date)
    filename = f"unmatched_lines_{file}.csv"
    print(f"Creating {filename}")
    date = datetime.strptime(date, "%Y-%m-%d")
    csv_columns = ["file", "line", "release_id", "isrc_id", "release_title", "track_title", "artists", "label_id", "service_id"]
    lines = 0
    try:
        with open(path+filename, 'w', encoding='utf-8') as csvfile:
            writer = csv.DictWriter(csvfile, fieldnames=csv_columns)
            writer.writeheader()
            fields = {"_id":0, "file": 1, "line": 1, "release_id": 1, "isrc_id": 1, "release_title": 1, "track_title": 1, "artists": 1, "label_id": 1, "service_id": 1}
            template_sales = collection.find({"accounting_date": {"$in": [date]}, "client_id": event["client_id"], "clean": 0},fields)
            for post in template_sales:
                writer.writerow(post)
                lines+=1
 
        num_files = get_num_files(path+filename)
        print("Compressing file")
        if os.path.isfile(f"{path+filename}.gz"): # Delete file if it exists to avoid duplicate files
            os.system("rm {}.gz".format(path+filename))
        command = f"gzip {path+filename}"
        os.system(command) # Compress file to gzip
        filename = filename+".gz"
        print(f"{filename} is ready!")
        url_to_download, final_path = upload_files_s3(bucket, s3_path, path, filename)
        print(send_unmatched_report(event["client_email"], file, url_to_download, lines, num_files))
        os.system("rm -r {}".format(path+filename))
    except Exception as e:
        print(sys.exc_info()[2])
        print(traceback.format_exc())