AmazonTemplate.py

Go to the documentation of this file.

import re
import awswrangler as wr
 
class AmazonTemplate:
    def preprocessing(self, filename, features, session, rel_col):
        """Loads file and do some fixes to fit it to our standard and make it able to be processed
 
        Args:
            filename (str): s3 full path of filename
            features (dict): contains delimiter, skip_rows and encoding required for current file
            session (boto3 obj): AWS client connection
            rel_col (str): column name used to identify upc/release_id
        Returns: df (pandas dataframe)
        """
        delimiter = features["delimiter"]
        skip_rows = features["skip_rows"]
        encoding = features["encoding"]
        df = wr.s3.read_csv(filename, sep=delimiter, skiprows=skip_rows, encoding=encoding, dtype={rel_col:str}, keep_default_na=False, low_memory=False, boto3_session=session)
        # df = read_csv(filename, sep=delimiter, skiprows=skip_rows, encoding=encoding, keep_default_na=False, low_memory=False)
        df.drop(df.index[-1], inplace=True)
        # df["Units"].fillna(0, inplace=True)
        return df
 
    def assetCurrency(self, df, filename):
        """Sets currency column given the filename (it contains the currency)
           For example ZQGRO_Monthly_ADS_Usage_202112_EU.txt
        Args:
            filename (str): current filename
            df (pandas dataframe): dataframe where changes will applied
        Returns: df (pandas dataframe)
        """
        currencies = {"AT": "EUR", "ES": "EUR", "FR": "EUR", "GB": "GBP", "IT": "EUR", "JP": "JPY", "EU": "EUR", "FE": "USD", "NA": "USD", "US": "USD", "IN": "USD", "DE": "EUR"}
        currency =  filename.split(".")
        currency = currency[0][-2:]
        df["currency_from_filename"] = currencies[currency]
        return df
 
 
    def date(self, filename):
        """Sets date column given the filename (it contains the date)
           For example ZQGRO_Monthly_ADS_Usage_202112_EU.txt
        Args:
            filename (str): current filename
        Returns: date_str (str)
        """
        # "ZQGRO_Monthly_ADS_Usage_202112_EU.txt"
        date = re.findall(r'_20\d{4}_', filename)[0]
        date = date.replace("_", "")
        date_str = date[:4]+"-"+date[4:]+"-01"
        # df["date_from_file"] = date_str
        return date_str