SecretlyCanadianTemplate.py

Go to the documentation of this file.

import re
import awswrangler as wr
 
class SecretlyCanadianTemplate:
    def preprocessing(self, filename, features, session, rel_col):
        """Loads file and do some fixes to fit it to our standard and make it able to be processed
 
        Args:
            filename (str): s3 full path of filename
            features (dict): contains delimiter, skip_rows and encoding required for current file
            session (boto3 obj): AWS client connection
            rel_col (str): column name used to identify upc/release_id
        Returns: df (pandas dataframe)
        """
        delimiter = features["delimiter"]
        skip_rows = features["skip_rows"]
        encoding = features["encoding"]
        df = wr.s3.read_csv(filename, sep=delimiter, skiprows=skip_rows, encoding=encoding, dtype={rel_col:str}, low_memory=False, boto3_session=session)
        # df = read_csv(filename, sep=delimiter, skiprows=skip_rows, encoding=encoding, low_memory=False)
        df.replace("?", 0, inplace=True)
        return df
 
    def type(self, transaction_type):
        """Maps the sale type with the name specified by the client according their requirements
        Args:
            transaction_type (str): sale type shown in the original file
        Returns: (sale type in a single one letter)
        """
        if transaction_type == "Radio (Direct)":
            return "R"
        elif transaction_type == "Cloud":
            return "C"
        elif transaction_type == "Track":
            return "T"
        elif transaction_type == "Album":
            return "A"
        elif transaction_type == "Not a Sale":
            return "O"
        elif transaction_type == "Radio (Broadcast)" or transaction_type == "Radio (broadcasting statutory)":
            return "R_B"
        return "S"
 
    def assetType(self, df):
        """Applies type functions to current dataframe
        Args:
            df (pandas dataframe): dataframe where changes will applied
        Returns: df (pandas dataframe)
        """
        # Add a new column to insert type
        try:
            df["gettypesecretly2"] = df.apply(lambda Row: self.type(Row['transaction type']), axis=1)
            return df["gettypesecretly2"]
        except: 
            df["gettypesecretly"] = df.apply(lambda Row: self.type(Row['digalbumtrackorstream']), axis=1)
            return df["gettypesecretly"]
    
    def date_dig(self, filename):
        """Sets date column given the filename (it contains the date)
           For example ADM083122digpd.csv
           It's used for digital formats
        Args:
            filename (str): current filename
        Returns: date_str (str)
        """
        try:
            date = re.findall(r'\d{6}dig', filename)[0]
        except:
            date = re.findall(r'\d{6}NonNR', filename)[0]
        year = "20"+date[4:6]
        month = date[:2]
        return year+month
 
    def date_phy(self, filename):
        """Sets date column given the filename (it contains the date)
           For example ABT083122physpd.csv
           It's used for physycal formats
        Args:
            filename (str): current filename
        Returns: date_str (str)
        """
        date = re.findall(r'\d{6}phy', filename)[0]
        year = "20"+date[4:6]
        month = date[:2]
        return year+month