TencentTemplate.py

Go to the documentation of this file.

import re
import awswrangler as wr
 
class TencentTemplate:
 
    def preprocessing(self, filename, features, session, rel_col):
        """Loads file and do some fixes to fit it to our standard and make it able to be processed
 
        Args:
            filename (str): s3 full path of filename
            features (dict): contains delimiter, skip_rows and encoding required for current file
            session (boto3 obj): AWS client connection
            rel_col (str): column name used to identify upc/release_id
        Returns: df (pandas dataframe)
        """
        delimiter = features["delimiter"]
        skip_rows = features["skip_rows"]
        encoding = features["encoding"]
        df = wr.s3.read_csv(filename, sep=delimiter, skiprows=skip_rows, encoding=encoding, dtype={rel_col:str}, low_memory=False, boto3_session=session)
        df[rel_col] = df.apply(lambda Row: self.fix_upc(Row[rel_col]), axis=1)
        return df
 
    def fix_upc(self, upc):
        """Deletes 'UPC-' from each upc in the original file
        Args:
            upc (str): upc shown in the original file
        Returns: upc (str)
        """
        upc = upc.replace("UPC-", "")
        return upc
 
    def fix_date(self, start_date):
        """Fits the date to YYYY-MM-DD format
        Args:
            start_date (str): sale date shown in the original file
        Returns: start_date_fix (str)
        """
        # print(start_date)
        start_date = str(start_date)
        start_date_fix = start_date.split("-")[0]
        return start_date_fix
 
    def date(self, df):
        """Applies type functions to current dataframe
        Args:
            df (pandas dataframe): dataframe where changes will applied
        Returns: df[iso_date] (pandas series)
        """
        # def assetType(self, df):
        # Add a new column to insert type
        df["iso_date"] = df.apply(lambda Row: self.fix_date(Row['start_date']), axis=1)
        return df["iso_date"]