DeezerTemplate.py

Go to the documentation of this file.

import re
import awswrangler as wr
from pandas import to_datetime
 
class DeezerTemplate:
    def preprocessing(self, filename, features, session, rel_col):
        """
        args:
        filename: -str- s3 full path of filename
        features: -dict- which contains delimiter, skip_rows and encoding
        session: -class- s3 connexion 
        """
        delimiter = features["delimiter"]
        skip_rows = features["skip_rows"]
        encoding = features["encoding"]
        df = wr.s3.read_csv(filename, sep=delimiter, skiprows=skip_rows, encoding=encoding, dtype={rel_col:str}, low_memory=False, boto3_session=session)
        df["Start Report"] = to_datetime(df["Start Report"], format="%d-%m-%Y")
        # df = read_csv(filename, sep=delimiter, skiprows=skip_rows, encoding=encoding, low_memory=False)#, keep_default_na=False, low_memory=False)
        return df
 
    def date(self, filename):
        """Sets date column given the filename (it contains the date)
           For example Deezer_GYROstreamMERLIN_20220201_20220228.txt
        Args:
            filename (str): current filename
        Returns: date_str (str)
        """
        # "Deezer_GYROstreamMERLIN_20220201_20220228.txt"
        date = re.findall(r'_\d{8}_', filename)[0]
        date = date.replace("_", "")
        date_str = date[:4]+"-"+date[4:6]+"-"+date[6:]
        # df["date_from_file"] = date_str
        return date_str