IKEA/code/preprocess.py

import os
import pandas as pd
import numpy as np
import us
from sklearn.metrics import fbeta_score
from sklearn.model_selection import train_test_split
def handle_time(X):
    X['utc_time'] = pd.to_datetime(X['utc_time'], unit='ms', utc=True)
    X['local_tz'] = X['user_state'].apply(lambda x: us.states.lookup(x).capital_tz)
    X['local_time'] = X.groupby('local_tz')['utc_time'].transform(lambda x: x.dt.tz_convert(x.name))
    X['utc_time'] = X['utc_time'].dt.tz_localize(None)
    X['hour'] = X['utc_time'].dt.hour
    X['day'] = X['utc_time'].dt.day
    X['month'] = X['utc_time'].dt.month
    return X

def preprocessing():
    X = pd.read_csv('data\\X_train.csv')
    y = pd.read_json('data\\y_train.json')
    X_test = pd.read_csv('data\\X_test.csv')
    for c in ['user_isp', 'device_maker', 'device_model']:
        X[c] = X[c].fillna('Unknown')
    features = handle_time(X).drop(['bidid', 'utc_time', 'marketplace',  'local_tz', 'local_time'], axis=1)
    target = y
    X_test = handle_time(X_test).drop(['bidid', 'utc_time', 'marketplace',  'local_tz', 'local_time'], axis=1)
    X_train, X_val, y_train, y_val = train_test_split(features, target)
    np.save('processed_data\\X_train_processed',X_train )
    np.save('processed_data\\X_val_processed', X_val)
    np.save('processed_data\\y_train_processed', y_train)
    np.save('processed_data\\y_val_processed', y_val)
    np.save('processed_data\\X_test', X_test)
if __name__ == "__main__":
    preprocessing()