aaa/code/processing.py

# Figure out way to get rid of sklearn deprecation warning

import numpy as np
import pandas as pd
import time
import json
import pickle
import sklearn as sk


def processing():
    """Module for data preprocessing and feature engineering for project
    predicting source star from radio astronomy data"""

    print("Reading in raw data...")
    s = pd.read_csv("./data/raw/stars.csv", low_memory=False)

    start_time = time.time()

    print("Removing unnecessary columns...")
    s = s[['DriftRate', 'Freq', 'SNR', 'Source']]

    print("Renaming columns to be friendlier...")
    s = s.rename(columns={"DriftRate": "Drift_rate(Hz/sec)", "Freq": "Frequency(MHz)",
                          "SNR": "Signal_to_noise_ratio"})

    print("Saving processed data...")
    s.to_csv("./data/processed/stars_proc.csv", )

    end_time = time.time()

    # Save processing time as a metric
    print("Saving processing time...\n")
    with open("./metrics/proc_time.json",'w') as f:
        json.dump({'Processing_time': end_time-start_time}, f)

    # Print processing time to terminal
    print("Processing time: " + str(end_time-start_time) + " seconds\n")
    print("Data featurization completed.\n")


if __name__ == '__main__':
    processing()