nl2ml/models_scripts/task2seq/piplines_creation.py

# -*- coding: utf-8 -*-
"""piplines_creation.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1k0KJokr1_jAQsAimwlt3W7zRVsuL2yDr
"""

import argparse
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

parser = argparse.ArgumentParser()
parser.add_argument("MURKUP_DATA_PATH", help="path to your input CSV", type=str)
parser.add_argument("GRAPH_ID2NAME", help="path to graph id2name data", type=str)
parser.add_argument("PIPELINES_PATH", help="path to save pipelines", type=str)
args = parser.parse_args()

DATASET_PATH = args.MURKUP_DATA_PATH
GRAPH_ID2NAME_PATH = args.GRAPH_ID2NAME
PIPELINES_PATH = args.PIPELINES_PATH

df = pd.read_csv(DATASET_PATH, sep = ';')
graph =  pd.read_csv(GRAPH_ID2NAME_PATH)

graph.rename(columns = {'id': 'graph_vertex_id'}, inplace = True)
df = pd.merge(df, graph)

#Смотрим, есть ли блоки кода, которые разные люди отнесли к разным вершинам графа:

duplicated_blocks = df[df["code_block_id"].duplicated(keep=False)]
conflicted_blocks = duplicated_blocks.groupby("code_block_id")["graph_vertex_id"].nunique() != 1
conflicted_blocks[conflicted_blocks ==True]

if len(conflicted_blocks) == 0:
  df = df.drop_duplicates('code_block_id')

#Составляем пайплайны для каждого ноутбука

def group_by_notebooks(data, vertex_col = 'graph_vertex_subclass', len_col = 'len', notebook_id_col = 'kaggle_id', 
                       competition_id_col = 'competition_id', competition_name_col = 'comp_name', code_block_col = 'code_block_id') -> pd.DataFrame:
    notebook_cols = [notebook_id_col, vertex_col, len_col, competition_id_col, competition_name_col, code_block_col]
    df = pd.DataFrame(columns=notebook_cols)
    for i, notebook_id in enumerate(data[notebook_id_col].unique()):
        if not pd.isnull(notebook_id):
            notebook = data[data[notebook_id_col] == notebook_id].reset_index(drop=True).sort_values('code_block_id')
            vertices_seq = " ".join(notebook[vertex_col])
            code_block_ids = " ".join(str(x) for x in notebook['code_block_id'])
            lenth = len(notebook['code_block_id'])
            competition_id = notebook[competition_id_col].unique()[0]
            competition_name = notebook[competition_name_col].unique()[0]
            row = [notebook_id, vertices_seq, lenth, competition_id, competition_name, code_block_ids]
            df.loc[i] = row
            print('notebook #{} done'.format(notebook_id))
    return df

piplines = group_by_notebooks(df)

piplines['len'].describe()

piplines.to_csv(PIPELINES_PATH)