22_graph_test_2/perfect.dvc.yaml

stages:
  etl_online_retail_ii:
    # convert xlsx file with two tabs into single .csv file
    # clean data (missing values if any in the target columns)
    cmd: python3 src/etl_online_retail_ii.py data/online_retail_II.xlsx
    deps:
      - data/raw/online_retail_II.xlsx
      - src/etl_online_retail_ii.py
      - src/tools/etl_tools.py
    outs:
      # output file contains columns: order_id, product_id, product_name
      - data/input/online_retail_transformed.csv
  etl_instacart:
    # blend multiple files into a single .csv file
    # clean data (missing values if any in the target columns)
    cmd: python3 src/etl_instacart.py data/order_products__prior.csv data/products.csv
    deps:
      - data/raw/order_products__prior.csv
      - data/raw/products.csv
      - src/etl_instacart.py
      - src/tools/etl_tools.py
    outs:
      # output file contains columns: order_id, product_id, product_name
      - data/input/instacart_transformed.csv
  product_name_normalization:
    # for every given file clean normalize products names
    # remove non-alphanumerics, to lower case, remove duplicating/trailing/leading whitespaces
    cmd: python3 src/names_cleaner.py data/input/
    deps:
      - data/input/
      - src/names_cleaner.py
      - src/tools/preprocess_tools.py
    outs:
      - data/input_names_normalized/
  drop_products_by_name:
    # remove products having the same name but different product_ids)
    # drop products with empty names
    cmd: python3 src/products_removers.py data/input_names_normalized/
    deps:
      - src/products_removers.py
      - src/tools/preprocess_tools.py
      - data/input_names_normalized/
    outs:
      - data/input_preprocessed/
  build_products_registry:
    # for each store data generate csv file with two columns: product_id, product_name
    # files contain unique products only
    cmd: python3 src/generate_products_registry.py data/input_preprocessed/ data/products_registry/
    deps:
      - data/input_preprocessed/
      - src/generate_products_registry.py
    outs:
      - data/products_registry/
  split_into_subsets_all_data:
    # split both datasets into the train and test parts
    # the first argument is the fraction of train data
    cmd: python3 src/train_test_split.py 0.8 data/input_preprocessed/
    deps:
      - data/input_preprocessed/
      - src/tools/preprocess_tools.py
      - src/train_test_split.py
    outs:
      # files are renamed according to the pattern: {original_file_name}_{test|train}.csv
      - data/train_data/
      - data/test_data/
  preprocess_test_add_dummy_orders:
    # recommendation should be given based on the basket content
    # test/validation orders must be augmented:
    # for each order create dummy orders consisting of random subsets of goods from the original one
    # thus if original contains 5 goods, we should have 4 dummy orders: with 1 (random) product, with 2 (previous one, + 1 random), etc.
    # dataframe should have columns: product_id, order_id, dummy_order_id, target_product
    # original order should be dropped.
    # files with dummy orders must have corresponding suffix
    # json with splits should be extended with validation_dummy and test_dummy key-value pairs.
    cmd: python3 src/prepare_dummy_orders.py data/test_data/ data/test_final/
    deps:
      - src/prepare_dummy_orders.py
      - src/tools/preprocess_tools.py
      - data/test_data/
    outs:
      - data/test_final/
  preprocess_train_drop_by_size:
    # from TRAIN data drop records which do not satisfy min and max size of orders
    # original files are just copied to the destination directory
    cmd: python3 src/drop_by_order_size.py data/train_data/ data/train_final/
    deps:
      - src/preprocess_tools.py
      - data/train_data/
    params:
      - order_size_train.min
      - order_size_train.max
    outs:
      - data/train_final/
      - data/cross_val_limit_order_size/
  input_data_for_random_model:
    # preprocess train data for random model: for each train file prepare plain txt file with list of order ids
    cmd: python3 src/prepare_train_for_random_model.py data/input_preprocessed/ data/train_for_random/
    deps:
      - src/tools/preprocess_tools.py
      - src/prepare_train_for_random_model.py
      - data/input_preprocessed/
    outs:
      - data/train_for_random/
  input_data_for_embedding_model:
    # preprocess train data for embeddings model: for each train file prepare textual file in starspace format (use trainmode=0 for https://github.com/facebookresearch/StarSpace)
    # dataframe should have columns: basket_product_ids, basket_product_descriptions, label
    # each example is string concatenation of product names in the basket.
    # label is missing product_id. Model is trained to predict missing products.
    # thus if order contains two products we train model to predict one product using another (we have two combinations in total)
    # if an order contains 5 products, then we get 5 combinations where we have 4 orders (string concatenation) in the left and 1 one order (order id) in the right.
    cmd: python3 src/prepare_train_for_embedding_model.py data/train_final/ data/train_for_embeddings/
    deps:
      - src/tools/preprocess_tools.py
      - data/train_final/
    outs:
      - data/train_for_embeddings/
  train_random_model:
    # for each training file prepare a model.
    # models are saved as pkl files
    # completely random sampling, but model should not recommend items which are already in the basket
    # model should get list of items which are already in the basket and return N recommendations (recommendations.n_items)
    cmd: python3 src/trainer_random_model.py data/train_for_random/ data/random_models/
    deps:
      - data/train_for_random/
      - src/models/trainer_random_model.py
    params:
      - random_model.random_seed
      - recommendations.n_items
    outs:
      - data/random_models/
  train_basket_tfidf_perceptron:
    # for each training file prepare a model
    # models are saved as pkl files
    # Algorithm idea: 1) extract tf-idf features from textual description of items in the basket;
    # 2) train multilayer perceptron to predict the missing item
    # 3) use the last perceptron layer as embedding. compute embedding for all products.
    # model should return embedding for any given non-empty list product descriptions/names in english
    cmd: python3 src/trainer_basket_tfidf_perceptron_embedding.py data/train_for_embeddings/ data/embedding_models/basket_tfidf_perceptron/
    deps:
      - data/train_for_embeddings/
      - src/models/trainer_basket_tfidf_perceptron_embedding.py
    params:
      - basket_tfidf_perceptron_model.random_seed
      - basket_tfidf_perceptron_model.params_grid
      - recommendations.n_items
    outs:
      - data/embedding_models/basket_tfidf_perceptron/
  train_basket_sent2vec_perceptron:
    # for each training file prepare a model
    # models are saved as pkl files
    # Algorithm idea: 1) get embeddings from textual description of items in the basket using fasttext sentence embeddings;
    # 2) train multilayer perceptron to predict the missing item
    # 3) use the last perceptron layer as embedding. compute embedding for all products.
    # model should return embedding for any given non-empty list product descriptions/names in english
    cmd: python3 src/trainer_basket_sent2vec_perceptron_embedding.py data/train_for_embeddings/ data/embedding_models/basket_sent2vec_perceptron/
    deps:
      - data/train_for_embeddings/
      - src/models/trainer_basket_sent2vec_perceptron_embedding.py
    params:
      - basket_sent2vec_perceptron_model.random_seed
      - basket_sent2vec_perceptron_model.params_grid
      - recommendations.n_items
    outs:
      - data/embedding_models/basket_sent2vec_perceptron/
  train_basket_starspace:
    # for each training file prepare a model
    # models are saved as pkl files
    # Algorithm idea: use trainmode=0 for https://github.com/facebookresearch/StarSpace
    # each example is string concatenation of product names in the basket.
    # label is missing product_id. Model is trained to predict missing products.
    # model should return embedding for any given non-empty list product descriptions/names in english
    cmd: python3 src/trainer_basket_starspace_embedding.py  data/train_for_embeddings/ data/embedding_models/basket_starspace/
    deps:
      - data/train_for_embeddings/
      - src/models/trainer_basket_starspace_embedding.py
    params:
      - basket_starspace_model.random_seed
      - basket_starspace_model.params_grid
    outs:
      - data/embedding_models/basket_starspace/
  compute_embeddings:
    # use every available model to compute embeddings of the existing (training) items
    # for every product add vector of embeddings
    cmd: python3 src/compute_embeddings.py data/embedding_models/ data/products_registry/ data/computed_embeddings/
    deps:
      - data/embedding_models/
      - data/products_registry/
    outs:
      - data/computed_embeddings/
  index_embeddings:
    # index embedding for each given file with computed embeddings
    # build kdtree searcher for each given embeddings file
    cmd: python3 src/embeddings_indexer.py data/computed_embeddings/ data/indexed_embeddings/
    deps:
      - src/embeddings_indexer.py
      - data/computed_embeddings/
      - embedding_indexer_performance_testing.md
    outs:
      - data/indexed_embeddings/
  recommend_products_basket_embedding_by_model:
    # given recommendation using indexed products embeddings, embeddings extractor
    cmd: python3 src/recommender_basket_embedding_by_model.py data/indexed_embeddings/ data/embedding_models/ data/test_final/
    deps:
      - src/recommender_basket_embedding_by_model.py
      - data/indexed_embeddings/
      - data/embedding_models/
      - data/test_final/
      - data/products_registry/
    params:
      - recommendations.n_items
    outs:
      - data/recommendations/basket_embeddings_by_model/
  recommend_products_basket_tfidf_classifier:
    # for each training file prepare a model
    # there is no reason to save the models.
    # Use the following classifiers: random forest, logistic regression
    # Algorithm idea: 1) extract tf-idf features from textual description of items in the basket;
    # For each algortihm vary it parameters (no more than 10-20 parameters combinations for each algo),
    # fit it to the training data and give recommendation (top N products by sorted probability descending) for the testing data.
    # save given recommendations into json.
    cmd: python3 src/recommender_basket_basket_tfidf_classifier.py data/train_for_embeddings/ data/test_final/
    deps:
      - src/recommender_basket_basket_tfidf_classifier.py
      - data/train_for_embeddings/
      - data/test_final/
      - data/products_registry/
    params:
      - recommendations.n_items
    outs:
      - data/metrics/basket_tfidf_classifier_accuracy.csv
      - data/recommendations/basket_tfidf_classifier_by_model/
  recommend_products_basket_sen2vec_classifier:
    # for each training file prepare a model
    # there is no reson to save the models.
    # Use the following classifiers: random forest, logistic regression
    # Algorithm idea: 1) extract sent2vec (use spacy large model) features from textual description of items in the basket;
    # For each algortihm vary it parameters (no more than 10-20 parameters combinations for each algo),
    # fit it to the training data and give recommendation (top N products by sorted probability descending) for the testing data.
    # save given recommendations into json.
    cmd: python3 src/recommender_basket_basket_sen2vec_classifier.py data/train_for_embeddings/ data/test_final/
    deps:
      - src/recommender_basket_basket_sen2vec_classifier.py
      - data/train_for_embeddings/
      - data/test_final/
      - data/products_registry/
    params:
      - recommendations.n_items
    outs:
      - data/metrics/basket_sen2vec_classifier_accuracy.csv
      - data/recommendations/basket_sen2vec_classifier_by_model/
  recommend_products_basket_embedding_avg:
    # given recommendation using indexed products embeddings, and embedding of individual products.
    # basket embedding is computed by averaging items embeddings
    cmd: python3 src/recommender_basket_embedding_avg.py data/indexed_embeddings/ data/computed_embeddings/ data/test_final/
    deps:
      - src/recommender_basket_embedding_avg.py
      - data/indexed_embeddings/
      - data/computed_embeddings/
      - data/test_final/
      - data/products_registry/
    params:
      - recommendations.n_items
    outs:
      - data/recommendations/basket_embeddings_avg/
  recommend_products_random_model:
    # given recommendation using random sampling from the list of known products
    cmd: python3 src/recommender_random_model.py data/random_models/ data/train_for_random/ data/test_final/
    deps:
      - src/recommender_random_model.py
      - data/test_final/
      - data/random_models/
    params:
      - recommendations.n_items
    outs:
      - data/recommendations/random_model/
  metric_models_comparison:
    # compute MAP7 and MAP10 metrics for each model at order_id level instead of customer level
    cmd: python3 src/estimate_recommendations.py data/recommendations/
    deps:
      - src/estimate_recommendations.py
      - src/tools/metrics.py
      - data/recommendations/
    metrics:
      - summary.json
### embeddings indexer preformance testing
  prepare_embeddings_sets:
    # embeddings indexer preformance testing (eipt)
    # use random model to compute embeddings of the training items
    # for set of vector lengths
    cmd: python3 src/prepare_embeddings_sets.py data/computed_embeddings_sets/
    deps:
      - src/prepare_embeddings_sets.py
    params:
      - embeddings.vector_size
    outs:
      - data/computed_embeddings_sets/train/
      - data/computed_embeddings_sets/test/
  prepare_eipt_params:
    # embeddings indexer preformance testing (eipt)
    # use paarams.yaml to create grid of parameters for indexing testing for each model
    cmd: python3 src/prepare_eipt_params.py data/computed_embeddings_sets/
    deps:
      - params.yaml
      - src/prepare_eipt_params.py
    params:
      - params.model
    outs:
      - data/eipt/testing_params/
# index building by different models
  annoy_eipt_index_embeddings:
    # embeddings indexer preformance testing (eipt)
    # index embedding for each given file with computed embeddings
    # build annoy searcher for each given embeddings file
    # calculates performance metrics for annoy model:
    # index_building_time, model_save_time(?), fittet_object_size(?)
    cmd: python3 src/eipt_annoy_embeddings_indexer.py
    deps:
      - src/annoy_embeddings_indexer/
      - src/eipt_annoy_embeddings_indexer.py
      - data/computed_embeddings_sets/train/
      - data/eipt/testing_params/annoy_eipt_params.csv
    params:
      - params.model.annoy
      - params.metrics.train
    outs:
      - data/eipt/indexed_embeddings/annoy/
      - data/eipt/indexer_models/annoy/
      - data/eipt/metrics/by_model/eipt_annoy_training_metrics.csv
  kdtree_eipt_index_embeddings:
    # embeddings indexer preformance testing (eipt)
    # index embedding for each given file with computed embeddings
    # build kdtree searcher for each given embeddings file
    # calculates performance metrics for kdtree model:
    # index_building_time, model_save_time, fittet_object_size
    cmd: python3 src/eipt_kdtree_embeddings_indexer.py
    deps:
      - src/kdtree_embeddings_indexer/
      - src/eipt_kdtree_embeddings_indexer.py
      - data/computed_embeddings_sets/train/
      - data/eipt/testing_params/kdtree_eipt_params.csv
    params:
      - params.model.kdtree
      - params.metrics.train
    outs:
      - data/eipt/indexed_embeddings/kdtree/
      - data/eipt/indexer_models/kdtree/
      - data/eipt/metrics/by_model/eipt_kdtree_training_metrics.csv
  faiss_eipt_index_embeddings:
    # embeddings indexer preformance testing (eipt)
    # index embedding for each given file with computed embeddings
    # build faiss searcher for each given embeddings file
    # calculates performance metrics for faiss model:
    # index_building_time, model_save_time(?), fittet_object_size(?)
    cmd: python3 src/eipt_faiss_embeddings_indexer.py
    deps:
      - src/faiss_embeddings_indexer/
      - src/eipt_faiss_embeddings_indexer.py
      - data/computed_embeddings_sets/train/
      - data/eipt/testing_params/faiss_eipt_params.csv
    params:
      - params.model.faiss
    outs:
      - data/eipt/indexed_embeddings/faiss/
      - data/eipt/indexer_models/faiss/
      - data/eipt/metrics/by_model/eipt_faiss_training_metrics.csv
  postgre_eipt_index_embeddings:
    # embeddings indexer preformance testing (eipt)
    # index embedding for each given file with computed embeddings
    # build postgre DB and index for each given embeddings file
    # calculates performance metrics:
    # index_building_time, model_save_time(?), fittet_object_size(?)
    cmd: python3 src/eipt_postgre_embeddings_indexer.py
    deps:
      - src/postgre_embeddings_indexer/
      - src/eipt_postgre_embeddings_indexer.py
      - data/computed_embeddings_sets/train/
      - data/eipt/testing_params/postgre_eipt_params.csv
    params:
      - params.model.postgre
      - params.metrics.train
    outs:
      - data/eipt/indexed_embeddings/postgre/
      - data/eipt/indexer_models/postgre/
      - data/eipt/metrics/by_model/eipt_postgre_training_metrics.csv
# index search by different models
  annoy_eipt_index_search:
    # embeddings indexer preformance testing (eipt)
    # use annoy searcher for each given embeddings file
    # calculates performance metrics for annoy model:
    # vector_search_time, vector_search_accuracy, model_load_time(?)
    cmd: python3 src/annoy_eipt_index_searcher.py
    deps:
      - src/annoy_embeddings_indexer/
      - src/annoy_eipt_index_searcher.py
      - data/eipt/indexer_models/annoy/
      - data/computed_embeddings_sets/test/
    params:
      - params.metrics.test
    outs:
      - data/eipt/recommendations/annoy/
      - data/eipt/metrics/by_model/eipt_annoy_testing_metrics.csv
  kdtree_eipt_index_search:
    # embeddings indexer preformance testing (eipt)
    # use kdtree searcher for each given embeddings file
    # calculates performance metrics for kdtree model:
    # vector_search_time, vector_search_accuracy, model_load_time(?)
    cmd: python3 src/kdtree_eipt_index_searcher.py
    deps:
      - src/kdtree_embeddings_indexer/
      - src/kdtree_eipt_index_searcher.py
      - data/eipt/indexer_models/kdtree
      - data/computed_embeddings_sets/test/
    params:
      - params.metrics.test
    outs:
      - data/eipt/recommendations/kdtree/
      - data/eipt/metrics/by_model/eipt_kdtree_testing_metrics.csv
  faiss_eipt_index_search:
    # embeddings indexer preformance testing (eipt)
    # use faiss searcher for each given embeddings file
    # calculates performance metrics for faiss model:
    # vector_search_time, vector_search_accuracy, model_load_time(?)
    cmd: python3 src/faiss_eipt_index_searcher.py
    deps:
      - src/faiss_embeddings_indexer/
      - src/faiss_eipt_index_searcher.py
      - data/eipt/indexer_models/faiss/
      - data/computed_embeddings_sets/test/
    params:
      - params.metrics.test
    outs:
      - data/eipt/recommendations/faiss/
      - data/eipt/metrics/by_model/eipt_faiss_testing_metrics.csv
  postgre_eipt_index_search:
    # embeddings indexer preformance testing (eipt)
    # use faiss searcher for each given embeddings file
    # calculates performance metrics for postgre DB:
    # vector_search_time, vector_search_accuracy, model_load_time(?)
    cmd: python3 src/faiss_eipt_index_searcher.py
    deps:
      - src/postgre_embeddings_indexer/
      - src/postgre_eipt_index_searcher.py
      - data/eipt/indexer_models/postgre/
      - data/computed_embeddings_sets/test/
    params:
      - params.metrics.test
    outs:
      - data/eipt/recommendations/postgre/
      - data/eipt/metrics/by_model/eipt_postgre_testing_metrics.csv
  fullscan_eipt_index_search:
    # embeddings indexer preformance testing (eipt)
    # use fullscan searcher for each given embeddings file
    # calculates performance metrics for fullscan search:
    # vector_search_time, vector_search_accuracy
    cmd: python3 src/fullscan_eipt_index_searcher.py
    deps:
      - src/fullscan_eipt_index_searcher.py
      - data/eipt/indexed_embeddings/fullscan/
      - data/computed_embeddings_sets/
    params:
      - params.metrics.test
    outs:
      - data/eipt/recommendations/fullscan/
      - data/eipt/metrics/by_model/eipt_fullscan_testing_metrics.csv
# metrics_analysis
  eipt_analysis:
    # embeddings indexer preformance testing (eipt)
    # How does search/index time depends on vector dimensionality, selected distance metric, number of vectors?
    # How does timing/accuracy depends on the selected algorithm parameters if any?
    # analyse the resulting metrics of all models, build useful graphics
    cmd: python3 src/eipt_analysis.py
    deps:
      - src/eipt_analysis.py
      - data/eipt/metrics/by_model/
      - data/eipt/testing_params/
      - data/computed_embeddings_sets/
      - template.md
    outs:
      - data/eipt/metrics/graphs/
      - data/eipt/metrics/eipt_metrics_summary.csv
      - embedding_indexer_performance_testing.md