get_assets/README.md

#Tips

• Stream a file from dagshub repo in order to use it localy

    from dagshub.streaming import DagsHubFilesystem
    import json
    fs = DagsHubFilesystem()
    file_path = 'assets/all.json'
    f = fs.open(file_path)
    data = json.load(f)
  

• Create a new experiment

    from dagshub import dagshub_logger, DAGsHubLogger
    # Option 1 - As a context manager:
    with dagshub_logger( metrics_path="logs/test_metrics.csv", hparams_path="logs/test_params.yml") as logger:
  
        # Metric logging: 
        logger.log_metrics(loss=3.14, step_num=1) 
        # OR: 
        logger.log_metrics({'loss': 3.14}, step_num=1) 
        # Hyperparameters logging: 
        logger.log_hyperparams(optimizer='sgd') 
        # OR: 
        logger.log_hyperparams({'optimizer': 'sgd'}) 
  
    # Option 2 - As a normal Python object: 
    logger = DAGsHubLogger(metrics_path="logs/test_metrics.csv", hparams_path="logs/test_params.yml") 
    logger.log_hyperparams(optimizer='sgd') 
    logger.log_metrics(loss=3.14, step_num=1) 
    # ... 
    logger.save() 
    logger.close()
  

• Create a pipeline

    dvc stage add -n featurization \
    -d code/featurization.py \
    -d data/test_data.csv \
    -d data/train_data.csv \
    -o data/norm_params.json \
    -o data/processed_test_data.npy \
    -o data/processed_train_data.npy \
    python3 code/featurization.py
  
  
    dvc stage add -n get_sample_pipe \
    -d scripts/get_train.py \
    -d source/labdoc_init_sample.jsonl\
    -o assets/all.spacy\
    python -m spacy run get_train