ML-Purdue
/
aqacf-ml-main


  
1

	
2

	
3

	
4

	
5

	
6

	
7

	
8

	
9

	
10

	
11

	
12

	
13

	
14

	
15

	
16

	
17

	
18

	
19

	
20

	
21

	
22

	
23

	
24

	
25

	
26

	
27

	
28

	
29

	
30

	
31

	
32

	
33

	
34

	
35

	
36

	
            #!/usr/bin/bash
#SBATCH --job-name=epa_ddp
#SBATCH --nodes=1
#SBATCH --cpus-per-task=8
#SBATCH --mem=24G
#SBATCH --account=standby
#SBATCH --gres=gpu:1
#SBATCH --time=4:00:00
#SBATCH --output=epa_ddp_%j.out
#SBATCH --error=epa_ddp_%j.err

module load cuda/12.1.1 cudnn/9.2.0.82-12 conda
#conda activate /scratch/gilbreth/rai53/fire

#export MLFLOW_TRACKING_USERNAME=$MLFLOW_USERNAME
#export MLFLOW_TRACKING_PASSWORD=$MLFLOW_TOKEN
export NCCL_NET_MERGE_LEVEL=LOC
export NCCL_NET_MERGE_LEVEL=LOC
export NCCL_SOCKET_IFNAME=^lo,docker
export TORCH_NCCL_BLOCKING_WAIT=1
export NCCL_TIMEOUT_SEC=1200  # 20 minutes timeout
export NCCL_IB_DISABLE=0      # Enable InfiniBand if available
export NCCL_P2P_DISABLE=0     # Enable P2P if available
export NCCL_SHM_DISABLE=0     # Enable shared memory

export OMP_NUM_THREADS=8
export NCCL_DEBUG=INFO
export NCCL_DEBUG_SUBSYS=ALL


torchrun --standalone --nnodes=1 --nproc_per_node=1 \
    epa_seq2seq.py \
    ../config/training_config_v1.json \
    ../data/group0.json \
    ../data/in_situ_prediction_dataset_full \
    ../cpts