Multi-GPU Distributed Training

runcrate instances create --name multi-gpu --gpu A100 --gpu-count 4 --template ubuntu-train
runcrate instances status multi-gpu

runcrate ssh multi-gpu -- "nvidia-smi"
runcrate ssh multi-gpu -- "python -c 'import torch; print(f\"GPUs: {torch.cuda.device_count()}\")'"

runcrate cp ./train_ddp.py multi-gpu:/workspace/

runcrate ssh multi-gpu -- "cd /workspace && torchrun \
  --nproc_per_node=4 \
  --master_port=29500 \
  train_ddp.py \
  --batch-size 32 \
  --epochs 10 \
  --lr 1e-4"

runcrate ssh multi-gpu -- "pip install deepspeed"

runcrate cp ./ds_config.json multi-gpu:/workspace/
runcrate cp ./train_deepspeed.py multi-gpu:/workspace/

runcrate ssh multi-gpu -- "cd /workspace && deepspeed \
  --num_gpus=4 \
  train_deepspeed.py \
  --deepspeed_config ds_config.json"

runcrate ssh multi-gpu -- nvidia-smi
runcrate ssh multi-gpu -- "nvidia-smi topo -m"
runcrate ssh multi-gpu -- "tail -30 /workspace/output/training.log"

runcrate cp multi-gpu:/workspace/output/ ./training-output/

runcrate instances delete multi-gpu