N_NODE=4

N_GPU_PER_NODE=8

CONFIG="configs/xxx_train_config.json"

export OMP_NUM_THREADS=4

export TOKENIZERS_PARALLELISM=False

TODAY=$(date +%Y-%m%d-%H%M)

accelerate launch \

--num_machines$N_NODE \

--num_processes$(($N_NODE*$N_GPU_PER_NODE)) \

--use_deepspeed \

--deepspeed_multinode_launcher'standard' \

--zero_stage 2 \

--offload_optimizer_device'cpu' \

--offload_param_device'none' \

--gradient_accumulation_steps 1 \

--gradient_clipping 1.0 \

--zero3_init_flagfalse \

--zero3_save_16bit_modelfalse \

--main_training_function'main' \

--mixed_precision'bf16' \

--dynamo_backend'no' \

--same_network \

--machine_rank$RANK \

--main_process_ip$MASTER_ADDR \

--main_process_port$MASTER_PORT \

--rdzv_backend'static' \

pefts/mft_accelerate.py --train_config"$CONFIG" --distributed_type"deepspeed"