multi_node_job.out

#!/bin/bash
#SBATCH --job-name=multi_node_job
#SBATCH --time=12:00:00
#SBATCH --account=com-304
#SBATCH --qos=com-304
#SBATCH --gres=gpu:2
#SBATCH --nodes=2
#SBATCH --ntasks-per-node=1
#SBATCH --cpus-per-task=4
#SBATCH --mem=64G
#SBATCH --output=multi_node_job.out
#SBATCH --error=multi_node_job.err

# === Accept arguments ===
CONFIG_FILE=$1       # First argument
WANDB_KEY=$2        # Second argument


# === Initialization ===
set -x
cat $0
export MASTER_PORT=25678
export MASTER_ADDR=$(hostname)
export WANDB_API_KEY=$WANDB_KEY
export NCCL_DEBUG=INFO

# === Run main script ===
srun bash -c "
  TORCHRUN_ARGS=\"--node-rank=\${SLURM_PROCID} \
     --master-addr=\${MASTER_ADDR} \
     --master-port=\${MASTER_PORT} \
     --nnodes=\${SLURM_NNODES} \
     --nproc-per-node=2\"

  echo \${SLURM_PROCID}
  echo \${TORCHRUN_ARGS}
  echo \${SLURMD_NODENAME}

  torchrun \${TORCHRUN_ARGS} run_training.py \
    --config $CONFIG_FILE
"

0
--node-rank=0 --master-addr=i44 --master-port=25678 --nnodes=2 --nproc-per-node=2
i44
1
--node-rank=1 --master-addr=i44 --master-port=25678 --nnodes=2 --nproc-per-node=2
ixl01
| distributed init (rank 0): env://, gpu 0
i44:1630820:1630820 [0] NCCL INFO Bootstrap : Using eno1:10.91.27.44<0>
i44:1630820:1630820 [0] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
i44:1630820:1630820 [0] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
i44:1630820:1630820 [0] NCCL INFO NET/Plugin: Using internal network plugin.
i44:1630820:1630820 [0] NCCL INFO cudaDriverVersion 12020
NCCL version 2.21.5+cuda12.4
i44:1630820:1630820 [0] NCCL INFO Comm config Blocking set to 1
| distributed init (rank 1): env://, gpu 1
i44:1630821:1630821 [1] NCCL INFO cudaDriverVersion 12020
i44:1630821:1630821 [1] NCCL INFO Bootstrap : Using eno1:10.91.27.44<0>
i44:1630821:1630821 [1] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
i44:1630821:1630821 [1] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
i44:1630821:1630821 [1] NCCL INFO NET/Plugin: Using internal network plugin.
i44:1630821:1630821 [1] NCCL INFO Comm config Blocking set to 1
i44:1630820:1630841 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [RO]; OOB eno1:10.91.27.44<0>
i44:1630820:1630841 [0] NCCL INFO Using non-device net plugin version 0
i44:1630820:1630841 [0] NCCL INFO Using network IB
i44:1630821:1630844 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [RO]; OOB eno1:10.91.27.44<0>
i44:1630821:1630844 [1] NCCL INFO Using non-device net plugin version 0
i44:1630821:1630844 [1] NCCL INFO Using network IB
| distributed init (rank 2): env://, gpu 0
| distributed init (rank 3): env://, gpu 1
ixl01:1589323:1589323 [0] NCCL INFO cudaDriverVersion 12020
ixl01:1589324:1589324 [1] NCCL INFO cudaDriverVersion 12020
ixl01:1589323:1589323 [0] NCCL INFO Bootstrap : Using eno5:10.91.28.1<0>
ixl01:1589324:1589324 [1] NCCL INFO Bootstrap : Using eno5:10.91.28.1<0>
ixl01:1589323:1589323 [0] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
ixl01:1589324:1589324 [1] NCCL INFO NET/Plugin: No plugin found (libnccl-net.so)
ixl01:1589323:1589323 [0] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
ixl01:1589323:1589323 [0] NCCL INFO NET/Plugin: Using internal network plugin.
ixl01:1589324:1589324 [1] NCCL INFO NET/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-net.so
ixl01:1589324:1589324 [1] NCCL INFO NET/Plugin: Using internal network plugin.
ixl01:1589323:1589323 [0] NCCL INFO Comm config Blocking set to 1
ixl01:1589324:1589324 [1] NCCL INFO Comm config Blocking set to 1
ixl01:1589324:1589346 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [RO]; OOB eno5:10.91.28.1<0>
ixl01:1589324:1589346 [1] NCCL INFO Using non-device net plugin version 0
ixl01:1589324:1589346 [1] NCCL INFO Using network IB
ixl01:1589323:1589347 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_1:1/IB [RO]; OOB eno5:10.91.28.1<0>
ixl01:1589323:1589347 [0] NCCL INFO Using non-device net plugin version 0
ixl01:1589323:1589347 [0] NCCL INFO Using network IB
i44:1630821:1630844 [1] NCCL INFO ncclCommInitRank comm 0x9fea810 rank 1 nranks 4 cudaDev 1 nvmlDev 1 busId d8000 commId 0xf74159e14e16d1a1 - Init START
ixl01:1589323:1589347 [0] NCCL INFO ncclCommInitRank comm 0x95b6710 rank 2 nranks 4 cudaDev 0 nvmlDev 0 busId b2000 commId 0xf74159e14e16d1a1 - Init START
ixl01:1589324:1589346 [1] NCCL INFO ncclCommInitRank comm 0x9bad760 rank 3 nranks 4 cudaDev 1 nvmlDev 1 busId b3000 commId 0xf74159e14e16d1a1 - Init START
i44:1630820:1630841 [0] NCCL INFO ncclCommInitRank comm 0x9365bc0 rank 0 nranks 4 cudaDev 0 nvmlDev 0 busId 86000 commId 0xf74159e14e16d1a1 - Init START
i44:1630821:1630844 [1] NCCL INFO Setting affinity for GPU 1 to 300000
i44:1630820:1630841 [0] NCCL INFO Setting affinity for GPU 0 to 300000
ixl01:1589323:1589347 [0] NCCL INFO Setting affinity for GPU 0 to f00000
ixl01:1589324:1589346 [1] NCCL INFO Setting affinity for GPU 1 to f00000
i44:1630821:1630844 [1] NCCL INFO comm 0x9fea810 rank 1 nRanks 4 nNodes 2 localRanks 2 localRank 1 MNNVL 0
i44:1630820:1630841 [0] NCCL INFO comm 0x9365bc0 rank 0 nRanks 4 nNodes 2 localRanks 2 localRank 0 MNNVL 0
i44:1630820:1630841 [0] NCCL INFO Channel 00/02 :    0   1   2   3
i44:1630820:1630841 [0] NCCL INFO Channel 01/02 :    0   1   2   3
i44:1630820:1630841 [0] NCCL INFO Trees [0] 1/2/-1->0->-1 [1] 1/-1/-1->0->2
i44:1630820:1630841 [0] NCCL INFO P2P Chunksize set to 131072
i44:1630821:1630844 [1] NCCL INFO Trees [0] -1/-1/-1->1->0 [1] -1/-1/-1->1->0
ixl01:1589324:1589346 [1] NCCL INFO comm 0x9bad760 rank 3 nRanks 4 nNodes 2 localRanks 2 localRank 1 MNNVL 0
ixl01:1589324:1589346 [1] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] -1/-1/-1->3->2
i44:1630821:1630844 [1] NCCL INFO P2P Chunksize set to 131072
ixl01:1589323:1589347 [0] NCCL INFO comm 0x95b6710 rank 2 nRanks 4 nNodes 2 localRanks 2 localRank 0 MNNVL 0
ixl01:1589324:1589346 [1] NCCL INFO P2P Chunksize set to 131072
ixl01:1589323:1589347 [0] NCCL INFO Trees [0] 3/-1/-1->2->0 [1] 3/0/-1->2->-1
ixl01:1589323:1589347 [0] NCCL INFO P2P Chunksize set to 131072
ixl01:1589323:1589347 [0] NCCL INFO Channel 00/0 : 1[1] -> 2[0] [receive] via NET/IB/0
ixl01:1589323:1589347 [0] NCCL INFO Channel 01/0 : 1[1] -> 2[0] [receive] via NET/IB/0
ixl01:1589323:1589347 [0] NCCL INFO Channel 00/0 : 2[0] -> 3[1] via P2P/CUMEM
ixl01:1589323:1589347 [0] NCCL INFO Channel 01/0 : 2[0] -> 3[1] via P2P/CUMEM
ixl01:1589324:1589346 [1] NCCL INFO Channel 00/0 : 3[1] -> 0[0] [send] via NET/IB/0
ixl01:1589324:1589346 [1] NCCL INFO Channel 01/0 : 3[1] -> 0[0] [send] via NET/IB/0
i44:1630821:1630844 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[0] [send] via NET/IB/1
i44:1630821:1630844 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[0] [send] via NET/IB/1
i44:1630820:1630841 [0] NCCL INFO Channel 00/0 : 3[1] -> 0[0] [receive] via NET/IB/1
i44:1630820:1630841 [0] NCCL INFO Channel 01/0 : 3[1] -> 0[0] [receive] via NET/IB/1
i44:1630820:1630841 [0] NCCL INFO Channel 00 : 0[0] -> 1[1] via SHM/direct/direct
i44:1630820:1630841 [0] NCCL INFO Channel 01 : 0[0] -> 1[1] via SHM/direct/direct
ixl01:1589324:1589346 [1] NCCL INFO Connected all rings
ixl01:1589324:1589346 [1] NCCL INFO Channel 00/0 : 3[1] -> 2[0] via P2P/CUMEM
ixl01:1589324:1589346 [1] NCCL INFO Channel 01/0 : 3[1] -> 2[0] via P2P/CUMEM
ixl01:1589323:1589347 [0] NCCL INFO Connected all rings
i44:1630821:1630844 [1] NCCL INFO Connected all rings
i44:1630820:1630841 [0] NCCL INFO Connected all rings
ixl01:1589323:1589347 [0] NCCL INFO Channel 00/0 : 0[0] -> 2[0] [receive] via NET/IB/0
ixl01:1589323:1589347 [0] NCCL INFO Channel 01/0 : 0[0] -> 2[0] [receive] via NET/IB/0
ixl01:1589323:1589347 [0] NCCL INFO Channel 00/0 : 2[0] -> 0[0] [send] via NET/IB/0
ixl01:1589323:1589347 [0] NCCL INFO Channel 01/0 : 2[0] -> 0[0] [send] via NET/IB/0
i44:1630821:1630844 [1] NCCL INFO Channel 00 : 1[1] -> 0[0] via SHM/direct/direct
i44:1630821:1630844 [1] NCCL INFO Channel 01 : 1[1] -> 0[0] via SHM/direct/direct
i44:1630820:1630841 [0] NCCL INFO Channel 00/0 : 2[0] -> 0[0] [receive] via NET/IB/1
i44:1630820:1630841 [0] NCCL INFO Channel 01/0 : 2[0] -> 0[0] [receive] via NET/IB/1
i44:1630820:1630841 [0] NCCL INFO Channel 00/0 : 0[0] -> 2[0] [send] via NET/IB/1
i44:1630820:1630841 [0] NCCL INFO Channel 01/0 : 0[0] -> 2[0] [send] via NET/IB/1
i44:1630821:1630844 [1] NCCL INFO Connected all trees
i44:1630820:1630841 [0] NCCL INFO Connected all trees
i44:1630821:1630844 [1] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512
i44:1630820:1630841 [0] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512
i44:1630820:1630841 [0] NCCL INFO 2 coll channels, 2 collnet channels, 0 nvls channels, 2 p2p channels, 2 p2p channels per peer
i44:1630821:1630844 [1] NCCL INFO 2 coll channels, 2 collnet channels, 0 nvls channels, 2 p2p channels, 2 p2p channels per peer
ixl01:1589323:1589347 [0] NCCL INFO Connected all trees
ixl01:1589324:1589346 [1] NCCL INFO Connected all trees
ixl01:1589323:1589347 [0] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512
ixl01:1589324:1589346 [1] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512
ixl01:1589323:1589347 [0] NCCL INFO 2 coll channels, 2 collnet channels, 0 nvls channels, 2 p2p channels, 2 p2p channels per peer
ixl01:1589324:1589346 [1] NCCL INFO 2 coll channels, 2 collnet channels, 0 nvls channels, 2 p2p channels, 2 p2p channels per peer
ixl01:1589323:1589347 [0] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
ixl01:1589324:1589346 [1] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
ixl01:1589323:1589347 [0] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
ixl01:1589324:1589346 [1] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
ixl01:1589323:1589347 [0] NCCL INFO ncclCommInitRank comm 0x95b6710 rank 2 nranks 4 cudaDev 0 nvmlDev 0 busId b2000 commId 0xf74159e14e16d1a1 - Init COMPLETE
ixl01:1589324:1589346 [1] NCCL INFO ncclCommInitRank comm 0x9bad760 rank 3 nranks 4 cudaDev 1 nvmlDev 1 busId b3000 commId 0xf74159e14e16d1a1 - Init COMPLETE
i44:1630820:1630841 [0] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
i44:1630821:1630844 [1] NCCL INFO TUNER/Plugin: Plugin load returned 2 : libnccl-net.so: cannot open shared object file: No such file or directory : when loading libnccl-tuner.so
i44:1630821:1630844 [1] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
i44:1630821:1630844 [1] NCCL INFO ncclCommInitRank comm 0x9fea810 rank 1 nranks 4 cudaDev 1 nvmlDev 1 busId d8000 commId 0xf74159e14e16d1a1 - Init COMPLETE
i44:1630820:1630841 [0] NCCL INFO TUNER/Plugin: Using internal tuner plugin.
i44:1630820:1630841 [0] NCCL INFO ncclCommInitRank comm 0x9365bc0 rank 0 nranks 4 cudaDev 0 nvmlDev 0 busId 86000 commId 0xf74159e14e16d1a1 - Init COMPLETE
Logging to wandb project COM304_nano4M, entity rayane-charifchefchaouni-epfl, run name multiclevr_caption_d6-6w512
Namespace(run_name='nano4M/multiclevr_caption_d6-6w512', model_config={'_target_': 'nanofm.models.fourm.FourM', 'enc_tokens_read_key': 'enc_tokens', 'dec_tokens_read_key': 'dec_tokens', 'enc_modalities_read_key': 'enc_modalities', 'dec_modalities_read_key': 'dec_modalities', 'enc_positions_read_key': 'enc_positions', 'dec_positions_read_key': 'dec_positions', 'enc_pad_mask_read_key': 'enc_pad_mask', 'dec_pad_mask_read_key': 'dec_pad_mask', 'modalities': ['tok_rgb@256', 'tok_depth@256', 'tok_normal@256', 'scene_desc', 'caption'], 'vocab_sizes': [64000, 64000, 64000, 50304, 50304], 'max_seq_lens': [256, 256, 256, 256, 256], 'dim': 512, 'enc_depth': 6, 'dec_depth': 6, 'head_dim': 64, 'per_modality_loss_avg': True}, train_loader_config={'_target_': 'nanofm.data.multimodal.create_multimodal_masked_dataloader', 'root_dir': '/work/com-304/datasets/clevr_com_304/', 'split': 'train', 'modalities': ['tok_rgb@256', 'tok_depth@256', 'tok_normal@256', 'scene_desc', 'caption'], 'vocab_sizes': [64000, 64000, 64000, 50304, 50304], 'max_seq_lens': [256, 256, 256, 256, 256], 'overlap_vocab': True, 'overlap_posembs': True, 'input_alphas': [1.0, 1.0, 1.0, 1.0, 1.0], 'target_alphas': [1.0, 1.0, 1.0, 1.0, 1.0], 'input_tokens_range': [1, 128], 'target_tokens_range': [1, 128], 'sample_from_k_augmentations': 10, 'text_tokenizer_path': 'gpt2', 'text_max_length': 256, 'batch_size': 128, 'infinite': True, 'num_workers': 10, 'pin_memory': True, 'shuffle': True, 'drop_last': True, 'distributed': True}, eval_loader_config={'_target_': 'nanofm.data.multimodal.create_multimodal_masked_dataloader', 'root_dir': '/work/com-304/datasets/clevr_com_304/', 'split': 'val', 'modalities': ['tok_rgb@256', 'tok_depth@256', 'tok_normal@256', 'scene_desc', 'caption'], 'vocab_sizes': [64000, 64000, 64000, 50304, 50304], 'max_seq_lens': [256, 256, 256, 256, 256], 'overlap_vocab': True, 'overlap_posembs': True, 'input_alphas': [1.0, 1.0, 1.0, 1.0, 1.0], 'target_alphas': [1.0, 1.0, 1.0, 1.0, 1.0], 'input_tokens_range': [1, 128], 'target_tokens_range': [1, 128], 'text_tokenizer_path': 'gpt2', 'text_max_length': 256, 'batch_size': 128, 'num_workers': 10, 'pin_memory': True, 'shuffle': False, 'drop_last': False, 'distributed': True}, batch_size=128, total_tokens=5000, warmup_tokens=500, num_tokens_per_sample=256, dtype='fp16', seed=0, opt_eps=1e-08, opt_betas=[0.9, 0.95], lr=0.0006, min_lr=1e-06, weight_decay=0.05, clip_grad=1.0, eval_freq=100, save_ckpt_freq=1000, output_dir='./outputs/nano4M/multiclevr_caption_d6-6w512', resume='', start_iteration=0, auto_resume=True, device='cuda', find_unused_params=False, dist_url='env://', log_wandb=True, wandb_project='COM304_nano4M', wandb_entity='rayane-charifchefchaouni-epfl', wandb_run_name='multiclevr_caption_d6-6w512', global_vars={'batch_size': 128, 'modalities': ['tok_rgb@256', 'tok_depth@256', 'tok_normal@256', 'scene_desc', 'caption'], 'vocab_sizes': [64000, 64000, 64000, 50304, 50304], 'max_seq_lens': [256, 256, 256, 256, 256], 'input_alphas': [1.0, 1.0, 1.0, 1.0, 1.0], 'target_alphas': [1.0, 1.0, 1.0, 1.0, 1.0], 'input_tokens_range': [1, 128], 'target_tokens_range': [1, 128]}, config_path='cfgs/nano4M/multiclevr_caption_d6-6w512.yaml', rank=0, world_size=4, gpu=0, distributed=True, dist_backend='nccl')
Model = FourM(
  (enc_tok_emb): Embedding(64000, 512)
  (enc_mod_emb): Embedding(5, 512)
  (dec_mod_emb): Embedding(5, 512)
  (encoder): TransformerTrunk(
    (blocks): ModuleList(
      (0-5): 6 x Block(
        (norm1): LayerNorm()
        (attn): Attention(
          (qkv): Linear(in_features=512, out_features=1536, bias=False)
          (attn_out_proj): Linear(in_features=512, out_features=512, bias=False)
        )
        (norm2): LayerNorm()
        (mlp): Mlp(
          (fc1): Linear(in_features=512, out_features=2048, bias=False)
          (fc2): Linear(in_features=2048, out_features=512, bias=False)
          (act): GELU(approximate='none')
        )
      )
    )
  )
  (decoder): TransformerDecoderTrunk(
    (blocks): ModuleList(
      (0-5): 6 x DecoderBlock(
        (norm1): LayerNorm()
        (query_norm): LayerNorm()
        (context_norm): LayerNorm()
        (norm2): LayerNorm()
        (self_attn): Attention(
          (qkv): Linear(in_features=512, out_features=1536, bias=False)
          (attn_out_proj): Linear(in_features=512, out_features=512, bias=False)
        )
        (cross_attn): CrossAttention(
          (q_proj): Linear(in_features=512, out_features=512, bias=False)
          (kv_proj): Linear(in_features=512, out_features=1024, bias=False)
          (attn_out_proj): Linear(in_features=512, out_features=512, bias=False)
        )
        (mlp): Mlp(
          (fc1): Linear(in_features=512, out_features=2048, bias=False)
          (fc2): Linear(in_features=2048, out_features=512, bias=False)
          (act): GELU(approximate='none')
        )
      )
    )
  )
  (dec_context_proj): Linear(in_features=512, out_features=512, bias=False)
  (to_logits): Linear(in_features=512, out_features=64000, bias=False)
  (enc_norm): LayerNorm()
  (dec_norm): LayerNorm()
)
Number of params: 109.862912 M
Total tokens: 5000M
Total iters: 38147
Warmup tokens: 500M
Warmup iters: 3815
Eval freq: every 763 iterations
Save ckpt freq: every 7630 iterations
Batch size per GPU = 128
Total (effective) batch size = 512
LR = 0.00060000
Min LR = 0.00000100
Param groups = {
  "decay": {
    "weight_decay": 0.05,
    "params": [
      "enc_tok_emb.weight",
      "enc_mod_emb.weight",
      "dec_mod_emb.weight",
      "encoder.blocks.0.attn.qkv.weight",
      "encoder.blocks.0.attn.attn_out_proj.weight",
      "encoder.blocks.0.mlp.fc1.weight",
      "encoder.blocks.0.mlp.fc2.weight",
      "encoder.blocks.1.attn.qkv.weight",
      "encoder.blocks.1.attn.attn_out_proj.weight",
      "encoder.blocks.1.mlp.fc1.weight",
      "encoder.blocks.1.mlp.fc2.weight",
      "encoder.blocks.2.attn.qkv.weight",
      "encoder.blocks.2.attn.attn_out_proj.weight",
      "encoder.blocks.2.mlp.fc1.weight",
      "encoder.blocks.2.mlp.fc2.weight",
      "encoder.blocks.3.attn.qkv.weight",
      "encoder.blocks.3.attn.attn_out_proj.weight",
      "encoder.blocks.3.mlp.fc1.weight",
      "encoder.blocks.3.mlp.fc2.weight",
      "encoder.blocks.4.attn.qkv.weight",
      "encoder.blocks.4.attn.attn_out_proj.weight",
      "encoder.blocks.4.mlp.fc1.weight",
      "encoder.blocks.4.mlp.fc2.weight",
      "encoder.blocks.5.attn.qkv.weight",
      "encoder.blocks.5.attn.attn_out_proj.weight",
      "encoder.blocks.5.mlp.fc1.weight",
      "encoder.blocks.5.mlp.fc2.weight",
      "decoder.blocks.0.self_attn.qkv.weight",
      "decoder.blocks.0.self_attn.attn_out_proj.weight",
      "decoder.blocks.0.cross_attn.q_proj.weight",
      "decoder.blocks.0.cross_attn.kv_proj.weight",
      "decoder.blocks.0.cross_attn.attn_out_proj.weight",
      "decoder.blocks.0.mlp.fc1.weight",
      "decoder.blocks.0.mlp.fc2.weight",
      "decoder.blocks.1.self_attn.qkv.weight",
      "decoder.blocks.1.self_attn.attn_out_proj.weight",
      "decoder.blocks.1.cross_attn.q_proj.weight",
      "decoder.blocks.1.cross_attn.kv_proj.weight",
      "decoder.blocks.1.cross_attn.attn_out_proj.weight",
      "decoder.blocks.1.mlp.fc1.weight",
      "decoder.blocks.1.mlp.fc2.weight",
      "decoder.blocks.2.self_attn.qkv.weight",
      "decoder.blocks.2.self_attn.attn_out_proj.weight",
      "decoder.blocks.2.cross_attn.q_proj.weight",
      "decoder.blocks.2.cross_attn.kv_proj.weight",
      "decoder.blocks.2.cross_attn.attn_out_proj.weight",
      "decoder.blocks.2.mlp.fc1.weight",
      "decoder.blocks.2.mlp.fc2.weight",
      "decoder.blocks.3.self_attn.qkv.weight",
      "decoder.blocks.3.self_attn.attn_out_proj.weight",
      "decoder.blocks.3.cross_attn.q_proj.weight",
      "decoder.blocks.3.cross_attn.kv_proj.weight",
      "decoder.blocks.3.cross_attn.attn_out_proj.weight",
      "decoder.blocks.3.mlp.fc1.weight",
      "decoder.blocks.3.mlp.fc2.weight",
      "decoder.blocks.4.self_attn.qkv.weight",
      "decoder.blocks.4.self_attn.attn_out_proj.weight",
      "decoder.blocks.4.cross_attn.q_proj.weight",
      "decoder.blocks.4.cross_attn.kv_proj.weight",
      "decoder.blocks.4.cross_attn.attn_out_proj.weight",
      "decoder.blocks.4.mlp.fc1.weight",
      "decoder.blocks.4.mlp.fc2.weight",
      "decoder.blocks.5.self_attn.qkv.weight",
      "decoder.blocks.5.self_attn.attn_out_proj.weight",
      "decoder.blocks.5.cross_attn.q_proj.weight",
      "decoder.blocks.5.cross_attn.kv_proj.weight",
      "decoder.blocks.5.cross_attn.attn_out_proj.weight",
      "decoder.blocks.5.mlp.fc1.weight",
      "decoder.blocks.5.mlp.fc2.weight",
      "dec_context_proj.weight",
      "to_logits.weight"
    ]
  },
  "no_decay": {
    "weight_decay": 0.0,
    "params": [
      "encoder.blocks.0.norm1.weight",
      "encoder.blocks.0.norm2.weight",
      "encoder.blocks.1.norm1.weight",
      "encoder.blocks.1.norm2.weight",
      "encoder.blocks.2.norm1.weight",
      "encoder.blocks.2.norm2.weight",
      "encoder.blocks.3.norm1.weight",
      "encoder.blocks.3.norm2.weight",
      "encoder.blocks.4.norm1.weight",
      "encoder.blocks.4.norm2.weight",
      "encoder.blocks.5.norm1.weight",
      "encoder.blocks.5.norm2.weight",
      "decoder.blocks.0.norm1.weight",
      "decoder.blocks.0.query_norm.weight",
      "decoder.blocks.0.context_norm.weight",
      "decoder.blocks.0.norm2.weight",
      "decoder.blocks.1.norm1.weight",
      "decoder.blocks.1.query_norm.weight",
      "decoder.blocks.1.context_norm.weight",
      "decoder.blocks.1.norm2.weight",
      "decoder.blocks.2.norm1.weight",
      "decoder.blocks.2.query_norm.weight",
      "decoder.blocks.2.context_norm.weight",
      "decoder.blocks.2.norm2.weight",
      "decoder.blocks.3.norm1.weight",
      "decoder.blocks.3.query_norm.weight",
      "decoder.blocks.3.context_norm.weight",
      "decoder.blocks.3.norm2.weight",
      "decoder.blocks.4.norm1.weight",
      "decoder.blocks.4.query_norm.weight",
      "decoder.blocks.4.context_norm.weight",
      "decoder.blocks.4.norm2.weight",
      "decoder.blocks.5.norm1.weight",
      "decoder.blocks.5.query_norm.weight",
      "decoder.blocks.5.context_norm.weight",
      "decoder.blocks.5.norm2.weight",
      "enc_norm.weight",
      "dec_norm.weight"
    ]
  }
}
optimizer settings: {'lr': 0.0006, 'weight_decay': 0.0, 'eps': 1e-08, 'betas': [0.9, 0.95]}
Set warmup iters = 3815
Start training for 5000M tokens = 38147 iterations
ixl01:1589324:1589352 [1] NCCL INFO [Service thread] Connection closed by localRank 1
ixl01:1589323:1589354 [0] NCCL INFO [Service thread] Connection closed by localRank 0
i44:1630821:1630850 [1] NCCL INFO [Service thread] Connection closed by localRank 1
ixl01:1589324:1589476 [1] NCCL INFO comm 0x9bad760 rank 3 nranks 4 cudaDev 1 busId b3000 - Abort COMPLETE
ixl01:1589323:1589475 [0] NCCL INFO comm 0x95b6710 rank 2 nranks 4 cudaDev 0 busId b2000 - Abort COMPLETE
i44:1630821:1631005 [1] NCCL INFO comm 0x9fea810 rank 1 nranks 4 cudaDev 1 busId d8000 - Abort COMPLETE