Unable to start distributed training

[lyangfan]

Hi:
I want to integrate monai package and mmengine to do medical image segmentation model distributed training. Here is code
utils.py

import os
import json

import torch
from mmengine.model import BaseModel, BaseDataPreprocessor
from monai.data import Dataset
from monai.networks.nets import SwinUNETR
from monai.losses import DiceCELoss
from monai.metrics import DiceHelper
from monai.transforms import (
    OneOf,
    LoadImaged,
    Orientationd,
    Compose,
    CropForegroundd,
    ToTensord,
    EnsureChannelFirstd,
    Spacingd,
    RandAffined,
    ScaleIntensityRanged,
    RandScaleIntensityd,
    RandAdjustContrastd,
    RandCropByPosNegLabeld,
    RandFlipd,
    RandGaussianNoised,
    RandGaussianSmoothd
)
from mmengine.evaluator import BaseMetric
from mmengine.registry import METRICS

def get_train_transforms(args):
    monai_transforms = [
            LoadImaged(keys=["image", "label"], image_only=False),
            EnsureChannelFirstd(keys=["image", "label"]),
            Orientationd(keys=["image", "label"], axcodes="RAS"),
            Spacingd(keys=["image", "label"], pixdim=(2.5, 2.5, 5.0), mode=("bilinear", "nearest")),
            CropForegroundd(keys=["image", "label"], source_key="image"),
            ScaleIntensityRanged(
                keys=["image"],
                a_min=-1000,
                a_max=1000,
                b_min=0.0,
                b_max=1.0,
                clip=True,
            ),
            RandCropByPosNegLabeld(
                keys=["image", "label"],
                label_key="label",
                spatial_size=(96, 96, 96),
                pos=1,
                neg=1,
                num_samples=args.num_samples,
                image_key="image",
                image_threshold=-1024,
            ),
        OneOf([
            RandAffined(
                keys=["image", "label"],
                prob=0.48,
                scale_range=(0.7, 1.4)),            
            RandAffined(
                keys=["image", "label"],
                prob=0.48,
                rotate_range =(-30, 30)),
            RandAffined(
                keys=["image", "label"],
                prob=0.24,
                scale_range=(0.7, 1.4),
                rotate_range =(-30, 30)),
            ]),
        RandGaussianNoised(keys=['image'], prob=0.15),
        RandGaussianSmoothd(keys=['image'], 
                prob=0.125,
                sigma_x=(0.5, 1.5),
                sigma_y=(0.5, 1.5),
                sigma_z=(0.5, 1.5)),
        RandScaleIntensityd(
            keys=["image"],
            factors=0.3,
            prob=0.15
        ),
        RandAdjustContrastd(
            keys=["image"],
            prob=0.15,
            gamma=(0.7, 1.5)
        ),
        RandFlipd(
            keys=["image", "label"],
            prob=0.5,
        ),
        ToTensord(keys=["image", "label"])
    ]
    train_transforms = Compose(monai_transforms)
    return  train_transforms

def get_val_transforms(args):
    monai_transforms = [
            LoadImaged(keys=["image", "label"], image_only=False),
            EnsureChannelFirstd(keys=["image", "label"]),
            Orientationd(keys=["image", "label"], axcodes="RAS"),
            Spacingd(keys=["image", "label"], pixdim=(2.5, 2.5, 5.0), mode=("bilinear", "nearest")),
            ScaleIntensityRanged(
                keys=["image"],
                a_min=-1000,
                a_max=1000,
                b_min=0.0,
                b_max=1.0,
                clip=True,
            ),
            RandCropByPosNegLabeld(
                keys=["image", "label"],
                label_key="label",
                spatial_size=(96, 96, 96),
                pos=1,
                neg=1,
                num_samples=args.num_samples,
                image_key="image",
                image_threshold=-1024,
            ),
        ToTensord(keys=["image", "label"])
    ]
    val_transforms = Compose(monai_transforms)
    return val_transforms

def get_path_dict(args):
    with open(args.train_json_path, 'r') as f:
        data_split = json.load(f)
    target_files_names_dict = data_split[args.percent_name]
    target_files_names_list = []
    for _, file_names in target_files_names_dict.items():
        target_files_names_list += file_names
    train_images_path = [os.path.join(args.train_image_dir, image_name) for image_name in target_files_names_list]
    train_labels_path = [os.path.join(args.train_label_dir, label_name) for label_name in target_files_names_list]
    train_data_dict = [{'image': image_path, 'label': label_path} for image_path, label_path in zip(train_images_path, train_labels_path)]
    val_files_names_list = os.listdir(args.val_image_dir)
    val_images_path = [os.path.join(args.val_image_dir, image_name) for image_name in val_files_names_list]
    val_labels_path = [os.path.join(args.val_label_dir, label_name) for label_name in val_files_names_list]
    val_data_dict = [{'image': image_path, 'label': label_path} for image_path, label_path in zip(val_images_path, val_labels_path)]
    return train_data_dict, val_data_dict

def get_monai_dataset(args):
    train_data_dict, val_data_dict = get_path_dict(args)
    train_transforms = get_train_transforms(args)
    val_transforms = get_val_transforms(args) 
    train_dataset = Dataset(data=train_data_dict, transform=train_transforms)
    val_dataset = Dataset(data=val_data_dict, transform=val_transforms)
    return train_dataset, val_dataset

class monai_data_processor(BaseDataPreprocessor):
    def __init__(self):
        super().__init__()
    def forward(self, data, training=False):
        data = super().forward(data=data, training=training)
        image, label = data['image'], data['label']
        return image, label

def get_model():
    class MMSwinUNETR(BaseModel):
        def __init__(self):
            super().__init__(data_preprocessor=monai_data_processor())
            self.model = SwinUNETR(
                img_size=(96, 96, 96),
                in_channels=1,
                out_channels=9,
                feature_size=48,
                drop_rate=0.0,
                attn_drop_rate=0.0,
                dropout_path_rate=0,
                use_checkpoint=False,
            )
        def forward(self, image, label, mode):
            x = self.model(image)
            if mode == 'loss':
                loss_func = DiceCELoss(to_onehot_y=True, softmax=True)
                return {'loss': loss_func(x, label)}
            elif mode == 'predict':
                return x, label

    model = MMSwinUNETR()
    return model

def load_ssl_weights(model, args):
    state_dict = torch.load(args.ssl_pretrained_path, map_location='cpu')
    if "module." in list(state_dict.keys())[0]:
        print("Tag 'module.' found in state dict - fixing!")
    for key in list(state_dict.keys()):
        state_dict[key.replace("module", "model")] = state_dict.pop(key)
    model.load_state_dict(state_dict, strict=False)
    print("Using pretrained self-supervised Swin UNETR backbone weights !")

    return model

@METRICS.register_module() 
class MeanDice(BaseMetric):
    def process(self, data_batch, data_samples):
        result = {
            'pred': data_samples[0],
            'gt': data_samples[1]
        }
        self.results.append(result)
    def compute_metrics(self, results):
        preds = torch.concatenate([res['pred'] for res in results], dim=0)
        gts = torch.concatenate([res['gt'] for res in results], dim=0)
        return {'mean_dice': DiceHelper(include_background=True, 
                                        softmax=True, 
                                        get_not_nans=False,
                                        reduction='mean')(preds, gts).item()}
    
def get_hook_config():
    default_hooks = dict(
        checkpoint=dict(type='CheckpointHook', 
                        interval=1, 
                        by_epoch=True,
                        save_best='mean_dice',
                        rule='greater',
                        max_keep_ckpts=5
                        ),
        logger=dict(type='LoggerHook'),
        sampler_seed=dict(type='DistSamplerSeedHook'),
    )
    return default_hooks

def get_log_processor():
    log_processor = dict(
                    window_size=10,
                    by_epoch=True,
                    custom_cfg=[
                        dict(data_src='loss',
                        log_name='global_loss',
                        method_name='mean',
                        window_size='global')])
    return log_processor

main.py

import argparse

import torch
from monai.data import DataLoader
from utils import get_monai_dataset, get_model, load_ssl_weights, MeanDice, get_hook_config, get_log_processor
from mmengine.runner import Runner
from mmengine.optim import AmpOptimWrapper
from monai.utils import set_determinism


def main(args):  
    if args.use_monai_determinism:
        set_determinism()
    train_dataset, val_dataset = get_monai_dataset(args)
    train_dataloader = DataLoader(
        train_dataset,
        batch_size=args.batch_size,
        shuffle=True,
        num_workers=args.num_workers,
        pin_memory=True,
    )
    val_dataloader = DataLoader(
        val_dataset,
        batch_size=args.batch_size,
        shuffle=False,
        num_workers=args.num_workers,
        pin_memory=True,
    )
    model = get_model()
    if args.ssl_pretrained:
        model = load_ssl_weights(model, args)
    optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=args.reg_weight)
    optim_wrapper = AmpOptimWrapper(optimizer=optimizer)
    param_scheduler = [
        dict(type='LinearLR', 
             by_epoch=True, 
             start_factor=0.1 * args.lr, 
             begin=0,
             end=args.max_epochs * 0.1, 
             convert_to_iter_based=True),
        dict(type='CosineAnnealingLR', 
             by_epoch=True, 
             T_max=args.max_epochs * 0.9, 
             convert_to_iter_based=True)
    ]
    if args.use_monai_determinism:
        runner = Runner(
            model=model,
            work_dir=args.work_dir,
            train_dataloader=train_dataloader,
            optim_wrapper=optim_wrapper,
            train_cfg=dict(by_epoch=True, max_epochs=args.max_epochs, val_interval=args.val_interval),
            val_dataloader=val_dataloader,
            val_cfg=dict(),
            val_evaluator=dict(type=MeanDice),
            param_scheduler=param_scheduler,
            default_hooks=get_hook_config(),
            log_processor=get_log_processor(),
            experiment_name=args.experiment_name,
        )     
    else:
        runner = Runner(
            model=model,
            work_dir=args.work_dir,
            train_dataloader=train_dataloader,
            optim_wrapper=optim_wrapper,
            train_cfg=dict(by_epoch=True, max_epochs=args.max_epochs, val_interval=args.val_interval),
            val_dataloader=val_dataloader,
            val_cfg=dict(),
            val_evaluator=dict(type=MeanDice),
            param_scheduler=param_scheduler,
            default_hooks=get_hook_config(),
            log_processor=get_log_processor(),
            experiment_name=args.experiment_name,
            randomness=dict(seed=0)
        )
    runner.train()

def parse_args():
    parser = argparse.ArgumentParser(description='SSL model performance Test')
    parser.add_argument('--work_dir', type=str)
    parser.add_argument('--experiment_name', type=str)
    parser.add_argument('--ssl_pretrained', action='store_true')
    parser.add_argument('--ssl_pretrained_path', type=str)
    parser.add_argument('--train_json_path', type=str)
    parser.add_argument('--percent_name', type=str)
    parser.add_argument('--train_image_dir', type=str)
    parser.add_argument('--train_label_dir', type=str)
    parser.add_argument('--val_image_dir', type=str)
    parser.add_argument('--val_label_dir', type=str)
    parser.add_argument('--num_samples', type=int)
    parser.add_argument('--batch_size', type=int)
    parser.add_argument('--num_workers', type=int)
    parser.add_argument('--max_epochs', type=int)
    parser.add_argument('--lr', type=float, default=1e-4)
    parser.add_argument('--reg_weight', type=float, default=1e-5)
    parser.add_argument('--val_interval', type=int, default=1)
    parser.add_argument('--use_monai_determinism', type=bool, default=False)
    parser.add_argument(
        '--launcher',
        choices=['none', 'pytorch', 'slurm', 'mpi'],
        default='none',
        help='job launcher')
    parser.add_argument('--local-rank', type=int, default=0)

    args = parser.parse_args()
    return args


if __name__ == '__main__':
    args = parse_args()
    main(args)

Here is the command

python -m torch.distributed.launch --nproc_per_node=2 main.py --work_dir /public/home/three_whyz123/yhfu/yfliu/run/02high_dimensional_phenotyping/CT/self_supervised_learning/Swin_UNETR/output/training_logs/ssl_model_performance_test/reduce_manual_labeling_efforts/percent_0.1 --experiment_name percent_0.1 --ssl_pretrained --ssl_pretrained_path /public/home/three_whyz123/yhfu/yfliu/run/02high_dimensional_phenotyping/CT/self_supervised_learning/Swin_UNETR/output/training_logs/ssl_training/log5/epoch_494_model.pt --train_json_path /public/home/three_whyz123/yhfu/yfliu/run/02high_dimensional_phenotyping/CT/self_supervised_learning/Swin_UNETR/output/training_logs/ssl_model_performance_test/reduce_manual_labeling_efforts/data_choose.json --percent_name 0.1 --train_image_dir /public/home/three_whyz123/yhfu/yfliu/data/TB/sequence_846/label/carcass_label_data/carcass_9_classes/carcass_9_classes_train/nifti/all --train_label_dir /public/home/three_whyz123/yhfu/yfliu/data/TB/sequence_846/label/carcass_label_data/carcass_9_classes/carcass_9_classes_train/carcass_9_classes_label --val_image_dir /public/home/three_whyz123/yhfu/yfliu/data/TB/sequence_846/label/carcass_label_data/carcass_9_classes/carcass_9_classes_test_mannual/nifti --val_label_dir /public/home/three_whyz123/yhfu/yfliu/data/TB/sequence_846/label/carcass_label_data/carcass_9_classes/carcass_9_classes_test_mannual/label --num_samples 4 --batch_size 4 --num_workers 12 --max_epochs 1000  --launcher pytorch

But I can't start the distributed learning. It always use 1 GPU to do multiprocess training. Here is a part of training log:

2023/08/25 11:50:17 - mmengine - INFO - 
------------------------------------------------------------
System environment:
    sys.platform: linux
    Python: 3.10.12 | packaged by conda-forge | (main, Jun 23 2023, 22:40:32) [GCC 12.3.0]
    CUDA available: True
    numpy_random_seed: 0
    GPU 0,1: NVIDIA A100-PCIE-40GB
    CUDA_HOME: /usr/local/cuda
    NVCC: Cuda compilation tools, release 11.3, V11.3.109
    GCC: gcc (GCC) 4.8.5
    PyTorch: 2.0.1+cu117
    PyTorch compiling details: PyTorch built with:
  - GCC 9.3
  - C++ Version: 201703
  - Intel(R) oneAPI Math Kernel Library Version 2022.2-Product Build 20220804 for Intel(R) 64 architecture applications
  - Intel(R) MKL-DNN v2.7.3 (Git Hash 6dbeffbae1f23cbbeae17adb7b5b13f1f37c080e)
  - OpenMP 201511 (a.k.a. OpenMP 4.5)
  - LAPACK is enabled (usually provided by MKL)
  - NNPACK is enabled
  - CPU capability usage: AVX2
  - CUDA Runtime 11.7
  - NVCC architecture flags: -gencode;arch=compute_37,code=sm_37;-gencode;arch=compute_50,code=sm_50;-gencode;arch=compute_60,code=sm_60;-gencode;arch=compute_70,code=sm_70;-gencode;arch=compute_75,code=sm_75;-gencode;arch=compute_80,code=sm_80;-gencode;arch=compute_86,code=sm_86
  - CuDNN 8.5
  - Magma 2.6.1
  - Build settings: BLAS_INFO=mkl, BUILD_TYPE=Release, CUDA_VERSION=11.7, CUDNN_VERSION=8.5.0, CXX_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/c++, CXX_FLAGS= -D_GLIBCXX_USE_CXX11_ABI=0 -fabi-version=11 -Wno-deprecated -fvisibility-inlines-hidden -DUSE_PTHREADPOOL -DNDEBUG -DUSE_KINETO -DLIBKINETO_NOROCTRACER -DUSE_FBGEMM -DUSE_QNNPACK -DUSE_PYTORCH_QNNPACK -DUSE_XNNPACK -DSYMBOLICATE_MOBILE_DEBUG_HANDLE -O2 -fPIC -Wall -Wextra -Werror=return-type -Werror=non-virtual-dtor -Werror=bool-operation -Wnarrowing -Wno-missing-field-initializers -Wno-type-limits -Wno-array-bounds -Wno-unknown-pragmas -Wunused-local-typedefs -Wno-unused-parameter -Wno-unused-function -Wno-unused-result -Wno-strict-overflow -Wno-strict-aliasing -Wno-error=deprecated-declarations -Wno-stringop-overflow -Wno-psabi -Wno-error=pedantic -Wno-error=redundant-decls -Wno-error=old-style-cast -fdiagnostics-color=always -faligned-new -Wno-unused-but-set-variable -Wno-maybe-uninitialized -fno-math-errno -fno-trapping-math -Werror=format -Werror=cast-function-type -Wno-stringop-overflow, LAPACK_INFO=mkl, PERF_WITH_AVX=1, PERF_WITH_AVX2=1, PERF_WITH_AVX512=1, TORCH_DISABLE_GPU_ASSERTS=ON, TORCH_VERSION=2.0.1, USE_CUDA=ON, USE_CUDNN=ON, USE_EXCEPTION_PTR=1, USE_GFLAGS=OFF, USE_GLOG=OFF, USE_MKL=ON, USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=1, USE_NNPACK=ON, USE_OPENMP=ON, USE_ROCM=OFF, 

    TorchVision: 0.15.2+cu117
    OpenCV: 4.8.0
    MMEngine: 0.8.4

Runtime environment:
    dist_cfg: {'backend': 'nccl'}
    seed: 0
    Distributed launcher: none
    Distributed training: False
    GPU number: 1
------------------------------------------------------------

2023/08/25 11:50:17 - mmengine - INFO - 
------------------------------------------------------------
System environment:
    sys.platform: linux
    Python: 3.10.12 | packaged by conda-forge | (main, Jun 23 2023, 22:40:32) [GCC 12.3.0]
    CUDA available: True
    numpy_random_seed: 0
    GPU 0,1: NVIDIA A100-PCIE-40GB
    CUDA_HOME: /usr/local/cuda
    NVCC: Cuda compilation tools, release 11.3, V11.3.109
    GCC: gcc (GCC) 4.8.5
    PyTorch: 2.0.1+cu117
    PyTorch compiling details: PyTorch built with:
  - GCC 9.3
  - C++ Version: 201703
  - Intel(R) oneAPI Math Kernel Library Version 2022.2-Product Build 20220804 for Intel(R) 64 architecture applications
  - Intel(R) MKL-DNN v2.7.3 (Git Hash 6dbeffbae1f23cbbeae17adb7b5b13f1f37c080e)
  - OpenMP 201511 (a.k.a. OpenMP 4.5)
  - LAPACK is enabled (usually provided by MKL)
  - NNPACK is enabled
  - CPU capability usage: AVX2
  - CUDA Runtime 11.7
  - NVCC architecture flags: -gencode;arch=compute_37,code=sm_37;-gencode;arch=compute_50,code=sm_50;-gencode;arch=compute_60,code=sm_60;-gencode;arch=compute_70,code=sm_70;-gencode;arch=compute_75,code=sm_75;-gencode;arch=compute_80,code=sm_80;-gencode;arch=compute_86,code=sm_86
  - CuDNN 8.5
  - Magma 2.6.1
  - Build settings: BLAS_INFO=mkl, BUILD_TYPE=Release, CUDA_VERSION=11.7, CUDNN_VERSION=8.5.0, CXX_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/c++, CXX_FLAGS= -D_GLIBCXX_USE_CXX11_ABI=0 -fabi-version=11 -Wno-deprecated -fvisibility-inlines-hidden -DUSE_PTHREADPOOL -DNDEBUG -DUSE_KINETO -DLIBKINETO_NOROCTRACER -DUSE_FBGEMM -DUSE_QNNPACK -DUSE_PYTORCH_QNNPACK -DUSE_XNNPACK -DSYMBOLICATE_MOBILE_DEBUG_HANDLE -O2 -fPIC -Wall -Wextra -Werror=return-type -Werror=non-virtual-dtor -Werror=bool-operation -Wnarrowing -Wno-missing-field-initializers -Wno-type-limits -Wno-array-bounds -Wno-unknown-pragmas -Wunused-local-typedefs -Wno-unused-parameter -Wno-unused-function -Wno-unused-result -Wno-strict-overflow -Wno-strict-aliasing -Wno-error=deprecated-declarations -Wno-stringop-overflow -Wno-psabi -Wno-error=pedantic -Wno-error=redundant-decls -Wno-error=old-style-cast -fdiagnostics-color=always -faligned-new -Wno-unused-but-set-variable -Wno-maybe-uninitialized -fno-math-errno -fno-trapping-math -Werror=format -Werror=cast-function-type -Wno-stringop-overflow, LAPACK_INFO=mkl, PERF_WITH_AVX=1, PERF_WITH_AVX2=1, PERF_WITH_AVX512=1, TORCH_DISABLE_GPU_ASSERTS=ON, TORCH_VERSION=2.0.1, USE_CUDA=ON, USE_CUDNN=ON, USE_EXCEPTION_PTR=1, USE_GFLAGS=OFF, USE_GLOG=OFF, USE_MKL=ON, USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=1, USE_NNPACK=ON, USE_OPENMP=ON, USE_ROCM=OFF, 

    TorchVision: 0.15.2+cu117
    OpenCV: 4.8.0
    MMEngine: 0.8.4

Runtime environment:
    dist_cfg: {'backend': 'nccl'}
    seed: 0
    Distributed launcher: none
    Distributed training: False
    GPU number: 1
------------------------------------------------------------

2023/08/25 11:50:18 - mmengine - INFO - Distributed training is not used, all SyncBatchNorm (SyncBN) layers in the model will be automatically reverted to BatchNormXd layers if they are used.
2023/08/25 11:50:18 - mmengine - INFO - Distributed training is not used, all SyncBatchNorm (SyncBN) layers in the model will be automatically reverted to BatchNormXd layers if they are used.
2023/08/25 11:50:18 - mmengine - INFO - Hooks will be executed in the following order:
before_run:
(VERY_HIGH   ) RuntimeInfoHook                    
(BELOW_NORMAL) LoggerHook                         
 -------------------- 
before_train:
(VERY_HIGH   ) RuntimeInfoHook                    
(NORMAL      ) IterTimerHook                      
(VERY_LOW    ) CheckpointHook                     
 -------------------- 
before_train_epoch:
(VERY_HIGH   ) RuntimeInfoHook                    
(NORMAL      ) IterTimerHook                      
(NORMAL      ) DistSamplerSeedHook                
 -------------------- 
before_train_iter:
(VERY_HIGH   ) RuntimeInfoHook                    
(NORMAL      ) IterTimerHook                      
 -------------------- 
after_train_iter:
(VERY_HIGH   ) RuntimeInfoHook                    
(NORMAL      ) IterTimerHook                      
(BELOW_NORMAL) LoggerHook                         
(LOW         ) ParamSchedulerHook                 
(VERY_LOW    ) CheckpointHook                     
 -------------------- 
after_train_epoch:
(NORMAL      ) IterTimerHook                      
(LOW         ) ParamSchedulerHook                 
(VERY_LOW    ) CheckpointHook                     
 -------------------- 
before_val:
(VERY_HIGH   ) RuntimeInfoHook                    
 -------------------- 
before_val_epoch:
(NORMAL      ) IterTimerHook                      
 -------------------- 
before_val_iter:
(NORMAL      ) IterTimerHook                      
 -------------------- 
after_val_iter:
(NORMAL      ) IterTimerHook                      
(BELOW_NORMAL) LoggerHook                         
 -------------------- 
after_val_epoch:
(VERY_HIGH   ) RuntimeInfoHook                    
(NORMAL      ) IterTimerHook                      
(BELOW_NORMAL) LoggerHook                         
(LOW         ) ParamSchedulerHook                 
(VERY_LOW    ) CheckpointHook                     
 -------------------- 
after_val:
(VERY_HIGH   ) RuntimeInfoHook                    
 -------------------- 
after_train:
(VERY_HIGH   ) RuntimeInfoHook                    
(VERY_LOW    ) CheckpointHook                     
 -------------------- 
before_test:
(VERY_HIGH   ) RuntimeInfoHook                    
 -------------------- 
before_test_epoch:
(NORMAL      ) IterTimerHook                      
 -------------------- 
before_test_iter:
(NORMAL      ) IterTimerHook                      
 -------------------- 
after_test_iter:
(NORMAL      ) IterTimerHook                      
(BELOW_NORMAL) LoggerHook                         
 -------------------- 
after_test_epoch:
(VERY_HIGH   ) RuntimeInfoHook                    
(NORMAL      ) IterTimerHook                      
(BELOW_NORMAL) LoggerHook                         
 -------------------- 
after_test:
(VERY_HIGH   ) RuntimeInfoHook                    
 -------------------- 
after_run:
(BELOW_NORMAL) LoggerHook                         
 -------------------- 
2023/08/25 11:50:18 - mmengine - WARNING - Dataset Dataset has no metainfo. ``dataset_meta`` in visualizer will be None.
2023/08/25 11:50:18 - mmengine - WARNING - The prefix is not set in metric class MeanDice.
2023/08/25 11:50:18 - mmengine - WARNING - Dataset Dataset has no metainfo. ``dataset_meta`` in evaluator, metric and visualizer will be None.
2023/08/25 11:50:18 - mmengine - INFO - Hooks will be executed in the following order:
before_run:
(VERY_HIGH   ) RuntimeInfoHook                    
(BELOW_NORMAL) LoggerHook                         
 -------------------- 
before_train:
(VERY_HIGH   ) RuntimeInfoHook                    
(NORMAL      ) IterTimerHook                      
(VERY_LOW    ) CheckpointHook                     
 -------------------- 
before_train_epoch:
(VERY_HIGH   ) RuntimeInfoHook                    
(NORMAL      ) IterTimerHook                      
(NORMAL      ) DistSamplerSeedHook                
 -------------------- 
before_train_iter:
(VERY_HIGH   ) RuntimeInfoHook                    
(NORMAL      ) IterTimerHook                      
 -------------------- 
after_train_iter:
(VERY_HIGH   ) RuntimeInfoHook                    
(NORMAL      ) IterTimerHook                      
(BELOW_NORMAL) LoggerHook                         
(LOW         ) ParamSchedulerHook                 
(VERY_LOW    ) CheckpointHook                     
 -------------------- 
after_train_epoch:
(NORMAL      ) IterTimerHook                      
(LOW         ) ParamSchedulerHook                 
(VERY_LOW    ) CheckpointHook                     
 -------------------- 
before_val:
(VERY_HIGH   ) RuntimeInfoHook                    
 -------------------- 
before_val_epoch:
(NORMAL      ) IterTimerHook                      
 -------------------- 
before_val_iter:
(NORMAL      ) IterTimerHook                      
 -------------------- 
after_val_iter:
(NORMAL      ) IterTimerHook                      
(BELOW_NORMAL) LoggerHook                         
 -------------------- 
after_val_epoch:
(VERY_HIGH   ) RuntimeInfoHook                    
(NORMAL      ) IterTimerHook                      
(BELOW_NORMAL) LoggerHook                         
(LOW         ) ParamSchedulerHook                 
(VERY_LOW    ) CheckpointHook                     
 -------------------- 
after_val:
(VERY_HIGH   ) RuntimeInfoHook                    
 -------------------- 
after_train:
(VERY_HIGH   ) RuntimeInfoHook                    
(VERY_LOW    ) CheckpointHook                     
 -------------------- 
before_test:
(VERY_HIGH   ) RuntimeInfoHook                    
 -------------------- 
before_test_epoch:
(NORMAL      ) IterTimerHook                      
 -------------------- 
before_test_iter:
(NORMAL      ) IterTimerHook                      
 -------------------- 
after_test_iter:
(NORMAL      ) IterTimerHook                      
(BELOW_NORMAL) LoggerHook                         
 -------------------- 
after_test_epoch:
(VERY_HIGH   ) RuntimeInfoHook                    
(NORMAL      ) IterTimerHook                      
(BELOW_NORMAL) LoggerHook                         
 -------------------- 
after_test:
(VERY_HIGH   ) RuntimeInfoHook                    
 -------------------- 
after_run:
(BELOW_NORMAL) LoggerHook                         
 -------------------- 
Name of parameter - Initialization information

model.swinViT.patch_embed.proj.weight - torch.Size([48, 1, 2, 2, 2]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.patch_embed.proj.bias - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.norm1.weight - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.norm1.bias - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.attn.relative_position_bias_table - torch.Size([2197, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.attn.qkv.weight - torch.Size([144, 48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.attn.qkv.bias - torch.Size([144]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.attn.proj.weight - torch.Size([48, 48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.attn.proj.bias - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.norm2.weight - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.norm2.bias - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.mlp.linear1.weight - torch.Size([192, 48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.mlp.linear1.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.mlp.linear2.weight - torch.Size([48, 192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.mlp.linear2.bias - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.norm1.weight - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.norm1.bias - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.attn.relative_position_bias_table - torch.Size([2197, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.attn.qkv.weight - torch.Size([144, 48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.attn.qkv.bias - torch.Size([144]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.attn.proj.weight - torch.Size([48, 48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.attn.proj.bias - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.norm2.weight - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.norm2.bias - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.mlp.linear1.weight - torch.Size([192, 48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.mlp.linear1.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.mlp.linear2.weight - torch.Size([48, 192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.mlp.linear2.bias - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.downsample.reduction.weight - torch.Size([96, 384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.downsample.norm.weight - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.downsample.norm.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.norm1.weight - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.norm1.bias - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.attn.relative_position_bias_table - torch.Size([2197, 6]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.attn.qkv.weight - torch.Size([288, 96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.attn.qkv.bias - torch.Size([288]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.attn.proj.weight - torch.Size([96, 96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.attn.proj.bias - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.norm2.weight - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.norm2.bias - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.mlp.linear1.weight - torch.Size([384, 96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.mlp.linear1.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.mlp.linear2.weight - torch.Size([96, 384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.mlp.linear2.bias - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.norm1.weight - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.norm1.bias - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.attn.relative_position_bias_table - torch.Size([2197, 6]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.attn.qkv.weight - torch.Size([288, 96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.attn.qkv.bias - torch.Size([288]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.attn.proj.weight - torch.Size([96, 96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.attn.proj.bias - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.norm2.weight - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.norm2.bias - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.mlp.linear1.weight - torch.Size([384, 96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.mlp.linear1.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.mlp.linear2.weight - torch.Size([96, 384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.mlp.linear2.bias - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.downsample.reduction.weight - torch.Size([192, 768]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.downsample.norm.weight - torch.Size([768]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.downsample.norm.bias - torch.Size([768]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.norm1.weight - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.norm1.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.attn.relative_position_bias_table - torch.Size([2197, 12]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.attn.qkv.weight - torch.Size([576, 192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.attn.qkv.bias - torch.Size([576]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.attn.proj.weight - torch.Size([192, 192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.attn.proj.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.norm2.weight - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.norm2.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.mlp.linear1.weight - torch.Size([768, 192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.mlp.linear1.bias - torch.Size([768]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.mlp.linear2.weight - torch.Size([192, 768]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.mlp.linear2.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.norm1.weight - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.norm1.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.attn.relative_position_bias_table - torch.Size([2197, 12]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.attn.qkv.weight - torch.Size([576, 192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.attn.qkv.bias - torch.Size([576]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.attn.proj.weight - torch.Size([192, 192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.attn.proj.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.norm2.weight - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.norm2.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.mlp.linear1.weight - torch.Size([768, 192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.mlp.linear1.bias - torch.Size([768]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.mlp.linear2.weight - torch.Size([192, 768]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.mlp.linear2.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.downsample.reduction.weight - torch.Size([384, 1536]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.downsample.norm.weight - torch.Size([1536]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.downsample.norm.bias - torch.Size([1536]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.norm1.weight - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.norm1.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.attn.relative_position_bias_table - torch.Size([2197, 24]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.attn.qkv.weight - torch.Size([1152, 384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.attn.qkv.bias - torch.Size([1152]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.attn.proj.weight - torch.Size([384, 384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.attn.proj.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.norm2.weight - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.norm2.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.mlp.linear1.weight - torch.Size([1536, 384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.mlp.linear1.bias - torch.Size([1536]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.mlp.linear2.weight - torch.Size([384, 1536]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.mlp.linear2.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.norm1.weight - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.norm1.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.attn.relative_position_bias_table - torch.Size([2197, 24]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.attn.qkv.weight - torch.Size([1152, 384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.attn.qkv.bias - torch.Size([1152]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.attn.proj.weight - torch.Size([384, 384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.attn.proj.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.norm2.weight - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.norm2.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.mlp.linear1.weight - torch.Size([1536, 384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.mlp.linear1.bias - torch.Size([1536]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.mlp.linear2.weight - torch.Size([384, 1536]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.mlp.linear2.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.downsample.reduction.weight - torch.Size([768, 3072]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.downsample.norm.weight - torch.Size([3072]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.downsample.norm.bias - torch.Size([3072]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder1.layer.conv1.conv.weight - torch.Size([48, 1, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder1.layer.conv2.conv.weight - torch.Size([48, 48, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder1.layer.conv3.conv.weight - torch.Size([48, 1, 1, 1, 1]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder2.layer.conv1.conv.weight - torch.Size([48, 48, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder2.layer.conv2.conv.weight - torch.Size([48, 48, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder3.layer.conv1.conv.weight - torch.Size([96, 96, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder3.layer.conv2.conv.weight - torch.Size([96, 96, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder4.layer.conv1.conv.weight - torch.Size([192, 192, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder4.layer.conv2.conv.weight - torch.Size([192, 192, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder10.layer.conv1.conv.weight - torch.Size([768, 768, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder10.layer.conv2.conv.weight - torch.Size([768, 768, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder5.transp_conv.conv.weight - torch.Size([768, 384, 2, 2, 2]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder5.conv_block.conv1.conv.weight - torch.Size([384, 768, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder5.conv_block.conv2.conv.weight - torch.Size([384, 384, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder5.conv_block.conv3.conv.weight - torch.Size([384, 768, 1, 1, 1]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder4.transp_conv.conv.weight - torch.Size([384, 192, 2, 2, 2]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder4.conv_block.conv1.conv.weight - torch.Size([192, 384, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder4.conv_block.conv2.conv.weight - torch.Size([192, 192, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder4.conv_block.conv3.conv.weight - torch.Size([192, 384, 1, 1, 1]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder3.transp_conv.conv.weight - torch.Size([192, 96, 2, 2, 2]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder3.conv_block.conv1.conv.weight - torch.Size([96, 192, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder3.conv_block.conv2.conv.weight - torch.Size([96, 96, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder3.conv_block.conv3.conv.weight - torch.Size([96, 192, 1, 1, 1]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder2.transp_conv.conv.weight - torch.Size([96, 48, 2, 2, 2]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder2.conv_block.conv1.conv.weight - torch.Size([48, 96, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder2.conv_block.conv2.conv.weight - torch.Size([48, 48, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder2.conv_block.conv3.conv.weight - torch.Size([48, 96, 1, 1, 1]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder1.transp_conv.conv.weight - torch.Size([48, 48, 2, 2, 2]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder1.conv_block.conv1.conv.weight - torch.Size([48, 96, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder1.conv_block.conv2.conv.weight - torch.Size([48, 48, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder1.conv_block.conv3.conv.weight - torch.Size([48, 96, 1, 1, 1]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.out.conv.conv.weight - torch.Size([9, 48, 1, 1, 1]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.out.conv.conv.bias - torch.Size([9]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  
2023/08/25 11:50:18 - mmengine - WARNING - Dataset Dataset has no metainfo. ``dataset_meta`` in visualizer will be None.
2023/08/25 11:50:18 - mmengine - WARNING - The prefix is not set in metric class MeanDice.
2023/08/25 11:50:18 - mmengine - WARNING - Dataset Dataset has no metainfo. ``dataset_meta`` in evaluator, metric and visualizer will be None.
2023/08/25 11:50:18 - mmengine - WARNING - "FileClient" will be deprecated in future. Please use io functions in https://mmengine.readthedocs.io/en/latest/api/fileio.html#file-io
2023/08/25 11:50:18 - mmengine - WARNING - "HardDiskBackend" is the alias of "LocalBackend" and the former will be deprecated in future.
2023/08/25 11:50:18 - mmengine - INFO - Checkpoints will be saved to /public/home/three_whyz123/yhfu/yfliu/run/02high_dimensional_phenotyping/CT/self_supervised_learning/Swin_UNETR/output/training_logs/ssl_model_performance_test/reduce_manual_labeling_efforts/percent_0.1_pretrained.
Name of parameter - Initialization information

model.swinViT.patch_embed.proj.weight - torch.Size([48, 1, 2, 2, 2]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.patch_embed.proj.bias - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.norm1.weight - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.norm1.bias - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.attn.relative_position_bias_table - torch.Size([2197, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.attn.qkv.weight - torch.Size([144, 48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.attn.qkv.bias - torch.Size([144]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.attn.proj.weight - torch.Size([48, 48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.attn.proj.bias - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.norm2.weight - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.norm2.bias - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.mlp.linear1.weight - torch.Size([192, 48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.mlp.linear1.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.mlp.linear2.weight - torch.Size([48, 192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.0.mlp.linear2.bias - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.norm1.weight - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.norm1.bias - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.attn.relative_position_bias_table - torch.Size([2197, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.attn.qkv.weight - torch.Size([144, 48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.attn.qkv.bias - torch.Size([144]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.attn.proj.weight - torch.Size([48, 48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.attn.proj.bias - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.norm2.weight - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.norm2.bias - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.mlp.linear1.weight - torch.Size([192, 48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.mlp.linear1.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.mlp.linear2.weight - torch.Size([48, 192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.blocks.1.mlp.linear2.bias - torch.Size([48]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.downsample.reduction.weight - torch.Size([96, 384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.downsample.norm.weight - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers1.0.downsample.norm.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.norm1.weight - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.norm1.bias - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.attn.relative_position_bias_table - torch.Size([2197, 6]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.attn.qkv.weight - torch.Size([288, 96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.attn.qkv.bias - torch.Size([288]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.attn.proj.weight - torch.Size([96, 96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.attn.proj.bias - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.norm2.weight - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.norm2.bias - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.mlp.linear1.weight - torch.Size([384, 96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.mlp.linear1.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.mlp.linear2.weight - torch.Size([96, 384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.0.mlp.linear2.bias - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.norm1.weight - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.norm1.bias - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.attn.relative_position_bias_table - torch.Size([2197, 6]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.attn.qkv.weight - torch.Size([288, 96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.attn.qkv.bias - torch.Size([288]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.attn.proj.weight - torch.Size([96, 96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.attn.proj.bias - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.norm2.weight - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.norm2.bias - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.mlp.linear1.weight - torch.Size([384, 96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.mlp.linear1.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.mlp.linear2.weight - torch.Size([96, 384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.blocks.1.mlp.linear2.bias - torch.Size([96]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.downsample.reduction.weight - torch.Size([192, 768]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.downsample.norm.weight - torch.Size([768]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers2.0.downsample.norm.bias - torch.Size([768]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.norm1.weight - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.norm1.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.attn.relative_position_bias_table - torch.Size([2197, 12]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.attn.qkv.weight - torch.Size([576, 192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.attn.qkv.bias - torch.Size([576]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.attn.proj.weight - torch.Size([192, 192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.attn.proj.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.norm2.weight - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.norm2.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.mlp.linear1.weight - torch.Size([768, 192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.mlp.linear1.bias - torch.Size([768]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.mlp.linear2.weight - torch.Size([192, 768]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.0.mlp.linear2.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.norm1.weight - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.norm1.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.attn.relative_position_bias_table - torch.Size([2197, 12]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.attn.qkv.weight - torch.Size([576, 192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.attn.qkv.bias - torch.Size([576]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.attn.proj.weight - torch.Size([192, 192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.attn.proj.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.norm2.weight - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.norm2.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.mlp.linear1.weight - torch.Size([768, 192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.mlp.linear1.bias - torch.Size([768]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.mlp.linear2.weight - torch.Size([192, 768]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.blocks.1.mlp.linear2.bias - torch.Size([192]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.downsample.reduction.weight - torch.Size([384, 1536]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.downsample.norm.weight - torch.Size([1536]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers3.0.downsample.norm.bias - torch.Size([1536]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.norm1.weight - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.norm1.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.attn.relative_position_bias_table - torch.Size([2197, 24]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.attn.qkv.weight - torch.Size([1152, 384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.attn.qkv.bias - torch.Size([1152]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.attn.proj.weight - torch.Size([384, 384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.attn.proj.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.norm2.weight - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.norm2.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.mlp.linear1.weight - torch.Size([1536, 384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.mlp.linear1.bias - torch.Size([1536]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.mlp.linear2.weight - torch.Size([384, 1536]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.0.mlp.linear2.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.norm1.weight - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.norm1.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.attn.relative_position_bias_table - torch.Size([2197, 24]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.attn.qkv.weight - torch.Size([1152, 384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.attn.qkv.bias - torch.Size([1152]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.attn.proj.weight - torch.Size([384, 384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.attn.proj.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.norm2.weight - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.norm2.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.mlp.linear1.weight - torch.Size([1536, 384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.mlp.linear1.bias - torch.Size([1536]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.mlp.linear2.weight - torch.Size([384, 1536]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.blocks.1.mlp.linear2.bias - torch.Size([384]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.downsample.reduction.weight - torch.Size([768, 3072]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.downsample.norm.weight - torch.Size([3072]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.swinViT.layers4.0.downsample.norm.bias - torch.Size([3072]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder1.layer.conv1.conv.weight - torch.Size([48, 1, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder1.layer.conv2.conv.weight - torch.Size([48, 48, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder1.layer.conv3.conv.weight - torch.Size([48, 1, 1, 1, 1]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder2.layer.conv1.conv.weight - torch.Size([48, 48, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder2.layer.conv2.conv.weight - torch.Size([48, 48, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder3.layer.conv1.conv.weight - torch.Size([96, 96, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder3.layer.conv2.conv.weight - torch.Size([96, 96, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder4.layer.conv1.conv.weight - torch.Size([192, 192, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder4.layer.conv2.conv.weight - torch.Size([192, 192, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder10.layer.conv1.conv.weight - torch.Size([768, 768, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.encoder10.layer.conv2.conv.weight - torch.Size([768, 768, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder5.transp_conv.conv.weight - torch.Size([768, 384, 2, 2, 2]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder5.conv_block.conv1.conv.weight - torch.Size([384, 768, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder5.conv_block.conv2.conv.weight - torch.Size([384, 384, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder5.conv_block.conv3.conv.weight - torch.Size([384, 768, 1, 1, 1]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder4.transp_conv.conv.weight - torch.Size([384, 192, 2, 2, 2]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder4.conv_block.conv1.conv.weight - torch.Size([192, 384, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder4.conv_block.conv2.conv.weight - torch.Size([192, 192, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder4.conv_block.conv3.conv.weight - torch.Size([192, 384, 1, 1, 1]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder3.transp_conv.conv.weight - torch.Size([192, 96, 2, 2, 2]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder3.conv_block.conv1.conv.weight - torch.Size([96, 192, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder3.conv_block.conv2.conv.weight - torch.Size([96, 96, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder3.conv_block.conv3.conv.weight - torch.Size([96, 192, 1, 1, 1]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder2.transp_conv.conv.weight - torch.Size([96, 48, 2, 2, 2]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder2.conv_block.conv1.conv.weight - torch.Size([48, 96, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder2.conv_block.conv2.conv.weight - torch.Size([48, 48, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder2.conv_block.conv3.conv.weight - torch.Size([48, 96, 1, 1, 1]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder1.transp_conv.conv.weight - torch.Size([48, 48, 2, 2, 2]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder1.conv_block.conv1.conv.weight - torch.Size([48, 96, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder1.conv_block.conv2.conv.weight - torch.Size([48, 48, 3, 3, 3]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.decoder1.conv_block.conv3.conv.weight - torch.Size([48, 96, 1, 1, 1]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.out.conv.conv.weight - torch.Size([9, 48, 1, 1, 1]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  

model.out.conv.conv.bias - torch.Size([9]): 
The value is the same before and after calling `init_weights` of MMSwinUNETR  
2023/08/25 11:50:18 - mmengine - WARNING - "FileClient" will be deprecated in future. Please use io functions in https://mmengine.readthedocs.io/en/latest/api/fileio.html#file-io
2023/08/25 11:50:18 - mmengine - WARNING - "HardDiskBackend" is the alias of "LocalBackend" and the former will be deprecated in future.
2023/08/25 11:50:18 - mmengine - INFO - Checkpoints will be saved to /public/home/three_whyz123/yhfu/yfliu/run/02high_dimensional_phenotyping/CT/self_supervised_learning/Swin_UNETR/output/training_logs/ssl_model_performance_test/reduce_manual_labeling_efforts/percent_0.1_pretrained.
2023/08/25 11:50:30 - mmengine - INFO - Exp name: percent_0.1_pretrained_20230825_115016
2023/08/25 11:50:30 - mmengine - INFO - Epoch(train)    [1][8/8]  lr: 4.3877e-07  eta: 6:32:39  time: 1.4732  data_time: 0.8846  memory: 9227  loss: 3.0262  global_loss: 3.0262
2023/08/25 11:50:30 - mmengine - INFO - Saving checkpoint at 1 epochs
2023/08/25 11:50:31 - mmengine - INFO - Exp name: percent_0.1_pretrained_20230825_115016
2023/08/25 11:50:31 - mmengine - INFO - Epoch(train)    [1][8/8]  lr: 4.3877e-07  eta: 6:58:41  time: 1.5709  data_time: 0.8989  memory: 9230  loss: 3.2593  global_loss: 3.2593
2023/08/25 11:50:31 - mmengine - INFO - Saving checkpoint at 1 epochs
2023/08/25 11:50:42 - mmengine - INFO - Epoch(val)    [1][10/24]    eta: 0:00:14  time: 1.0446  data_time: 0.9265  memory: 3786  
2023/08/25 11:50:44 - mmengine - INFO - Epoch(val)    [1][10/24]    eta: 0:00:15  time: 1.0759  data_time: 0.9410  memory: 3789  
2023/08/25 11:50:48 - mmengine - INFO - Epoch(val)    [1][20/24]    eta: 0:00:03  time: 0.5986  data_time: 0.4830  memory: 4461  
2023/08/25 11:50:49 - mmengine - INFO - Epoch(val)    [1][20/24]    eta: 0:00:03  time: 0.5713  data_time: 0.4533  memory: 4464  
2023/08/25 11:50:59 - mmengine - INFO - Epoch(val) [1][24/24]    mean_dice: 0.1435  data_time: 0.5874  time: 0.7059
2023/08/25 11:51:00 - mmengine - INFO - The best checkpoint with 0.1435 mean_dice at 1 epoch is saved to best_mean_dice_epoch_1.pth.
2023/08/25 11:51:00 - mmengine - INFO - Epoch(val) [1][24/24]    mean_dice: 0.0878  data_time: 0.5810  time: 0.7022
2023/08/25 11:51:01 - mmengine - INFO - The best checkpoint with 0.0878 mean_dice at 1 epoch is saved to best_mean_dice_epoch_1.pth.
2023/08/25 11:51:11 - mmengine - INFO - Exp name: percent_0.1_pretrained_20230825_115016
2023/08/25 11:51:11 - mmengine - INFO - Epoch(train)    [2][8/8]  lr: 9.3907e-07  eta: 5:40:15  time: 0.9588  data_time: 0.6123  memory: 9227  loss: 2.9758  global_loss: 2.9969

I can run successfully the distributed training examples
python -m torch.distributed.launch --nproc_per_node=8 examples/distributed_training.py --launcher pytorch

Looking forward to your reply~
Yangfan Liu
2023.8.25