How to train Detectron2: Instance Segmentation (R101-FPN 3x) with custom dataset

[winter2897]

Instructions To Reproduce the Issue:

Hi @ppwwyyxx,

I conducted Detectron2 training with TTPLA dataset using Resnet 101 FPN 3x architecture and model zoo respectively. However, after the training is completed, the model cannot detect any object. Did I set up the wrong configuration file or the environment problem?

1. Full Training Code:

Config YAML: https://github.com/facebookresearch/detectron2/blob/master/configs/COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x.yaml
Model Zoo: https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x/138205316/model_final_a3ec72.pkl

from detectron2.engine import DefaultTrainer
from detectron2.config import get_cfg
from detectron2.data import MetadataCatalog, DatasetCatalog
from detectron2.data.datasets import register_coco_instances
import os


register_coco_instances("RTR_dataset", {}, "./data/geo/trainval.json", "./data/geo/images")
RTR_dataset_metadata = MetadataCatalog.get("RTR_dataset")
dataset_dicts = DatasetCatalog.get("RTR_dataset")

cfg = get_cfg()
cfg.merge_from_file("./model/mask_rcnn_R_101_FPN_3x.yaml")
cfg.DATASETS.TRAIN = ("RTR_dataset",)
cfg.DATASETS.TEST = ()  # no metrics implemented for this dataset
cfg.DATALOADER.NUM_WORKERS = 2
cfg.MODEL.WEIGHTS = "./model/model_final_r101fpn3x.pkl"  # initialize from model zoo
cfg.SOLVER.IMS_PER_BATCH = 2
cfg.SOLVER.BASE_LR = 0.00025
cfg.SOLVER.MAX_ITER = 300 # 300 iterations seems good enough, but you can certainly train longer
cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 512 # faster, and good enough for this toy dataset
cfg.MODEL.ROI_HEADS.NUM_CLASSES = 4  # 4 classes 

os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)
trainer = DefaultTrainer(cfg)
trainer.resume_or_load(resume=False)
trainer.train()

2. Training command:

python train.py

3. Training Log:

[03/30 10:00:07 d2.engine.defaults]: Model:
GeneralizedRCNN(
  (backbone): FPN(
    (fpn_lateral2): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1))
    (fpn_output2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (fpn_lateral3): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1))
    (fpn_output3): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (fpn_lateral4): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))
    (fpn_output4): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (fpn_lateral5): Conv2d(2048, 256, kernel_size=(1, 1), stride=(1, 1))
    (fpn_output5): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (top_block): LastLevelMaxPool()
.........
(mask_fcn4): Conv2d(
        256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)
        (activation): ReLU()
      )
      (deconv): ConvTranspose2d(256, 256, kernel_size=(2, 2), stride=(2, 2))
      (deconv_relu): ReLU()
      (predictor): Conv2d(256, 4, kernel_size=(1, 1), stride=(1, 1))
    )
  )
)

[03/30 10:00:07 d2.data.datasets.coco]: Loaded 1240 images in COCO format from ./data/geo/trainval.json
[03/30 10:00:07 d2.data.build]: Removed 0 images with no usable annotations. 1240 images left.
[03/30 10:00:07 d2.data.build]: Distribution of instances among all 4 categories:
|   category   | #instances   |   category   | #instances   |   category    | #instances   |
|:------------:|:-------------|:------------:|:-------------|:-------------:|:-------------|
|    cable     | 10067        | tower_wooden | 333          | tower_lattice | 401          |
| tower_tucohy | 231          |              |              |               |              |
|    total     | 11032        |              |              |               |              |
[03/30 10:00:07 d2.data.dataset_mapper]: [DatasetMapper] Augmentations used in training: [ResizeShortestEdge(short_edge_length=(640, 672, 704, 736, 768, 800), max_size=1333, sample_style='choice'), RandomFlip()]
[03/30 10:00:07 d2.data.build]: Using training sampler TrainingSampler
[03/30 10:00:07 d2.data.common]: Serializing 1240 elements to byte tensors and concatenating them all ...
[03/30 10:00:07 d2.data.common]: Serialized dataset takes 4.16 MiB
WARNING [03/30 10:00:07 d2.solver.build]: SOLVER.STEPS contains values larger than SOLVER.MAX_ITER. These values will be ignored.
Skip loading parameter 'roi_heads.box_predictor.cls_score.weight' to the model due to incompatible shapes: (81, 1024) in the checkpoint but (5, 1024) in the model! You might want to double check if this is expected.
Skip loading parameter 'roi_heads.box_predictor.cls_score.bias' to the model due to incompatible shapes: (81,) in the checkpoint but (5,) in the model! You might want to double check if this is expected.
Skip loading parameter 'roi_heads.box_predictor.bbox_pred.weight' to the model due to incompatible shapes: (320, 1024) in the checkpoint but (16, 1024) in the model! You might want to double check if this is expected.
Skip loading parameter 'roi_heads.box_predictor.bbox_pred.bias' to the model due to incompatible shapes: (320,) in the checkpoint but (16,) in the model! You might want to double check if this is expected.
Skip loading parameter 'roi_heads.mask_head.predictor.weight' to the model due to incompatible shapes: (80, 256, 1, 1) in the checkpoint but (4, 256, 1, 1) in the model! You might want to double check if this is expected.
Skip loading parameter 'roi_heads.mask_head.predictor.bias' to the model due to incompatible shapes: (80,) in the checkpoint but (4,) in the model! You might want to double check if this is expected.
[03/30 10:00:08 d2.engine.train_loop]: Starting training from iteration 0
/home/quanrtr/miniconda3/envs/kun/lib/python3.6/site-packages/detectron2/structures/masks.py:348: UserWarning: This overload of nonzero is deprecated:
        nonzero()
Consider using one of the following signatures instead:
        nonzero(*, bool as_tuple) (Triggered internally at  /pytorch/torch/csrc/utils/python_arg_parser.cpp:882.)
  item = item.nonzero().squeeze(1).cpu().numpy().tolist()
/home/quanrtr/miniconda3/envs/kun/lib/python3.6/site-packages/detectron2/structures/masks.py:348: UserWarning: This overload of nonzero is deprecated:
        nonzero()
Consider using one of the following signatures instead:
        nonzero(*, bool as_tuple) (Triggered internally at  /pytorch/torch/csrc/utils/python_arg_parser.cpp:882.)
  item = item.nonzero().squeeze(1).cpu().numpy().tolist()
/home/quanrtr/miniconda3/envs/kun/lib/python3.6/site-packages/detectron2/modeling/roi_heads/fast_rcnn.py:103: UserWarning: This overload of nonzero is deprecated:
        nonzero()
Consider using one of the following signatures instead:
        nonzero(*, bool as_tuple) (Triggered internally at  /pytorch/torch/csrc/utils/python_arg_parser.cpp:882.)
  num_fg = fg_inds.nonzero().numel()
[03/30 10:00:20 d2.utils.events]:  eta: 0:02:46  iter: 19  total_loss: 4.103  loss_cls: 1.396  loss_box_reg: 0.1013  loss_mask: 0.6894  loss_rpn_cls: 1.479  loss_rpn_loc: 0.524  time: 0.5892  data_time: 0.0208  lr: 4.9953e-06  max_mem: 3623M
[03/30 10:00:32 d2.utils.events]:  eta: 0:02:36  iter: 39  total_loss: 4.407  loss_cls: 1.305  loss_box_reg: 0.09074  loss_mask: 0.6666  loss_rpn_cls: 1.759  loss_rpn_loc: 0.6491  time: 0.5962  data_time: 0.0025  lr: 9.9902e-06  max_mem: 3623M
[03/30 10:00:44 d2.utils.events]:  eta: 0:02:25  iter: 59  total_loss: 3.348  loss_cls: 1.131  loss_box_reg: 0.1867  loss_mask: 0.6406  loss_rpn_cls: 1.099  loss_rpn_loc: 0.37  time: 0.6007  data_time: 0.0026  lr: 1.4985e-05  max_mem: 4255M
[03/30 10:00:57 d2.utils.events]:  eta: 0:02:13  iter: 79  total_loss: 2.718  loss_cls: 0.8822  loss_box_reg: 0.2123  loss_mask: 0.5814  loss_rpn_cls: 0.5217  loss_rpn_loc: 0.3537  time: 0.6042  data_time: 0.0026  lr: 1.998e-05  max_mem: 4255M
[03/30 10:01:09 d2.utils.events]:  eta: 0:02:01  iter: 99  total_loss: 2.382  loss_cls: 0.6559  loss_box_reg: 0.2144  loss_mask: 0.547  loss_rpn_cls: 0.4541  loss_rpn_loc: 0.3723  time: 0.6044  data_time: 0.0026  lr: 2.4975e-05  max_mem: 4255M
[03/30 10:01:21 d2.utils.events]:  eta: 0:01:49  iter: 119  total_loss: 1.601  loss_cls: 0.5031  loss_box_reg: 0.2091  loss_mask: 0.4781  loss_rpn_cls: 0.1126  loss_rpn_loc: 0.3208  time: 0.6061  data_time: 0.0027  lr: 2.997e-05  max_mem: 4255M
[03/30 10:01:34 d2.utils.events]:  eta: 0:01:37  iter: 139  total_loss: 1.705  loss_cls: 0.4886  loss_box_reg: 0.3325  loss_mask: 0.4104  loss_rpn_cls: 0.106  loss_rpn_loc: 0.2543  time: 0.6069  data_time: 0.0027  lr: 3.4965e-05  max_mem: 4311M
[03/30 10:01:46 d2.utils.events]:  eta: 0:01:25  iter: 159  total_loss: 1.466  loss_cls: 0.3541  loss_box_reg: 0.2419  loss_mask: 0.3701  loss_rpn_cls: 0.1793  loss_rpn_loc: 0.2974  time: 0.6092  data_time: 0.0025  lr: 3.996e-05  max_mem: 4311M
[03/30 10:01:59 d2.utils.events]:  eta: 0:01:13  iter: 179  total_loss: 1.618  loss_cls: 0.4474  loss_box_reg: 0.3361  loss_mask: 0.3178  loss_rpn_cls: 0.09992  loss_rpn_loc: 0.355  time: 0.6123  data_time: 0.0025  lr: 4.4955e-05  max_mem: 4320M
[03/30 10:02:11 d2.utils.events]:  eta: 0:01:01  iter: 199  total_loss: 1.409  loss_cls: 0.3286  loss_box_reg: 0.2704  loss_mask: 0.2794  loss_rpn_cls: 0.07947  loss_rpn_loc: 0.3234  time: 0.6134  data_time: 0.0026  lr: 4.995e-05  max_mem: 4320M
[03/30 10:02:24 d2.utils.events]:  eta: 0:00:49  iter: 219  total_loss: 1.418  loss_cls: 0.3585  loss_box_reg: 0.3242  loss_mask: 0.2447  loss_rpn_cls: 0.09826  loss_rpn_loc: 0.2729  time: 0.6131  data_time: 0.0025  lr: 5.4945e-05  max_mem: 4320M
[03/30 10:02:36 d2.utils.events]:  eta: 0:00:37  iter: 239  total_loss: 1.27  loss_cls: 0.3096  loss_box_reg: 0.3652  loss_mask: 0.2166  loss_rpn_cls: 0.09202  loss_rpn_loc: 0.23  time: 0.6146  data_time: 0.0025  lr: 5.994e-05  max_mem: 4338M
[03/30 10:02:49 d2.utils.events]:  eta: 0:00:24  iter: 259  total_loss: 1.107  loss_cls: 0.2624  loss_box_reg: 0.3041  loss_mask: 0.2128  loss_rpn_cls: 0.08282  loss_rpn_loc: 0.2305  time: 0.6151  data_time: 0.0027  lr: 6.4935e-05  max_mem: 4338M
[03/30 10:03:02 d2.utils.events]:  eta: 0:00:12  iter: 279  total_loss: 1.448  loss_cls: 0.3245  loss_box_reg: 0.4043  loss_mask: 0.211  loss_rpn_cls: 0.07957  loss_rpn_loc: 0.4302  time: 0.6161  data_time: 0.0027  lr: 6.993e-05  max_mem: 4338M
[03/30 10:03:17 d2.utils.events]:  eta: 0:00:00  iter: 299  total_loss: 1.31  loss_cls: 0.2739  loss_box_reg: 0.3201  loss_mask: 0.2107  loss_rpn_cls: 0.1004  loss_rpn_loc: 0.2654  time: 0.6194  data_time: 0.0027  lr: 7.4925e-05  max_mem: 4338M
[03/30 10:03:20 d2.engine.hooks]: Overall training speed: 298 iterations in 0:03:04 (0.6194 s / it)
[03/30 10:03:20 d2.engine.hooks]: Total training time: 0:03:10 (0:00:06 on hooks)

4. Detect Code:

import numpy as np
import os, cv2
from detectron2.engine import DefaultPredictor
from detectron2.config import get_cfg
from detectron2.utils.visualizer import Visualizer
from detectron2.utils.visualizer import ColorMode
from detectron2.data import MetadataCatalog
os.environ['DISPLAY'] = ':0'

cfg = get_cfg()
cfg.merge_from_file("./model/mask_rcnn_R_101_FPN_3x.yaml")
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5  # set threshold for this model
cfg.MODEL.WEIGHTS = ("./output/model_final.pth")

img_path="./imgs_test/geo/31_7785.jpg"

im = cv2.imread(img_path)
predictor = DefaultPredictor(cfg)
outputs = predictor(im)

print(outputs["instances"].pred_classes)
print(outputs["instances"].pred_boxes)
print(outputs["instances"].pred_masks)

v = Visualizer(im[:, :, ::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=0.2, instance_mode=ColorMode.IMAGE)
out = v.draw_instance_predictions(outputs["instances"].to("cpu"))

cv2.imshow('show solar', out.get_image()[:, :, ::-1])
cv2.waitKey(0)
cv2.imwrite('./output.jpg', out.get_image()[:, :, ::-1])   
cv2.destroyAllWindows()

5. Run Detect Log

Skip loading parameter 'roi_heads.box_predictor.cls_score.weight' to the model due to incompatible shapes: (5, 1024) in the checkpoint but (81, 1024) in the model! You might want to double check if this is expected.
Skip loading parameter 'roi_heads.box_predictor.cls_score.bias' to the model due to incompatible shapes: (5,) in the checkpoint but (81,) in the model! You might want to double check if this is expected.
Skip loading parameter 'roi_heads.box_predictor.bbox_pred.weight' to the model due to incompatible shapes: (16, 1024) in the checkpoint but (320, 1024) in the model! You might want to double check if this is expected.
Skip loading parameter 'roi_heads.box_predictor.bbox_pred.bias' to the model due to incompatible shapes: (16,) in the checkpoint but (320,) in the model! You might want to double check if this is expected.
Skip loading parameter 'roi_heads.mask_head.predictor.weight' to the model due to incompatible shapes: (4, 256, 1, 1) in the checkpoint but (80, 256, 1, 1) in the model! You might want to double check if this is expected.
Skip loading parameter 'roi_heads.mask_head.predictor.bias' to the model due to incompatible shapes: (4,) in the checkpoint but (80,) in the model! You might want to double check if this is expected.
/home/quanrtr/miniconda3/envs/kun/lib/python3.6/site-packages/detectron2/modeling/roi_heads/fast_rcnn.py:154: UserWarning: This overload of nonzero is deprecated:
        nonzero()
Consider using one of the following signatures instead:
        nonzero(*, bool as_tuple) (Triggered internally at  /pytorch/torch/csrc/utils/python_arg_parser.cpp:882.)
  filter_inds = filter_mask.nonzero()
tensor([], device='cuda:0', dtype=torch.int64)
Boxes(tensor([], device='cuda:0', size=(0, 4)))
tensor([], device='cuda:0', size=(0, 2160, 3840), dtype=torch.uint8)

6. Visualization

The trained model cannot detect any object:

Environment:

RTX 3060 with CUDA : 11.0, cuDNN: 8.0.5 and Pytorch: 1.7.1

Thanks for any insight!

[winter2897]

Update :

I fixed this according to instruction #1974. However it still exists.

[04/01 11:40:06 d2.engine.train_loop]: Starting training from iteration 0
/home/quanrtr/miniconda3/envs/kun/lib/python3.6/site-packages/detectron2/structures/masks.py:348: UserWarning: This overload of nonzero is deprecated:
        nonzero()
Consider using one of the following signatures instead:
        nonzero(*, bool as_tuple) (Triggered internally at  /pytorch/torch/csrc/utils/python_arg_parser.cpp:882.)
  item = item.nonzero().squeeze(1).cpu().numpy().tolist()
/home/quanrtr/miniconda3/envs/kun/lib/python3.6/site-packages/detectron2/structures/masks.py:348: UserWarning: This overload of nonzero is deprecated:
        nonzero()
Consider using one of the following signatures instead:
        nonzero(*, bool as_tuple) (Triggered internally at  /pytorch/torch/csrc/utils/python_arg_parser.cpp:882.)
  item = item.nonzero().squeeze(1).cpu().numpy().tolist()
/home/quanrtr/miniconda3/envs/kun/lib/python3.6/site-packages/detectron2/modeling/roi_heads/fast_rcnn.py:103: UserWarning: This overload of nonzero is deprecated:
        nonzero()
Consider using one of the following signatures instead:
        nonzero(*, bool as_tuple) (Triggered internally at  /pytorch/torch/csrc/utils/python_arg_parser.cpp:882.)
  num_fg = fg_inds.nonzero().numel()

[winter2897]

I have solved the problem. The reason is that I haven't defined a new CLASS NAME and NUMBER CLASS for my custom dataset.

[winter2897]

import detectron2
from detectron2.utils.logger import setup_logger
setup_logger()
import os
# import some common libraries
import matplotlib.pyplot as plt
import numpy as np
import cv2

# import some common detectron2 utilities
from detectron2.engine import DefaultPredictor
from detectron2.config import get_cfg
from detectron2.utils.visualizer import Visualizer
from detectron2.data import MetadataCatalog, DatasetCatalog

from detectron2.data.datasets import register_coco_instances
register_coco_instances("RTR", {}, "./rtr_crop/trainval.json", "./rtr_crop/images")

RTR_metadata = MetadataCatalog.get("RTR")
dataset_dicts = DatasetCatalog.get("RTR")

import random
import cv2
from detectron2 import model_zoo
from detectron2.engine import DefaultTrainer
from detectron2.config import get_cfg


cfg = get_cfg()
cfg.merge_from_file(model_zoo.get_config_file("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml"))
cfg.DATASETS.TRAIN = ("RTR",)
cfg.DATASETS.TEST = ()   # no metrics implemented for this dataset
cfg.MODEL.ROI_HEADS.NUM_CLASSES = 4  # 3 classes (data, fig, hazelnut)

os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)

# Prediction
cfg.MODEL.WEIGHTS = os.path.join("./output/", "model_0009999.pth")
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5   # set the testing threshold for this model
cfg.DATASETS.TEST = ("RTR", )
predictor = DefaultPredictor(cfg)

from detectron2.utils.visualizer import ColorMode

test_img_path = './rtr_crop/images/DM21-001_Sametel-031121__0000_A_split_0.png'
im = cv2.imread(test_img_path)
outputs = predictor(im)
v = Visualizer(im[:, :, ::-1],
                   metadata=RTR_metadata, 
                   scale=0.8, 
                   instance_mode=ColorMode.IMAGE_BW)   # remove the colors of unsegmented pixels

v = v.draw_instance_predictions(outputs["instances"].to("cpu"))
# cv2_imshow(v.get_image()[:, :, ::-1])
save = cv2.imwrite('predict1.png', v.get_image()[:, :, ::-1])
cv2.imshow('Img', v.get_image()[:, :, ::-1])
cv2.waitKey(0)
cv2.destroyAllWindows()