diff --git a/detectron2/data/dataset_mapper.py b/detectron2/data/dataset_mapper.py
index a8714f7990..da00a87a20 100644
--- a/detectron2/data/dataset_mapper.py
+++ b/detectron2/data/dataset_mapper.py
@@ -156,7 +156,7 @@ def __call__(self, dataset_dict):
 
         # USER: Remove if you don't do semantic/panoptic segmentation.
         if "sem_seg_file_name" in dataset_dict:
-            sem_seg_gt = utils.read_image(dataset_dict.pop("sem_seg_file_name"), "L").squeeze(2)
+            sem_seg_gt = utils.read_image(dataset_dict.pop("sem_seg_file_name")).astype("double")
         else:
             sem_seg_gt = None