Merge pull request opencv#17489 from Lapshin:ffmpeg_cap_consider_rotation_metadata_3.4

Author: mshabunin

modules/videoio/include/opencv2/videoio.hpp

@@ -177,6 +177,8 @@ enum VideoCaptureProperties {
        CAP_PROP_WB_TEMPERATURE=45, //!< white-balance color temperature
        CAP_PROP_CODEC_PIXEL_FORMAT =46,    //!< (read-only) codec's pixel format. 4-character code - see VideoWriter::fourcc . Subset of [AV_PIX_FMT_*](https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/raw.c) or -1 if unknown
        CAP_PROP_BITRATE       =47, //!< (read-only) Video bitrate in kbits/s
+       CAP_PROP_ORIENTATION_META=48, //!< (read-only) Frame rotation defined by stream meta (applicable for FFmpeg back-end only)
+       CAP_PROP_ORIENTATION_AUTO=49, //!< if true - rotates output frames of CvCapture considering video file's metadata  (applicable for FFmpeg back-end only) (https://github.com/opencv/opencv/issues/15499)
 #ifndef CV_DOXYGEN
        CV__CAP_PROP_LATEST
 #endif

modules/videoio/src/cap_ffmpeg.cpp

@@ -235,7 +235,11 @@ class CvCapture_FFMPEG_proxy CV_FINAL : public cv::IVideoCapture
         if (!ffmpegCapture ||
            !icvRetrieveFrame_FFMPEG_p(ffmpegCapture, &data, &step, &width, &height, &cn))
             return false;
-        cv::Mat(height, width, CV_MAKETYPE(CV_8U, cn), data, step).copyTo(frame);
+
+        cv::Mat tmp(height, width, CV_MAKETYPE(CV_8U, cn), data, step);
+        this->rotateFrame(tmp);
+        tmp.copyTo(frame);
+
         return true;
     }
     virtual bool open( const cv::String& filename )
@@ -262,6 +266,30 @@ class CvCapture_FFMPEG_proxy CV_FINAL : public cv::IVideoCapture
 
 protected:
     CvCapture_FFMPEG* ffmpegCapture;
+
+    void rotateFrame(cv::Mat &mat) const
+    {
+        bool rotation_auto = 0 != getProperty(CV_FFMPEG_CAP_PROP_ORIENTATION_AUTO);
+        int rotation_angle = static_cast<int>(getProperty(CV_FFMPEG_CAP_PROP_ORIENTATION_META));
+
+        if(!rotation_auto || rotation_angle%360 == 0)
+        {
+            return;
+        }
+
+        cv::RotateFlags flag;
+        if(rotation_angle == 90 || rotation_angle == -270) { // Rotate clockwise 90 degrees
+            flag = cv::ROTATE_90_CLOCKWISE;
+        } else if(rotation_angle == 270 || rotation_angle == -90) { // Rotate clockwise 270 degrees
+            flag = cv::ROTATE_90_COUNTERCLOCKWISE;
+        } else if(rotation_angle == 180 || rotation_angle == -180) { // Rotate clockwise 180 degrees
+            flag = cv::ROTATE_180;
+        } else { // Unsupported rotation
+            return;
+        }
+
+        cv::rotate(mat, mat, flag);
+    }
 };
 
 } // namespace

modules/videoio/src/cap_ffmpeg_api.hpp

@@ -28,7 +28,9 @@ enum
     CV_FFMPEG_CAP_PROP_SAR_NUM=40,
     CV_FFMPEG_CAP_PROP_SAR_DEN=41,
     CV_FFMPEG_CAP_PROP_CODEC_PIXEL_FORMAT=46,
-    CV_FFMPEG_CAP_PROP_BITRATE=47
+    CV_FFMPEG_CAP_PROP_BITRATE=47,
+    CV_FFMPEG_CAP_PROP_ORIENTATION_META=48,
+    CV_FFMPEG_CAP_PROP_ORIENTATION_AUTO=49
 };
 
 typedef struct CvCapture_FFMPEG CvCapture_FFMPEG;

modules/videoio/src/cap_ffmpeg_impl.hpp

@@ -482,6 +482,7 @@ struct CvCapture_FFMPEG
     bool setProperty(int, double);
     bool grabFrame();
     bool retrieveFrame(int, unsigned char** data, int* step, int* width, int* height, int* cn);
+    void rotateFrame(cv::Mat &mat) const;
 
     void init();
 
@@ -497,6 +498,7 @@ struct CvCapture_FFMPEG
     double  r2d(AVRational r) const;
     int64_t dts_to_frame_number(int64_t dts);
     double  dts_to_sec(int64_t dts) const;
+    void    get_rotation_angle();
 
     AVFormatContext * ic;
     AVCodec         * avcodec;
@@ -512,6 +514,8 @@ struct CvCapture_FFMPEG
 
     int64_t frame_number, first_frame_number;
 
+    bool   rotation_auto;
+    int    rotation_angle; // valid 0, 90, 180, 270
     double eps_zero;
 /*
    'filename' contains the filename of the videosource,
@@ -560,8 +564,17 @@ void CvCapture_FFMPEG::init()
     frame_number = 0;
     eps_zero = 0.000025;
 
-#if LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(52, 111, 0)
+    rotation_angle = 0;
+
+#if (LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(52, 111, 0))
+#if (LIBAVUTIL_BUILD >= CALC_FFMPEG_VERSION(52, 92, 100))
+    rotation_auto = true;
+#else
+    rotation_auto = false;
+#endif
     dict = NULL;
+#else
+    rotation_auto = false;
 #endif
 
     rawMode = false;
@@ -1032,6 +1045,7 @@ bool CvCapture_FFMPEG::open( const char* _filename )
             frame.cn = 3;
             frame.step = 0;
             frame.data = NULL;
+            get_rotation_angle();
             break;
         }
     }
@@ -1290,7 +1304,6 @@ bool CvCapture_FFMPEG::grabFrame()
     return valid;
 }
 
-
 bool CvCapture_FFMPEG::retrieveFrame(int, unsigned char** data, int* step, int* width, int* height, int* cn)
 {
     if (!video_st)
@@ -1376,7 +1389,6 @@ bool CvCapture_FFMPEG::retrieveFrame(int, unsigned char** data, int* step, int*
     return true;
 }
 
-
 double CvCapture_FFMPEG::getProperty( int property_id ) const
 {
     if( !video_st ) return 0;
@@ -1400,9 +1412,9 @@ double CvCapture_FFMPEG::getProperty( int property_id ) const
     case CV_FFMPEG_CAP_PROP_FRAME_COUNT:
         return (double)get_total_frames();
     case CV_FFMPEG_CAP_PROP_FRAME_WIDTH:
-        return (double)frame.width;
+        return (double)((rotation_auto && rotation_angle%180) ? frame.height : frame.width);
     case CV_FFMPEG_CAP_PROP_FRAME_HEIGHT:
-        return (double)frame.height;
+        return (double)((rotation_auto && rotation_angle%180) ? frame.width : frame.height);
     case CV_FFMPEG_CAP_PROP_FPS:
         return get_fps();
     case CV_FFMPEG_CAP_PROP_FOURCC:
@@ -1446,6 +1458,15 @@ double CvCapture_FFMPEG::getProperty( int property_id ) const
         break;
     case CV_FFMPEG_CAP_PROP_BITRATE:
         return static_cast<double>(get_bitrate());
+    case CV_FFMPEG_CAP_PROP_ORIENTATION_META:
+        return static_cast<double>(rotation_angle);
+    case CV_FFMPEG_CAP_PROP_ORIENTATION_AUTO:
+#if ((LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(52, 111, 0)) && \
+     (LIBAVUTIL_BUILD >= CALC_FFMPEG_VERSION(52, 94, 100)))
+        return static_cast<double>(rotation_auto);
+#else
+        return 0;
+#endif
     default:
         break;
     }
@@ -1524,6 +1545,17 @@ double CvCapture_FFMPEG::dts_to_sec(int64_t dts) const
         r2d(ic->streams[video_stream]->time_base);
 }
 
+void CvCapture_FFMPEG::get_rotation_angle()
+{
+    rotation_angle = 0;
+#if ((LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(52, 111, 0)) && \
+     (LIBAVUTIL_BUILD >= CALC_FFMPEG_VERSION(52, 94, 100)))
+    AVDictionaryEntry *rotate_tag = av_dict_get(video_st->metadata, "rotate", NULL, 0);
+    if (rotate_tag != NULL)
+        rotation_angle = atoi(rotate_tag->value);
+#endif
+}
+
 void CvCapture_FFMPEG::seek(int64_t _frame_number)
 {
     _frame_number = std::min(_frame_number, get_total_frames());
@@ -1619,6 +1651,16 @@ bool CvCapture_FFMPEG::setProperty( int property_id, double value )
         if (value == -1)
             return setRaw();
         return false;
+    case CV_FFMPEG_CAP_PROP_ORIENTATION_AUTO:
+#if ((LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(52, 111, 0)) && \
+     (LIBAVUTIL_BUILD >= CALC_FFMPEG_VERSION(52, 94, 100)))
+        rotation_auto = static_cast<bool>(value);
+        return true;
+#else
+        rotation_auto = 0;
+        return false;
+#endif
+        break;
     default:
         return false;
     }

modules/videoio/test/test_ffmpeg.cpp

@@ -636,5 +636,57 @@ const ffmpeg_cap_properties_param_t videoio_ffmpeg_properties[] = {
 
 INSTANTIATE_TEST_CASE_P(videoio, ffmpeg_cap_properties, testing::ValuesIn(videoio_ffmpeg_properties));
 
+// related issue: https://github.com/opencv/opencv/issues/15499
+TEST(videoio, mp4_orientation_meta_auto)
+{
+    string video_file = string(cvtest::TS::ptr()->get_data_path()) + "video/big_buck_bunny_rotated.mp4";
+
+    VideoCapture cap;
+    EXPECT_NO_THROW(cap.open(video_file, CAP_FFMPEG));
+    ASSERT_TRUE(cap.isOpened()) << "Can't open the video: " << video_file << " with backend " << CAP_FFMPEG << std::endl;
+
+    cap.set(CAP_PROP_ORIENTATION_AUTO, true);
+    if (cap.get(CAP_PROP_ORIENTATION_AUTO) == 0)
+        throw SkipTestException("FFmpeg frame rotation metadata is not supported");
+
+    Size actual;
+    EXPECT_NO_THROW(actual = Size((int)cap.get(CAP_PROP_FRAME_WIDTH),
+                                    (int)cap.get(CAP_PROP_FRAME_HEIGHT)));
+    EXPECT_EQ(384, actual.width);
+    EXPECT_EQ(672, actual.height);
+
+    Mat frame;
+
+    cap >> frame;
+
+    ASSERT_EQ(384, frame.cols);
+    ASSERT_EQ(672, frame.rows);
+}
+
+// related issue: https://github.com/opencv/opencv/issues/15499
+TEST(videoio, mp4_orientation_no_rotation)
+{
+    string video_file = string(cvtest::TS::ptr()->get_data_path()) + "video/big_buck_bunny_rotated.mp4";
+
+    VideoCapture cap;
+    EXPECT_NO_THROW(cap.open(video_file, CAP_FFMPEG));
+    cap.set(CAP_PROP_ORIENTATION_AUTO, 0);
+    ASSERT_TRUE(cap.isOpened()) << "Can't open the video: " << video_file << " with backend " << CAP_FFMPEG << std::endl;
+    ASSERT_FALSE(cap.get(CAP_PROP_ORIENTATION_AUTO));
+
+    Size actual;
+    EXPECT_NO_THROW(actual = Size((int)cap.get(CAP_PROP_FRAME_WIDTH),
+                                    (int)cap.get(CAP_PROP_FRAME_HEIGHT)));
+    EXPECT_EQ(672, actual.width);
+    EXPECT_EQ(384, actual.height);
+
+    Mat frame;
+
+    cap >> frame;
+
+    ASSERT_EQ(672, frame.cols);
+    ASSERT_EQ(384, frame.rows);
+}
+
 #endif
 }} // namespace