diff --git a/demos/classification_demo/python/classification_demo.py b/demos/classification_demo/python/classification_demo.py index 0efa96d0628..968e90ebce7 100755 --- a/demos/classification_demo/python/classification_demo.py +++ b/demos/classification_demo/python/classification_demo.py @@ -34,6 +34,7 @@ import monitors from images_capture import open_images_capture from helpers import resolution, log_latency_per_stage +from video_writer import LazyVideoWriter log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.DEBUG, stream=sys.stdout) @@ -187,7 +188,7 @@ def main(): render_metrics = PerformanceMetrics() presenter = None output_transform = None - video_writer = cv2.VideoWriter() + video_writer = LazyVideoWriter() ESC_KEY = 27 key = -1 while True: @@ -209,8 +210,7 @@ def main(): render_metrics.update(rendering_start_time) metrics.update(start_time, frame) - if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1): - video_writer.write(frame) + video_writer.write(frame) next_frame_id_to_show += 1 if not args.no_show: @@ -239,7 +239,7 @@ def main(): presenter = monitors.Presenter(args.utilization_monitors, 55, (round(output_resolution[0] / 4), round(output_resolution[1] / 8))) if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'), - cap.fps(), output_resolution): + cap.fps(), args.output_limit, output_resolution): raise RuntimeError("Can't open video writer") # Submit for inference async_pipeline.submit_data(frame, next_frame_id, {'frame': frame, 'start_time': start_time}) @@ -270,8 +270,7 @@ def main(): render_metrics.update(rendering_start_time) metrics.update(start_time, frame) - if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1): - video_writer.write(frame) + video_writer.write(frame) if not args.no_show: cv2.imshow('Classification Results', frame) diff --git a/demos/common/cpp/models/include/models/image_model.h b/demos/common/cpp/models/include/models/image_model.h index aa8ab609b9e..53a9a237879 100644 --- a/demos/common/cpp/models/include/models/image_model.h +++ b/demos/common/cpp/models/include/models/image_model.h @@ -44,6 +44,8 @@ class ImageModel : public ModelBase { size_t netInputHeight = 0; size_t netInputWidth = 0; + size_t netOutputHeight = 0; + size_t netOutputWidth = 0; cv::InterpolationFlags interpolationMode = cv::INTER_LINEAR; RESIZE_MODE resizeMode = RESIZE_FILL; }; diff --git a/demos/common/cpp/models/src/super_resolution_model.cpp b/demos/common/cpp/models/src/super_resolution_model.cpp index 7df6c74475a..5be9065da64 100644 --- a/demos/common/cpp/models/src/super_resolution_model.cpp +++ b/demos/common/cpp/models/src/super_resolution_model.cpp @@ -35,6 +35,8 @@ #include "models/internal_model_data.h" #include "models/results.h" +static constexpr unsigned log_throttle_interval_frames_count = 500; + SuperResolutionModel::SuperResolutionModel(const std::string& modelFileName, const cv::Size& inputImgSize, const std::string& layout) @@ -106,9 +108,10 @@ void SuperResolutionModel::prepareInputsOutputs(std::shared_ptr& mode const ov::Shape& outShape = model->output().get_shape(); const ov::Layout outputLayout("NCHW"); - const auto outWidth = outShape[ov::layout::width_idx(outputLayout)]; + netOutputWidth = outShape[ov::layout::width_idx(outputLayout)]; + netOutputHeight = outShape[ov::layout::height_idx(outputLayout)]; const auto inWidth = lrShape[ov::layout::width_idx(outputLayout)]; - changeInputSize(model, static_cast(outWidth / inWidth)); + changeInputSize(model, static_cast(netOutputWidth / inWidth)); } void SuperResolutionModel::changeInputSize(std::shared_ptr& model, int coeff) { @@ -155,8 +158,15 @@ std::shared_ptr SuperResolutionModel::preprocess(const InputD cv::cvtColor(img, img, cv::COLOR_BGR2GRAY); } - if (static_cast(img.cols) != netInputWidth || static_cast(img.rows) != netInputHeight) { - slog::warn << "\tChosen model aspect ratio doesn't match image aspect ratio" << slog::endl; + if (static_cast(img.cols) != netInputWidth || static_cast(img.rows) != netInputHeight || + !is_aspect_ratio_equal(std::make_tuple(img.cols, img.rows), + std::make_tuple(netOutputWidth, netOutputHeight))) { + static unsigned counter = 0; + if (counter++ % log_throttle_interval_frames_count == 0) { + slog::warn << "\tChosen model aspect ratio for resolution: " << netOutputWidth << "x" << netOutputHeight + << " doesn't match initial image aspect ratio for resolution: " << img.cols << "x" << img.rows + << ". You may observe video disproportions. To avoid this please use a suitable model" << slog::endl; + } } const size_t height = lrInputTensor.get_shape()[ov::layout::height_idx(layout)]; const size_t width = lrInputTensor.get_shape()[ov::layout::width_idx(layout)]; @@ -237,8 +247,15 @@ std::shared_ptr SuperResolutionChannelJoint::preprocess(const const ov::Tensor lrInputTensor = request.get_tensor(inputsNames[0]); const ov::Layout layout("NCHW"); - if (static_cast(img.cols) != netInputWidth || static_cast(img.rows) != netInputHeight) { - slog::warn << "\tChosen model aspect ratio doesn't match image aspect ratio" << slog::endl; + if (static_cast(img.cols) != netInputWidth || static_cast(img.rows) != netInputHeight || + !is_aspect_ratio_equal(std::make_tuple(img.cols, img.rows), + std::make_tuple(netOutputWidth, netOutputHeight))) { + static unsigned counter = 0; + if (counter++ % log_throttle_interval_frames_count == 0) { + slog::warn << "\tChosen model aspect ratio for resolution: " << netOutputWidth << "x" << netOutputHeight + << " doesn't match initial image aspect ratio for resolution: " << img.cols << "x" << img.rows + << ". You may observe video disproportions. To avoid this please use a suitable model" << slog::endl; + } } const size_t height = lrInputTensor.get_shape()[ov::layout::height_idx(layout)]; @@ -297,10 +314,11 @@ void SuperResolutionChannelJoint::prepareInputsOutputs(std::shared_ptroutput().get_shape(); const ov::Layout outputLayout("NCHW"); - const auto outWidth = outShape[ov::layout::width_idx(outputLayout)]; + netOutputWidth = outShape[ov::layout::width_idx(outputLayout)]; + netOutputHeight = outShape[ov::layout::height_idx(outputLayout)]; const auto inWidth = lrShape[ov::layout::width_idx(outputLayout)]; - changeInputSize(model, static_cast(outWidth / inWidth)); + changeInputSize(model, static_cast(netOutputWidth / inWidth)); ov::set_batch(model, 3); } diff --git a/demos/common/cpp/utils/include/utils/image_utils.h b/demos/common/cpp/utils/include/utils/image_utils.h index 5dce63d27f7..a14277156b0 100644 --- a/demos/common/cpp/utils/include/utils/image_utils.h +++ b/demos/common/cpp/utils/include/utils/image_utils.h @@ -15,6 +15,7 @@ */ #pragma once +#include #include @@ -27,3 +28,6 @@ enum RESIZE_MODE { cv::Mat resizeImageExt(const cv::Mat& mat, int width, int height, RESIZE_MODE resizeMode = RESIZE_FILL, cv::InterpolationFlags interpolationMode = cv::INTER_LINEAR, cv::Rect* roi = nullptr, cv::Scalar BorderConstant = cv::Scalar(0, 0, 0)); + + +bool is_aspect_ratio_equal(const std::tuple &lhs_res,const std::tuple &rhs_res); diff --git a/demos/common/cpp/utils/include/utils/ocv_common.hpp b/demos/common/cpp/utils/include/utils/ocv_common.hpp index 93e116de9cd..a4986493ef3 100644 --- a/demos/common/cpp/utils/include/utils/ocv_common.hpp +++ b/demos/common/cpp/utils/include/utils/ocv_common.hpp @@ -321,20 +321,39 @@ class InputTransform { cv::Scalar stdScales; }; +using OnLimitCallbackType = std::function; +static void on_limit_reached_default(const cv::Mat&, unsigned frame_index, unsigned limit) { + + static const unsigned throttle_interval_frames_count = 500; + if ((frame_index - limit) % throttle_interval_frames_count == 0) { + slog::warn << "VideoWriter will skip writing further frames due to frame limit applied: " + << limit << "\nIf you want to turn off this limitation please set `-limit 0` as the demo parameter\n" + << slog::endl; + } +} + class LazyVideoWriter { cv::VideoWriter writer; unsigned nwritten; + unsigned nframes; public: const std::string filenames; const double fps; const unsigned lim; + OnLimitCallbackType on_limit_callback; - LazyVideoWriter(const std::string& filenames, double fps, unsigned lim) : - nwritten{1}, filenames{filenames}, fps{fps}, lim{lim} {} + LazyVideoWriter(const std::string& filenames, double fps, unsigned lim, + OnLimitCallbackType callback = on_limit_reached_default) : + nwritten{1}, nframes{1}, filenames{filenames}, fps{fps}, lim{lim}, on_limit_callback{callback} {} void write(const cv::Mat& im) { - if (writer.isOpened() && (nwritten < lim || 0 == lim)) { - writer.write(im); - ++nwritten; + if (writer.isOpened()) { + if((nwritten < lim || 0 == lim)) { + writer.write(im); + ++nwritten; + } else { + on_limit_callback(im, nframes, lim); + } + ++nframes; return; } if (!writer.isOpened() && !filenames.empty()) { diff --git a/demos/common/cpp/utils/src/image_utils.cpp b/demos/common/cpp/utils/src/image_utils.cpp index b6d4c08975c..21ec0f1ed14 100644 --- a/demos/common/cpp/utils/src/image_utils.cpp +++ b/demos/common/cpp/utils/src/image_utils.cpp @@ -53,3 +53,9 @@ cv::Mat resizeImageExt(const cv::Mat& mat, int width, int height, RESIZE_MODE re } return dst; } + +bool is_aspect_ratio_equal(const std::tuple &lhs_res, const std::tuple &rhs_res) { + float leftAspectRation = std::get<0>(lhs_res) / static_cast(std::get<1>(lhs_res)); + float rightAspectRation = std::get<0>(rhs_res) / static_cast(std::get<1>(rhs_res)); + return fabs(leftAspectRation - rightAspectRation) <= std::numeric_limits::epsilon(); +} diff --git a/demos/common/python/video_writer.py b/demos/common/python/video_writer.py new file mode 100644 index 00000000000..a81bbd4f28d --- /dev/null +++ b/demos/common/python/video_writer.py @@ -0,0 +1,47 @@ +""" + Copyright (C) 2020-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +from cv2 import VideoWriter +import logging as log + +DEFAULT_LIMIT_WARN_THROTTLE_INTERVAL_FRAMES = 500 + +def on_limit_reached_default(frame, number, limit): + if (number - limit) % DEFAULT_LIMIT_WARN_THROTTLE_INTERVAL_FRAMES == 1: + log.warning("VideoWriter will skip writing next frame due to frame limit applied: {0}.\nIf you want to turn off this limitation please set `-limit 0`".format(limit)) + +class LazyVideoWriter(VideoWriter): + def __init__(self): + VideoWriter.__init__(self) + self.frames_processed = 0 + self.output_limit = 0 + self.on_limit_callback = None + + def open(self, output, fourcc, fps, output_limit, output_resolution, on_limit_callback = on_limit_reached_default): + super().open(output, fourcc, fps, output_resolution) + + self.output_limit = output_limit + self.frames_processed = 0 + self.on_limit_callback = on_limit_callback + + def write(self, frame): + if super().isOpened(): + if self.output_limit <= 0 or self.frames_processed <= self.output_limit: + super().write(frame) + else: + if self.on_limit_callback is not None: + self.on_limit_callback(frame, self.frames_processed, self.output_limit) + self.frames_processed += 1