Add update on change feature and threshold setting (#4)

royshil · web-flow · commit a7a5f79fb5ee · 2024-01-11T12:15:59.000-05:00
* Add update on change feature and threshold setting

This commit adds the "UpdateOnChange" feature and "UpdateOnChangeThreshold" setting to the application. The "UpdateOnChange" feature allows the application to update only when there is a change in the input image. The "UpdateOnChangeThreshold" setting determines the percentage of change required to trigger an update. These new features improve the efficiency of the application by reducing unnecessary updates.

* lint

* Remove unnecessary log statements and add error handling in tesseract_thread function

* Refactor tesseract-ocr-utils.cpp

* Add Output Formatting option to en-US.ini
diff --git a/cmake/FetchOpenCV.cmake b/cmake/FetchOpenCV.cmake
@@ -75,7 +75,8 @@ if(MSVC)
               ${opencv_SOURCE_DIR}/x64/vc17/staticlib/zlib.lib)
   target_include_directories(OpenCV SYSTEM INTERFACE ${opencv_SOURCE_DIR}/include)
 else()
-  target_link_libraries(OpenCV INTERFACE ${opencv_SOURCE_DIR}/lib/libopencv_imgproc.a
-                                         ${opencv_SOURCE_DIR}/lib/libopencv_core.a)
+  target_link_libraries(
+    OpenCV INTERFACE ${opencv_SOURCE_DIR}/lib/libopencv_imgproc.a ${opencv_SOURCE_DIR}/lib/libopencv_core.a
+                     ${opencv_SOURCE_DIR}/lib/opencv4/3rdparty/libzlib.a)
   target_include_directories(OpenCV SYSTEM INTERFACE ${opencv_SOURCE_DIR}/include/opencv4)
 endif()
diff --git a/data/locale/en-US.ini b/data/locale/en-US.ini
@@ -11,3 +11,6 @@ OutputTextSource="Output Text Source"
 NoOutput="No Output"
 UpdateTimer="Update Timer (ms)"
 AdvancedSettings="Advanced Settings"
+UpdateOnChange="Update Only on Image Change"
+UpdateOnChangeThreshold="Change Threshold %"
+OutputFormatting="Output Formatting"
diff --git a/src/filter-data.h b/src/filter-data.h
@@ -27,6 +27,7 @@ struct filter_data {
 	gs_stagesurf_t *stagesurface;
 
 	cv::Mat inputBGRA;
+	cv::Mat lastInputBGRA;
 	tesseract::TessBaseAPI *tesseract_model;
 	std::string language;
 	int pageSegmentationMode;
@@ -39,6 +40,8 @@ struct filter_data {
 	size_t window_size;
 	uint32_t update_timer_ms;
 	std::string output_format_template;
+	bool update_on_change;
+	int update_on_change_threshold;
 
 	bool isDisabled;
 
diff --git a/src/ocr-filter.cpp b/src/ocr-filter.cpp
@@ -28,6 +28,16 @@ const char *ocr_filter_getname(void *unused)
 
 /**                   PROPERTIES                     */
 
+bool update_on_change_modified(obs_properties_t *props, obs_property_t *property,
+			       obs_data_t *settings)
+{
+	bool update_on_change = obs_data_get_bool(settings, "update_on_change");
+	obs_property_set_visible(obs_properties_get(props, "update_on_change_threshold"),
+				 update_on_change);
+	UNUSED_PARAMETER(property);
+	return true;
+}
+
 // Change the type of enable_smoothing_modified to obs_property_modified_t
 bool enable_smoothing_modified(obs_properties_t *props, obs_property_t *property,
 			       obs_data_t *settings)
@@ -67,19 +77,30 @@ obs_properties_t *ocr_filter_properties(void *data)
 	// add advanced settings checkbox
 	obs_properties_add_bool(props, "advanced_settings", obs_module_text("AdvancedSettings"));
 
+	// Add property for "update on change" checkbox
+	obs_properties_add_bool(props, "update_on_change", obs_module_text("UpdateOnChange"));
+	// Add update threshold property
+	obs_properties_add_int_slider(props, "update_on_change_threshold",
+				      obs_module_text("UpdateOnChangeThreshold"), 1, 100, 1);
+	// Add a callback to enable or disable the update threshold property
+	obs_property_set_modified_callback(obs_properties_get(props, "update_on_change"),
+					   update_on_change_modified);
+
 	obs_property_set_modified_callback(
 		obs_properties_get(props, "advanced_settings"),
 		[](obs_properties_t *props_modified, obs_property_t *property,
 		   obs_data_t *settings) {
 			bool advanced_settings = obs_data_get_bool(settings, "advanced_settings");
 			for (const char *prop :
 			     {"page_segmentation_mode", "char_whitelist", "conf_threshold",
-			      "user_patterns", "enable_smoothing", "word_length", "window_size"}) {
+			      "user_patterns", "enable_smoothing", "word_length", "window_size",
+			      "update_on_change_threshold", "update_on_change"}) {
 				obs_property_set_visible(obs_properties_get(props_modified, prop),
 							 advanced_settings);
 			}
 			if (advanced_settings) {
 				enable_smoothing_modified(props_modified, nullptr, settings);
+				update_on_change_modified(props_modified, nullptr, settings);
 			}
 			UNUSED_PARAMETER(property);
 			return true;
@@ -154,6 +175,8 @@ obs_properties_t *ocr_filter_properties(void *data)
 void ocr_filter_defaults(obs_data_t *settings)
 {
 	obs_data_set_default_int(settings, "update_timer", 100);
+	obs_data_set_default_bool(settings, "update_on_change", true);
+	obs_data_set_default_int(settings, "update_on_change_threshold", 15);
 	obs_data_set_default_string(settings, "language", "eng");
 	obs_data_set_default_bool(settings, "advanced_settings", false);
 	obs_data_set_default_int(settings, "page_segmentation_mode", tesseract::PSM_SINGLE_WORD);
@@ -176,8 +199,16 @@ void ocr_filter_update(void *data, obs_data_t *settings)
 	// Update the output text source
 	update_text_source_on_settings(tf, settings);
 
+	bool hard_tesseract_init_required = false;
+
+	std::string new_language = obs_data_get_string(settings, "language");
+	if (new_language != tf->language) {
+		// if the language changed, we need to reinitialize the tesseract model
+		hard_tesseract_init_required = true;
+	}
+	tf->language = new_language;
+
 	tf->pageSegmentationMode = (int)obs_data_get_int(settings, "page_segmentation_mode");
-	tf->language = obs_data_get_string(settings, "language");
 	tf->char_whitelist = obs_data_get_string(settings, "char_whitelist");
 	tf->user_patterns = obs_data_get_string(settings, "user_patterns");
 	tf->conf_threshold = (int)obs_data_get_int(settings, "conf_threshold");
@@ -186,9 +217,12 @@ void ocr_filter_update(void *data, obs_data_t *settings)
 	tf->window_size = obs_data_get_int(settings, "window_size");
 	tf->update_timer_ms = (uint32_t)obs_data_get_int(settings, "update_timer");
 	tf->output_format_template = obs_data_get_string(settings, "output_formatting");
+	tf->update_on_change = obs_data_get_bool(settings, "update_on_change");
+	tf->update_on_change_threshold =
+		(int)obs_data_get_int(settings, "update_on_change_threshold");
 
 	// Initialize the Tesseract OCR model
-	initialize_tesseract_ocr(tf);
+	initialize_tesseract_ocr(tf, hard_tesseract_init_required);
 }
 
 void ocr_filter_activate(void *data)
@@ -242,6 +276,8 @@ void ocr_filter_destroy(void *data)
 
 		stop_and_join_tesseract_thread(tf);
 
+		cleanup_config_files(tf->unique_id);
+
 		if (tf->tesseractTraineddataFilepath != nullptr) {
 			bfree(tf->tesseractTraineddataFilepath);
 		}
diff --git a/src/tesseract-ocr-utils.cpp b/src/tesseract-ocr-utils.cpp
@@ -4,7 +4,8 @@
 
 #include <obs-module.h>
 
-#include <opencv2/core/mat.hpp>
+#include <opencv2/core.hpp>
+#include <opencv2/imgproc.hpp>
 
 #include <tesseract/baseapi.h>
 
@@ -24,17 +25,34 @@ inline uint64_t get_time_ns(void)
 		.count();
 }
 
-void initialize_tesseract_ocr(filter_data *tf)
+void cleanup_config_files(const std::string &unique_id)
+{
+	check_plugin_config_folder_exists();
+
+	// delete the user patterns file
+	std::string filename = "user-patterns-" + unique_id + ".txt";
+	std::string user_patterns_filepath = obs_module_config_path(filename.c_str());
+	std::remove(user_patterns_filepath.c_str());
+
+	// delete the user patterns config file
+	filename = "user-patterns" + unique_id + ".config";
+	std::string patterns_config_filepath = obs_module_config_path(filename.c_str());
+	std::remove(patterns_config_filepath.c_str());
+}
+
+void initialize_tesseract_ocr(filter_data *tf, bool hard_tesseract_init_required)
 {
 	// Load model
 	obs_log(LOG_INFO, "Loading tesseract model from: %s", tf->tesseractTraineddataFilepath);
 	try {
 		stop_and_join_tesseract_thread(tf);
 
-		if (tf->tesseract_model != nullptr) {
-			tf->tesseract_model->End();
-			delete tf->tesseract_model;
-			tf->tesseract_model = nullptr;
+		if (hard_tesseract_init_required) {
+			if (tf->tesseract_model != nullptr) {
+				tf->tesseract_model->End();
+				delete tf->tesseract_model;
+				tf->tesseract_model = nullptr;
+			}
 		}
 
 		char **configs = nullptr;
@@ -71,16 +89,20 @@ void initialize_tesseract_ocr(filter_data *tf)
 			strcpy(configs[0], patterns_config_filepath.c_str());
 		}
 
-		tf->tesseract_model = new tesseract::TessBaseAPI();
+		if (hard_tesseract_init_required) {
+			tf->tesseract_model = new tesseract::TessBaseAPI();
 
-		// set tesseract page segmentation mode to single word
-		int retval = tf->tesseract_model->Init(tf->tesseractTraineddataFilepath,
-						       tf->language.c_str(),
-						       tesseract::OEM_LSTM_ONLY, configs,
-						       configs_size, nullptr, nullptr, false);
-		if (retval != 0) {
-			throw std::runtime_error("Failed to initialize tesseract model");
+			int retval = tf->tesseract_model->Init(tf->tesseractTraineddataFilepath,
+							       tf->language.c_str(),
+							       tesseract::OEM_LSTM_ONLY, configs,
+							       configs_size, nullptr, nullptr,
+							       false);
+			if (retval != 0) {
+				throw std::runtime_error("Failed to initialize tesseract model");
+			}
 		}
+
+		// set tesseract page segmentation mode
 		tf->tesseract_model->SetPageSegMode(
 			static_cast<tesseract::PageSegMode>(tf->pageSegmentationMode));
 
@@ -139,8 +161,6 @@ std::string run_tesseract_ocr(filter_data *tf, const cv::Mat &imageBGRA)
 		recognitionResult = tf->smoothing_filter->add_reading(recognitionResult);
 	}
 
-	obs_log(LOG_DEBUG, "OCR result: %s", recognitionResult.c_str());
-
 	return recognitionResult;
 }
 
@@ -237,14 +257,34 @@ void tesseract_thread(void *data)
 		cv::Mat imageBGRA;
 		{
 			std::unique_lock<std::mutex> lock(tf->inputBGRALock, std::try_to_lock);
-			if (!lock.owns_lock()) {
-				return;
+			if (lock.owns_lock()) {
+				imageBGRA = tf->inputBGRA.clone();
 			}
-			imageBGRA = tf->inputBGRA.clone();
 		}
 
 		if (!imageBGRA.empty()) {
 			try {
+				// if update on change is true check if the image has changed
+				if (tf->update_on_change &&
+				    imageBGRA.size() == tf->lastInputBGRA.size()) {
+					const int change_threshold_from_image_area =
+						(int)((float)tf->update_on_change_threshold /
+						      100.0f *
+						      (float)(imageBGRA.cols * imageBGRA.rows));
+					// if the image has not changed, skip the processing
+					// take the absolute difference between the images, convert to gray and count the non-zero pixels
+					cv::Mat diff;
+					cv::absdiff(imageBGRA, tf->lastInputBGRA, diff);
+					cv::cvtColor(diff, diff, cv::COLOR_BGRA2GRAY);
+					if (cv::countNonZero(diff) <
+					    change_threshold_from_image_area) {
+						// obs_log(LOG_INFO, "Image has not changed, skipping processing");
+						// skip the processing
+						continue;
+					}
+				}
+				tf->lastInputBGRA = imageBGRA.clone();
+
 				// Process the image
 				std::string ocr_result = run_tesseract_ocr(tf, imageBGRA);
 
diff --git a/src/tesseract-ocr-utils.h b/src/tesseract-ocr-utils.h
@@ -6,7 +6,8 @@
 #include <deque>
 #include <string>
 
-void initialize_tesseract_ocr(filter_data *tf);
+void cleanup_config_files(const std::string &unique_id);
+void initialize_tesseract_ocr(filter_data *tf, bool hard_tesseract_init_required = false);
 std::string run_tesseract_ocr(filter_data *tf, const cv::Mat &imageBGRA);
 std::string strip(const std::string &str);
 void stop_and_join_tesseract_thread(struct filter_data *tf);
