Quick-fix of segfaulting async (Python 3.12) (PolusAI#263)

* initial

* dropped test_nonimq_wsi_scalability to please Python 3.12

Author: friskluft

CMakeLists.txt

@@ -188,7 +188,9 @@ set(SOURCE
 	src/nyx/features/texture_feature.cpp
 	src/nyx/features/zernike.cpp
 	src/nyx/features/zernike_nontriv.cpp
+	src/nyx/helpers/diag.cpp
 	src/nyx/helpers/timing.cpp
+	src/nyx/arrow_helpers.cpp
 	src/nyx/cli_fpimage_options.cpp
 	src/nyx/cli_gabor_options.cpp
 	src/nyx/cli_glcm_options.cpp
@@ -221,10 +223,10 @@ set(SOURCE
 	src/nyx/roi_blacklist.cpp
 	src/nyx/roi_cache.cpp
 	src/nyx/roi_cache_basic.cpp
-	src/nyx/scan_fastloader_way.cpp
 	src/nyx/slideprops.cpp
 	src/nyx/strpat.cpp
 	src/nyx/workflow_3d.cpp
+	src/nyx/workflow_pythonapi.cpp
 	src/nyx/workflow_segmented.cpp
 	src/nyx/workflow_wholeslide.cpp
 )

src/nyx/arrow_helpers.cpp

@@ -0,0 +1,65 @@
+#include <string>
+#include "environment.h"
+#include "helpers/fsystem.h"
+#include "helpers/helpers.h"
+
+namespace Nyxus
+{
+
+	std::string get_arrow_filename(const std::string& output_path, const std::string& default_filename, const SaveOption& arrow_file_type) {
+
+		/*
+					output_path			condition			verdict
+		Case 1: 	/foo/bar		exist in fs				is a directory, append default filename with proper ext
+					/foo/bar/		or ends with / or \
+					\foo\bar\
+
+		Case 2:		/foo/bar		does not exist in fs	assume the extension is missing, append proper ext
+									but /foo exists
+
+		Case 3: 	/foo/bar		neither /foo nor 		treat as directory, append default filename with proper ext
+									/foo/bar exists in fs
+
+		Case 4: 	/foo/bar.ext	exists in fs and is a 	append default filename with proper ext
+									directory
+
+		Case 5: 	/foo/bar.ext	does not exist in fs  	this is a file, check if ext is correct and modify if needed
+
+		Case 6:		empty									default filename with proper ext
+
+
+		*/
+		std::string valid_ext = [&arrow_file_type]() {
+			if (arrow_file_type == Nyxus::SaveOption::saveArrowIPC) {
+				return ".arrow";
+			}
+			else if (arrow_file_type == Nyxus::SaveOption::saveParquet) {
+				return ".parquet";
+			}
+			else { return ""; }
+		}();
+
+		if (output_path != "") {
+			auto arrow_path = fs::path(output_path);
+			if (fs::is_directory(arrow_path) // case 1, 4
+				|| Nyxus::ends_with_substr(output_path, "/")
+				|| Nyxus::ends_with_substr(output_path, "\\")) {
+				arrow_path = arrow_path / default_filename;
+			}
+			else if (!arrow_path.has_extension()) {
+				if (!fs::is_directory(arrow_path.parent_path())) { // case 3
+					arrow_path = arrow_path / default_filename;
+				}
+				// else case 2, do nothing here	
+			}
+			// case 5 here, but also for 1-4, update extenstion here
+			arrow_path.replace_extension(valid_ext);
+			return arrow_path.string();
+		}
+		else { // case 6
+			return default_filename + valid_ext;
+		}
+	}
+
+}
+

src/nyx/helpers/diag.cpp

@@ -0,0 +1,122 @@
+#include <fstream>
+#include <iostream>
+#include "../environment.h"
+#include "../globals.h"
+#include "../roi_cache.h"
+#include "../helpers/fsystem.h"
+
+namespace Nyxus
+{
+
+void dump_roi_metrics(const std::string& label_fpath)
+{
+	// are we amidst a 3D scenario ?
+	bool dim3 = theEnvironment.dim();
+
+	// prepare the file name
+	fs::path pseg(label_fpath);
+	std::string fpath = theEnvironment.output_dir + "/roi_metrics_" + pseg.stem().string() + ".csv";
+
+	// fix the special 3D file name character if needed
+	if (dim3)
+		for (auto& ch : fpath)
+			if (ch == '*')
+				ch = '~';
+
+	std::cout << "Dumping ROI metrics to " << fpath << '\n';
+
+	std::ofstream f(fpath);
+	if (f.fail())
+	{
+		std::cerr << "Error: cannot create file " << fpath << '\n';
+		return;
+	}
+
+	// header
+	f << "label, area, minx, miny, maxx, maxy, width, height, min_intens, max_intens, size_bytes, size_class \n";
+	// sort labels
+	std::vector<int>  sortedLabs{ uniqueLabels.begin(), uniqueLabels.end() };
+	std::sort(sortedLabs.begin(), sortedLabs.end());
+	// body
+	for (auto lab : sortedLabs)
+	{
+		LR& r = roiData[lab];
+		auto szb = r.get_ram_footprint_estimate();
+		std::string ovsz = szb < theEnvironment.get_ram_limit() ? "TRIVIAL" : "OVERSIZE";
+		f << lab << ", "
+			<< r.aux_area << ", "
+			<< r.aabb.get_xmin() << ", "
+			<< r.aabb.get_ymin() << ", "
+			<< r.aabb.get_xmax() << ", "
+			<< r.aabb.get_ymax() << ", "
+			<< r.aabb.get_width() << ", "
+			<< r.aabb.get_height() << ", "
+			<< r.aux_min << ", "
+			<< r.aux_max << ", "
+			<< szb << ", "
+			<< ovsz << ", ";
+		f << "\n";
+	}
+
+	f.flush();
+}
+
+void dump_roi_pixels(const std::vector<int>& batch_labels, const std::string& label_fpath)
+{
+	// no data ?
+	if (batch_labels.size() == 0)
+	{
+		std::cerr << "Error: no ROI pixel data for file " << label_fpath << '\n';
+		return;
+	}
+
+	// sort labels for reader's comfort
+	std::vector<int>  srt_L{ batch_labels.begin(), batch_labels.end() };
+	std::sort(srt_L.begin(), srt_L.end());
+
+	// are we amidst a 3D scenario ?
+	bool dim3 = theEnvironment.dim();
+
+	// prepare the file name
+	fs::path pseg(label_fpath);
+	std::string fpath = theEnvironment.output_dir + "/roi_pixels_" + pseg.stem().string() + "_batch" + std::to_string(srt_L[0]) + '-' + std::to_string(srt_L[srt_L.size() - 1]) + ".csv";
+
+	// fix the special 3D file name character if needed
+	if (dim3)
+		for (auto& ch : fpath)
+			if (ch == '*')
+				ch = '~';
+
+	std::cout << "Dumping ROI pixels to " << fpath << '\n';
+
+	std::ofstream f(fpath);
+	if (f.fail())
+	{
+		std::cerr << "Error: cannot create file " << fpath << '\n';
+		return;
+	}
+
+	// header
+	f << "label,x,y,z,intensity, \n";
+
+	// body
+	for (auto lab : srt_L)
+	{
+		LR& r = roiData[lab];
+		if (dim3)
+			for (auto& plane : r.zplanes)
+				for (auto idx : plane.second)
+				{
+					auto& pxl = r.raw_pixels_3D[idx];
+					f << lab << "," << pxl.x << ',' << pxl.y << ',' << pxl.z << ',' << pxl.inten << ',' << '\n';
+
+				}
+		else
+			for (auto pxl : r.raw_pixels)
+				f << lab << "," << pxl.x << ',' << pxl.y << ',' << pxl.inten << ',' << '\n';
+	}
+
+	f.flush();
+}
+
+}