Merge pull request #550 from astrorama/develop

marcschefer · web-flow · commit 2cfb39eb1422 · 2023-03-16T14:44:48.000+01:00
Changes need to build latest version
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -6,4 +6,4 @@ find_package(ElementsProject)
 #---------------------------------------------------------------
 
 # Declare project name and version
-elements_project(SourceXtractorPlusPlus 0.19.2 USE Alexandria 2.28.2 DESCRIPTION "SourceXtractor++, the next generation SExtractor")
+elements_project(SourceXtractorPlusPlus 0.20 USE Alexandria 2.28.2 DESCRIPTION "SourceXtractor++, the next generation SExtractor")
diff --git a/SEFramework/SEFramework/Aperture/TransformedAperture.h b/SEFramework/SEFramework/Aperture/TransformedAperture.h
@@ -25,6 +25,7 @@
 #define _SEFRAMEWORK_SEFRAMEWORK_APERTURE_TRANSFORMEDAPERTURE_H
 
 #include "Aperture.h"
+#include <array>
 
 namespace SourceXtractor {
 
diff --git a/SEFramework/SEFramework/Image/Image.h b/SEFramework/SEFramework/Image/Image.h
@@ -23,6 +23,7 @@
 #ifndef _SEFRAMEWORK_IMAGE_IMAGE_H
 #define _SEFRAMEWORK_IMAGE_IMAGE_H
 
+#include <cstdint>
 #include <memory>
 
 #include "SEUtils/Types.h"
diff --git a/SEFramework/SEFramework/Image/ImageChunk.h b/SEFramework/SEFramework/Image/ImageChunk.h
@@ -24,9 +24,11 @@
 #ifndef _SEFRAMEWORK_IMAGE_IMAGECHUNK_H_
 #define _SEFRAMEWORK_IMAGE_IMAGECHUNK_H_
 
+#include "SEFramework/Image/Image.h"
+
 #include <cassert>
+#include <vector>
 
-#include "SEFramework/Image/Image.h"
 
 namespace SourceXtractor {
 
diff --git a/SEFramework/SEFramework/Image/ImageSource.h b/SEFramework/SEFramework/Image/ImageSource.h
@@ -25,11 +25,13 @@
 #ifndef _SEFRAMEWORK_IMAGE_IMAGESOURCE_H_
 #define _SEFRAMEWORK_IMAGE_IMAGESOURCE_H_
 
-#include <boost/variant.hpp>
-
 #include "SEFramework/Image/Image.h"
 #include "SEFramework/Image/ImageTile.h"
 
+#include <boost/variant.hpp>
+#include <map>
+
+
 namespace SourceXtractor {
 
 /**
diff --git a/SEFramework/SEFramework/Output/OutputRegistry.h b/SEFramework/SEFramework/Output/OutputRegistry.h
@@ -23,10 +23,11 @@
 #define _SEFRAMEWORK_OUTPUTREGISTRY_H
 
 #include <functional>
-#include <vector>
-#include <string>
 #include <map>
+#include <set>
+#include <string>
 #include <typeindex>
+#include <vector>
 
 #include "Table/Row.h"
 #include "SEFramework/Source/SourceInterface.h"
diff --git a/SEFramework/SEFramework/Pipeline/Partition.h b/SEFramework/SEFramework/Pipeline/Partition.h
@@ -25,6 +25,7 @@
 
 #include "SEFramework/Pipeline/PipelineStage.h"
 #include "SEFramework/Source/SourceInterface.h"
+#include <vector>
 
 namespace SourceXtractor {
 
diff --git a/SEFramework/SEFramework/Pipeline/SourceGrouping.h b/SEFramework/SEFramework/Pipeline/SourceGrouping.h
@@ -25,6 +25,7 @@
 
 #include <list>
 #include <memory>
+#include <set>
 
 #include "SEFramework/Pipeline/PipelineStage.h"
 #include "SEFramework/Source/SourceGroupFactory.h"
diff --git a/SEFramework/SEFramework/Source/SourceGroupInterface.h b/SEFramework/SEFramework/Source/SourceGroupInterface.h
@@ -23,6 +23,7 @@
 #define _SEFRAMEWORK_SOURCEGROUPINTERFACE_H
 
 #include "SEFramework/Source/SourceInterface.h"
+#include <list>
 
 namespace SourceXtractor {
 
diff --git a/SEFramework/SEFramework/Source/SourceGroupWithOnDemandProperties.h b/SEFramework/SEFramework/Source/SourceGroupWithOnDemandProperties.h
@@ -23,8 +23,9 @@
 #ifndef _SEFRAMEWORK_SOURCE_SOURCEGROUP_H
 #define _SEFRAMEWORK_SOURCE_SOURCEGROUP_H
 
-#include <set>
+#include <list>
 #include <iterator>
+#include <set>
 #include <type_traits>
 
 #include "SEFramework/Source/SourceInterface.h"
diff --git a/SEFramework/src/lib/Pipeline/SourceGrouping.cpp b/SEFramework/src/lib/Pipeline/SourceGrouping.cpp
@@ -21,6 +21,7 @@
  */
 
 #include "SEFramework/Pipeline/SourceGrouping.h"
+#include <vector>
 
 
 namespace SourceXtractor {
diff --git a/SEImplementation/SEImplementation/Background/SE/MedianFilter.h b/SEImplementation/SEImplementation/Background/SE/MedianFilter.h
@@ -20,6 +20,7 @@
 
 #include "SEFramework/Image/VectorImage.h"
 #include "SEFramework/Image/WriteableImage.h"
+#include <array>
 
 namespace SourceXtractor {
 
diff --git a/SEImplementation/SEImplementation/Plugin/FlexibleModelFitting/FlexibleModelFittingParameterManager.h b/SEImplementation/SEImplementation/Plugin/FlexibleModelFitting/FlexibleModelFittingParameterManager.h
@@ -24,13 +24,14 @@
 #ifndef _SEIMPLEMENTATION_PLUGIN_FLEXIBLEMODELFITTING_FLEXIBLEMODELFITTINGPARAMETERMANAGER_H_
 #define _SEIMPLEMENTATION_PLUGIN_FLEXIBLEMODELFITTING_FLEXIBLEMODELFITTINGPARAMETERMANAGER_H_
 
-#include <map>
-#include <vector>
-
 #include "ModelFitting/Engine/EngineParameterManager.h"
 #include "SEFramework/Source/SourceInterface.h"
 #include "SEImplementation/Plugin/FlexibleModelFitting/FlexibleModelFittingParameter.h"
 
+#include <map>
+#include <set>
+#include <vector>
+
 namespace SourceXtractor {
 
 // Needed to store the source in a reference_wrapper
diff --git a/SEImplementation/SEImplementation/Plugin/FluxRadius/FluxRadiusTask.h b/SEImplementation/SEImplementation/Plugin/FluxRadius/FluxRadiusTask.h
@@ -20,6 +20,7 @@
 
 #include "SEUtils/Types.h"
 #include "SEFramework/Task/SourceTask.h"
+#include <vector>
 
 namespace SourceXtractor {
 
diff --git a/SEImplementation/SEImplementation/Plugin/SourceIDs/SourceIDTask.h b/SEImplementation/SEImplementation/Plugin/SourceIDs/SourceIDTask.h
@@ -24,11 +24,13 @@
 #ifndef _SEIMPLEMENTATION_PLUGIN_SOURCEIDS_SOURCEIDTASK_H_
 #define _SEIMPLEMENTATION_PLUGIN_SOURCEIDS_SOURCEIDTASK_H_
 
-#include <atomic>
 #include "SEFramework/Task/SourceTask.h"
 #include "SEImplementation/Property/SourceId.h"
 #include "SEImplementation/Plugin/SourceIDs/SourceID.h"
 
+#include <atomic>
+#include <cstdint>
+
 namespace SourceXtractor {
 
 class SourceIDTask : public SourceTask {
diff --git a/SEImplementation/SEImplementation/Plugin/Vignet/VignetSourceTask.h b/SEImplementation/SEImplementation/Plugin/Vignet/VignetSourceTask.h
@@ -26,6 +26,7 @@
 #define _SEIMPLEMENTATION_PLUGIN_VIGNETSOURCETASK_H_
 
 #include "SEFramework/Task/SourceTask.h"
+#include <array>
 
 namespace SourceXtractor {
 class VignetSourceTask : public SourceTask {
diff --git a/SEImplementation/SEImplementation/Property/PixelCoordinateList.h b/SEImplementation/SEImplementation/Property/PixelCoordinateList.h
@@ -22,10 +22,12 @@
 #ifndef _SEIMPLEMENTATION_PIXELCOORDINATELIST_H
 #define _SEIMPLEMENTATION_PIXELCOORDINATELIST_H
 
-#include <algorithm>
 #include "SEUtils/PixelCoordinate.h"
 #include "SEFramework/Property/Property.h"
 
+#include <algorithm>
+#include <vector>
+
 namespace SourceXtractor {
 
 class PixelCoordinateList : public Property {
diff --git a/SEImplementation/src/lib/Plugin/FluxRadius/FluxRadiusTask.cpp b/SEImplementation/src/lib/Plugin/FluxRadius/FluxRadiusTask.cpp
@@ -16,6 +16,8 @@
  */
 
 #include <algorithm>
+#include <functional>
+
 #include <NdArray/NdArray.h>
 #include <SEFramework/Image/Image.h>
 #include "SEImplementation/Plugin/FluxRadius/FluxRadius.h"
@@ -50,7 +52,7 @@ void FluxRadiusTask::computeProperties(SourceInterface& source) const {
       // strictly increasing, so we search for the first bin where the accumulated flux is
       // >= the target flux, and interpolate with the previous one
       auto next = std::find_if(std::begin(growth_curve), std::end(growth_curve),
-                               std::bind2nd(std::greater_equal<double>(), target_flux));
+                               std::bind(std::greater_equal<double>(), std::placeholders::_1, target_flux));
       if (next == std::end(growth_curve)) {
         --next;
       }
diff --git a/SEImplementation/src/lib/Plugin/GroupInfo/GroupInfoTask.cpp b/SEImplementation/src/lib/Plugin/GroupInfo/GroupInfoTask.cpp
@@ -22,9 +22,10 @@
  */
 
 
-#include <atomic>
 #include "SEImplementation/Plugin/GroupInfo/GroupInfo.h"
 #include "SEImplementation/Plugin/GroupInfo/GroupInfoTask.h"
+#include <atomic>
+#include <cstdint>
 
 namespace SourceXtractor {
 
diff --git a/SEMain/SEMain/Sorter.h b/SEMain/SEMain/Sorter.h
@@ -26,6 +26,7 @@
 
 #include "SEFramework/Pipeline/PipelineStage.h"
 #include "SEFramework/Source/SourceGroupInterface.h"
+#include <map>
 
 namespace SourceXtractor {
 
diff --git a/SEMain/src/lib/Sorter.cpp b/SEMain/src/lib/Sorter.cpp
@@ -14,9 +14,11 @@
  * along with this library; if not, write to the Free Software Foundation, Inc.,
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
+
+#include "SEMain/Sorter.h"
 #include <SEImplementation/Plugin/SourceIDs/SourceID.h>
 #include <algorithm>
-#include "SEMain/Sorter.h"
+#include <vector>
 
 namespace SourceXtractor {
 
diff --git a/doc/src/Troubles.rst b/doc/src/Troubles.rst
@@ -73,4 +73,61 @@ or directing the operating system to use a partition with sufficient disk space
 
 Note that right now |SourceXtractor++| does **not** have a parameter to directly specify the directory to store
 temporary data.
- 
+
+
+.. index::
+   single: sourcextractor++ stops suddenly
+
+|SourceXtractor++| stops shortly after the first measurements are done
+----------------------------------------------------------------------
+
+It might happen that |SourceXtractor++| stops or dumps off when the first measurements are due or when the first few measurements have been done.
+The log file then ends similar to this example:
+
+.. code-block:: console
+
+	2023-02-09T16:06:50CET Progress  INFO : Segmentation: 487 / 7200 (6.76%)
+	2023-02-09T16:06:50CET Progress  INFO : Detected: 1200
+	2023-02-09T16:06:50CET Progress  INFO : Deblended: 0
+	2023-02-09T16:06:50CET Progress  INFO : Measured: 0
+	2023-02-09T16:06:50CET Progress  INFO : Elapsed: 00:01:36
+	2023-02-09T16:06:58CET Progress  INFO : Segmentation: 502 / 7200 (6.97%)
+	2023-02-09T16:06:58CET Progress  INFO : Detected: 1234
+	2023-02-09T16:06:58CET Progress  INFO : Deblended: 1
+	2023-02-09T16:06:58CET Progress  INFO : Measured: 1 / 1 (100.00%)
+	2023-02-09T16:06:58CET Progress  INFO : Elapsed: 00:01:43
+	2023-02-09T16:07:04CET Progress  INFO : Segmentation: 502 / 7200 (6.97%)
+	2023-02-09T16:07:04CET Progress  INFO : Detected: 1235
+	2023-02-09T16:07:04CET Progress  INFO : Deblended: 4
+	2023-02-09T16:07:04CET Progress  INFO : Measured: 4 / 4 (100.00%)
+	2023-02-09T16:07:04CET Progress  INFO : Elapsed: 00:01:50
+	2023-02-09T16:07:09CET Progress  INFO : Segmentation: 504 / 7200 (7.00%)
+	2023-02-09T16:07:09CET Progress  INFO : Detected: 1240
+	2023-02-09T16:07:09CET Progress  INFO : Deblended: 7
+	2023-02-09T16:07:09CET Progress  INFO : Measured: 7 / 7 (100.00%)
+	2023-02-09T16:07:09CET Progress  INFO : Elapsed: 00:01:55
+
+A possible cause could be a race condition created  by the concurrent calls to lower level libraries such as openMP or openBLAS.
+This can happen in already multi-threaded applications such as |SourceXtractor++|
+(see `here <https://github.com/xianyi/OpenBLAS/wiki/faq#multi-threaded>`_ for details). 
+
+A confirmation that the dump is caused by a race condition in lower level libraries is if |SourceXtractor++| runs with multi-threading
+switched off (`--thread-count=0`), meaning that |SourceXtractor++| does not dump at least beyound a couple of dozen measurements in.
+
+A solution is to switch off the multi-threading of all lower level libraries by setting the following environmental variables
+*before the call* to |SourceXtractor++|:
+
+.. code-block:: console
+
+	export MKL_NUM_THREADS=1
+	export NUMEXPR_NUM_THREADS=1
+	export OMP_NUM_THREADS=1
+	export OPENBLAS_NUM_THREADS=1
+	export VECLIB_MAXIMUM_THREADS=1
+	export NTHREADS=1
+	sourcextractor++ --conf ...
+
+This switches off the multi-threading in the lower level libraries and should prevent the race conditions. In version 0.20 we
+are trying to make these settings inside |SourceXtractor++|
+(see `here <https://gitlab.euclid-sgs.uk/EuclidLibs/SourceXtractorPlusPlus/-/blob/develop/SEMain/src/program/SourceXtractor.cpp#L136>`_),
+however we are not sure whether this will work in all situations and modes.
