Do not capture python stdout and stderr

When calling sourcextractor++ from Python

Author: ayllon

SEImplementation/SEImplementation/Configuration/PythonConfig.h

@@ -1,4 +1,4 @@
-/*
+/**
  * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université
  *
  * This library is free software; you can redistribute it and/or modify it under
@@ -15,7 +15,7 @@
  * along with this library; if not, write to the Free Software Foundation, Inc.,
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
-/* 
+/*
  * @file PythonConfig.h
  * @author Nikolaos Apostolakos <[email protected]>
  */
@@ -29,20 +29,21 @@
 namespace SourceXtractor {
 
 class PythonConfig : public Euclid::Configuration::Configuration {
-  
+
 public:
-  
+
   explicit PythonConfig(long manager_id);
-  
+
   std::map<std::string, Configuration::OptionDescriptionList> getProgramOptions() override;
 
   void preInitialize(const UserValues& args) override;
 
   void initialize(const UserValues& args) override;
-  
+
   PythonInterpreter& getInterpreter() const;
 
 private:
+  bool                  m_capture_output = true;
   boost::python::object m_measurement_config;
 };
 

SEImplementation/SEImplementation/PythonConfig/PythonInterpreter.h

@@ -1,4 +1,4 @@
-/*
+/**
  * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université
  *
  * This library is free software; you can redistribute it and/or modify it under
@@ -15,7 +15,7 @@
  * along with this library; if not, write to the Free Software Foundation, Inc.,
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
-/* 
+/*
  * @file PythonInterpreter.h
  * @author Nikolaos Apostolakos <[email protected]>
  */
@@ -34,17 +34,20 @@
 namespace SourceXtractor {
 
 class PythonInterpreter {
-  
+
 public:
-  
+
   static PythonInterpreter& getSingleton();
-  
+
   void runFile(const std::string& filename, const std::vector<std::string>& argv);
 
+  /// Capture Python's stdout and stderr and pass the output through the logging
+  void captureOutput();
+
   void setupContext(boost::python::object config = {});
 
   virtual ~PythonInterpreter();
-  
+
   std::map<int, PyMeasurementImage> getMeasurementImages();
 
   std::map<int, PyAperture> getApertures();
@@ -54,31 +57,31 @@ class PythonInterpreter {
   std::map<std::string, std::vector<int>> getApertureOutputColumns();
 
   std::map<int, boost::python::object> getConstantParameters();
-  
+
   std::map<int, boost::python::object> getFreeParameters();
-  
+
   std::map<int, boost::python::object> getDependentParameters();
-  
+
   std::map<int, boost::python::object> getPriors();
-  
+
   std::map<int, boost::python::object> getConstantModels();
 
   std::map<int, boost::python::object> getPointSourceModels();
-  
+
   std::map<int, boost::python::object> getSersicModels();
-  
+
   std::map<int, boost::python::object> getExponentialModels();
-  
+
   std::map<int, boost::python::object> getDeVaucouleursModels();
-  
+
   std::map<int, boost::python::object> getOnnxModels();
 
   std::map<int, std::vector<int>> getFrameModelsMap();
 
   std::map<std::string, boost::python::object> getModelFittingParams();
 
 private:
-  
+
   PythonInterpreter();
 
   std::map<int, boost::python::object> getMapFromDict(const char* object_name,

SEImplementation/src/lib/Configuration/PythonConfig.cpp

@@ -1,4 +1,4 @@
-/*
+/**
  * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université
  *
  * This library is free software; you can redistribute it and/or modify it under
@@ -31,7 +31,9 @@ namespace {
 
 const std::string PYTHON_CONFIG_FILE{"python-config-file"};
 const std::string PYTHON_ARGV{"python-arg"};
+// These are internal and not exposed to the configuration manager
 const std::string PYTHON_CONFIG_OBJ{"python-config-object"};
+const std::string PYTHON_CAPTURE_OUTPUT{"python-capture-output"};
 
 }  // namespace
 
@@ -58,13 +60,19 @@ void PythonConfig::preInitialize(const UserValues& args) {
   } else if (!filename.empty() && !fs::exists(filename)) {
     throw Elements::Exception() << "Python configuration file " << filename << " does not exist";
   }
+
+  auto py_capture_output = args.find(PYTHON_CAPTURE_OUTPUT);
+  if (py_capture_output != args.end()) {
+    m_capture_output = py_capture_output->second.as<bool>();
+  }
 }
 
 void PythonConfig::initialize(const UserValues& args) {
   auto& singleton = PythonInterpreter::getSingleton();
-  if (m_measurement_config) {
-    singleton.setupContext(m_measurement_config);
-  } else {
+  if (m_capture_output) {
+    singleton.captureOutput();
+  }
+  if (!m_measurement_config) {
     auto filename = args.find(PYTHON_CONFIG_FILE)->second.as<std::string>();
     if (!filename.empty()) {
       std::vector<std::string> argv;

SEImplementation/src/lib/PythonConfig/PythonInterpreter.cpp

@@ -51,23 +51,31 @@ PythonInterpreter& PythonInterpreter::getSingleton() {
 
 PythonInterpreter::PythonInterpreter() : m_out_wrapper(stdout_logger), m_err_wrapper(stderr_logger) {
   // We may be called *from* Python!
-  if (Py_IsInitialized()) {
-    return;
-  }
-  // Python sets its own signal handler for SIGINT (Ctrl+C), so it can throw a KeyboardInterrupt
-  // Here we are not interested on this behaviour, so we get whatever handler we've got (normally
-  // the default one) and restore it after initializing the interpreter
-  struct sigaction sigint_handler;
-  sigaction(SIGINT, nullptr, &sigint_handler);
-
-  PyImport_AppendInittab("pyston", PYSTON_MODULE_INIT);
-  Py_Initialize();
+  if (!Py_IsInitialized()) {
+    struct sigaction sigint_handler;
+
+    // Python sets its own signal handler for SIGINT (Ctrl+C), so it can throw a KeyboardInterrupt
+    // Here we are not interested on this behaviour, so we get whatever handler we've got (normally
+    // the default one) and restore it after initializing the interpreter
+    sigaction(SIGINT, nullptr, &sigint_handler);
+
+    // Make pyston importable
+    PyImport_AppendInittab("pyston", PYSTON_MODULE_INIT);
+
+    // Initialize Python
+    Py_Initialize();
 #if PY_VERSION_HEX < 3090000
-  PyEval_InitThreads();
+    PyEval_InitThreads();
 #endif
-  PyEval_SaveThread();
+    PyEval_SaveThread();
+
+    // Restore SIGINT handler
+    sigaction(SIGINT, &sigint_handler, nullptr);
+  }
 
-  sigaction(SIGINT, &sigint_handler, nullptr);
+  // Import ourselves so the conversions are registered
+  Pyston::GILLocker locker;
+  py::import("_SEPythonConfig");
 }
 
 PythonInterpreter::~PythonInterpreter() {
@@ -97,13 +105,6 @@ void PythonInterpreter::runFile(const std::string& filename, const std::vector<s
     }
     PySys_SetArgv(argv.size() + 1, py_argv);
 
-    // Import ourselves so the conversions are registered
-    py::import("_SEPythonConfig");
-
-    // Setup stdout and stderr
-    PySys_SetObject("stdout", py::object(boost::ref(m_out_wrapper)).ptr());
-    PySys_SetObject("stderr", py::object(boost::ref(m_err_wrapper)).ptr());
-
     // Run the file
     py::object main_module = py::import("__main__");
     py::setattr(main_module, "__file__", py::object(filename));
@@ -125,6 +126,12 @@ void PythonInterpreter::runFile(const std::string& filename, const std::vector<s
   }
 }
 
+void PythonInterpreter::captureOutput() {
+  Pyston::GILLocker locker;
+  PySys_SetObject("stdout", py::object(boost::ref(m_out_wrapper)).ptr());
+  PySys_SetObject("stderr", py::object(boost::ref(m_err_wrapper)).ptr());
+}
+
 void PythonInterpreter::setupContext(boost::python::object config) {
   Pyston::GILLocker locker;
   try {

SEPythonModule/src/lib/Context.cpp

@@ -26,6 +26,7 @@
 #include "SEImplementation/Configuration/DetectionFrameConfig.h"
 #include "SEImplementation/Configuration/MultiThreadingConfig.h"
 #include "SEImplementation/Configuration/OutputConfig.h"
+#include "SEImplementation/Configuration/PythonConfig.h"
 #include "SEImplementation/Deblending/DeblendingFactory.h"
 #include "SEImplementation/Grouping/GroupingFactory.h"
 #include "SEImplementation/Measurement/MeasurementFactory.h"
@@ -88,8 +89,10 @@ Context::Context(const py::dict& global_config, const py::object& measurement_co
   config_wrapper.fromPython(global_config);
   auto options = config_wrapper.getOptions();
 
-  // Override the python object to use for the measurement configuration
-  options["python-config-object"].value() = boost::any(measurement_config);
+  // Override the python object to use for the measurement configuration and disable output capture
+  options["python-config-object"].value()  = boost::any(measurement_config);
+  options["python-capture-output"].value() = boost::any(false);
+
   m_config_manager->initialize(options);
 
   m_segmentation_factory->configure(*m_config_manager);