Merge pull request #143 from astrorama/develop

marcschefer · web-flow · commit b4e1e14b2cc8 · 2019-10-03T16:20:20.000+02:00
Develop
diff --git a/.travis.yml b/.travis.yml
@@ -0,0 +1,66 @@
+language: minimal
+os: linux
+
+branches:
+  only:
+  - develop
+  - /^\d+.\d+(.\d+)?(-\S*)?$/
+
+env:
+  global:
+    - OS_ARCH=x86_64
+
+services:
+  - docker
+
+stages:
+  - build
+  - buildapp
+
+jobs:
+  include:
+  - &build
+    stage: build
+    env: OS_TYPE=fedora OS_VERSION=29
+    script:
+      - docker run --name builder --volume "$(pwd):/src" "${OS_TYPE}:${OS_VERSION}" bash "/src/.travis/build-rpm.sh"
+      - docker cp "builder:/build/Packages/RPM/RPMS/x86_64" "./packages"   
+    after_success: |
+      if [ ! -z "${BINTRAY_REPO}" ] && [ -z "${TRAVIS_PULL_REQUEST_BRANCH}" ]; then
+        package=$(grep elements_project CMakeLists.txt | sed 's/.*(//'| cut -d' ' -f1)
+        
+        if [ ! -z "${TRAVIS_TAG}" ]; then
+            target_repo="master"
+            version="${TRAVIS_TAG}"
+        else
+            target_repo="${TRAVIS_BRANCH}"
+            version=$(grep elements_project CMakeLists.txt | sed 's/.*(//' | sed 's/[^ ]* //' | sed 's/ .*//')
+        fi
+        
+        cd packages
+        for p in *.rpm; do
+          curl -T "${p}" "-u${BINTRAY_USER}:${BINTRAY_TOKEN}" "${BINTRAY_REPO}/${target_repo}/${OS_TYPE}/${OS_VERSION}/${OS_ARCH}/rpms/${p};bt_package=${package};bt_version=${version}?override=1&publish=1"
+        done;
+      fi
+  - <<: *build
+    env: OS_TYPE=fedora OS_VERSION=30
+  - <<: *build
+    env: OS_TYPE=fedora OS_VERSION=31
+  - <<: *build
+    env: OS_TYPE=fedora OS_VERSION=32
+  - <<: *build
+    env: OS_TYPE=centos OS_VERSION=7
+  - stage: buildapp
+    if: tag IS present
+    env: OS_TYPE=fedora OS_VERSION=29
+    script:
+      - docker run --name appbuilder --device /dev/fuse --privileged --volume "$(pwd):/src" "${OS_TYPE}:${OS_VERSION}" bash "/src/.travis/build-appimage.sh"
+      - docker cp "appbuilder:/sextractor++-x86_64.AppImage" "."
+    deploy:
+      provider: releases
+      on:
+        tags: true
+      api_key: $GITHUB_OAUTH_TOKEN
+      file: sextractor++-x86_64.AppImage
+      skip_cleanup: true
+
diff --git a/.travis/build-appimage.sh b/.travis/build-appimage.sh
@@ -0,0 +1,89 @@
+#!/bin/bash
+set -ex
+
+# Platform-specific configuration
+source /etc/os-release
+
+CMAKEFLAGS="-DINSTALL_DOC=ON -DRPM_NO_CHECK=OFF"
+
+if [ $NAME == 'Fedora' ] && [ $VERSION_ID -ge 30 ]; then
+  PYTHON="python3"
+  PIP="pip3" 
+else
+  PYTHON="python2"
+  PIP="pip2"
+fi
+
+
+# Astrorama repository
+cat > /etc/yum.repos.d/astrorama.repo << EOF
+[bintray--astrorama-fedora]
+name=bintray--astrorama-fedora
+baseurl=https://dl.bintray.com/astrorama/travis/master/fedora/\$releasever/\$basearch
+gpgcheck=0
+repo_gpgcheck=0
+enabled=1
+EOF
+
+# Install pip, virtualenv, fuse and which
+yum install -y ${PYTHON}-pip python3-virtualenv fuse-libs which
+
+# Install sextractor from rpms
+yum install -y SExtractorxx
+
+# Download linuxdeploy
+curl -OL 'https://github.com/linuxdeploy/linuxdeploy/releases/download/continuous/linuxdeploy-x86_64.AppImage'
+chmod a+x 'linuxdeploy-x86_64.AppImage'
+
+# Create a virtualenv with the dependencies
+# are self-contained
+APPDIR=`mktemp -d`
+virtualenv --clear --no-pip --no-wheel --no-setuptools --python="${PYTHON}" "${APPDIR}"
+source "${APPDIR}/bin/activate"
+
+"${PIP}" --isolated install --ignore-installed --prefix "/usr" --root "${APPDIR}" astropy
+
+deactivate
+
+virtualenv --relocatable "${APPDIR}"
+
+sed -i 's/VIRTUAL_ENV=.*/VIRTUAL_ENV="${APPIMAGE_MOUNT}"/g' "${APPDIR}/bin/activate"
+
+# virtualenv create symlinks to the system Python installation
+# Replace them with the actual content
+# virtualenv has --always-copy, but it is quite buggy and the result ends being unusable
+PYTHONDIRNAME="$(ls -1 ${APPDIR}/usr/lib64/ | grep '^python')"
+for symlink in $(find "${APPDIR}/lib/${PYTHONDIRNAME}" -type l); do
+    original="$(readlink "${symlink}")"
+    echo "Replacing symlink with copy ${symlink} => ${original}"
+    rm -f "${symlink}"
+    cp -rfv "${original}" "${symlink}"
+done
+
+# These files are missing for some reason ¯\_(ツ)_/¯
+DISTUTILSDIR="$(dirname "$(${PYTHON} -c 'import distutils; print(distutils.__file__)')")"
+cp -vf ${DISTUTILSDIR}/*.py "${APPDIR}/lib/${PYTHONDIRNAME}/distutils"
+
+# Bundle
+SRCDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../"
+RSCDIR="${SRCDIR}/SEMain/auxdir/SEMain"
+SCRIPTDIR="${SRCDIR}/SEMain/scripts/"
+
+./linuxdeploy-x86_64.AppImage \
+    --appdir "${APPDIR}" \
+    -d "${RSCDIR}/sextractor++.desktop" -i "${RSCDIR}/sextractor++.png" \
+    --custom-apprun="${SCRIPTDIR}/AppRun" \
+    -e $(which sextractor++) \
+    -o appimage
+
+# Try it
+unset PYTHONPATH
+./sextractor++-x86_64.AppImage --version
+if [[ $? != 0 ]]; then
+    echo "Failed to build the AppImage!"
+    exit 1
+else
+    echo "Success!"
+    exit 0
+fi
+
diff --git a/.travis/build-rpm.sh b/.travis/build-rpm.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+set -ex
+
+# Environment
+export VERBOSE=0
+export CTEST_OUTPUT_ON_FAILURE=1
+
+# Platform-specific configuration
+source /etc/os-release
+
+CMAKEFLAGS="-DINSTALL_DOC=ON -DRPM_NO_CHECK=OFF"
+
+if [ "$ID" == "fedora" ]; then
+  if [ "$VERSION_ID" -ge 30 ]; then
+    PYTHON="python3"
+    CMAKEFLAGS="$CMAKEFLAGS -DPYTHON_EXPLICIT_VERSION=3"
+  else
+    PYTHON="python2"
+  fi
+elif [ "$ID" == "centos" ]; then
+  yum install -y -q epel-release
+  PYTHON="python2"
+fi
+
+# Astrorama repository
+cat > /etc/yum.repos.d/astrorama.repo << EOF
+[bintray--astrorama-fedora]
+name=bintray--astrorama-fedora
+baseurl=https://dl.bintray.com/astrorama/travis/master/${ID}/\$releasever/\$basearch
+gpgcheck=0
+repo_gpgcheck=0
+enabled=1
+
+[bintray--astrorama-fedora-develop]
+name=bintray--astrorama-fedora-develop
+baseurl=https://dl.bintray.com/astrorama/travis/develop/${ID}/\$releasever/\$basearch
+gpgcheck=0
+repo_gpgcheck=0
+enabled=1
+EOF
+
+# From the CMakeLists.txt, retrieve the list of dependencies
+cmake_deps=$(grep -oP 'elements_project\(\S+\s+\S+ USE \K(\S+ \S+)*(?=\))' /src/CMakeLists.txt)
+rpm_dev_deps=$(echo ${cmake_deps} | awk '{for(i=1;i<NF;i+=2){print $i "-devel-" $(i+1)}}')
+rpm_doc_deps=$(echo ${cmake_deps} | awk '{for(i=1;i<NF;i+=2){print $i "-doc-" $(i+1)}}')
+yum install -y -q ${rpm_dev_deps} ${rpm_doc_deps}
+
+# Dependencies
+yum install -y -q cmake make gcc-c++ rpm-build
+yum install -y -q boost-devel $PYTHON-pytest log4cpp-devel doxygen CCfits-devel
+yum install -y -q graphviz $PYTHON-sphinx $PYTHON-sphinxcontrib-apidoc
+yum install -y -q gmock-devel gtest-devel
+yum install -y -q ${PYTHON}-devel boost-${PYTHON}-devel fftw-devel levmar-devel wcslib-devel
+
+# Build
+mkdir -p /build
+cd /build
+cmake -DCMAKE_INSTALL_PREFIX=/usr $CMAKEFLAGS /src
+make $MAKEFLAGS rpm
+
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -6,4 +6,4 @@ find_package(ElementsProject)
 #---------------------------------------------------------------
 
 # Declare project name and version
-elements_project(SExtractorxx 0.1 USE Alexandria 2.14 Elements 5.8)
+elements_project(SExtractorxx 0.1 USE Alexandria 2.14.1 Elements 5.8)
diff --git a/SEImplementation/CMakeLists.txt b/SEImplementation/CMakeLists.txt
@@ -36,9 +36,9 @@ find_package(PythonLibs ${PYTHON_EXPLICIT_VERSION} REQUIRED)
 find_package(BoostPython ${PYTHON_EXPLICIT_VERSION})
 find_package(Boost 1.53 REQUIRED)
 
-if (${Boost_VERSION} LESS "106600")
+if (${Boost_VERSION} LESS "106700")
     message(WARNING "
-Numerical differentiation is available starting from Boost 106600 (${Boost_VERSION} found instead).
+Numerical differentiation is available starting from Boost 106700 (${Boost_VERSION} found instead).
 The model fitting plugin will use a potentially less robust fallback method for computing error margins!")
 endif ()
 
diff --git a/SEImplementation/SEImplementation/Plugin/FlexibleModelFitting/FlexibleModelFittingTask.h b/SEImplementation/SEImplementation/Plugin/FlexibleModelFitting/FlexibleModelFittingTask.h
@@ -65,11 +65,10 @@ class FlexibleModelFittingTask : public GroupTask {
   void updateCheckImages(SourceGroupInterface& group,
       double pixel_scale, FlexibleModelFittingParameterManager& manager) const;
 
-  SeFloat computeReducedChiSquaredForFrame(std::shared_ptr<const Image<SeFloat>> image,
-      std::shared_ptr<const Image<SeFloat>> model, std::shared_ptr<const Image<SeFloat>> weights,
-      int nb_of_free_params) const;
-  SeFloat computeReducedChiSquared(SourceGroupInterface& group,
-      double pixel_scale, FlexibleModelFittingParameterManager& manager) const;
+  SeFloat computeChiSquaredForFrame(std::shared_ptr<const Image<SeFloat>> image,
+      std::shared_ptr<const Image<SeFloat>> model, std::shared_ptr<const Image<SeFloat>> weights, int& data_points) const;
+  SeFloat computeChiSquared(SourceGroupInterface& group,
+      double pixel_scale, FlexibleModelFittingParameterManager& manager, int& total_data_points) const;
 
   // Task configuration
   unsigned int m_max_iterations;
diff --git a/SEImplementation/src/lib/CheckImages/CheckImages.cpp b/SEImplementation/src/lib/CheckImages/CheckImages.cpp
@@ -258,7 +258,7 @@ void CheckImages::saveImages() {
       filename += "_" + ci.first->getLabel();
       filename.replace_extension(m_residual_filename.extension());
       auto frame_filename = m_residual_filename.parent_path() / filename;
-      FitsWriter::writeFile(*residual_image, filename.native(), ci.first->getCoordinateSystem());
+      FitsWriter::writeFile(*residual_image, frame_filename.native(), ci.first->getCoordinateSystem());
     }
   }
 
diff --git a/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingParameter.cpp b/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingParameter.cpp
@@ -24,7 +24,7 @@
 #include <iostream>
 
 #include <boost/version.hpp>
-#if BOOST_VERSION >= 106600
+#if BOOST_VERSION >= 106700
 
 #if BOOST_VERSION >= 107000
 #include <boost/math/differentiation/finite_difference.hpp>
@@ -182,7 +182,7 @@ std::vector<double> FlexibleModelFittingDependentParameter::getPartialDerivative
         return m_value_calculator(cs, params);
     };
 
-#if BOOST_VERSION >= 106600
+#if BOOST_VERSION >= 106700
     result[i] = bmd::finite_difference_derivative(f, param_values[i]);
 #else
     // if boost's function is unavailable use our own function
diff --git a/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingTask.cpp b/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingTask.cpp
@@ -307,7 +307,21 @@ void FlexibleModelFittingTask::computeProperties(SourceGroupInterface& group) co
   LevmarEngine engine{m_max_iterations, 1E-6, 1E-6, 1E-6, 1E-6, 1E-4};
   auto solution = engine.solveProblem(engine_parameter_manager, res_estimator);
   size_t iterations = (size_t) boost::any_cast<std::array<double, 10>>(solution.underlying_framework_info)[5];
-  SeFloat avg_reduced_chi_squared = computeReducedChiSquared(group, pixel_scale, parameter_manager);
+
+  int total_data_points = 0;
+  SeFloat avg_reduced_chi_squared = computeChiSquared(group, pixel_scale, parameter_manager, total_data_points);
+
+  int nb_of_free_parameters = 0;
+  for (auto& source : group) {
+    for (auto parameter : m_parameters) {
+      bool is_free_parameter = std::dynamic_pointer_cast<FlexibleModelFittingFreeParameter>(parameter).get();
+      bool accessed_by_modelfitting = parameter_manager.isParamAccessed(source, parameter);
+      if (is_free_parameter && accessed_by_modelfitting) {
+        nb_of_free_parameters++;
+      }
+    }
+  }
+  avg_reduced_chi_squared /= (total_data_points - nb_of_free_parameters);
 
   // Collect parameters for output
   for (auto& source : group) {
@@ -316,10 +330,10 @@ void FlexibleModelFittingTask::computeProperties(SourceGroupInterface& group) co
 
     for (auto parameter : m_parameters) {
       bool is_dependent_parameter = std::dynamic_pointer_cast<FlexibleModelFittingDependentParameter>(parameter).get();
-      bool accesed_by_modelfitting = parameter_manager.isParamAccessed(source, parameter);
+      bool accessed_by_modelfitting = parameter_manager.isParamAccessed(source, parameter);
       auto modelfitting_parameter = parameter_manager.getParameter(source, parameter);
 
-      if (is_dependent_parameter || accesed_by_modelfitting) {
+      if (is_dependent_parameter || accessed_by_modelfitting) {
         parameter_values[parameter->getId()] = modelfitting_parameter->getValue();
         parameter_sigmas[parameter->getId()] = parameter->getSigma(parameter_manager, source, solution.parameter_sigmas);
       }
@@ -334,6 +348,7 @@ void FlexibleModelFittingTask::computeProperties(SourceGroupInterface& group) co
         source_flags |= Flags::PARTIAL_FIT;
       }
     }
+
     source.setProperty<FlexibleModelFitting>(iterations, avg_reduced_chi_squared, source_flags, parameter_values,
                                              parameter_sigmas);
   }
@@ -374,11 +389,10 @@ void FlexibleModelFittingTask::updateCheckImages(SourceGroupInterface& group,
   }
 }
 
-SeFloat FlexibleModelFittingTask::computeReducedChiSquaredForFrame(std::shared_ptr<const Image<SeFloat>> image,
-    std::shared_ptr<const Image<SeFloat>> model, std::shared_ptr<const Image<SeFloat>> weights,
-    int nb_of_free_params) const {
+SeFloat FlexibleModelFittingTask::computeChiSquaredForFrame(std::shared_ptr<const Image<SeFloat>> image,
+    std::shared_ptr<const Image<SeFloat>> model, std::shared_ptr<const Image<SeFloat>> weights, int& data_points) const {
   double reduced_chi_squared = 0.0;
-  int data_points = 0;
+  data_points = 0;
   for (int y=0; y < image->getHeight(); y++) {
     for (int x=0; x < image->getWidth(); x++) {
       double tmp = image->getValue(x, y) - model->getValue(x, y);
@@ -388,13 +402,14 @@ SeFloat FlexibleModelFittingTask::computeReducedChiSquaredForFrame(std::shared_p
       }
     }
   }
-  return reduced_chi_squared / (data_points - nb_of_free_params);
+  return reduced_chi_squared;
 }
 
-SeFloat FlexibleModelFittingTask::computeReducedChiSquared(SourceGroupInterface& group,
-    double pixel_scale, FlexibleModelFittingParameterManager& manager) const {
+SeFloat FlexibleModelFittingTask::computeChiSquared(SourceGroupInterface& group,
+    double pixel_scale, FlexibleModelFittingParameterManager& manager, int& total_data_points) const {
 
-  SeFloat avg_reduced_chi_squared = 0;
+  SeFloat total_chi_squared = 0;
+  total_data_points = 0;
   int valid_frames = 0;
   for (auto frame : m_frames) {
     int frame_index = frame->getFrameNb();
@@ -408,13 +423,16 @@ SeFloat FlexibleModelFittingTask::computeReducedChiSquared(SourceGroupInterface&
       auto image = createImageCopy(group, frame_index);
       auto weight = createWeightImage(group, frame_index);
 
-      SeFloat reduced_chi_squared = computeReducedChiSquaredForFrame(
-          image, final_stamp, weight, manager.getParameterNb());
-      avg_reduced_chi_squared += reduced_chi_squared;
+      int data_points = 0;
+      SeFloat chi_squared = computeChiSquaredForFrame(
+          image, final_stamp, weight, data_points);
+
+      total_data_points += data_points;
+      total_chi_squared += chi_squared;
     }
   }
 
-  return avg_reduced_chi_squared / valid_frames;
+  return total_chi_squared;
 }
 
 FlexibleModelFittingTask::~FlexibleModelFittingTask() {
diff --git a/SEMain/auxdir/SEMain/sextractor++.desktop b/SEMain/auxdir/SEMain/sextractor++.desktop
@@ -5,4 +5,4 @@ Terminal=true
 NoDisplay=true
 Exec=sextractor++
 Icon=sextractor++
-
+Categories=Education;
diff --git a/SEMain/src/program/SExtractor.cpp b/SEMain/src/program/SExtractor.cpp
diff --git a/SEUtils/tests/src/NumericalDerivative_test.cpp b/SEUtils/tests/src/NumericalDerivative_test.cpp
diff --git a/build-appimage.sh b/build-appimage.sh
diff --git a/cmake/auxdir/package/Elements-squeeze.spec.in b/cmake/auxdir/package/Elements-squeeze.spec.in
diff --git a/cmake/doc/Doxyfile.in b/cmake/doc/Doxyfile.in

Original file line number	Diff line number	Diff line change
`@@ -258,7 +258,7 @@ void CheckImages::saveImages() {`
`258`	`258`	`filename += "_" + ci.first->getLabel();`
`259`	`259`	`filename.replace_extension(m_residual_filename.extension());`
`260`	`260`	`auto frame_filename = m_residual_filename.parent_path() / filename;`
`261`		`- FitsWriter::writeFile(*residual_image, filename.native(), ci.first->getCoordinateSystem());`
	`261`	`+ FitsWriter::writeFile(*residual_image, frame_filename.native(), ci.first->getCoordinateSystem());`
`262`	`262`	`}`
`263`	`263`	`}`
`264`	`264`