Scale from depth

Author: servantftransperfect

meshroom/aliceVision/SfMTransform.py

@@ -1,4 +1,4 @@
-__version__ = "3.1"
+__version__ = "3.2"
 
 from meshroom.core import desc
 from meshroom.core.utils import DESCRIBER_TYPES, VERBOSE_LEVEL
@@ -23,6 +23,7 @@ class SfMTransform(desc.AVCommandLineNode):
  * from_gps: Align with the gps positions from the image metadata
  * align_ground: Detect ground level and align to it
  * from_lineup: Align using a camera pose (json line up file), tracks and a mesh
+ * from_depthmaps: Rescale scene to match metric depth in tracks
 """
 
     inputs = [
@@ -47,9 +48,10 @@ class SfMTransform(desc.AVCommandLineNode):
                         " - from_markers: Defines the coordinate system from markers specified by --markers.\n"
                         " - from_gps: Defines coordinate system from GPS metadata.\n"
                         " - from_lineup: Defines coordinate system using lineup json file.\n"
-                        " - align_ground: Defines ground level from the point cloud density. It assumes that the scene is oriented.",
+                        " - align_ground: Defines ground level from the point cloud density. It assumes that the scene is oriented.\n"
+                        " - from_depthmaps: Scale given depthmaps\n",
             value="auto",
-            values=["transformation", "manual", "auto", "auto_from_cameras", "auto_from_cameras_x_axis", "auto_from_landmarks", "from_single_camera", "from_center_camera", "from_markers", "from_gps", "from_lineup", "align_ground"],
+            values=["transformation", "manual", "auto", "auto_from_cameras", "auto_from_cameras_x_axis", "auto_from_landmarks", "from_single_camera", "from_center_camera", "from_markers", "from_gps", "from_lineup", "align_ground", "from_depthmaps"],
         ),
         desc.File(
             name="lineUp",
@@ -63,7 +65,7 @@ class SfMTransform(desc.AVCommandLineNode):
             label="Tracks File",
             description="Tracks file for lineup.",
             value="",
-            enabled=lambda node: node.method.value == "from_lineup"
+            enabled=lambda node: node.method.value in ["from_lineup", "from_depthmaps"]
         ),
         desc.File(
             name="objectFile",

src/aliceVision/sfm/pipeline/expanding/ExpansionProcess.hpp

@@ -66,22 +66,21 @@ class ExpansionProcess{
         _postProcessHandler = std::move(expansionPostProcess);
     }
 
-private:
-
     /**
-     * @brief Process sfmData if something exists inside (previous sfm)
+     * @brief Remap the sfmData landmarks to id compatible with trackmap
      * @param[in] sfmData the object to update
      * @param[in] tracks the tracks for this scene
      */
-    bool prepareExisting(sfmData::SfMData & sfmData, const track::TracksHandler & tracksHandler);
+    static void remapExistingLandmarks(sfmData::SfMData & sfmData, const track::TracksHandler & tracksHandler);
+
+private:
 
     /**
-     * @brief Remap the sfmData landmarks to id compatible with trackmap
+     * @brief Process sfmData if something exists inside (previous sfm)
      * @param[in] sfmData the object to update
      * @param[in] tracks the tracks for this scene
      */
-    void remapExistingLandmarks(sfmData::SfMData & sfmData, const track::TracksHandler & tracksHandler);
-
+    bool prepareExisting(sfmData::SfMData & sfmData, const track::TracksHandler & tracksHandler);
 
 
 private:

src/software/utils/main_sfmTransform.cpp

@@ -14,6 +14,7 @@
 #include <aliceVision/track/TracksHandler.hpp>
 #include <aliceVision/geometry/Pose3.hpp>
 #include <aliceVision/mesh/MeshIntersection.hpp>
+#include <aliceVision/sfm/pipeline/expanding/ExpansionProcess.hpp>
 #include <aliceVision/config.hpp>
 #include <fstream>
 
@@ -30,10 +31,11 @@
 
 
 
+
 // These constants define the current software version.
 // They must be updated when the command line is changed.
-#define ALICEVISION_SOFTWARE_VERSION_MAJOR 1
-#define ALICEVISION_SOFTWARE_VERSION_MINOR 1
+#define ALICEVISION_SOFTWARE_VERSION_MAJOR 3
+#define ALICEVISION_SOFTWARE_VERSION_MINOR 2
 
 using namespace aliceVision;
 
@@ -57,7 +59,8 @@ enum class EAlignmentMethod : unsigned char
     FROM_MARKERS,
     FROM_GPS,
     FROM_LINEUP,
-    ALIGN_GROUND
+    ALIGN_GROUND,
+    FROM_DEPTHMAPS
 };
 
 /**
@@ -93,6 +96,8 @@ std::string EAlignmentMethod_enumToString(EAlignmentMethod alignmentMethod)
             return "from_lineup";
         case EAlignmentMethod::ALIGN_GROUND:
             return "align_ground";
+        case EAlignmentMethod::FROM_DEPTHMAPS:
+            return "from_depthmaps";
     }
     throw std::out_of_range("Invalid EAlignmentMethod enum");
 }
@@ -131,6 +136,8 @@ EAlignmentMethod EAlignmentMethod_stringToEnum(const std::string& alignmentMetho
         return EAlignmentMethod::FROM_LINEUP;
     if (method == "align_ground")
         return EAlignmentMethod::ALIGN_GROUND;
+    if (method == "from_depthmaps")
+        return EAlignmentMethod::FROM_DEPTHMAPS;
     throw std::out_of_range("Invalid SfM alignment method : " + alignmentMethod);
 }
 
@@ -316,6 +323,62 @@ bool getPoseFromJson(const std::string & lineUpFilename, Eigen::Matrix4d & T, In
     return true;
 }
 
+bool computeNewScaleFromDepths(const std::string & tracksFilename, sfmData::SfMData& sfmData, double & S)
+{
+    // Load tracks
+    ALICEVISION_LOG_INFO("Load tracks");
+    track::TracksHandler tracksHandler;
+    if (!tracksHandler.load(tracksFilename, sfmData.getValidViews()))
+    {
+        ALICEVISION_LOG_ERROR("The input tracks file '" + tracksFilename + "' cannot be read.");
+        return false;
+    }
+
+    sfm::ExpansionProcess::remapExistingLandmarks(sfmData, tracksHandler);
+
+    const track::TracksMap & tracks = tracksHandler.getAllTracks();
+
+    std::vector<double> ratios;
+    for (const auto & [lid, landmark] : sfmData.getLandmarks())
+    {
+        const track::Track & track = tracks.at(lid);
+
+        for (const auto & [viewId, observation] : landmark.getObservations())
+        {
+            const sfmData::View & v = sfmData.getView(viewId);
+            const sfmData::CameraPose cp = sfmData.getPose(v);
+
+            Vec3 pt = cp.getTransform()(landmark.X);
+            if (pt.z() < 1e-12)
+            {
+                continue;
+            }
+
+            double fz = track.featPerView.at(viewId).depth;
+            if (fz < 1e-12)
+            {
+                continue;
+            }
+            
+            //Ratio between the depth from sfm, and the depth from depthmap
+            double ratio = fz / pt.z();
+            ratios.push_back(ratio);
+        }
+    }
+
+    if (ratios.size() == 0)
+    {
+        return false;
+    }
+
+    // Compute median
+    const auto medianIterator = ratios.begin() + ratios.size() / 2 - 1;
+    std::nth_element(ratios.begin(), medianIterator, ratios.end());
+    S = *medianIterator;
+
+    return true;
+}
+
 
 bool parseLineUp(const std::string & lineUpFilename, const std::string & tracksFilename, const std::string & objectFilename, sfmData::SfMData& sfmData, double & S, Eigen::Matrix3d &  R, Eigen::Vector3d & t)
 {
@@ -478,7 +541,8 @@ int aliceVision_main(int argc, char** argv)
          "\t- from_single_camera: Refines the coordinate system from the camera specified by --tranformation.\n"
          "\t- from_markers: Refines the coordinate system from markers specified by --markers.\n"
          "\t- from_gps: Redefines coordinate system from GPS metadata.\n"
-         "\t- align_ground: defines ground level from the point cloud density. It assumes that the scene is oriented.\n")
+         "\t- align_ground: defines ground level from the point cloud density. It assumes that the scene is oriented.\n"
+         "\t- from_depthmaps: Scale given injected depthmaps.\n")
         ("transformation", po::value<std::string>(&transform)->default_value(transform),
          "Required only for 'transformation' and 'single camera' methods:\n"
          "Transformation: Align [X,Y,Z] to +Y-axis, rotate around Y by R deg, scale by S; syntax: X,Y,Z;R;S.\n"
@@ -687,6 +751,18 @@ int aliceVision_main(int argc, char** argv)
             }
             break;
         }
+        case EAlignmentMethod::FROM_DEPTHMAPS:
+        {
+            R.setIdentity();
+            t.fill(0.0);
+
+            if (!computeNewScaleFromDepths(tracksFilename, sfmData, S))
+            {
+                ALICEVISION_LOG_ERROR("Failed to use depths");
+                return EXIT_FAILURE;
+            }
+            break;
+        }
         case EAlignmentMethod::ALIGN_GROUND:
         {
             sfm::computeNewCoordinateSystemGroundAuto(sfmData, t);