From e017ff346ff2c809466445ee229ee00ed34a257e Mon Sep 17 00:00:00 2001 From: cDc Date: Sun, 10 Feb 2019 22:35:59 +0200 Subject: [PATCH 01/70] dense: improve PatchMatch algorithm --- apps/DensifyPointCloud/DensifyPointCloud.cpp | 22 +- .../InterfaceVisualSFM/InterfaceVisualSFM.cpp | 6 +- build/Utils.cmake | 11 +- libs/Common/Config.h | 4 + libs/Common/List.h | 42 +- libs/Common/Random.h | 164 ++++++ libs/Common/Rotation.h | 6 +- libs/Common/Rotation.inl | 83 ++- libs/Common/Types.h | 95 +-- libs/Common/Types.inl | 32 + libs/MVS/DepthMap.cpp | 548 +++++++++++++++--- libs/MVS/DepthMap.h | 163 +++++- libs/MVS/Image.cpp | 6 +- libs/MVS/Image.h | 4 +- libs/MVS/Scene.cpp | 8 +- libs/MVS/SceneDensify.cpp | 113 +++- 16 files changed, 1023 insertions(+), 284 deletions(-) create mode 100644 libs/Common/Random.h diff --git a/apps/DensifyPointCloud/DensifyPointCloud.cpp b/apps/DensifyPointCloud/DensifyPointCloud.cpp index bcace9c5e..beb89e973 100644 --- a/apps/DensifyPointCloud/DensifyPointCloud.cpp +++ b/apps/DensifyPointCloud/DensifyPointCloud.cpp @@ -85,22 +85,26 @@ bool Initialize(size_t argc, LPCTSTR* argv) // group of options allowed both on command line and in config file unsigned nResolutionLevel; + unsigned nMaxResolution; unsigned nMinResolution; unsigned nNumViews; unsigned nMinViewsFuse; + unsigned nOptimize; unsigned nEstimateColors; unsigned nEstimateNormals; boost::program_options::options_description config("Densify options"); config.add_options() ("input-file,i", boost::program_options::value(&OPT::strInputFileName), "input filename containing camera poses and image list") ("output-file,o", boost::program_options::value(&OPT::strOutputFileName), "output filename for storing the dense point-cloud") - ("resolution-level", boost::program_options::value(&nResolutionLevel)->default_value(1), "how many times to scale down the images before point cloud computation") - ("min-resolution", boost::program_options::value(&nMinResolution)->default_value(640), "do not scale images lower than this resolution") - ("number-views", boost::program_options::value(&nNumViews)->default_value(4), "number of views used for depth-map estimation (0 - all neighbor views available)") - ("number-views-fuse", boost::program_options::value(&nMinViewsFuse)->default_value(3), "minimum number of images that agrees with an estimate during fusion in order to consider it inlier") - ("estimate-colors", boost::program_options::value(&nEstimateColors)->default_value(1), "estimate the colors for the dense point-cloud") - ("estimate-normals", boost::program_options::value(&nEstimateNormals)->default_value(0), "estimate the normals for the dense point-cloud") - ("sample-mesh", boost::program_options::value(&OPT::fSampleMesh)->default_value(0.f), "uniformly samples points on a mesh (0 - disabled, <0 - number of points, >0 - sample density per square unit)") + ("resolution-level", boost::program_options::value(&nResolutionLevel)->default_value(1), "how many times to scale down the images before point cloud computation") + ("max-resolution", boost::program_options::value(&nMaxResolution)->default_value(3200), "do not scale images higher than this resolution") + ("min-resolution", boost::program_options::value(&nMinResolution)->default_value(640), "do not scale images lower than this resolution") + ("number-views", boost::program_options::value(&nNumViews)->default_value(8), "number of views used for depth-map estimation (0 - all neighbor views available)") + ("number-views-fuse", boost::program_options::value(&nMinViewsFuse)->default_value(3), "minimum number of images that agrees with an estimate during fusion in order to consider it inlier") + ("optimize", boost::program_options::value(&nOptimize)->default_value(7), "filter used after depth-map estimation (0 - disabled, 1 - remove speckles, 2 - fill gaps, 4 - cross-adjust)") + ("estimate-colors", boost::program_options::value(&nEstimateColors)->default_value(2), "estimate the colors for the dense point-cloud") + ("estimate-normals", boost::program_options::value(&nEstimateNormals)->default_value(0), "estimate the normals for the dense point-cloud") + ("sample-mesh", boost::program_options::value(&OPT::fSampleMesh)->default_value(0.f), "uniformly samples points on a mesh (0 - disabled, <0 - number of points, >0 - sample density per square unit)") ; // hidden options, allowed both on command line and @@ -161,13 +165,17 @@ bool Initialize(size_t argc, LPCTSTR* argv) OPT::strOutputFileName = Util::getFileFullName(OPT::strInputFileName) + _T("_dense.mvs"); // init dense options + if (!Util::isFullPath(OPT::strDenseConfigFileName)) + OPT::strDenseConfigFileName = MAKE_PATH(OPT::strDenseConfigFileName); OPTDENSE::init(); const bool bValidConfig(OPTDENSE::oConfig.Load(OPT::strDenseConfigFileName)); OPTDENSE::update(); OPTDENSE::nResolutionLevel = nResolutionLevel; + OPTDENSE::nMaxResolution = nMaxResolution; OPTDENSE::nMinResolution = nMinResolution; OPTDENSE::nNumViews = nNumViews; OPTDENSE::nMinViewsFuse = nMinViewsFuse; + OPTDENSE::nOptimize = nOptimize; OPTDENSE::nEstimateColors = nEstimateColors; OPTDENSE::nEstimateNormals = nEstimateNormals; if (!bValidConfig) diff --git a/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp b/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp index bfb563e66..97ff984b9 100644 --- a/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp +++ b/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp @@ -248,7 +248,6 @@ void UndistortImage(const Camera& camera, const REAL& k1, const Image8U3 imgIn, typedef Sampler::Cubic Sampler; const Sampler sampler; Point2f pt; - Pixel32F clr; for (int v=0; v(sampler, pt); - col.r = CLAMP(ROUND2INT(clr.r), 0, 255); - col.g = CLAMP(ROUND2INT(clr.g), 0, 255); - col.b = CLAMP(ROUND2INT(clr.b), 0, 255); + col = Cast(imgIn.sample(sampler, pt)); } else { // set to black col = Pixel8U::BLACK; diff --git a/build/Utils.cmake b/build/Utils.cmake index 6c33c1f74..fc713bdf1 100644 --- a/build/Utils.cmake +++ b/build/Utils.cmake @@ -421,8 +421,15 @@ macro(optimize_default_compiler_settings) set(BUILD_EXTRA_EXE_LINKER_FLAGS_RELEASE "") set(BUILD_EXTRA_EXE_LINKER_FLAGS_DEBUG "") - # enable C++11 support - set(CMAKE_CXX_STANDARD 11) + # try to enable C++14/C++11 support + check_cxx_compiler_flag(--std=c++14 SUPPORTS_STD_CXX14) + check_cxx_compiler_flag(--std=c++11 SUPPORTS_STD_CXX11) + if(SUPPORTS_STD_CXX14) + set(CMAKE_CXX_STANDARD 14) + elseif(SUPPORTS_STD_CXX11) + set(CMAKE_CXX_STANDARD 11) + endif() + message("Compiling with C++${CMAKE_CXX_STANDARD}") if(MINGW) # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=40838 diff --git a/libs/Common/Config.h b/libs/Common/Config.h index 60783fe46..779b2bb3e 100644 --- a/libs/Common/Config.h +++ b/libs/Common/Config.h @@ -194,6 +194,10 @@ # define FORCEINLINE inline #endif +#ifndef _SUPPORT_CPP11 +# define constexpr inline +#endif + #define SAFE_DELETE(p) { if (p!=NULL) { delete (p); (p)=NULL; } } #define SAFE_DELETE_ARR(p) { if (p!=NULL) { delete [] (p); (p)=NULL; } } #define SAFE_FREE(p) { if (p!=NULL) { free(p); (p)=NULL; } } diff --git a/libs/Common/List.h b/libs/Common/List.h index e94672fd1..794b0a2a4 100644 --- a/libs/Common/List.h +++ b/libs/Common/List.h @@ -70,8 +70,9 @@ #define CLISTDEF0(TYPE) SEACAVE::cList< TYPE, const TYPE&, 0 > #define CLISTDEF2(TYPE) SEACAVE::cList< TYPE, const TYPE&, 2 > -#define CLISTDEFIDX(TYPE,IDXTYPE) SEACAVE::cList< TYPE, const TYPE&, 1, 16, IDXTYPE > #define CLISTDEF0IDX(TYPE,IDXTYPE) SEACAVE::cList< TYPE, const TYPE&, 0, 16, IDXTYPE > +#define CLISTDEFIDX(TYPE,IDXTYPE) SEACAVE::cList< TYPE, const TYPE&, 1, 16, IDXTYPE > +#define CLISTDEF2IDX(TYPE,IDXTYPE) SEACAVE::cList< TYPE, const TYPE&, 2, 16, IDXTYPE > namespace SEACAVE { @@ -109,7 +110,6 @@ class cList typedef ARG_TYPE ArgType; typedef IDX_TYPE IDX; typedef int (STCALL *TFncCompare)(const void* elem, const void* key); //returns 0 if equal, otherwise <0 or >0 respectively - typedef bool (STCALL *TFncCompareBool)(ARG_TYPE elem, ARG_TYPE key); //returns true if the two elements are strict in weak ordering // construct an empty list inline cList() : _size(0), _vectorSize(0), _vector(NULL) @@ -627,40 +627,57 @@ class cList return std::accumulate(Begin(), End(), TYPE(0)) / _size; } + inline TYPE& PartialSort(IDX index) + { + TYPE* const nth(Begin()+index); + std::partial_sort(Begin(), nth, End()); + return *nth; + } + inline void Sort() { std::sort(Begin(), End()); } - inline void Sort(TFncCompareBool xCompare) + template + inline void Sort(const Functor& functor) { - std::sort(Begin(), End(), xCompare); + std::sort(Begin(), End(), functor); } inline bool IsSorted() const { + #ifdef _SUPPORT_CPP11 + return std::is_sorted(Begin(), End()); + #else if (_size < 2) return true; IDX i = _size-1; do { ARG_TYPE elem1 = _vector[i]; ARG_TYPE elem0 = _vector[--i]; - if (!(elem0 < elem1 || elem0 == elem1)) + if (elem1 < elem0) return false; } while (i > 0); return true; + #endif } - inline bool IsSorted(TFncCompareBool xCompare) const + template + inline bool IsSorted(const Functor& functor) const { + #ifdef _SUPPORT_CPP11 + return std::is_sorted(Begin(), End(), functor); + #else if (_size < 2) return true; IDX i = _size-1; do { ARG_TYPE elem1 = _vector[i]; ARG_TYPE elem0 = _vector[--i]; - if (!xCompare(elem0, elem1)) + if (functor(elem1, elem0)) return false; } while (i > 0); return true; + #endif } inline std::pair InsertSortUnique(ARG_TYPE elem) @@ -1273,7 +1290,7 @@ class cList TYPE* _vector; public: - static const IDX NO_INDEX; + enum : IDX { NO_INDEX = DECLARE_NO_INDEX(IDX) }; #if _USE_VECTORINTERFACE != 0 public: @@ -1284,6 +1301,9 @@ class cList typedef const value_type& const_reference; typedef std::vector VectorType; inline cList(const VectorType& rList) { CopyOf(&rList[0], rList.size()); } + #ifdef _SUPPORT_CPP11 + inline cList(std::initializer_list l) : _size(0), _vectorSize((IDX)l.size()), _vector(NULL) { ASSERT(l.size() -const typename cList::IDX -cList::NO_INDEX(DECLARE_NO_INDEX(IDX)); /*----------------------------------------------------------------*/ template @@ -1346,7 +1363,6 @@ inline bool ValidIDX(const IDX_TYPE& idx) { // some test functions // run cListTest(99999); #if 0 -static bool SortIntTest(int a, int b) { return a arrR; cList arr0; @@ -1362,7 +1378,7 @@ inline bool cListTestIter(unsigned elems) { arrC.Insert(e); } std::sort(arrR.begin(), arrR.end()); - arrC.Sort(SortIntTest); + arrC.Sort([](int a, int b) { return a(); +} +FORCEINLINE double randomd() { + return RANDOM(); +} +FORCEINLINE long double randomld() { + return RANDOM(); +} +STATIC_ASSERT(RAND_MAX < 2147483648); // integer randomRange assumes this is capped +template +FORCEINLINE T randomRange(T nMin, T nMax) { + ASSERT(nMin <= nMax && nMax-nMin+1 < 8589934596); // not to overflow a uint64_t + return nMin + T((uint64_t(nMax-nMin)*RAND()+RAND_MAX/2)/RAND_MAX); +} +template<> +FORCEINLINE float randomRange(float fMin, float fMax) { + return fMin + (fMax - fMin) * random(); +} +template<> +FORCEINLINE double randomRange(double fMin, double fMax) { + return fMin + (fMax - fMin) * randomd(); +} +template<> +FORCEINLINE long double randomRange(long double fMin, long double fMax) { + return fMin + (fMax - fMin) * randomld(); +} +template +FORCEINLINE T randomMeanRange(T mean, T delta/*=(max-min)/2*/) { + ASSERT(delta >= 0 && delta*2+1 < 8589934596); // not to overflow a uint64_t + return (mean + T((uint64_t(delta)*2*RAND()+RAND_MAX/2)/RAND_MAX)) - delta; +} +template<> +FORCEINLINE float randomMeanRange(float mean, float delta/*=(max-min)/2*/) { + return mean + delta * (2.f * random() - 1.f); +} +template<> +FORCEINLINE double randomMeanRange(double mean, double delta/*=(max-min)/2*/) { + return mean + delta * (2.0 * randomd() - 1.0); +} +template<> +FORCEINLINE long double randomMeanRange(long double mean, long double delta/*=(max-min)/2*/) { + return mean + delta * (2.0L * randomld() - 1.0L); +} +// gaussian random number generation +template +FORCEINLINE T gaussian(T val, T sigma) { + return EXP(-SQUARE(val/sigma)/2)/(SQRT(T(M_PI*2))*sigma); +} +template +FORCEINLINE T randomGaussian(T mean, T sigma) { + T x, y, r2; + do { + x = T(-1) + T(2) * RANDOM(); + y = T(-1) + T(2) * RANDOM(); + r2 = x * x + y * y; + } while (r2 > T(1) || r2 == T(0)); + return mean + sigma * y * SQRT(T(-2) * LOGN(r2) / r2); +} +template +FORCEINLINE T randomGaussian(T sigma) { + return randomGaussian(T(0), sigma); +} +FORCEINLINE float randomGaussian() { + return randomGaussian(0.f, 1.f); +} +FORCEINLINE double randomGaussiand() { + return randomGaussian(0.0, 1.0); +} +FORCEINLINE long double randomGaussianld() { + return randomGaussian(0.0L, 1.0L); +} +/*----------------------------------------------------------------*/ + + +// Encapsulates state for random number generation +// based on C++11 random number generator functionality +struct Random : std::mt19937 { + typedef std::mt19937 generator_type; + + Random() : generator_type(std::random_device()()) {} + Random(result_type seed) : generator_type(seed) {} + + // integer randomRange assumes this is capped + STATIC_ASSERT(max() < 4294967296); + + // returns a uniform random number in the range [0, 1] + template + FORCEINLINE typename std::enable_if::value, T>::type random() { + return (T)random()/(T)max(); + } + // returns a uniform random number in the range [0, max()] + template + FORCEINLINE typename std::enable_if::value, T>::type random() { + return (T)operator()(); + } + + // returns a uniform random number in the range [nMin, nMax] + template + FORCEINLINE typename std::enable_if::value, T>::type randomRange(T nMin, T nMax) { + return nMin + (nMax-nMin) * random(); + } + template + FORCEINLINE typename std::enable_if::value, T>::type randomRange(T nMin, T nMax) { + ASSERT(nMin <= nMax && nMax-nMin+1 < 4294967297); // not to overflow a uint64_t + return nMin + (T)(((uint64_t)(nMax-nMin) * random() + max()/2)/max()); + } + + // returns a uniform random number in the range [mean-delta, mean+delta] + template + FORCEINLINE typename std::enable_if::value, T>::type randomMeanRange(T mean, T delta/*=(max-min)/2*/) { + return mean + delta * (T(2) * random() - T(1)); + } + template + FORCEINLINE typename std::enable_if::value, T>::type randomMeanRange(T mean, T delta/*=(max-min)/2*/) { + ASSERT(delta >= 0 && delta*T(2)+1 < 4294967297); // not to overflow a uint64_t + return mean + (T)(((uint64_t)delta*2 * random() + max()/2)/max()) - delta; + } + + // returns a uniform random number in the range [nMin, nMax] using std implementation + template + FORCEINLINE typename std::enable_if::value, T>::type randomUniform(T nMin, T nMax) { + return std::uniform_real_distribution(nMin, nMax)(*this); + } + template + FORCEINLINE typename std::enable_if::value, T>::type randomUniform(T nMin, T nMax) { + return std::uniform_int_distribution(nMin, nMax)(*this); + } + + // returns a gaussian random number using std implementation + template + FORCEINLINE T randomGaussian(T mean, T stddev) { + return std::normal_distribution(mean, stddev)(*this); + } +}; +/*----------------------------------------------------------------*/ + +} // namespace SEACAVE + +#endif // __SEACAVE_RANDOM_H__ diff --git a/libs/Common/Rotation.h b/libs/Common/Rotation.h index 7959e7ed6..87761e56e 100644 --- a/libs/Common/Rotation.h +++ b/libs/Common/Rotation.h @@ -290,13 +290,15 @@ class TRMatrixBase : public TMatrix /** @brief Initialization from parametrized rotation (axis-angle) */ inline TRMatrixBase(const Vec& rot); - /** @brief Initialization from rotation axis w and angle phi (in rad) - using Rodrigues' formula */ + /** @brief Initialization from rotation axis w and angle phi (in rad) using Rodrigues' formula */ inline TRMatrixBase(const Vec& w, const TYPE phi); /** @brief Initialization from quaternion */ inline TRMatrixBase(const Quat& q); + /** @brief Initialization with the rotation from roll/pitch/yaw (in rad) */ + inline TRMatrixBase(TYPE roll, TYPE pitch, TYPE yaw); + inline ~TRMatrixBase(); template inline TRMatrixBase& operator = (const cv::Matx& rhs) { BaseBase::operator = (rhs); return *this; } diff --git a/libs/Common/Rotation.inl b/libs/Common/Rotation.inl index e8a82a50f..212d6c075 100644 --- a/libs/Common/Rotation.inl +++ b/libs/Common/Rotation.inl @@ -553,6 +553,12 @@ inline TRMatrixBase::TRMatrixBase(const Quat& q) SetFromQuaternion(q); } +template +inline TRMatrixBase::TRMatrixBase(TYPE roll, TYPE pitch, TYPE yaw) +{ + SetXYZ(roll, pitch, yaw); +} + template inline TRMatrixBase::~TRMatrixBase() {} @@ -561,33 +567,48 @@ inline TRMatrixBase::~TRMatrixBase() template void TRMatrixBase::SetXYZ(TYPE PhiX, TYPE PhiY, TYPE PhiZ) { - TRMatrixBase Rx(TRMatrixBase::IDENTITY); - TRMatrixBase Ry(TRMatrixBase::IDENTITY); - TRMatrixBase Rz(TRMatrixBase::IDENTITY); - - /* set Rz, Ry, Rx as rotation matrices */ - Rz(0,0) = cos(PhiZ); - Rz(0,1) = -sin(PhiZ); - Rz(1,0) = -Rz(0,1); - Rz(1,1) = Rz(0,0); - //Rz(2,2) = 1; - - Ry(0,0) = cos(PhiY); - Ry(0,2) = sin(PhiY); - //Ry(1,1) = 1; - Ry(2,0) = -Ry(0,2); //-sin(PhiY); - Ry(2,2) = Ry(0,0); //cos(PhiY); - - //Rx(0,0) = 1; - Rx(1,1) = cos(PhiX); - Rx(1,2) = -sin(PhiX); - Rx(2,1) = -Rx(1,2); //sin(PhiX); - Rx(2,2) = Rx(1,1); //cos(PhiX); - -// Mat Result=Rx*Ry*Rz; -// for (int i =0; i<9; i++) -// this->val[i]=Result.GetData()[i]; - (*this) = Rx*Ry*Rz; + #if 0 + TRMatrixBase Rx(TRMatrixBase::IDENTITY); + TRMatrixBase Ry(TRMatrixBase::IDENTITY); + TRMatrixBase Rz(TRMatrixBase::IDENTITY); + + /* set Rz, Ry, Rx as rotation matrices */ + Rz(0,0) = cos(PhiZ); + Rz(0,1) = -sin(PhiZ); + Rz(1,0) = -Rz(0,1); + Rz(1,1) = Rz(0,0); + //Rz(2,2) = 1; + + Ry(0,0) = cos(PhiY); + Ry(0,2) = sin(PhiY); + //Ry(1,1) = 1; + Ry(2,0) = -Ry(0,2); //-sin(PhiY); + Ry(2,2) = Ry(0,0); //cos(PhiY); + + //Rx(0,0) = 1; + Rx(1,1) = cos(PhiX); + Rx(1,2) = -sin(PhiX); + Rx(2,1) = -Rx(1,2); //sin(PhiX); + Rx(2,2) = Rx(1,1); //cos(PhiX); + + (*this) = Rx*Ry*Rz; + #else + const TYPE sin_x = sin(PhiX); + const TYPE sin_y = sin(PhiY); + const TYPE sin_z = sin(PhiZ); + const TYPE cos_x = cos(PhiX); + const TYPE cos_y = cos(PhiY); + const TYPE cos_z = cos(PhiZ); + val[0] = cos_y * cos_z; + val[1] = -cos_y * sin_z; + val[2] = sin_y; + val[3] = cos_x * sin_z + cos_z * sin_x * sin_y; + val[4] = cos_x * cos_z - sin_x * sin_y * sin_z; + val[5] = -cos_y * sin_x; + val[6] = sin_x * sin_z - cos_x * cos_z * sin_y; + val[7] = cos_z * sin_x + cos_x * sin_y * sin_z; + val[8] = cos_x * cos_y; + #endif } @@ -692,21 +713,21 @@ void TRMatrixBase::Set(const Vec& wa, TYPE phi) } const TYPE wnorm(norm(wa)); - if (wnorm < TYPE(1e-7)) { + if (wnorm < std::numeric_limits::epsilon()) { CPC_ERROR("Vector "< -FORCEINLINE T RANDOM() { return (T(1)/RAND_MAX)*RAND(); } +FORCEINLINE T RANDOM() { return T(RAND())/RAND_MAX; } template union TAliasCast @@ -598,15 +598,15 @@ namespace SEACAVE { // F U N C T I O N S /////////////////////////////////////////////// template -inline T& NEGATE(T& a) { +constexpr T& NEGATE(T& a) { return (a = -a); } template -inline T SQUARE(const T& a) { +constexpr T SQUARE(const T& a) { return (a * a); } template -inline T CUBE(const T& a) { +constexpr T CUBE(const T& a) { return (a * a * a); } template @@ -626,7 +626,7 @@ inline T LOG10(const T& a) { return T(log10(a)); } template -inline T powi(T base, int exp) { +constexpr T powi(T base, int exp) { T result(1); while (exp) { if (exp & 1) @@ -636,7 +636,7 @@ inline T powi(T base, int exp) { } return result; } -inline int log2i(int val) { +constexpr int log2i(int val) { int ret = -1; while (val > 0) { val >>= 1; @@ -654,14 +654,14 @@ inline T arithmeticSeries(T n, T a1=1, T d=1) { return (n*(a1*2+(n-1)*d))/2; } template -inline T factorial(T n) { +constexpr T factorial(T n) { T ret = 1; while (n > 1) ret *= n--; return ret; } template -inline T combinations(const T& n, const T& k) { +constexpr T combinations(const T& n, const T& k) { ASSERT(n >= k); #if 1 T num = n; @@ -675,6 +675,24 @@ inline T combinations(const T& n, const T& k) { #endif } +// adapted from https://github.com/whackashoe/fastapprox.git +// (set bSafe to true if the values might be smaller than -126) +template +inline float FPOW2(float p) { + if (bSafe && p < -126.f) { + return 0.f; + } else { + ASSERT(p >= -126.f); + CastF2I v; + v.i = static_cast((1 << 23) * (p + 126.94269504f)); + return v.f; + } +} +template +inline float FEXP(float v) { + return FPOW2(1.44269504f * v); +} + // Inverse of the square root // Compute a fast 1 / sqrtf(v) approximation inline float RSQRT(float v) { @@ -839,66 +857,6 @@ FORCEINLINE int Round2Int(double x) { /*----------------------------------------------------------------*/ -// Random number generation -// uniform random number generation -FORCEINLINE float random() { - return RANDOM(); -} -FORCEINLINE double randomd() { - return RANDOM(); -} -template -FORCEINLINE T randomRange(T nMin, T nMax) { - return nMin + ((nMax - nMin) * RAND())/RAND_MAX; -} -template<> -FORCEINLINE float randomRange(float fMin, float fMax) { - return fMin + (fMax - fMin) * random(); -} -template<> -FORCEINLINE double randomRange(double fMin, double fMax) { - return fMin + (fMax - fMin) * randomd(); -} -template -FORCEINLINE T randomMeanRange(T mean, T delta/*=(max-min)/2*/) { - return mean-delta + (delta*T(2) * RAND())/RAND_MAX; -} -template<> -FORCEINLINE float randomMeanRange(float mean, float delta/*=(max-min)/2*/) { - return mean + delta * (2.f * random() - 1.f); -} -template<> -FORCEINLINE double randomMeanRange(double mean, double delta/*=(max-min)/2*/) { - return mean + delta * (2.0 * randomd() - 1.0); -} -// gaussian random number generation -template -FORCEINLINE T gaussian(T val, T sigma) { - return EXP(-SQUARE(val/sigma)/2)/(SQRT(T(M_PI*2))*sigma); -} -template -FORCEINLINE T randomGaussian(T mean, T sigma) { - T x, y, r2; - do { - x = T(-1) + T(2) * RANDOM(); - y = T(-1) + T(2) * RANDOM(); - r2 = x * x + y * y; - } while (r2 > T(1) || r2 == T(0)); - return mean + sigma * y * SQRT(T(-2) * LOGN(r2) / r2); -} -template -FORCEINLINE T randomGaussian(T sigma) { - return randomGaussian(T(0), sigma); -} -FORCEINLINE float randomGaussian() { - return randomGaussian(0.f, 1.f); -} -FORCEINLINE double randomGaussiand() { - return randomGaussian(0.0, 1.0); -} -/*----------------------------------------------------------------*/ - - // INTERPOLATION // Linear interpolation @@ -1173,6 +1131,7 @@ inline _Tp SAFEDIVIDE(_Tp x, _Tp y) { return (y==_Tp(0) ? INVZERO(y) : x/ } // namespace SEACAVE +#include "Random.h" #include "HalfFloat.h" diff --git a/libs/Common/Types.inl b/libs/Common/Types.inl index 0dc277ffb..a4d0429aa 100644 --- a/libs/Common/Types.inl +++ b/libs/Common/Types.inl @@ -1556,6 +1556,38 @@ template inline TPoint3 Cast(const TPoint3& pt) { return pt; } +// Pixel +template +inline TPixel Cast(const TPixel::value, T>::type>& pt) { + return TPixel( + (uint8_t)CLAMP(ROUND2INT(pt.r), 0, 255), + (uint8_t)CLAMP(ROUND2INT(pt.g), 0, 255), + (uint8_t)CLAMP(ROUND2INT(pt.b), 0, 255) + ); +} +template +inline TPixel Cast(const TPixel& pt) { + return pt; +} +// Color +template +inline TColor Cast(const TColor::value, T>::type>& pt) { + return TColor( + (uint8_t)CLAMP(ROUND2INT(pt.r), 0, 255), + (uint8_t)CLAMP(ROUND2INT(pt.g), 0, 255), + (uint8_t)CLAMP(ROUND2INT(pt.b), 0, 255), + (uint8_t)CLAMP(ROUND2INT(pt.a), 0, 255) + ); +} +template +inline TColor Cast(const TColor& pt) { + return pt; +} +// Matrix +template +inline TMatrix Cast(const TMatrix& v) { + return v; +} /*----------------------------------------------------------------*/ diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index ed2350524..0b881a538 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -61,44 +61,46 @@ using namespace MVS; namespace MVS { DEFOPT_SPACE(OPTDENSE, _T("Dense")) -MDEFVAR_OPTDENSE_uint32(nMinResolution, "Min Resolution", "Do not scale images lower than this resolution", "640") DEFVAR_OPTDENSE_uint32(nResolutionLevel, "Resolution Level", "How many times to scale down the images before dense reconstruction", "1") +MDEFVAR_OPTDENSE_uint32(nMaxResolution, "Max Resolution", "Do not scale images lower than this resolution", "3200") +MDEFVAR_OPTDENSE_uint32(nMinResolution, "Min Resolution", "Do not scale images lower than this resolution", "640") DEFVAR_OPTDENSE_uint32(nMinViews, "Min Views", "minimum number of agreeing views to validate a depth", "2") MDEFVAR_OPTDENSE_uint32(nMaxViews, "Max Views", "maximum number of neighbor images used to compute the depth-map for the reference image", "12") DEFVAR_OPTDENSE_uint32(nMinViewsFuse, "Min Views Fuse", "minimum number of images that agrees with an estimate during fusion in order to consider it inlier", "2") DEFVAR_OPTDENSE_uint32(nMinViewsFilter, "Min Views Filter", "minimum number of images that agrees with an estimate in order to consider it inlier", "2") MDEFVAR_OPTDENSE_uint32(nMinViewsFilterAdjust, "Min Views Filter Adjust", "minimum number of images that agrees with an estimate in order to consider it inlier (0 - disabled)", "1") MDEFVAR_OPTDENSE_uint32(nMinViewsTrustPoint, "Min Views Trust Point", "min-number of views so that the point is considered for approximating the depth-maps (<2 - random initialization)", "2") -MDEFVAR_OPTDENSE_uint32(nNumViews, "Num Views", "Number of views used for depth-map estimation (0 - all views available)", "1", "0") +MDEFVAR_OPTDENSE_uint32(nNumViews, "Num Views", "Number of views used for depth-map estimation (0 - all views available)", "0", "1", "4") MDEFVAR_OPTDENSE_bool(bFilterAdjust, "Filter Adjust", "adjust depth estimates during filtering", "1") MDEFVAR_OPTDENSE_bool(bAddCorners, "Add Corners", "add support points at image corners with nearest neighbor disparities", "1") MDEFVAR_OPTDENSE_float(fViewMinScore, "View Min Score", "Min score to consider a neighbor images (0 - disabled)", "2.0") MDEFVAR_OPTDENSE_float(fViewMinScoreRatio, "View Min Score Ratio", "Min score ratio to consider a neighbor images", "0.3") -MDEFVAR_OPTDENSE_float(fMinArea, "Min Area", "Min shared area for accepting the depth triangulation", "0.1") +MDEFVAR_OPTDENSE_float(fMinArea, "Min Area", "Min shared area for accepting the depth triangulation", "0.05") MDEFVAR_OPTDENSE_float(fMinAngle, "Min Angle", "Min angle for accepting the depth triangulation", "3.0") MDEFVAR_OPTDENSE_float(fOptimAngle, "Optim Angle", "Optimal angle for computing the depth triangulation", "10.0") -MDEFVAR_OPTDENSE_float(fMaxAngle, "Max Angle", "Max angle for accepting the depth triangulation", "45.0") +MDEFVAR_OPTDENSE_float(fMaxAngle, "Max Angle", "Max angle for accepting the depth triangulation", "65.0") MDEFVAR_OPTDENSE_float(fDescriptorMinMagnitudeThreshold, "Descriptor Min Magnitude Threshold", "minimum texture variance accepted when matching two patches (0 - disabled)", "0.01") MDEFVAR_OPTDENSE_float(fDepthDiffThreshold, "Depth Diff Threshold", "maximum variance allowed for the depths during refinement", "0.01") +MDEFVAR_OPTDENSE_float(fNormalDiffThreshold, "Normal Diff Threshold", "maximum variance allowed for the normal during fusion (degrees)", "25") MDEFVAR_OPTDENSE_float(fPairwiseMul, "Pairwise Mul", "pairwise cost scale to match the unary cost", "0.3") MDEFVAR_OPTDENSE_float(fOptimizerEps, "Optimizer Eps", "MRF optimizer stop epsilon", "0.005") MDEFVAR_OPTDENSE_int32(nOptimizerMaxIters, "Optimizer Max Iters", "MRF optimizer max number of iterations", "80") MDEFVAR_OPTDENSE_uint32(nSpeckleSize, "Speckle Size", "maximal size of a speckle (small speckles get removed)", "100") MDEFVAR_OPTDENSE_uint32(nIpolGapSize, "Interpolate Gap Size", "interpolate small gaps (left<->right, top<->bottom)", "7") -MDEFVAR_OPTDENSE_uint32(nOptimize, "Optimize", "should we filter the extracted depth-maps?", "7") // see OPTIMIZE_FLAGS -MDEFVAR_OPTDENSE_uint32(nEstimateColors, "Estimate Colors", "should we estimate the colors for the dense point-cloud?", "1", "0") +MDEFVAR_OPTDENSE_uint32(nOptimize, "Optimize", "should we filter the extracted depth-maps?", "7") // see DepthFlags +MDEFVAR_OPTDENSE_uint32(nEstimateColors, "Estimate Colors", "should we estimate the colors for the dense point-cloud?", "2", "0", "1") MDEFVAR_OPTDENSE_uint32(nEstimateNormals, "Estimate Normals", "should we estimate the normals for the dense point-cloud?", "0", "1", "2") MDEFVAR_OPTDENSE_float(fNCCThresholdKeep, "NCC Threshold Keep", "Maximum 1-NCC score accepted for a match", "0.5", "0.3") MDEFVAR_OPTDENSE_float(fNCCThresholdRefine, "NCC Threshold Refine", "1-NCC score under which a match is not refined anymore", "0.03") MDEFVAR_OPTDENSE_uint32(nEstimationIters, "Estimation Iters", "Number of iterations for depth-map refinement", "4") MDEFVAR_OPTDENSE_uint32(nRandomIters, "Random Iters", "Number of iterations for random assignment per pixel", "6") MDEFVAR_OPTDENSE_uint32(nRandomMaxScale, "Random Max Scale", "Maximum number of iterations to skip during random assignment", "2") -MDEFVAR_OPTDENSE_float(fRandomDepthRatio, "Random Depth Ratio", "Depth range ratio of the current estimate for random plane assignment", "0.01") -MDEFVAR_OPTDENSE_float(fRandomAngle1Range, "Random Angle1 Range", "Angle 1 range for random plane assignment (in degrees)", "90.0") -MDEFVAR_OPTDENSE_float(fRandomAngle2Range, "Random Angle2 Range", "Angle 2 range for random plane assignment (in degrees)", "15.0") -MDEFVAR_OPTDENSE_float(fRandomSmoothDepth, "Random Smooth Depth", "Depth variance used during neighbor smoothness assignment (ratio)", "0.006") -MDEFVAR_OPTDENSE_float(fRandomSmoothNormal, "Random Smooth Normal", "Normal variance used during neighbor smoothness assignment (degrees)", "8.5") -MDEFVAR_OPTDENSE_float(fRandomSmoothBonus, "Random Smooth Bonus", "Score factor used to encourage smoothness (1 - disabled)", "0.9") +MDEFVAR_OPTDENSE_float(fRandomDepthRatio, "Random Depth Ratio", "Depth range ratio of the current estimate for random plane assignment", "0.004") +MDEFVAR_OPTDENSE_float(fRandomAngle1Range, "Random Angle1 Range", "Angle 1 range for random plane assignment (in degrees)", "20.0") +MDEFVAR_OPTDENSE_float(fRandomAngle2Range, "Random Angle2 Range", "Angle 2 range for random plane assignment (in degrees)", "12.0") +MDEFVAR_OPTDENSE_float(fRandomSmoothDepth, "Random Smooth Depth", "Depth variance used during neighbor smoothness assignment (ratio)", "0.02") +MDEFVAR_OPTDENSE_float(fRandomSmoothNormal, "Random Smooth Normal", "Normal variance used during neighbor smoothness assignment (degrees)", "13") +MDEFVAR_OPTDENSE_float(fRandomSmoothBonus, "Random Smooth Bonus", "Score factor used to encourage smoothness (1 - disabled)", "0.93") } @@ -260,29 +262,55 @@ void DepthEstimator::MapMatrix2ZigzagIdx(const Image8U::Size& size, DepthEstimat // replace POWI(0.5f, (int)invScaleRange): 0 1 2 3 4 5 6 7 8 9 10 11 const float DepthEstimator::scaleRanges[12] = {1.f, 0.5f, 0.25f, 0.125f, 0.0625f, 0.03125f, 0.015625f, 0.0078125f, 0.00390625f, 0.001953125f, 0.0009765625f, 0.00048828125f}; -DepthEstimator::DepthEstimator(DepthData& _depthData0, volatile Thread::safe_t& _idx, const Image64F& _image0Sum, const MapRefArr& _coords, ENDIRECTION _dir) +DepthEstimator::DepthEstimator( + unsigned nIter, DepthData& _depthData0, volatile Thread::safe_t& _idx, + #if DENSE_NCC == DENSE_NCC_WEIGHTED + WeightMap& _weightMap0, + #else + const Image64F& _image0Sum, + #endif + const MapRefArr& _coords) : - neighborsData(0,4), neighbors(0,2), + #ifndef _RELEASE + rnd(SEACAVE::Random::default_seed), + #endif idxPixel(_idx), + neighbors(0,nMaxNeighbors), + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + neighborsClose(0,4), + #endif scores(_depthData0.images.size()-1), depthMap0(_depthData0.depthMap), normalMap0(_depthData0.normalMap), confMap0(_depthData0.confMap), + #if DENSE_NCC == DENSE_NCC_WEIGHTED + weightMap0(_weightMap0), + #endif + nIteration(nIter), images(InitImages(_depthData0)), image0(_depthData0.images[0]), - image0Sum(_image0Sum), coords(_coords), size(_depthData0.images.First().image.size()), + #if DENSE_NCC != DENSE_NCC_WEIGHTED + image0Sum(_image0Sum), + #endif + coords(_coords), size(_depthData0.images.First().image.size()), + dMin(_depthData0.dMin), dMax(_depthData0.dMax), + dir(nIter%2 ? RB2LT : LT2RB), #if DENSE_AGGNCC == DENSE_AGGNCC_NTH idxScore((_depthData0.images.size()-1)/3), + #elif DENSE_AGGNCC == DENSE_AGGNCC_MINMEAN + idxScore(_depthData0.images.size()<=2 ? 0u : 1u), #endif - dir(_dir), dMin(_depthData0.dMin), dMax(_depthData0.dMax), smoothBonusDepth(1.f-OPTDENSE::fRandomSmoothBonus), smoothBonusNormal((1.f-OPTDENSE::fRandomSmoothBonus)*0.96f), - smoothSigmaDepth(-1.f/(2.f*SQUARE(OPTDENSE::fRandomSmoothDepth))), // used in exp(-x^2 / (2*(0.006^2))) - smoothSigmaNormal(-1.f/(2.f*SQUARE(FD2R(OPTDENSE::fRandomSmoothNormal)))), // used in exp(-x^2 / (2*(0.15^2))) + smoothSigmaDepth(-1.f/(2.f*SQUARE(OPTDENSE::fRandomSmoothDepth))), // used in exp(-x^2 / (2*(0.01^2))) + smoothSigmaNormal(-1.f/(2.f*SQUARE(FD2R(OPTDENSE::fRandomSmoothNormal)))), // used in exp(-x^2 / (2*(0.22^2))) thMagnitudeSq(OPTDENSE::fDescriptorMinMagnitudeThreshold>0?SQUARE(OPTDENSE::fDescriptorMinMagnitudeThreshold):-1.f), angle1Range(FD2R(OPTDENSE::fRandomAngle1Range)), angle2Range(FD2R(OPTDENSE::fRandomAngle2Range)), thConfSmall(OPTDENSE::fNCCThresholdKeep*0.25f), thConfBig(OPTDENSE::fNCCThresholdKeep*0.5f), thRobust(OPTDENSE::fNCCThresholdKeep*1.2f) + #if DENSE_REFINE == DENSE_REFINE_EXACT + , thPerturbation(1.f/POW(2.f,float(nIter+1))) + #endif { - ASSERT(_depthData0.images.size() >= 2); + ASSERT(_depthData0.images.size() >= 1); } // center a patch of given size on the segment @@ -295,83 +323,190 @@ bool DepthEstimator::PreparePixelPatch(const ImageRef& x) // fetch the patch pixel values in the main image bool DepthEstimator::FillPixelPatch() { + #if DENSE_NCC != DENSE_NCC_WEIGHTED const float mean(GetImage0Sum(x0)/nTexels); normSq0 = 0; float* pTexel0 = texels0.data(); - for (int i=-nSizeHalfWindow; i<=nSizeHalfWindow; ++i) - for (int j=-nSizeHalfWindow; j<=nSizeHalfWindow; ++j) + for (int i=-nSizeHalfWindow; i<=nSizeHalfWindow; i+=nSizeStep) + for (int j=-nSizeHalfWindow; j<=nSizeHalfWindow; j+=nSizeStep) normSq0 += SQUARE(*pTexel0++ = image0.image(x0.y+i, x0.x+j)-mean); - X0 = (const Vec3&)image0.camera.TransformPointI2C(Cast(x0)); - return normSq0 > thMagnitudeSq; + #else + Weight& w = weightMap0[x0.y*image0.image.width()+x0.x]; + if (w.normSq0 == 0) { + w.sumWeights = 0; + int n = 0; + const float colCenter = image0.image(x0); + for (int i=-nSizeHalfWindow; i<=nSizeHalfWindow; i+=nSizeStep) { + for (int j=-nSizeHalfWindow; j<=nSizeHalfWindow; j+=nSizeStep) { + Weight::Pixel& pw = w.weights[n++]; + w.normSq0 += + (pw.tempWeight = image0.image(x0.y+i, x0.x+j)) * + (pw.weight = GetWeight(ImageRef(j,i), colCenter)); + w.sumWeights += pw.weight; + } + } + ASSERT(n == nTexels); + const float tm(w.normSq0/w.sumWeights); + w.normSq0 = 0; + n = 0; + do { + Weight::Pixel& pw = w.weights[n]; + const float t(pw.tempWeight - tm); + w.normSq0 += (pw.tempWeight = pw.weight * t) * t; + } while (++n < nTexels); + } + normSq0 = w.normSq0; + #endif + if (normSq0 < thMagnitudeSq) + return false; + reinterpret_cast(X0) = image0.camera.TransformPointI2C(Cast(x0)); + return true; } -// compute pixel's NCC score -float DepthEstimator::ScorePixel(Depth depth, const Normal& normal) +#if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE +// compute plane defined by current depth and normal estimate +void DepthEstimator::InitPlane(Depth depth, const Normal& normal) { - ASSERT(depth > 0 && normal.dot(Cast(static_cast(X0))) < 0); - FOREACH(idx, images) { - // center a patch of given size on the segment and fetch the pixel values in the target image - const ViewData& image1 = images[idx]; - float& score = scores[idx]; - const Matrix3x3f H(ComputeHomographyMatrix(image1, depth, normal)); - Point2f pt; - int n(0); - float sum(0); - #if DENSE_NCC != DENSE_NCC_DEFAULT - float sumSq(0), num(0); - #endif - for (int i=-nSizeHalfWindow; i<=nSizeHalfWindow; ++i) { - for (int j=-nSizeHalfWindow; j<=nSizeHalfWindow; ++j) { - ProjectVertex_3x3_2_2(H.val, Point2f((float)(x0.x+j), (float)(x0.y+i)).ptr(), pt.ptr()); - if (!image1.view.image.isInsideWithBorder(pt)) { - score = thRobust; - goto NEXT_IMAGE; - } - #if DENSE_NCC == DENSE_NCC_FAST - const float v(image1.view.image.sample(pt)); - sum += v; - sumSq += SQUARE(v); - num += texels0(n++)*v; - #else - sum += texels1(n++) = image1.view.image.sample(pt); - #endif - } + #if 0 + plane.Set(reinterpret_cast(normal), Vec3f(depth*Cast(X0))); + #else + plane.m_vN = reinterpret_cast(normal); + plane.m_fD = -depth*reinterpret_cast(normal).dot(Cast(X0)); + #endif +} +#endif + +// compute pixel's NCC score in the given target image +float DepthEstimator::ScorePixelImage(const ViewData& image1, Depth depth, const Normal& normal) +{ + // center a patch of given size on the segment and fetch the pixel values in the target image + Matrix3x3f H(ComputeHomographyMatrix(image1, depth, normal)); + Point3f X; + ProjectVertex_3x3_2_3(H.val, Point2f(float(x0.x-nSizeHalfWindow),float(x0.y-nSizeHalfWindow)).ptr(), X.ptr()); + Point3f baseX(X); + H *= float(nSizeStep); + int n(0); + float sum(0); + #if DENSE_NCC != DENSE_NCC_DEFAULT + float sumSq(0), num(0); + #endif + #if DENSE_NCC == DENSE_NCC_WEIGHTED + const Weight& w = weightMap0[x0.y*image0.image.width()+x0.x]; + #endif + for (int i=-nSizeHalfWindow; i<=nSizeHalfWindow; i+=nSizeStep) { + for (int j=-nSizeHalfWindow; j<=nSizeHalfWindow; j+=nSizeStep) { + const Point2f pt(X); + if (!image1.view.image.isInsideWithBorder(pt)) + return thRobust; + const float v(image1.view.image.sample(pt)); + #if DENSE_NCC == DENSE_NCC_FAST + sum += v; + sumSq += SQUARE(v); + num += texels0(n++)*v; + #elif DENSE_NCC == DENSE_NCC_WEIGHTED + const Weight::Pixel& pw = w.weights[n++]; + const float vw(v*pw.weight); + sum += vw; + sumSq += v*vw; + num += v*pw.tempWeight; + #else + sum += texels1(n++)=v; + #endif + X.x += H[0]; X.y += H[3]; X.z += H[6]; } - { - ASSERT(n == nTexels); - // score similarity of the reference and target texture patches - #if DENSE_NCC == DENSE_NCC_FAST - const float normSq1(sumSq-SQUARE(sum/nSizeWindow)); + baseX.x += H[1]; baseX.y += H[4]; baseX.z += H[7]; + X = baseX; + } + ASSERT(n == nTexels); + // score similarity of the reference and target texture patches + #if DENSE_NCC == DENSE_NCC_FAST + const float normSq1(sumSq-SQUARE(sum/nSizeWindow)); + #elif DENSE_NCC == DENSE_NCC_WEIGHTED + const float normSq1(sumSq-SQUARE(sum)/w.sumWeights); + #else + const float normSq1(normSqDelta(texels1.data(), sum/(float)nTexels)); + #endif + const float nrmSq(normSq0*normSq1); + if (nrmSq <= 0.f) + return thRobust; + #if DENSE_NCC == DENSE_NCC_DEFAULT + const float num(texels0.dot(texels1)); + #endif + const float ncc(CLAMP(num/SQRT(nrmSq), -1.f, 1.f)); + float score = 1.f - ncc; + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + // encourage smoothness + for (const NeighborEstimate& neighbor: neighborsClose) { + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + score *= 1.f - smoothBonusDepth * DENSE_EXP(SQUARE(plane.Distance(neighbor.X)/depth) * smoothSigmaDepth); #else - const float normSq1(normSqDelta(texels1.data(), sum/(float)nTexels)); - #endif - const float nrm(normSq0*normSq1); - if (nrm <= 0.f) { - score = thRobust; - continue; - } - #if DENSE_NCC == DENSE_NCC_DEFAULT - const float num(texels0.dot(texels1)); + score *= 1.f - smoothBonusDepth * DENSE_EXP(SQUARE((depth-neighbor.depth)/depth) * smoothSigmaDepth); #endif - const float ncc(CLAMP(num/SQRT(nrm), -1.f, 1.f)); - score = 1.f - ncc; - // encourage smoothness - for (const NeighborData& neighbor: neighborsData) { - score *= 1.f - smoothBonusDepth * EXP(SQUARE((depth-neighbor.depth)/depth) * smoothSigmaDepth); - score *= 1.f - smoothBonusNormal * EXP(SQUARE(ACOS(ComputeAngle(normal.ptr(), neighbor.normal.ptr()))) * smoothSigmaNormal); - } - } - NEXT_IMAGE:; + score *= 1.f - smoothBonusNormal * DENSE_EXP(SQUARE(ACOS(ComputeAngle(normal.ptr(), neighbor.normal.ptr()))) * smoothSigmaNormal); } + #endif + return score; +} + +// compute pixel's NCC score +float DepthEstimator::ScorePixel(Depth depth, const Normal& normal) +{ + ASSERT(depth > 0 && normal.dot(Cast(static_cast(X0))) < 0); + // compute score for this pixel as seen in each view + ASSERT(scores.size() == images.size()); + FOREACH(idxView, images) + scores[idxView] = ScorePixelImage(images[idxView], depth, normal); #if DENSE_AGGNCC == DENSE_AGGNCC_NTH // set score as the nth element - return scores.size() > 1 ? scores.GetNth(idxScore) : scores.front(); + return scores.GetNth(idxScore); #elif DENSE_AGGNCC == DENSE_AGGNCC_MEAN // set score as the average similarity + #if 1 return scores.mean(); #else + const float* pscore(scores.data()); + const float* pescore(pscore+scores.rows()); + float score(0); + do { + score += MINF(*pscore, thRobust); + } while (++pscore <= pescore); + return score/scores.rows(); + #endif + #elif DENSE_AGGNCC == DENSE_AGGNCC_MIN // set score as the min similarity return scores.minCoeff(); + #else + // set score as the min-mean similarity + if (idxScore == 0) + return *std::min_element(scores.cbegin(), scores.cend()); + #if 0 + return std::accumulate(scores.begin(), &scores.PartialSort(idxScore), 0.f) / idxScore; + #elif 1 + const float* pescore(&scores.PartialSort(idxScore)); + const float* pscore(scores.cbegin()); + int n(0); float score(0); + do { + const float s(*pscore); + if (s < thRobust) { + score += s; + ++n; + } + } while (++pscore <= pescore); + return n ? score/n : thRobust; + #else + const float thScore(MAXF(*std::min_element(scores.cbegin(), scores.cend()), 0.05f)*2); + const float* pscore(scores.cbegin()); + const float* pescore(pscore+scores.size()); + int n(0); float score(0); + do { + const float s(*pscore); + if (s <= thScore) { + score += s; + ++n; + } + } while (++pscore <= pescore); + return score/n; + #endif #endif } @@ -389,37 +524,65 @@ void DepthEstimator::ProcessPixel(IDX idx) if ((invScaleRange <= 2 || conf > OPTDENSE::fNCCThresholdRefine) && FillPixelPatch()) { // find neighbors neighbors.Empty(); - neighborsData.Empty(); + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + neighborsClose.Empty(); + #endif if (dir == LT2RB) { // direction from left-top to right-bottom corner if (x0.x > nSizeHalfWindow) { const ImageRef nx(x0.x-1, x0.y); const Depth ndepth(depthMap0(nx)); if (ndepth > 0) { + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA neighbors.emplace_back(nx); - neighborsData.emplace_back(ndepth,normalMap0(nx)); + neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) + #endif + }); + #else + neighbors.emplace_back(NeighborData{nx,ndepth,normalMap0(nx)}); + #endif } } if (x0.y > nSizeHalfWindow) { const ImageRef nx(x0.x, x0.y-1); const Depth ndepth(depthMap0(nx)); if (ndepth > 0) { + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA neighbors.emplace_back(nx); - neighborsData.emplace_back(ndepth,normalMap0(nx)); + neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) + #endif + }); + #else + neighbors.emplace_back(NeighborData{nx,ndepth,normalMap0(nx)}); + #endif } } + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA if (x0.x < size.width-nSizeHalfWindow) { const ImageRef nx(x0.x+1, x0.y); const Depth ndepth(depthMap0(nx)); if (ndepth > 0) - neighborsData.emplace_back(ndepth,normalMap0(nx)); + neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) + #endif + }); } if (x0.y < size.height-nSizeHalfWindow) { const ImageRef nx(x0.x, x0.y+1); const Depth ndepth(depthMap0(nx)); if (ndepth > 0) - neighborsData.emplace_back(ndepth,normalMap0(nx)); + neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) + #endif + }); } + #endif } else { ASSERT(dir == RB2LT); // direction from right-bottom to left-top corner @@ -427,49 +590,88 @@ void DepthEstimator::ProcessPixel(IDX idx) const ImageRef nx(x0.x+1, x0.y); const Depth ndepth(depthMap0(nx)); if (ndepth > 0) { + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA neighbors.emplace_back(nx); - neighborsData.emplace_back(ndepth,normalMap0(nx)); + neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) + #endif + }); + #else + neighbors.emplace_back(NeighborData{nx,ndepth,normalMap0(nx)}); + #endif } } if (x0.y < size.height-nSizeHalfWindow) { const ImageRef nx(x0.x, x0.y+1); const Depth ndepth(depthMap0(nx)); if (ndepth > 0) { + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA neighbors.emplace_back(nx); - neighborsData.emplace_back(ndepth,normalMap0(nx)); + neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) + #endif + }); + #else + neighbors.emplace_back(NeighborData{nx,ndepth,normalMap0(nx)}); + #endif } } + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA if (x0.x > nSizeHalfWindow) { const ImageRef nx(x0.x-1, x0.y); const Depth ndepth(depthMap0(nx)); if (ndepth > 0) - neighborsData.emplace_back(ndepth,normalMap0(nx)); + neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) + #endif + }); } if (x0.y > nSizeHalfWindow) { const ImageRef nx(x0.x, x0.y-1); const Depth ndepth(depthMap0(nx)); if (ndepth > 0) - neighborsData.emplace_back(ndepth,normalMap0(nx)); + neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) + #endif + }); } + #endif } Depth& depth = depthMap0(x0); Normal& normal = normalMap0(x0); const Normal viewDir(Cast(static_cast(X0))); ASSERT(depth > 0 && normal.dot(viewDir) < 0); + #if DENSE_REFINE == DENSE_REFINE_ITER // check if any of the neighbor estimates are better then the current estimate + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA FOREACH(n, neighbors) { - float nconf(confMap0(neighbors[n])); + const ImageRef& nx = neighbors[n]; + #else + for (NeighborData& neighbor: neighbors) { + const ImageRef& nx = neighbor.x; + #endif + float nconf(confMap0(nx)); const unsigned ninvScaleRange(DecodeScoreScale(nconf)); if (nconf >= OPTDENSE::fNCCThresholdKeep) continue; - const NeighborData& neighbor = neighborsData[n]; - ASSERT(neighbor.depth > 0); + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + NeighborEstimate& neighbor = neighborsClose[n]; + #endif if (neighbor.normal.dot(viewDir) >= 0) continue; - const float newconf(ScorePixel(neighbor.depth, neighbor.normal)); - ASSERT(newconf >= 0 && newconf <= 2); - if (conf > newconf) { - conf = newconf; + neighbor.depth = InterpolatePixel(nx, neighbor.depth, neighbor.normal); + ASSERT(neighbor.depth > 0); + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + InitPlane(neighbor.depth, neighbor.normal); + #endif + nconf = ScorePixel(neighbor.depth, neighbor.normal); + ASSERT(nconf >= 0 && nconf <= 2); + if (conf > nconf) { + conf = nconf; depth = neighbor.depth; normal = neighbor.normal; invScaleRange = (ninvScaleRange>1 ? ninvScaleRange-1 : ninvScaleRange); @@ -490,13 +692,16 @@ void DepthEstimator::ProcessPixel(IDX idx) Normal2Dir(normal, p); Normal nnormal; for (unsigned iter=invScaleRange; iter= 0) continue; + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + InitPlane(ndepth, nnormal); + #endif const float nconf(ScorePixel(ndepth, nnormal)); ASSERT(nconf >= 0); if (conf > nconf) { @@ -508,9 +713,162 @@ void DepthEstimator::ProcessPixel(IDX idx) ++invScaleRange; } } + #else + // current pixel estimate + PixelEstimate currEstimate{depth, normal}; + // propagate depth estimate from the best neighbor estimate + PixelEstimate prevEstimate; float prevCost(FLT_MAX); + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + FOREACH(n, neighbors) { + const ImageRef& nx = neighbors[n]; + #else + for (const NeighborData& neighbor: neighbors) { + const ImageRef& nx = neighbor.x; + #endif + float nconf(confMap0(nx)); + const unsigned ninvScaleRange(DecodeScoreScale(nconf)); + ASSERT(nconf >= 0 && nconf <= 2); + if (nconf >= OPTDENSE::fNCCThresholdKeep) + continue; + if (prevCost <= nconf) + continue; + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + const NeighborEstimate& neighbor = neighborsClose[n]; + #endif + if (neighbor.normal.dot(viewDir) >= 0) + continue; + prevEstimate.depth = InterpolatePixel(nx, neighbor.depth, neighbor.normal); + prevEstimate.normal = neighbor.normal; + prevCost = nconf; + } + if (prevCost == FLT_MAX) + prevEstimate = PerturbEstimate(currEstimate, thPerturbation); + // randomly sampled estimate + PixelEstimate randEstimate(PerturbEstimate(currEstimate, thPerturbation)); + // select best pixel estimate + const int numCosts = 5; + float costs[numCosts] = {0,0,0,0,0}; + const Depth depths[numCosts] = { + currEstimate.depth, prevEstimate.depth, randEstimate.depth, + currEstimate.depth, randEstimate.depth}; + const Normal normals[numCosts] = { + currEstimate.normal, prevEstimate.normal, + randEstimate.normal, randEstimate.normal, + currEstimate.normal}; + conf = FLT_MAX; + for (int idxCost=0; idxCost= 0); + if (conf > nconf) { + conf = nconf; + depth = ndepth; + normal = nnormal; + } + } + #endif } conf = EncodeScoreScale(conf, invScaleRange); } + +// interpolate given pixel's estimate to the current position +Depth DepthEstimator::InterpolatePixel(const ImageRef& nx, Depth depth, const Normal& normal) const +{ + ASSERT(depth > 0 && normal.dot(image0.camera.TransformPointI2C(Cast(nx))) < 0); + Depth depthNew; + #if 1 + // compute as intersection of the lines + // {(x1, y1), (x2, y2)} from neighbor's 3D point towards normal direction + // and + // {(0, 0), (x4, 1)} from camera center towards current pixel direction + // in the x or y plane + if (x0.x == nx.x) { + const float fy = (float)image0.camera.K[4]; + const float cy = (float)image0.camera.K[5]; + const float x1 = depth * (nx.y - cy) / fy; + const float y1 = depth; + const float x4 = (x0.y - cy) / fy; + const float denom = normal.z + x4 * normal.y; + if (ISZERO(denom)) + return depth; + const float x2 = x1 + normal.z; + const float y2 = y1 - normal.y; + const float nom = y1 * x2 - x1 * y2; + depthNew = nom / denom; + } + else { + ASSERT(x0.y == nx.y); + const float fx = (float)image0.camera.K[0]; + const float cx = (float)image0.camera.K[2]; + ASSERT(image0.camera.K[1] == 0); + const float x1 = depth * (nx.x - cx) / fx; + const float y1 = depth; + const float x4 = (x0.x - cx) / fx; + const float denom = normal.z + x4 * normal.x; + if (ISZERO(denom)) + return depth; + const float x2 = x1 + normal.z; + const float y2 = y1 - normal.x; + const float nom = y1 * x2 - x1 * y2; + depthNew = nom / denom; + } + #else + // compute as the ray - plane intersection + { + #if 0 + const Plane plane(Cast(normal), image0.camera.TransformPointI2C(Point3(nx, depth))); + const Ray3 ray(Point3::ZERO, normalized(X0)); + depthNew = (Depth)ray.Intersects(plane).z(); + #else + const Point3 planeN(normal); + const REAL planeD(planeN.dot(image0.camera.TransformPointI2C(Point3(nx, depth)))); + depthNew = (Depth)(planeD / planeN.dot(reinterpret_cast(X0))); + #endif + } + #endif + return ISINSIDE(depthNew,dMin,dMax) ? depthNew : depth; +} + +#if DENSE_REFINE == DENSE_REFINE_EXACT +DepthEstimator::PixelEstimate DepthEstimator::PerturbEstimate(const PixelEstimate& est, float perturbation) +{ + PixelEstimate ptbEst; + + // perturb depth + const float minDepth = est.depth * (1.f-perturbation); + const float maxDepth = est.depth * (1.f+perturbation); + ptbEst.depth = CLAMP(rnd.randomUniform(minDepth, maxDepth), dMin, dMax); + + // perturb normal + const Normal viewDir(Cast(static_cast(X0))); + std::uniform_real_distribution urd(-1.f, 1.f); + const int numMaxTrials = 3; + int numTrials = 0; + perturbation *= FHALF_PI; + while(true) { + // generate random perturbation rotation + const RMatrixBaseF R(urd(rnd)*perturbation, urd(rnd)*perturbation, urd(rnd)*perturbation); + // perturb normal vector + ptbEst.normal = R * est.normal; + // make sure the perturbed normal is still looking towards the camera, + // otherwise try again with a smaller perturbation + if (ptbEst.normal.dot(viewDir) < 0.f) + break; + if (++numTrials == numMaxTrials) { + ptbEst.normal = est.normal; + return ptbEst; + } + perturbation *= 0.5f; + } + ASSERT(ISEQUAL(norm(ptbEst.normal), 1.f)); + + return ptbEst; +} +#endif /*----------------------------------------------------------------*/ diff --git a/libs/MVS/DepthMap.h b/libs/MVS/DepthMap.h index 915e93685..e98affe80 100644 --- a/libs/MVS/DepthMap.h +++ b/libs/MVS/DepthMap.h @@ -44,13 +44,31 @@ // NCC type used for patch-similarity computation during depth-map estimation #define DENSE_NCC_DEFAULT 0 #define DENSE_NCC_FAST 1 -#define DENSE_NCC DENSE_NCC_FAST +#define DENSE_NCC_WEIGHTED 2 +#define DENSE_NCC DENSE_NCC_WEIGHTED // NCC score aggregation type used during depth-map estimation #define DENSE_AGGNCC_NTH 0 #define DENSE_AGGNCC_MEAN 1 #define DENSE_AGGNCC_MIN 2 -#define DENSE_AGGNCC DENSE_AGGNCC_MEAN +#define DENSE_AGGNCC_MINMEAN 3 +#define DENSE_AGGNCC DENSE_AGGNCC_MINMEAN + +// type of smoothness used during depth-map estimation +#define DENSE_SMOOTHNESS_NA 0 +#define DENSE_SMOOTHNESS_FAST 1 +#define DENSE_SMOOTHNESS_PLANE 2 +#define DENSE_SMOOTHNESS DENSE_SMOOTHNESS_PLANE + +// type of refinement used during depth-map estimation +#define DENSE_REFINE_ITER 0 +#define DENSE_REFINE_EXACT 1 +#define DENSE_REFINE DENSE_REFINE_ITER + +// exp function type used during depth estimation +#define DENSE_EXP_DEFUALT EXP +#define DENSE_EXP_FAST FEXP // ~10% faster, but slightly less precise +#define DENSE_EXP DENSE_EXP_DEFUALT #define ComposeDepthFilePathBase(b, i, e) MAKE_PATH(String::FormatString((b + "%04u." e).c_str(), i)) #define ComposeDepthFilePath(i, e) MAKE_PATH(String::FormatString("depth%04u." e, i)) @@ -69,8 +87,9 @@ enum DepthFlags { ADJUST_FILTER = (1 << 2), OPTIMIZE = (REMOVE_SPECKLES|FILL_GAPS) }; -extern unsigned nMinResolution; extern unsigned nResolutionLevel; +extern unsigned nMaxResolution; +extern unsigned nMinResolution; extern unsigned nMinViews; extern unsigned nMaxViews; extern unsigned nMinViewsFuse; @@ -88,6 +107,7 @@ extern float fOptimAngle; extern float fMaxAngle; extern float fDescriptorMinMagnitudeThreshold; extern float fDepthDiffThreshold; +extern float fNormalDiffThreshold; extern float fPairwiseMul; extern float fOptimizerEps; extern int nOptimizerMaxIters; @@ -111,6 +131,18 @@ extern float fRandomSmoothBonus; /*----------------------------------------------------------------*/ +template +struct WeightedPatchFix { + struct Pixel { + float weight; + float tempWeight; + }; + Pixel weights[nTexels]; + float sumWeights; + float normSq0; + WeightedPatchFix() : normSq0(0) {} +}; + struct MVS_API DepthData { struct ViewData { float scale; // image scale relative to the reference image @@ -126,7 +158,7 @@ struct MVS_API DepthData { return true; } }; - typedef SEACAVE::cList ViewDataArr; + typedef CLISTDEF2IDX(ViewData,IIndex) ViewDataArr; ViewDataArr images; // array of images used to compute this depth-map (reference image is the first) ViewScoreArr neighbors; // array of all images seeing this depth-map (ordered by decreasing importance) @@ -186,10 +218,12 @@ typedef MVS_API CLISTDEFIDX(DepthData,IIndex) DepthDataArr; struct MVS_API DepthEstimator { - enum { TexelChannels = 1 }; - enum { nSizeHalfWindow = 3 }; + enum { nSizeHalfWindow = 5 }; enum { nSizeWindow = nSizeHalfWindow*2+1 }; - enum { nTexels = nSizeWindow*nSizeWindow*TexelChannels }; + enum { nSizeStep = 2 }; + enum { TexelChannels = 1 }; + enum { nTexels = SQUARE((nSizeHalfWindow*2+nSizeStep)/nSizeStep)*TexelChannels }; + enum { nMaxNeighbors = 8 }; enum ENDIRECTION { LT2RB = 0, @@ -199,14 +233,34 @@ struct MVS_API DepthEstimator { typedef TPoint2 MapRef; typedef CLISTDEF0(MapRef) MapRefArr; + typedef CLISTDEF0IDX(ImageRef,unsigned) DirSamples; typedef Eigen::Matrix TexelVec; struct NeighborData { + ImageRef x; + Depth depth; + Normal normal; + }; + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + struct NeighborEstimate { + Depth depth; + Normal normal; + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + Planef::POINT X; + #endif + }; + #endif + #if DENSE_REFINE == DENSE_REFINE_EXACT + struct PixelEstimate { Depth depth; Normal normal; - inline NeighborData() {} - inline NeighborData(Depth d, const Normal& n) : depth(d), normal(n) {} }; + #endif + + #if DENSE_NCC == DENSE_NCC_WEIGHTED + typedef WeightedPatchFix Weight; + typedef CLISTDEFIDX(Weight,int) WeightMap; + #endif struct ViewData { const DepthData::ViewData& view; @@ -221,43 +275,78 @@ struct MVS_API DepthEstimator { Hr(image0.camera.K.inv()) {} }; - CLISTDEF0IDX(NeighborData,IIndex) neighborsData; // neighbor pixel depths to be used for smoothing - CLISTDEF0IDX(ImageRef,IIndex) neighbors; // neighbor pixels coordinates to be processed + SEACAVE::Random rnd; + volatile Thread::safe_t& idxPixel; // current image index to be processed + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_NA + CLISTDEF0IDX(NeighborData,IIndex) neighbors; // neighbor pixels coordinates to be processed + #else + CLISTDEF0IDX(ImageRef,IIndex) neighbors; // neighbor pixels coordinates to be processed + #endif + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + CLISTDEF0IDX(NeighborEstimate,IIndex) neighborsClose; // close neighbor pixel depths to be used for smoothing + #endif Vec3 X0; // ImageRef x0; // constants during one pixel loop float normSq0; // + #if DENSE_NCC != DENSE_NCC_WEIGHTED TexelVec texels0; // + #endif #if DENSE_NCC == DENSE_NCC_DEFAULT TexelVec texels1; #endif - #if DENSE_AGGNCC == DENSE_AGGNCC_NTH + #if DENSE_AGGNCC == DENSE_AGGNCC_NTH || DENSE_AGGNCC == DENSE_AGGNCC_MINMEAN FloatArr scores; #else Eigen::VectorXf scores; #endif + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + Planef plane; // plane defined by current depth and normal estimate + #endif DepthMap& depthMap0; NormalMap& normalMap0; ConfidenceMap& confMap0; + #if DENSE_NCC == DENSE_NCC_WEIGHTED + WeightMap& weightMap0; + #endif - const CLISTDEF0(ViewData) images; // neighbor images used + const unsigned nIteration; // current PatchMatch iteration + const CLISTDEF0IDX(ViewData,IIndex) images; // neighbor images used const DepthData::ViewData& image0; + #if DENSE_NCC != DENSE_NCC_WEIGHTED const Image64F& image0Sum; // integral image used to fast compute patch mean intensity + #endif const MapRefArr& coords; const Image8U::Size size; - #if DENSE_AGGNCC == DENSE_AGGNCC_NTH + const Depth dMin, dMax; + const ENDIRECTION dir; + #if DENSE_AGGNCC == DENSE_AGGNCC_NTH || DENSE_AGGNCC == DENSE_AGGNCC_MINMEAN const IDX idxScore; #endif - const ENDIRECTION dir; - const Depth dMin, dMax; - DepthEstimator(DepthData& _depthData0, volatile Thread::safe_t& _idx, const Image64F& _image0Sum, const MapRefArr& _coords, ENDIRECTION _dir); + DepthEstimator( + unsigned nIter, DepthData& _depthData0, volatile Thread::safe_t& _idx, + #if DENSE_NCC == DENSE_NCC_WEIGHTED + WeightMap& _weightMap0, + #else + const Image64F& _image0Sum, + #endif + const MapRefArr& _coords); bool PreparePixelPatch(const ImageRef&); bool FillPixelPatch(); + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + void InitPlane(Depth, const Normal&); + #endif + float ScorePixelImage(const ViewData& image1, Depth, const Normal&); float ScorePixel(Depth, const Normal&); void ProcessPixel(IDX idx); - + Depth InterpolatePixel(const ImageRef&, Depth, const Normal&) const; + #if DENSE_REFINE == DENSE_REFINE_EXACT + PixelEstimate PerturbEstimate(const PixelEstimate&, float perturbation); + #endif + + #if DENSE_NCC != DENSE_NCC_WEIGHTED inline float GetImage0Sum(const ImageRef& p) const { const ImageRef p0(p.x-nSizeHalfWindow, p.y-nSizeHalfWindow); const ImageRef p1(p0.x+nSizeWindow, p0.y); @@ -265,6 +354,19 @@ struct MVS_API DepthEstimator { const ImageRef p3(p0.x+nSizeWindow, p0.y+nSizeWindow); return (float)(image0Sum(p3) - image0Sum(p2) - image0Sum(p1) + image0Sum(p0)); } + #endif + + #if DENSE_NCC == DENSE_NCC_WEIGHTED + float GetWeight(const ImageRef& x, float center) const { + // color weight [0..1] + const float sigmaColor(-1.f/(2.f*SQUARE(0.2f))); + const float wColor(SQUARE(image0.image(x0+x)-center) * sigmaColor); + // spatial weight [0..1] + const float sigmaSpatial(-1.f/(2.f*SQUARE((int)nSizeHalfWindow))); + const float wSpatial(float(SQUARE(x.x) + SQUARE(x.y)) * sigmaSpatial); + return DENSE_EXP(wColor+wSpatial); + } + #endif inline Matrix3x3f ComputeHomographyMatrix(const ViewData& img, Depth depth, const Normal& normal) const { #if 0 @@ -277,10 +379,10 @@ struct MVS_API DepthEstimator { #endif } - static inline CLISTDEF0(ViewData) InitImages(const DepthData& depthData) { - CLISTDEF0(ViewData) images(0, depthData.images.GetSize()-1); + static inline CLISTDEF0IDX(ViewData,IIndex) InitImages(const DepthData& depthData) { + CLISTDEF0IDX(ViewData,IIndex) images(0, depthData.images.GetSize()-1); const DepthData::ViewData& image0(depthData.images.First()); - for (IDX i=1; i 0); - return randomRange(dMin, dMax); + return rnd.randomRange(dMin, dMax); } - static inline Normal RandomNormal(const Point3f& viewRay) { + inline Normal RandomNormal(const Point3f& viewRay) { Normal normal; - Dir2Normal(Point2f(randomRange(FD2R(0.f),FD2R(360.f)), randomRange(FD2R(120.f),FD2R(180.f))), normal); + Dir2Normal(Point2f(rnd.randomRange(FD2R(0.f),FD2R(360.f)), rnd.randomRange(FD2R(120.f),FD2R(180.f))), normal); return normal.dot(viewRay) > 0 ? -normal : normal; } + // adjust normal such that it makes at most 90 degrees with the viewing angle + inline void CorrectNormal(Normal& normal) const { + const Normal viewDir(Cast(X0)); + const float cosAngLen(normal.dot(viewDir)); + if (cosAngLen >= 0) + normal = RMatrixBaseF(normal.cross(viewDir), MINF((ACOS(cosAngLen/norm(viewDir))-FD2R(90.f))*1.01f, -0.001f)) * normal; + } + // encode/decode NCC score and refinement level in one float static inline float EncodeScoreScale(float score, unsigned invScaleRange=0) { ASSERT(score >= 0.f && score <= 2.01f); @@ -350,6 +460,9 @@ struct MVS_API DepthEstimator { const float angle1Range, angle2Range; const float thConfSmall, thConfBig; const float thRobust; + #if DENSE_REFINE == DENSE_REFINE_EXACT + const float thPerturbation; + #endif static const float scaleRanges[12]; }; /*----------------------------------------------------------------*/ diff --git a/libs/MVS/Image.cpp b/libs/MVS/Image.cpp index 133da424f..90d69a5a0 100644 --- a/libs/MVS/Image.cpp +++ b/libs/MVS/Image.cpp @@ -161,15 +161,15 @@ float Image::ResizeImage(unsigned nMaxResolution) /*----------------------------------------------------------------*/ // compute image scale for a given max and min resolution, using the current image file data -unsigned Image::RecomputeMaxResolution(unsigned& level, unsigned minImageSize) const +unsigned Image::RecomputeMaxResolution(unsigned& level, unsigned minImageSize, unsigned maxImageSize) const { IMAGEPTR pImage(ReadImageHeader(name)); if (pImage == NULL) { // something went wrong, use the current known size (however it will most probably fail later) - return Image8U3::computeMaxResolution(width, height, level, minImageSize); + return Image8U3::computeMaxResolution(width, height, level, minImageSize, maxImageSize); } // re-compute max image size - return Image8U3::computeMaxResolution(pImage->GetWidth(), pImage->GetHeight(), level, minImageSize); + return Image8U3::computeMaxResolution(pImage->GetWidth(), pImage->GetHeight(), level, minImageSize, maxImageSize); } // RecomputeMaxResolution /*----------------------------------------------------------------*/ diff --git a/libs/MVS/Image.h b/libs/MVS/Image.h index d071842b7..09b26bad5 100644 --- a/libs/MVS/Image.h +++ b/libs/MVS/Image.h @@ -105,7 +105,7 @@ class MVS_API Image bool ReloadImage(unsigned nMaxResolution=0, bool bLoadPixels=true); void ReleaseImage(); float ResizeImage(unsigned nMaxResolution=0); - unsigned RecomputeMaxResolution(unsigned& level, unsigned minImageSize) const; + unsigned RecomputeMaxResolution(unsigned& level, unsigned minImageSize, unsigned maxImageSize=INT_MAX) const; Camera GetCamera(const PlatformArr& platforms, const Image8U::Size& resolution) const; void UpdateCamera(const PlatformArr& platforms); @@ -142,7 +142,7 @@ class MVS_API Image BOOST_SERIALIZATION_SPLIT_MEMBER() #endif }; -typedef MVS_API SEACAVE::cList ImageArr; +typedef MVS_API CLISTDEF2IDX(Image,IIndex) ImageArr; /*----------------------------------------------------------------*/ } // namespace MVS diff --git a/libs/MVS/Scene.cpp b/libs/MVS/Scene.cpp index 6520b5cc6..60ae51019 100644 --- a/libs/MVS/Scene.cpp +++ b/libs/MVS/Scene.cpp @@ -422,9 +422,13 @@ bool Scene::Save(const String& fileName, ARCHIVE_TYPE type) const inline float Footprint(const Camera& camera, const Point3f& X) { + #if 0 const REAL fSphereRadius(1); const Point3 cX(camera.TransformPointW2C(Cast(X))); return (float)norm(camera.TransformPointC2I(Point3(cX.x+fSphereRadius,cX.y,cX.z))-camera.TransformPointC2I(cX))+std::numeric_limits::epsilon(); + #else + return (float)(camera.GetFocalLength()/camera.PointDepth(Cast(X))); + #endif } // compute visibility for the reference image @@ -472,10 +476,10 @@ bool Scene::SelectNeighborViews(uint32_t ID, IndexArr& points, unsigned nMinView continue; const Image& imageData2 = images[view]; const Point3f V2(imageData2.camera.C - Cast(point)); - const float footprint2(Footprint(imageData2.camera, point)); const float fAngle(ACOS(ComputeAngle(V1.ptr(), V2.ptr()))); - const float fScaleRatio(footprint1/footprint2); const float wAngle(MINF(POW(fAngle/fOptimAngle, 1.5f), 1.f)); + const float footprint2(Footprint(imageData2.camera, point)); + const float fScaleRatio(footprint1/footprint2); float wScale; if (fScaleRatio > 1.6f) wScale = SQUARE(1.6f/fScaleRatio); diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index 03d5006c8..73a088e92 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -138,7 +138,7 @@ class DepthMapsData bool GapInterpolation(DepthData& depthData); bool FilterDepthMap(DepthData& depthData, const IIndexArr& idxNeighbors, bool bAdjust=true); - void FuseDepthMaps(PointCloud& pointcloud, bool bEstimateNormal); + void FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, bool bEstimateNormal); protected: static void* STCALL ScoreDepthMapTmp(void*); @@ -373,7 +373,7 @@ bool DepthMapsData::InitViews(DepthData& depthData, IIndex idxNeighbor, IIndex n // print selected views if (g_nVerbosityLevel > 2) { String msg; - for (IDX i=1; i(static_cast(estimator.X0))); if (depth <= 0) { // init with random values - depth = DepthEstimator::RandomDepth(estimator.dMin, estimator.dMax); - normal = DepthEstimator::RandomNormal(viewDir); + depth = estimator.RandomDepth(estimator.dMin, estimator.dMax); + normal = estimator.RandomNormal(viewDir); } else if (normal.dot(viewDir) >= 0) { // replace invalid normal with random values - normal = DepthEstimator::RandomNormal(viewDir); + normal = estimator.RandomNormal(viewDir); } estimator.confMap0(x) = DepthEstimator::EncodeScoreScale(estimator.ScorePixel(depth, normal)); } @@ -609,15 +609,22 @@ void* STCALL DepthMapsData::EndDepthMapTmp(void* arg) #endif depth = 0; } else { + #if 1 + conf = 1.f/(MAXF(conf,1e-2f)*depth); + #else #if 1 FOREACH(i, estimator.images) estimator.scores[i] = ComputeAngle(estimator.image0.camera.TransformPointI2W(Point3(x,depth)).ptr(), estimator.image0.camera.C.ptr(), estimator.images[i].view.camera.C.ptr()); #if DENSE_AGGNCC == DENSE_AGGNCC_NTH - const float fCosAngle(estimator.scores.size() > 1 ? estimator.scores.GetNth(estimator.idxScore) : estimator.scores.front()); + const float fCosAngle(estimator.scores.GetNth(estimator.idxScore)); #elif DENSE_AGGNCC == DENSE_AGGNCC_MEAN const float fCosAngle(estimator.scores.mean()); - #else + #elif DENSE_AGGNCC == DENSE_AGGNCC_MIN const float fCosAngle(estimator.scores.minCoeff()); + #else + const float fCosAngle(estimator.idxScore ? + std::accumulate(estimator.scores.begin(), &estimator.scores.PartialSort(estimator.idxScore), 0.f) / estimator.idxScore : + *std::min_element(estimator.scores.cbegin(), estimator.scores.cend())); #endif const float wAngle(MINF(POW(ACOS(fCosAngle)/fOptimAngle,1.5f),1.f)); #else @@ -630,6 +637,7 @@ void* STCALL DepthMapsData::EndDepthMapTmp(void* arg) #else conf = SQRT((float)invScaleRange)*wAngle/(depth*SQUARE(MAXF(conf,1e-2f))); #endif + #endif } } return NULL; @@ -665,7 +673,7 @@ bool DepthMapsData::EstimateDepthMap(IIndex idxImage) // initialize the depth-map if (OPTDENSE::nMinViewsTrustPoint < 2) { // compute depth range and initialize known depths - const int nPixelArea(3); // half windows size around a pixel to be initialize with the known depth + const int nPixelArea(2); // half windows size around a pixel to be initialize with the known depth const Camera& camera = depthData.images.First().camera; depthData.dMin = FLT_MAX; depthData.dMax = 0; @@ -676,9 +684,12 @@ bool DepthMapsData::EstimateDepthMap(IIndex idxImage) const float d((float)camX.z); const ImageRef sx(MAXF(x.x-nPixelArea,0), MAXF(x.y-nPixelArea,0)); const ImageRef ex(MINF(x.x+nPixelArea,size.width-1), MINF(x.y+nPixelArea,size.height-1)); - for (int y=sx.y; y<=ex.y; ++y) - for (int x=sx.x; x<=ex.x; ++x) + for (int y=sx.y; y<=ex.y; ++y) { + for (int x=sx.x; x<=ex.x; ++x) { depthData.depthMap(y,x) = d; + depthData.normalMap(y,x) = Normal::ZERO; + } + } if (depthData.dMin > d) depthData.dMin = d; if (depthData.dMax < d) @@ -700,8 +711,12 @@ bool DepthMapsData::EstimateDepthMap(IIndex idxImage) } // init integral images and index to image-ref map for the reference data + #if DENSE_NCC == DENSE_NCC_WEIGHTED + DepthEstimator::WeightMap weightMap0(size.area()-(size.width+1)*DepthEstimator::nSizeHalfWindow); + #else Image64F imageSum0; cv::integral(image.image, imageSum0, CV_64F); + #endif if (prevDepthMapSize != size) { prevDepthMapSize = size; BitMatrix mask; @@ -723,7 +738,13 @@ bool DepthMapsData::EstimateDepthMap(IIndex idxImage) idxPixel = -1; ASSERT(estimators.IsEmpty()); while (estimators.GetSize() < nMaxThreads) - estimators.AddConstruct(depthData, idxPixel, imageSum0, coords, DepthEstimator::RB2LT); + estimators.AddConstruct(0, depthData, idxPixel, + #if DENSE_NCC == DENSE_NCC_WEIGHTED + weightMap0, + #else + imageSum0, + #endif + coords); ASSERT(estimators.GetSize() == threads.GetSize()+1); FOREACH(i, threads) threads[i].start(ScoreDepthMapTmp, &estimators[i]); @@ -745,11 +766,16 @@ bool DepthMapsData::EstimateDepthMap(IIndex idxImage) // run propagation and random refinement cycles on the reference data for (unsigned iter=0; iter ProjArr; typedef SEACAVE::cList ProjsArr; -void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateNormal) +void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, bool bEstimateNormal) { TD_TIMER_STARTD(); @@ -1329,6 +1361,7 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateNormal) // fuse all depth-maps, processing the best connected images first const unsigned nMinViewsFuse(MINF(OPTDENSE::nMinViewsFuse, scene.images.GetSize())); + const float normalError(COS(FD2R(OPTDENSE::fNormalDiffThreshold))); CLISTDEF0(Depth*) invalidDepths(0, 32); size_t nDepths(0); typedef TImage DepthIndex; @@ -1338,6 +1371,10 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateNormal) pointcloud.points.Reserve(nPointsEstimate); pointcloud.pointViews.Reserve(nPointsEstimate); pointcloud.pointWeights.Reserve(nPointsEstimate); + if (bEstimateColor) + pointcloud.colors.Reserve(nPointsEstimate); + if (bEstimateNormal) + pointcloud.normals.Reserve(nPointsEstimate); Util::Progress progress(_T("Fused depth-maps"), connections.GetSize()); GET_LOGCONSOLE().Pause(); FOREACHPTR(pConnection, connections) { @@ -1384,9 +1421,13 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateNormal) weights.Insert(depthData.confMap(x)); ProjArr& pointProjs = projs.AddEmpty(); pointProjs.Insert(Proj(x)); + const PointCloud::Normal normal(imageData.camera.R.t()*Cast(depthData.normalMap(x))); + ASSERT(ISEQUAL(norm(normal), 1.f)); // check the projection in the neighbor depth-maps REAL confidence(weights.First()); Point3 X(point*confidence); + Pixel32F C(Cast(imageData.image(x))*confidence); + PointCloud::Normal N(normal*confidence); invalidDepths.Empty(); FOREACHPTR(pNeighbor, depthData.neighbors) { const IIndex idxImageB(pNeighbor->idx.ID); @@ -1406,16 +1447,26 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateNormal) if (idxPointB != NO_ID) continue; if (IsDepthSimilar(pt.z, depthB, OPTDENSE::fDepthDiffThreshold)) { - // add view to the 3D point - ASSERT(views.FindFirst(idxImageB) == PointCloud::ViewArr::NO_INDEX); - const float confidenceB(depthDataB.confMap(xB)); - const IIndex idx(views.InsertSort(idxImageB)); - weights.InsertAt(idx, confidenceB); - pointProjs.InsertAt(idx, Proj(xB)); - idxPointB = idxPoint; - X += imageDataB.camera.TransformPointI2W(Point3(Point2f(xB),depthB))*REAL(confidenceB); - confidence += confidenceB; - } else + // check if normals agree + const PointCloud::Normal normalB(imageDataB.camera.R.t()*Cast(depthDataB.normalMap(xB))); + ASSERT(ISEQUAL(norm(normalB), 1.f)); + if (normal.dot(normalB) > normalError) { + // add view to the 3D point + ASSERT(views.FindFirst(idxImageB) == PointCloud::ViewArr::NO_INDEX); + const float confidenceB(depthDataB.confMap(xB)); + const IIndex idx(views.InsertSort(idxImageB)); + weights.InsertAt(idx, confidenceB); + pointProjs.InsertAt(idx, Proj(xB)); + idxPointB = idxPoint; + X += imageDataB.camera.TransformPointI2W(Point3(Point2f(xB),depthB))*REAL(confidenceB); + if (bEstimateColor) + C += Cast(imageDataB.image(xB))*confidenceB; + if (bEstimateNormal) + N += normalB*confidenceB; + confidence += confidenceB; + continue; + } + } if (pt.z < depthB) { // discard depth invalidDepths.Insert(&depthB); @@ -1437,6 +1488,10 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateNormal) // this point is valid, store it point = X*(REAL(1)/confidence); ASSERT(ISFINITE(point)); + if (bEstimateColor) + pointcloud.colors.AddConstruct(Cast(C*(1.f/confidence))); + if (bEstimateNormal) + pointcloud.normals.AddConstruct(normalized(N*(1.f/confidence))); // invalidate all neighbor depths that do not agree with it for (Depth* pDepth: invalidDepths) *pDepth = 0; @@ -1453,7 +1508,7 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateNormal) DEBUG_EXTRA("Depth-maps fused and filtered: %u depth-maps, %u depths, %u points (%d%%%%) (%s)", connections.GetSize(), nDepths, pointcloud.points.GetSize(), ROUND2INT((100.f*pointcloud.points.GetSize())/nDepths), TD_TIMER_GET_FMT().c_str()); - if (bEstimateNormal && !pointcloud.points.IsEmpty()) { + if (bEstimateNormal && !pointcloud.points.IsEmpty() && pointcloud.normals.IsEmpty()) { // estimate normal also if requested (quite expensive if normal-maps not available) TD_TIMER_STARTD(); pointcloud.normals.Resize(pointcloud.points.GetSize()); @@ -1556,7 +1611,7 @@ bool Scene::DenseReconstruction() data.images.Insert(idxImage); } // reload image at the appropriate resolution - const unsigned nMaxResolution(imageData.RecomputeMaxResolution(OPTDENSE::nResolutionLevel, OPTDENSE::nMinResolution)); + const unsigned nMaxResolution(imageData.RecomputeMaxResolution(OPTDENSE::nResolutionLevel, OPTDENSE::nMinResolution, OPTDENSE::nMaxResolution)); if (!imageData.ReloadImage(nMaxResolution)) { #ifdef DENSE_USE_OPENMP bAbort = true; @@ -1673,7 +1728,7 @@ bool Scene::DenseReconstruction() // fuse all depth-maps pointcloud.Release(); - data.detphMaps.FuseDepthMaps(pointcloud, OPTDENSE::nEstimateNormals == 2); + data.detphMaps.FuseDepthMaps(pointcloud, OPTDENSE::nEstimateColors == 2, OPTDENSE::nEstimateNormals == 2); #if TD_VERBOSE != TD_VERBOSE_OFF if (g_nVerbosityLevel > 2) { // print number of points with 3+ views @@ -1740,7 +1795,7 @@ void Scene::DenseReconstructionEstimate(void* pData) } // try to load already compute depth-map for this image if (depthData.Load(ComposeDepthFilePath(idx, "dmap"))) { - if (OPTDENSE::nOptimize & (OPTDENSE::OPTIMIZE)) { + if (OPTDENSE::nOptimize & OPTDENSE::OPTIMIZE) { // optimize depth-map data.events.AddEventFirst(new EVTOptimizeDepthMap(evtImage.idxImage)); } else { From 7d87728dc93bb33f8e2f87c2e03661b595254c70 Mon Sep 17 00:00:00 2001 From: cDc Date: Mon, 11 Feb 2019 21:54:43 +0200 Subject: [PATCH 02/70] viewer: improve pick-tool --- apps/Viewer/Scene.cpp | 103 ++++++++++++++++------------------------- apps/Viewer/Scene.h | 3 +- apps/Viewer/Window.cpp | 2 +- libs/Common/Ray.h | 87 +++++++++++++++++++++++++++++++++- libs/Common/Ray.inl | 78 +++++++++++++++++++++++++++++++ 5 files changed, 205 insertions(+), 68 deletions(-) diff --git a/apps/Viewer/Scene.cpp b/apps/Viewer/Scene.cpp index 703ce2625..be627b4e8 100644 --- a/apps/Viewer/Scene.cpp +++ b/apps/Viewer/Scene.cpp @@ -42,68 +42,39 @@ using namespace VIEWER; // S T R U C T S /////////////////////////////////////////////////// -template -struct LeafAvgSize { - typedef typename OCTREE::IDX_TYPE IDX; - const IDX maxNumItems; - double radius; - size_t count; - LeafAvgSize(IDX cellNumItems) : maxNumItems(cellNumItems/2), radius(0), count(0) {} - void operator () (const typename OCTREE::CELL_TYPE& cell, typename OCTREE::Type r) { - if (cell.GetNumItems() > maxNumItems) { - radius += r; - count++; - } - } - operator float () const { - return (float)(radius / (double)count); - } -}; +struct IndexDist { + IDX idx; + REAL dist; -template -struct TIntersectRay { - typedef SCENE Scene; - typedef OCTREE Octree; - typedef typename OCTREE::IDX_TYPE IDX; - - struct IndexDist { - IDX idx; - REAL dist; + inline IndexDist() : dist(REAL(FLT_MAX)) {} + inline bool IsValid() const { return dist < REAL(FLT_MAX); } +}; - inline IndexDist() : dist(REAL(FLT_MAX)) {} - inline bool IsValid() const { return dist < REAL(FLT_MAX); } - }; +struct IntersectRayPoints { + typedef MVS::PointCloud Scene; + typedef VIEWER::Scene::OctreePoints Octree; + typedef typename Octree::IDX_TYPE IDX; + typedef TCone Cone3; + typedef TConeIntersect Cone3Intersect; const Scene& scene; const Octree& octree; - const Ray3& ray; + const Cone3 cone; + const Cone3Intersect coneIntersect; + const unsigned minViews; IndexDist pick; - TIntersectRay(const Scene& _scene, const Octree& _octree, const Ray3& _ray) + IntersectRayPoints(const Scene& _scene, const Octree& _octree, const Ray3& _ray, unsigned _minViews) : - scene(_scene), - octree(_octree), - ray(_ray) + scene(_scene), octree(_octree), + cone(_ray, D2R(REAL(0.5))), coneIntersect(cone), + minViews(_minViews) { + octree.Collect(*this, *this); } inline bool Intersects(const typename Octree::POINT_TYPE& center, typename Octree::Type radius) const { - return ray.Intersects(AABB3f(center, radius)); - } -}; - -struct IntersectRayPoints : public TIntersectRay { - using typename TIntersectRay::Scene; - using typename TIntersectRay::Octree; - using typename TIntersectRay::IDX; - - const REAL pointSize; - const unsigned minViews; - - IntersectRayPoints(const Scene& _scene, const Octree& _octree, REAL _pointSize, unsigned _minViews, const Ray3& _ray) - : TIntersectRay(_scene, _octree, _ray), pointSize(_pointSize), minViews(_minViews) - { - octree.Collect(*this, *this); + return coneIntersect(Sphere3(center.cast(), REAL(radius)*M_SQRT2)); } void operator () (const IDX* idices, IDX size) { @@ -114,7 +85,7 @@ struct IntersectRayPoints : public TIntersectRay(X), pointSize), dist)) { + if (coneIntersect.Classify(Cast(X), dist) == VISIBLE) { ASSERT(dist >= 0); if (pick.dist > dist) { pick.dist = dist; @@ -125,17 +96,28 @@ struct IntersectRayPoints : public TIntersectRay { - using typename TIntersectRay::Scene; - using typename TIntersectRay::Octree; - using typename TIntersectRay::IDX; +struct IntersectRayMesh { + typedef MVS::Mesh Scene; + typedef VIEWER::Scene::OctreeMesh Octree; + typedef typename Octree::IDX_TYPE IDX; + + const Scene& scene; + const Octree& octree; + const Ray3& ray; + IndexDist pick; IntersectRayMesh(const Scene& _scene, const Octree& _octree, const Ray3& _ray) - : TIntersectRay(_scene, _octree, _ray) + : + scene(_scene), octree(_octree), + ray(_ray) { octree.Collect(*this, *this); } + inline bool Intersects(const typename Octree::POINT_TYPE& center, typename Octree::Type radius) const { + return ray.Intersects(AABB3f(center, radius)); + } + void operator () (const IDX* idices, IDX size) { // store all intersected faces only once typedef std::unordered_set FaceSet; @@ -204,18 +186,12 @@ class EVTComputeOctree : public Event MVS::Scene& scene = pScene->scene; if (!scene.mesh.IsEmpty()) { Scene::OctreeMesh octMesh(scene.mesh.vertices); - LeafAvgSize size(128); - octMesh.ParseCells(size); scene.mesh.ListIncidenteFaces(); pScene->octMesh.Swap(octMesh); - pScene->avgScale = size; } else if (!scene.pointcloud.IsEmpty()) { Scene::OctreePoints octPoints(scene.pointcloud.points); - LeafAvgSize size(256); - octPoints.ParseCells(size); pScene->octPoints.Swap(octPoints); - pScene->avgScale = size; } return true; } @@ -265,7 +241,6 @@ void Scene::Empty() images.Release(); scene.Release(); sceneName.clear(); - avgScale = 0; } void Scene::Release() { @@ -675,7 +650,7 @@ void Scene::CastRay(const Ray3& ray, int action) } else if (!octPoints.IsEmpty()) { // find ray intersection with the points - const IntersectRayPoints intRay(scene.pointcloud, octPoints, avgScale*0.1f, window.minViews, ray); + const IntersectRayPoints intRay(scene.pointcloud, octPoints, ray, window.minViews); if (intRay.pick.IsValid()) { window.selectionPoints[0] = window.selectionPoints[3] = scene.pointcloud.points[intRay.pick.idx]; window.selectionType = Window::SEL_POINT; diff --git a/apps/Viewer/Scene.h b/apps/Viewer/Scene.h index 043495273..0d31fa076 100644 --- a/apps/Viewer/Scene.h +++ b/apps/Viewer/Scene.h @@ -62,7 +62,6 @@ class Scene OctreePoints octPoints; OctreeMesh octMesh; - float avgScale; GLuint listPointCloud; GLuint listMesh; @@ -81,7 +80,7 @@ class Scene void ReleaseMesh(); inline bool IsValid() const { return window.IsValid(); } inline bool IsOpen() const { return IsValid() && !scene.IsEmpty(); } - inline bool IsOctreeValid() const { return avgScale > 0; } + inline bool IsOctreeValid() const { return !octPoints.IsEmpty() || !octMesh.IsEmpty(); } bool Init(int width, int height, LPCTSTR windowName, LPCTSTR fileName=NULL, LPCTSTR meshFileName=NULL); bool Open(LPCTSTR fileName, LPCTSTR meshFileName=NULL); diff --git a/apps/Viewer/Window.cpp b/apps/Viewer/Window.cpp index f39784f61..6a9ada4f1 100644 --- a/apps/Viewer/Window.cpp +++ b/apps/Viewer/Window.cpp @@ -292,7 +292,7 @@ void Window::MouseButton(int button, int action, int /*mods*/) // 4d World Coordinates const Mat4 invV(V.inverse()); ASSERT(ISEQUAL(invV(3,3),1.0)); - Eigen::Vector3d start(invV.topRightCorner<3,1>()); + const Eigen::Vector3d start(invV.topRightCorner<3,1>()); const Eigen::Vector4d ray_wor(invV*ray_eye); const Eigen::Vector3d dir(ray_wor.topRows<3>().normalized()); clbkRayScene(Ray3d(start, dir), action); diff --git a/libs/Common/Ray.h b/libs/Common/Ray.h index 802778e52..5f0f9cb65 100644 --- a/libs/Common/Ray.h +++ b/libs/Common/Ray.h @@ -30,7 +30,7 @@ class TTriangle typedef Eigen::Matrix POINT; typedef SEACAVE::TAABB AABB; typedef SEACAVE::TPlane PLANE; - enum { numScalar = (2*DIMS) }; + enum { numScalar = (3*DIMS) }; POINT a, b, c; // triangle vertices @@ -129,6 +129,91 @@ class TRay /*----------------------------------------------------------------*/ +// Basic cylinder class +template +class TCylinder +{ + STATIC_ASSERT(DIMS > 1 && DIMS <= 3); + +public: + typedef Eigen::Matrix VECTOR; + typedef Eigen::Matrix POINT; + typedef SEACAVE::TRay RAY; + typedef SEACAVE::TSphere SPHERE; + enum { numScalar = (2*DIMS+2) }; + + RAY ray; // central ray defining starting the point and direction + TYPE radius; // cylinder radius + TYPE height; // cylinder length measured from the origin in the ray direction + + //--------------------------------------- + + inline TCylinder() {} + + inline TCylinder(const RAY& _ray, TYPE _radius, TYPE _height=std::numeric_limits::max()) + : ray(_ray), radius(_radius), height(_height) { ASSERT(TYPE(0) < height); } + + bool Intersects(const SPHERE& sphere) const; + + inline GCLASS Classify(const POINT&) const; +}; // class TCylinder +/*----------------------------------------------------------------*/ + + +// Basic cone class +template +class TCone +{ + STATIC_ASSERT(DIMS > 1 && DIMS <= 3); + +public: + typedef Eigen::Matrix VECTOR; + typedef Eigen::Matrix POINT; + typedef SEACAVE::TRay RAY; + typedef SEACAVE::TSphere SPHERE; + enum { numScalar = (2*DIMS+3) }; + + RAY ray; // ray origin is the cone vertex and ray direction is the cone axis direction + TYPE angle; // cone angle, in radians, must be inside [0, pi/2] + TYPE minHeight; // heights satisfying: + TYPE maxHeight; // 0 <= minHeight < maxHeight <= std::numeric_limits::max() + + //--------------------------------------- + + inline TCone() {} + inline TCone(const RAY& _ray, TYPE _angle, TYPE _minHeight=TYPE(0), TYPE _maxHeight=std::numeric_limits::max()) + : ray(_ray), angle(_angle), minHeight(_minHeight), maxHeight(_maxHeight) { ASSERT(TYPE(0) <= minHeight && minHeight < maxHeight); } +}; // class TCone + +// Structure used to compute the intersection between a cone and the given sphere/point +template +class TConeIntersect +{ +public: + typedef Eigen::Matrix VECTOR; + typedef Eigen::Matrix POINT; + typedef SEACAVE::TCone CONE; + typedef SEACAVE::TSphere SPHERE; + + const CONE& cone; + + // cache angle derivatives, to avoid calling trigonometric functions in geometric queries + const TYPE cosAngle, sinAngle, sinAngleSq, cosAngleSq, invSinAngle; + + //--------------------------------------- + + inline TConeIntersect(const CONE& _cone) + : cone(_cone), cosAngle(COS(cone.angle)), sinAngle(SIN(cone.angle)), + sinAngleSq(SQUARE(sinAngle)), cosAngleSq(SQUARE(cosAngle)), + invSinAngle(TYPE(1)/sinAngle) {} + + bool operator()(const SPHERE& sphere) const; + + GCLASS Classify(const POINT&, TYPE& t) const; +}; // class TConeIntersect +/*----------------------------------------------------------------*/ + + #include "Ray.inl" /*----------------------------------------------------------------*/ diff --git a/libs/Common/Ray.inl b/libs/Common/Ray.inl index cfc2908f1..c89fbb6b5 100644 --- a/libs/Common/Ray.inl +++ b/libs/Common/Ray.inl @@ -896,3 +896,81 @@ TYPE TRay::Distance(const POINT& pt) const return SQRT(DistanceSq(pt)); } // Distance(POINT) /*----------------------------------------------------------------*/ + + +// S T R U C T S /////////////////////////////////////////////////// + +template +bool TCylinder::Intersects(const SPHERE& sphere) const +{ + const VECTOR CmV(sphere.center - ray.m_pOrig); + const TYPE t(ray.m_vDir.dot(CmV)); + // sphere behind the cylinder origin + if (t < TYPE(0)) + return false; + // sphere after the cylinder end + if (t > height) + return false; + const TYPE lenSq((CmV - ray.m_vDir * t).squaredNorm()); + return (lenSq <= SQUARE(sphere.radius + radius)); +} +/*----------------------------------------------------------------*/ + +// Classify point to cylinder. +template +inline GCLASS TCylinder::Classify(const POINT& p) const +{ + ASSERT(ISEQUAL(ray.m_vDir.norm(), TYPE(1))); + const VECTOR D(p - ray.m_pOrig); + const TYPE t(ray.m_vDir.dot(D)); + if (ISZERO(t)) + return PLANAR; + if (t < TYPE(0)) return BACK; + if (t > height) return FRONT; + const TYPE rSq((D - ray.m_vDir*t).squaredNorm()); + const TYPE radiusSq(SQUARE(radius)); + if (rSq > radiusSq) return CULLED; + if (rSq < radiusSq) return VISIBLE; + return PLANAR; +} +/*----------------------------------------------------------------*/ + + +// S T R U C T S /////////////////////////////////////////////////// + +template +bool TConeIntersect::operator()(const SPHERE& sphere) const +{ + const VECTOR CmV(sphere.center - cone.ray.m_pOrig); + const VECTOR D(CmV + cone.ray.m_vDir * (sphere.radius * invSinAngle)); + TYPE lenSq = D.squaredNorm(); + TYPE e = D.dot(cone.ray.m_vDir); + if (e > TYPE(0) && e*e >= lenSq*cosAngleSq) { + lenSq = CmV.squaredNorm(); + e = CmV.dot(cone.ray.m_vDir); + if (e < TYPE(0) && e*e >= lenSq*sinAngleSq) + return (lenSq <= SQUARE(sphere.radius)); + return true; + } + return false; +} +/*----------------------------------------------------------------*/ + +// Classify point to cone. +template +GCLASS TConeIntersect::Classify(const POINT& p, TYPE& t) const +{ + ASSERT(ISEQUAL(cone.ray.m_vDir.norm(), TYPE(1))); + const VECTOR D(p - cone.ray.m_pOrig); + t = cone.ray.m_vDir.dot(D); + if (ISZERO(t)) + return PLANAR; + if (t < cone.minHeight) return BACK; + if (t > cone.maxHeight) return FRONT; + ASSERT(!ISZERO(D.norm())); + const TYPE d(cosAngle*D.norm()); + if (t < d) return CULLED; + if (t > d) return VISIBLE; + return PLANAR; +} +/*----------------------------------------------------------------*/ From 0145e1ce3818800850ca653a54e9a2626a45e9e1 Mon Sep 17 00:00:00 2001 From: cDc Date: Wed, 13 Feb 2019 18:02:05 +0200 Subject: [PATCH 03/70] dense: increase patch size --- libs/MVS/DepthMap.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/MVS/DepthMap.h b/libs/MVS/DepthMap.h index e98affe80..3b361bea3 100644 --- a/libs/MVS/DepthMap.h +++ b/libs/MVS/DepthMap.h @@ -218,7 +218,7 @@ typedef MVS_API CLISTDEFIDX(DepthData,IIndex) DepthDataArr; struct MVS_API DepthEstimator { - enum { nSizeHalfWindow = 5 }; + enum { nSizeHalfWindow = 9 }; enum { nSizeWindow = nSizeHalfWindow*2+1 }; enum { nSizeStep = 2 }; enum { TexelChannels = 1 }; From c3dcbb7be1cfbd9b4f4a6a41e5a1bc386ed3a014 Mon Sep 17 00:00:00 2001 From: cDc Date: Wed, 13 Feb 2019 18:09:37 +0200 Subject: [PATCH 04/70] match: extend cylinder/cone functionality --- apps/Viewer/Scene.cpp | 4 +- libs/Common/Ray.h | 36 ++++++++++------- libs/Common/Ray.inl | 93 ++++++++++++++++++++++++++++--------------- 3 files changed, 83 insertions(+), 50 deletions(-) diff --git a/apps/Viewer/Scene.cpp b/apps/Viewer/Scene.cpp index be627b4e8..d4f49c93d 100644 --- a/apps/Viewer/Scene.cpp +++ b/apps/Viewer/Scene.cpp @@ -55,7 +55,7 @@ struct IntersectRayPoints { typedef VIEWER::Scene::OctreePoints Octree; typedef typename Octree::IDX_TYPE IDX; typedef TCone Cone3; - typedef TConeIntersect Cone3Intersect; + typedef TConeIntersect Cone3Intersect; const Scene& scene; const Octree& octree; @@ -74,7 +74,7 @@ struct IntersectRayPoints { } inline bool Intersects(const typename Octree::POINT_TYPE& center, typename Octree::Type radius) const { - return coneIntersect(Sphere3(center.cast(), REAL(radius)*M_SQRT2)); + return coneIntersect(Sphere3(center.cast(), REAL(radius)*SQRT_3)); } void operator () (const IDX* idices, IDX size) { diff --git a/libs/Common/Ray.h b/libs/Common/Ray.h index 5f0f9cb65..fd1989e6a 100644 --- a/libs/Common/Ray.h +++ b/libs/Common/Ray.h @@ -144,18 +144,22 @@ class TCylinder RAY ray; // central ray defining starting the point and direction TYPE radius; // cylinder radius - TYPE height; // cylinder length measured from the origin in the ray direction + TYPE minHeight; // cylinder heights satisfying: + TYPE maxHeight; // std::numeric_limits::lowest() <= minHeight <= 0 < maxHeight <= std::numeric_limits::max() //--------------------------------------- inline TCylinder() {} - inline TCylinder(const RAY& _ray, TYPE _radius, TYPE _height=std::numeric_limits::max()) - : ray(_ray), radius(_radius), height(_height) { ASSERT(TYPE(0) < height); } + inline TCylinder(const RAY& _ray, TYPE _radius, TYPE _minHeight=TYPE(0), TYPE _maxHeight=std::numeric_limits::max()) + : ray(_ray), radius(_radius), minHeight(_minHeight), maxHeight(_maxHeight) { ASSERT(TYPE(0) >= minHeight && minHeight < maxHeight); } - bool Intersects(const SPHERE& sphere) const; + inline bool IsFinite() const { return maxHeight < std::numeric_limits::max() && minHeight > std::numeric_limits::lowest(); } + inline TYPE GetLength() const { return maxHeight - minHeight; } + + bool Intersects(const SPHERE&) const; - inline GCLASS Classify(const POINT&) const; + inline GCLASS Classify(const POINT&, TYPE& t) const; }; // class TCylinder /*----------------------------------------------------------------*/ @@ -173,10 +177,10 @@ class TCone typedef SEACAVE::TSphere SPHERE; enum { numScalar = (2*DIMS+3) }; - RAY ray; // ray origin is the cone vertex and ray direction is the cone axis direction - TYPE angle; // cone angle, in radians, must be inside [0, pi/2] - TYPE minHeight; // heights satisfying: - TYPE maxHeight; // 0 <= minHeight < maxHeight <= std::numeric_limits::max() + RAY ray; // ray origin is the cone vertex and ray direction is the cone axis direction + TYPE angle; // cone angle, in radians, must be inside [0, pi/2] + TYPE minHeight; // heights satisfying: + TYPE maxHeight; // 0 <= minHeight < maxHeight <= std::numeric_limits::max() //--------------------------------------- @@ -186,14 +190,16 @@ class TCone }; // class TCone // Structure used to compute the intersection between a cone and the given sphere/point -template +template class TConeIntersect { + STATIC_ASSERT(DIMS > 1 && DIMS <= 3); + public: - typedef Eigen::Matrix VECTOR; - typedef Eigen::Matrix POINT; - typedef SEACAVE::TCone CONE; - typedef SEACAVE::TSphere SPHERE; + typedef Eigen::Matrix VECTOR; + typedef Eigen::Matrix POINT; + typedef SEACAVE::TCone CONE; + typedef SEACAVE::TSphere SPHERE; const CONE& cone; @@ -207,7 +213,7 @@ class TConeIntersect sinAngleSq(SQUARE(sinAngle)), cosAngleSq(SQUARE(cosAngle)), invSinAngle(TYPE(1)/sinAngle) {} - bool operator()(const SPHERE& sphere) const; + bool operator()(const SPHERE&) const; GCLASS Classify(const POINT&, TYPE& t) const; }; // class TConeIntersect diff --git a/libs/Common/Ray.inl b/libs/Common/Ray.inl index c89fbb6b5..05b1271d6 100644 --- a/libs/Common/Ray.inl +++ b/libs/Common/Ray.inl @@ -903,62 +903,88 @@ TYPE TRay::Distance(const POINT& pt) const template bool TCylinder::Intersects(const SPHERE& sphere) const { + struct Check { + static bool Intersection(const RAY& ray, const SPHERE& sphere, const VECTOR& CmV, TYPE t, TYPE radius, TYPE height) { + const POINT O(ray.m_pOrig + ray.m_vDir * height); + const VECTOR a(sphere.center - O); + if (a.squaredNorm() <= SQUARE(sphere.radius)) + return true; // ray origin inside the sphere + const POINT D((CmV - ray.m_vDir * t).normalized()); + const TYPE d = a.dot(D); + if (d < TYPE(0)) + return false; // intersection behind the ray origin + const TYPE dSq((a - D*d).squaredNorm()); + const TYPE srSq = SQUARE(sphere.radius); + if (dSq <= srSq) { + const TYPE r = d - SQRT(srSq-dSq); + ASSERT(r >= TYPE(0)); + return r <= radius; // intersection before the ray end + } + return false; + } + }; const VECTOR CmV(sphere.center - ray.m_pOrig); const TYPE t(ray.m_vDir.dot(CmV)); - // sphere behind the cylinder origin - if (t < TYPE(0)) - return false; - // sphere after the cylinder end - if (t > height) - return false; + // sphere projects behind the cylinder origin + if (t < minHeight) { + if (t+sphere.radius < minHeight) + return false; + return Check::Intersection(ray, sphere, CmV, t, radius, minHeight); + } + // sphere projects after the cylinder end + if (t > maxHeight) { + if (t-sphere.radius > maxHeight) + return false; + return Check::Intersection(ray, sphere, CmV, t, radius, maxHeight); + } const TYPE lenSq((CmV - ray.m_vDir * t).squaredNorm()); - return (lenSq <= SQUARE(sphere.radius + radius)); -} + return lenSq <= SQUARE(sphere.radius + radius); +} // Intersects /*----------------------------------------------------------------*/ // Classify point to cylinder. template -inline GCLASS TCylinder::Classify(const POINT& p) const +GCLASS TCylinder::Classify(const POINT& p, TYPE& t) const { ASSERT(ISEQUAL(ray.m_vDir.norm(), TYPE(1))); const VECTOR D(p - ray.m_pOrig); - const TYPE t(ray.m_vDir.dot(D)); - if (ISZERO(t)) - return PLANAR; - if (t < TYPE(0)) return BACK; - if (t > height) return FRONT; + t = ray.m_vDir.dot(D); + if (t < minHeight) return BACK; + if (t > maxHeight) return FRONT; const TYPE rSq((D - ray.m_vDir*t).squaredNorm()); const TYPE radiusSq(SQUARE(radius)); if (rSq > radiusSq) return CULLED; if (rSq < radiusSq) return VISIBLE; return PLANAR; -} +} // Classify /*----------------------------------------------------------------*/ // S T R U C T S /////////////////////////////////////////////////// -template -bool TConeIntersect::operator()(const SPHERE& sphere) const +template +bool TConeIntersect::operator()(const SPHERE& sphere) const { const VECTOR CmV(sphere.center - cone.ray.m_pOrig); - const VECTOR D(CmV + cone.ray.m_vDir * (sphere.radius * invSinAngle)); - TYPE lenSq = D.squaredNorm(); + const VECTOR D(CmV + cone.ray.m_vDir * (sphere.radius*invSinAngle)); TYPE e = D.dot(cone.ray.m_vDir); - if (e > TYPE(0) && e*e >= lenSq*cosAngleSq) { - lenSq = CmV.squaredNorm(); - e = CmV.dot(cone.ray.m_vDir); - if (e < TYPE(0) && e*e >= lenSq*sinAngleSq) - return (lenSq <= SQUARE(sphere.radius)); - return true; + if (e <= TYPE(0) || e*e < D.squaredNorm()*cosAngleSq) + return false; + e = CmV.dot(cone.ray.m_vDir); + if (e-sphere.radius > cone.maxHeight) + return false; + if (e < cone.minHeight) { + const TYPE lenSq = CmV.squaredNorm(); + if (e*e >= lenSq*sinAngleSq) + return lenSq <= SQUARE(sphere.radius); } - return false; -} + return true; +} // Intersect /*----------------------------------------------------------------*/ // Classify point to cone. -template -GCLASS TConeIntersect::Classify(const POINT& p, TYPE& t) const +template +GCLASS TConeIntersect::Classify(const POINT& p, TYPE& t) const { ASSERT(ISEQUAL(cone.ray.m_vDir.norm(), TYPE(1))); const VECTOR D(p - cone.ray.m_pOrig); @@ -968,9 +994,10 @@ GCLASS TConeIntersect::Classify(const POINT& p, TYPE& t) const if (t < cone.minHeight) return BACK; if (t > cone.maxHeight) return FRONT; ASSERT(!ISZERO(D.norm())); - const TYPE d(cosAngle*D.norm()); - if (t < d) return CULLED; - if (t > d) return VISIBLE; + const TYPE tSq(SQUARE(t)); + const TYPE dSq(cosAngleSq*D.squaredNorm()); + if (tSq < dSq) return CULLED; + if (tSq > dSq) return VISIBLE; return PLANAR; -} +} // Classify /*----------------------------------------------------------------*/ From 51d9f73f3cd25da6e4e200e0cc53487c2ebf3abb Mon Sep 17 00:00:00 2001 From: cDc Date: Tue, 19 Feb 2019 10:39:46 +0200 Subject: [PATCH 05/70] dense: visibility filter --- apps/DensifyPointCloud/DensifyPointCloud.cpp | 11 ++ apps/Viewer/Scene.cpp | 19 +-- libs/Common/Util.inl | 1 + libs/MVS/Image.cpp | 13 ++ libs/MVS/Image.h | 1 + libs/MVS/Scene.cpp | 4 +- libs/MVS/Scene.h | 1 + libs/MVS/SceneDensify.cpp | 147 ++++++++++++++++++- 8 files changed, 177 insertions(+), 20 deletions(-) diff --git a/apps/DensifyPointCloud/DensifyPointCloud.cpp b/apps/DensifyPointCloud/DensifyPointCloud.cpp index beb89e973..752156e72 100644 --- a/apps/DensifyPointCloud/DensifyPointCloud.cpp +++ b/apps/DensifyPointCloud/DensifyPointCloud.cpp @@ -49,6 +49,7 @@ String strOutputFileName; String strMeshFileName; String strDenseConfigFileName; float fSampleMesh; +int thFilterPointCloud; int nArchiveType; int nProcessPriority; unsigned nMaxThreads; @@ -105,6 +106,7 @@ bool Initialize(size_t argc, LPCTSTR* argv) ("estimate-colors", boost::program_options::value(&nEstimateColors)->default_value(2), "estimate the colors for the dense point-cloud") ("estimate-normals", boost::program_options::value(&nEstimateNormals)->default_value(0), "estimate the normals for the dense point-cloud") ("sample-mesh", boost::program_options::value(&OPT::fSampleMesh)->default_value(0.f), "uniformly samples points on a mesh (0 - disabled, <0 - number of points, >0 - sample density per square unit)") + ("filter-point-cloud", boost::program_options::value(&OPT::thFilterPointCloud)->default_value(0), "filter dense point-cloud based on visibility (0 - disabled)") ; // hidden options, allowed both on command line and @@ -241,6 +243,15 @@ int main(int argc, LPCTSTR* argv) VERBOSE("error: empty initial point-cloud"); return EXIT_FAILURE; } + if (OPT::thFilterPointCloud < 0) { + // filter point-cloud based on camera-point visibility intersections + scene.PointCloudFilter(OPT::thFilterPointCloud); + const String baseFileName(MAKE_PATH_SAFE(Util::getFileFullName(OPT::strOutputFileName))+_T("_filtered")); + scene.Save(baseFileName+_T(".mvs"), (ARCHIVE_TYPE)OPT::nArchiveType); + scene.pointcloud.Save(baseFileName+_T(".ply")); + Finalize(); + return EXIT_SUCCESS; + } if ((ARCHIVE_TYPE)OPT::nArchiveType != ARCHIVE_MVS) { TD_TIMER_START(); if (!scene.DenseReconstruction()) diff --git a/apps/Viewer/Scene.cpp b/apps/Viewer/Scene.cpp index d4f49c93d..8e63dbf9a 100644 --- a/apps/Viewer/Scene.cpp +++ b/apps/Viewer/Scene.cpp @@ -58,17 +58,13 @@ struct IntersectRayPoints { typedef TConeIntersect Cone3Intersect; const Scene& scene; - const Octree& octree; const Cone3 cone; const Cone3Intersect coneIntersect; const unsigned minViews; IndexDist pick; - IntersectRayPoints(const Scene& _scene, const Octree& _octree, const Ray3& _ray, unsigned _minViews) - : - scene(_scene), octree(_octree), - cone(_ray, D2R(REAL(0.5))), coneIntersect(cone), - minViews(_minViews) + IntersectRayPoints(const Octree& octree, const Ray3& _ray, const Scene& _scene, unsigned _minViews) + : scene(_scene), cone(_ray, D2R(REAL(0.5))), coneIntersect(cone), minViews(_minViews) { octree.Collect(*this, *this); } @@ -102,14 +98,11 @@ struct IntersectRayMesh { typedef typename Octree::IDX_TYPE IDX; const Scene& scene; - const Octree& octree; const Ray3& ray; IndexDist pick; - IntersectRayMesh(const Scene& _scene, const Octree& _octree, const Ray3& _ray) - : - scene(_scene), octree(_octree), - ray(_ray) + IntersectRayMesh(const Octree& octree, const Ray3& _ray, const Scene& _scene) + : scene(_scene), ray(_ray) { octree.Collect(*this, *this); } @@ -629,7 +622,7 @@ void Scene::CastRay(const Ray3& ray, int action) } else if (!octMesh.IsEmpty()) { // find ray intersection with the mesh - const IntersectRayMesh intRay(scene.mesh, octMesh, ray); + const IntersectRayMesh intRay(octMesh, ray, scene.mesh); if (intRay.pick.IsValid()) { const MVS::Mesh::Face& face = scene.mesh.faces[(MVS::Mesh::FIndex)intRay.pick.idx]; window.selectionPoints[0] = scene.mesh.vertices[face[0]]; @@ -650,7 +643,7 @@ void Scene::CastRay(const Ray3& ray, int action) } else if (!octPoints.IsEmpty()) { // find ray intersection with the points - const IntersectRayPoints intRay(scene.pointcloud, octPoints, ray, window.minViews); + const IntersectRayPoints intRay(octPoints, ray, scene.pointcloud, window.minViews); if (intRay.pick.IsValid()) { window.selectionPoints[0] = window.selectionPoints[3] = scene.pointcloud.points[intRay.pick.idx]; window.selectionType = Window::SEL_POINT; diff --git a/libs/Common/Util.inl b/libs/Common/Util.inl index 484c5313d..74b484b15 100644 --- a/libs/Common/Util.inl +++ b/libs/Common/Util.inl @@ -772,6 +772,7 @@ inline T MaxDepthDifference(T d, T threshold) { } template inline T DepthSimilarity(T d0, T d1) { + ASSERT(d0 > 0 && d1 > 0); #if 0 return ABS(d0-d1)*T(2)/(d0+d1); #else diff --git a/libs/MVS/Image.cpp b/libs/MVS/Image.cpp index 90d69a5a0..c5454b665 100644 --- a/libs/MVS/Image.cpp +++ b/libs/MVS/Image.cpp @@ -195,4 +195,17 @@ void Image::UpdateCamera(const PlatformArr& platforms) { camera = GetCamera(platforms, Image8U::Size(width, height)); } // UpdateCamera +// computes camera's field of view for the given direction +REAL Image::ComputeFOV(int dir) const +{ + switch (dir) { + case 0: // width + return 2*ATAN(REAL(width)/(camera.K(0,0)*2)); + case 1: // height + return 2*ATAN(REAL(height)/(camera.K(1,1)*2)); + case 2: // diagonal + return 2*ATAN(SQRT(REAL(SQUARE(width)+SQUARE(height)))/(camera.K(0,0)+camera.K(1,1))); + } + return 0; +} // ComputeFOV /*----------------------------------------------------------------*/ diff --git a/libs/MVS/Image.h b/libs/MVS/Image.h index 09b26bad5..20d68065e 100644 --- a/libs/MVS/Image.h +++ b/libs/MVS/Image.h @@ -109,6 +109,7 @@ class MVS_API Image Camera GetCamera(const PlatformArr& platforms, const Image8U::Size& resolution) const; void UpdateCamera(const PlatformArr& platforms); + REAL ComputeFOV(int dir) const; float GetNormalizationScale() const { ASSERT(width > 0 && height > 0); diff --git a/libs/MVS/Scene.cpp b/libs/MVS/Scene.cpp index 60ae51019..23c48e7a5 100644 --- a/libs/MVS/Scene.cpp +++ b/libs/MVS/Scene.cpp @@ -507,7 +507,6 @@ bool Scene::SelectNeighborViews(uint32_t ID, IndexArr& points, unsigned nMinView if (score.points == 0) continue; ASSERT(ID != IDB); - ViewScore& neighbor = neighbors.AddEmpty(); // compute how well the matched features are spread out (image covered area) const Point2f boundsA(imageData.GetSize()); const Point2f boundsB(imageDataB.GetSize()); @@ -527,11 +526,14 @@ bool Scene::SelectNeighborViews(uint32_t ID, IndexArr& points, unsigned nMinView } } ASSERT(pointsA.GetSize() == pointsB.GetSize() && pointsA.GetSize() <= score.points); + if (pointsA.IsEmpty()) + continue; const float areaA(ComputeCoveredArea((const float*)pointsA.Begin(), pointsA.GetSize(), boundsA.ptr())); const float areaB(ComputeCoveredArea((const float*)pointsB.Begin(), pointsB.GetSize(), boundsB.ptr())); const float area(MINF(areaA, areaB)); pointsA.Empty(); pointsB.Empty(); // store image score + ViewScore& neighbor = neighbors.AddEmpty(); neighbor.idx.ID = IDB; neighbor.idx.points = score.points; neighbor.idx.scale = score.avgScale/score.points; diff --git a/libs/MVS/Scene.h b/libs/MVS/Scene.h index a84b98928..581ad120e 100644 --- a/libs/MVS/Scene.h +++ b/libs/MVS/Scene.h @@ -81,6 +81,7 @@ class MVS_API Scene bool DenseReconstruction(); void DenseReconstructionEstimate(void*); void DenseReconstructionFilter(void*); + void PointCloudFilter(int thRemove=-1); // Mesh reconstruction bool ReconstructMesh(float distInsert=2, bool bUseFreeSpaceSupport=true, unsigned nItersFixNonManifold=4, diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index 73a088e92..4c349817d 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -1758,7 +1758,7 @@ bool Scene::DenseReconstruction() EstimatePointNormals(images, pointcloud); } return true; -} // DenseReconstructionDepthMap +} // DenseReconstruction /*----------------------------------------------------------------*/ void* DenseReconstructionEstimateTmp(void* arg) { @@ -1794,14 +1794,14 @@ void Scene::DenseReconstructionEstimate(void* pData) break; } // try to load already compute depth-map for this image - if (depthData.Load(ComposeDepthFilePath(idx, "dmap"))) { + if (File::access(ComposeDepthFilePath(idx, "dmap"))) { if (OPTDENSE::nOptimize & OPTDENSE::OPTIMIZE) { + if (!depthData.Load(ComposeDepthFilePath(idx, "dmap"))) { + VERBOSE("error: invalid depth-map '%s'", ComposeDepthFilePath(idx, "dmap").c_str()); + exit(EXIT_FAILURE); + } // optimize depth-map data.events.AddEventFirst(new EVTOptimizeDepthMap(evtImage.idxImage)); - } else { - // release image data - depthData.ReleaseImages(); - depthData.Release(); } // process next image data.events.AddEvent(new EVTProcessImage((uint32_t)Thread::safeInc(data.idxImage))); @@ -1980,3 +1980,138 @@ void Scene::DenseReconstructionFilter(void* pData) } } // DenseReconstructionFilter /*----------------------------------------------------------------*/ + +// filter point-cloud based on camera-point visibility intersections +void Scene::PointCloudFilter(int thRemove) +{ + TD_TIMER_STARTD(); + + typedef TOctree Octree; + struct Collector { + typedef Octree::IDX_TYPE IDX; + typedef PointCloud::Point::Type Real; + typedef TCone Cone; + typedef TSphere Sphere; + typedef TConeIntersect ConeIntersect; + + Cone cone; + const ConeIntersect coneIntersect; + const PointCloud& pointcloud; + IntArr& visibility; + PointCloud::Index idxPoint; + Real distance; + int weight; + #ifdef DENSE_USE_OPENMP + uint8_t pcs[sizeof(CriticalSection)]; + #endif + + Collector(const Cone::RAY& ray, Real angle, const PointCloud& _pointcloud, IntArr& _visibility) + : cone(ray, angle), coneIntersect(cone), pointcloud(_pointcloud), visibility(_visibility) + #ifdef DENSE_USE_OPENMP + { new(pcs) CriticalSection; } + ~Collector() { reinterpret_cast(pcs)->~CriticalSection(); } + inline CriticalSection& GetCS() { return *reinterpret_cast(pcs); } + #else + {} + #endif + inline void Init(PointCloud::Index _idxPoint, const PointCloud::Point& X, int _weight) { + const Real thMaxDepth(1.02f); + idxPoint =_idxPoint; + const PointCloud::Point::EVec D((PointCloud::Point::EVec&)X-cone.ray.m_pOrig); + distance = D.norm(); + cone.ray.m_vDir = D/distance; + cone.maxHeight = MaxDepthDifference(distance, thMaxDepth); + weight = _weight; + } + inline bool Intersects(const Octree::POINT_TYPE& center, Octree::Type radius) const { + return coneIntersect(Sphere(center, radius*Real(SQRT_3))); + } + inline void operator () (const IDX* idices, IDX size) { + const Real thSimilar(0.01f); + Real dist; + FOREACHRAWPTR(pIdx, idices, size) { + const PointCloud::Index idx(*pIdx); + if (coneIntersect.Classify(pointcloud.points[idx], dist) == VISIBLE && !IsDepthSimilar(distance, dist, thSimilar)) { + if (dist > distance) + visibility[idx] += pointcloud.pointViews[idx].size(); + else + visibility[idx] -= weight; + } + } + } + }; + typedef CLISTDEF2(Collector) Collectors; + + // create octree to speed-up search + Octree octree(pointcloud.points); + IntArr visibility(pointcloud.GetSize()); visibility.Memset(0); + Collectors collectors; collectors.reserve(images.size()); + FOREACH(idxView, images) { + const Image& image = images[idxView]; + const Ray3f ray(Cast(image.camera.C), Cast(image.camera.Direction())); + const float angle(float(image.ComputeFOV(0)/image.width)); + collectors.emplace_back(ray, angle, pointcloud, visibility); + } + + // run all camera-point visibility intersections + Util::Progress progress(_T("Point visibility checks"), pointcloud.GetSize()); + #ifdef DENSE_USE_OPENMP + #pragma omp parallel for //schedule(dynamic) + for (int64_t i=0; i<(int64_t)pointcloud.GetSize(); ++i) { + const PointCloud::Index idxPoint((PointCloud::Index)i); + #else + FOREACH(idxPoint, pointcloud.points) { + #endif + const PointCloud::Point& X = pointcloud.points[idxPoint]; + const PointCloud::ViewArr& views = pointcloud.pointViews[idxPoint]; + for (PointCloud::View idxView: views) { + Collector& collector = collectors[idxView]; + #ifdef DENSE_USE_OPENMP + Lock l(collector.GetCS()); + #endif + collector.Init(idxPoint, X, (int)views.size()); + octree.Collect(collector, collector); + } + ++progress; + } + progress.close(); + + #if TD_VERBOSE != TD_VERBOSE_OFF + if (g_nVerbosityLevel > 2) { + // print visibility stats + UnsignedArr counts(0, 64); + for (int views: visibility) { + if (views > 0) + continue; + while (counts.size() <= -views) + counts.push_back(0); + ++counts[-views]; + } + String msg; + msg.reserve(64*counts.size()); + FOREACH(c, counts) + if (counts[c]) + msg += String::FormatString("\n\t% 3u - % 9u", c, counts[c]); + VERBOSE("Visibility lengths (%u points):%s", pointcloud.GetSize(), msg.c_str()); + // save outlier points + PointCloud pc; + RFOREACH(idxPoint, pointcloud.points) { + if (visibility[idxPoint] <= thRemove) { + pc.points.push_back(pointcloud.points[idxPoint]); + pc.colors.push_back(pointcloud.colors[idxPoint]); + } + } + pc.Save(MAKE_PATH("scene_dense_outliers.ply")); + } + #endif + + // filter points + const size_t numInitPoints(pointcloud.GetSize()); + RFOREACH(idxPoint, pointcloud.points) { + if (visibility[idxPoint] <= thRemove) + pointcloud.RemovePoint(idxPoint); + } + + DEBUG_EXTRA("Point-cloud filtered: %u/%u points (%d%%%%) (%s)", pointcloud.points.size(), numInitPoints, ROUND2INT((100.f*pointcloud.points.GetSize())/numInitPoints), TD_TIMER_GET_FMT().c_str()); +} // PointCloudFilter +/*----------------------------------------------------------------*/ From d1d26bad93ee89227eda6fbf1f5a665226f8e7e5 Mon Sep 17 00:00:00 2001 From: cDc Date: Tue, 19 Feb 2019 10:42:42 +0200 Subject: [PATCH 06/70] dense: adjust patch size for average scene types --- libs/MVS/DepthMap.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/MVS/DepthMap.h b/libs/MVS/DepthMap.h index 3b361bea3..e98affe80 100644 --- a/libs/MVS/DepthMap.h +++ b/libs/MVS/DepthMap.h @@ -218,7 +218,7 @@ typedef MVS_API CLISTDEFIDX(DepthData,IIndex) DepthDataArr; struct MVS_API DepthEstimator { - enum { nSizeHalfWindow = 9 }; + enum { nSizeHalfWindow = 5 }; enum { nSizeWindow = nSizeHalfWindow*2+1 }; enum { nSizeStep = 2 }; enum { TexelChannels = 1 }; From 3e023b378b692768e98c2a69050267f9e9ef4d23 Mon Sep 17 00:00:00 2001 From: cDc Date: Fri, 22 Feb 2019 12:24:10 +0200 Subject: [PATCH 07/70] dense: correct normal --- apps/DensifyPointCloud/DensifyPointCloud.cpp | 2 +- libs/MVS/DepthMap.cpp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/DensifyPointCloud/DensifyPointCloud.cpp b/apps/DensifyPointCloud/DensifyPointCloud.cpp index 752156e72..f3101f2e9 100644 --- a/apps/DensifyPointCloud/DensifyPointCloud.cpp +++ b/apps/DensifyPointCloud/DensifyPointCloud.cpp @@ -100,7 +100,7 @@ bool Initialize(size_t argc, LPCTSTR* argv) ("resolution-level", boost::program_options::value(&nResolutionLevel)->default_value(1), "how many times to scale down the images before point cloud computation") ("max-resolution", boost::program_options::value(&nMaxResolution)->default_value(3200), "do not scale images higher than this resolution") ("min-resolution", boost::program_options::value(&nMinResolution)->default_value(640), "do not scale images lower than this resolution") - ("number-views", boost::program_options::value(&nNumViews)->default_value(8), "number of views used for depth-map estimation (0 - all neighbor views available)") + ("number-views", boost::program_options::value(&nNumViews)->default_value(5), "number of views used for depth-map estimation (0 - all neighbor views available)") ("number-views-fuse", boost::program_options::value(&nMinViewsFuse)->default_value(3), "minimum number of images that agrees with an estimate during fusion in order to consider it inlier") ("optimize", boost::program_options::value(&nOptimize)->default_value(7), "filter used after depth-map estimation (0 - disabled, 1 - remove speckles, 2 - fill gaps, 4 - cross-adjust)") ("estimate-colors", boost::program_options::value(&nEstimateColors)->default_value(2), "estimate the colors for the dense point-cloud") diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index 0b881a538..ba1391490 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -664,6 +664,7 @@ void DepthEstimator::ProcessPixel(IDX idx) if (neighbor.normal.dot(viewDir) >= 0) continue; neighbor.depth = InterpolatePixel(nx, neighbor.depth, neighbor.normal); + CorrectNormal(neighbor.normal); ASSERT(neighbor.depth > 0); #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE InitPlane(neighbor.depth, neighbor.normal); @@ -739,6 +740,7 @@ void DepthEstimator::ProcessPixel(IDX idx) continue; prevEstimate.depth = InterpolatePixel(nx, neighbor.depth, neighbor.normal); prevEstimate.normal = neighbor.normal; + CorrectNormal(prevEstimate.normal); prevCost = nconf; } if (prevCost == FLT_MAX) From 15050eea2a5abdddedb3433327ad74b4def0dfa1 Mon Sep 17 00:00:00 2001 From: cDc Date: Sat, 23 Feb 2019 19:32:59 +0200 Subject: [PATCH 08/70] common: log number of cores --- libs/Common/Util.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/Common/Util.cpp b/libs/Common/Util.cpp index 53df906b2..91c08797f 100644 --- a/libs/Common/Util.cpp +++ b/libs/Common/Util.cpp @@ -616,7 +616,7 @@ void Util::LogBuild() #else LOG(_T("Build date: ") __DATE__ _T(", ") __TIME__); #endif - LOG((_T("CPU: ") + Util::GetCPUInfo()).c_str()); + LOG(_T("CPU: %s (%u cores)"), Util::GetCPUInfo().c_str(), Thread::hardwareConcurrency()); LOG((_T("RAM: ") + Util::GetRAMInfo()).c_str()); LOG((_T("OS: ") + Util::GetOSInfo()).c_str()); if (!SIMD_ENABLED.isSet(Util::SSE)) LOG(_T("warning: no SSE compatible CPU or OS detected")); From f71ae6f0eff7030a5a884227d449b6757a375747 Mon Sep 17 00:00:00 2001 From: cDc Date: Mon, 4 Mar 2019 16:17:43 +0200 Subject: [PATCH 09/70] dense: various tweaks --- apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp | 2 +- libs/Common/Types.h | 17 +++++++++++++++++ libs/Common/Types.inl | 17 ----------------- libs/MVS/DepthMap.cpp | 17 ++++++++--------- libs/MVS/DepthMap.h | 4 +--- libs/MVS/SceneDensify.cpp | 7 ++++--- 6 files changed, 31 insertions(+), 33 deletions(-) diff --git a/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp b/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp index 97ff984b9..da13a8b1c 100644 --- a/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp +++ b/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp @@ -260,7 +260,7 @@ void UndistortImage(const Camera& camera, const REAL& k1, const Image8U3 imgIn, Pixel8U& col = imgOut(v,u); if (imgIn.isInside(pt)) { // get pixel color - col = Cast(imgIn.sample(sampler, pt)); + col = imgIn.sample(sampler, pt).cast(); } else { // set to black col = Pixel8U::BLACK; diff --git a/libs/Common/Types.h b/libs/Common/Types.h index 6caec5c5a..2355c8fd3 100644 --- a/libs/Common/Types.h +++ b/libs/Common/Types.h @@ -1769,6 +1769,14 @@ struct TPixel { inline void set(const ALT* clr) { c[0] = TYPE(clr[0]); c[1] = TYPE(clr[1]); c[2] = TYPE(clr[2]); } inline void get(ALT& _r, ALT& _g, ALT& _b) const { _r = ALT(r); _g = ALT(g); _b = ALT(b); } inline void get(ALT* clr) const { clr[0] = ALT(c[0]); clr[1] = ALT(c[1]); clr[2] = ALT(c[2]); } + template inline TPixel::value || !std::is_same::value,T>::type> cast() const { return TPixel(T(r), T(g), T(b)); } + template inline TPixel::value && std::is_same::value,T>::type> cast() const { + return TPixel( + (uint8_t)CLAMP(ROUND2INT(r), 0, 255), + (uint8_t)CLAMP(ROUND2INT(g), 0, 255), + (uint8_t)CLAMP(ROUND2INT(b), 0, 255) + ); + } // set/get as vector inline const TYPE& operator[](size_t i) const { ASSERT(i<3); return c[i]; } inline TYPE& operator[](size_t i) { ASSERT(i<3); return c[i]; } @@ -1880,6 +1888,15 @@ struct TColor { inline void set(const ALT* clr) { c[0] = TYPE(clr[0]); c[1] = TYPE(clr[1]); c[2] = TYPE(clr[2]); c[3] = TYPE(clr[3]); } inline void get(ALT& _r, ALT& _g, ALT& _b, ALT& _a) const { _r = ALT(r); _g = ALT(g); _b = ALT(b); _a = ALT(a); } inline void get(ALT* clr) const { clr[0] = ALT(c[0]); clr[1] = ALT(c[1]); clr[2] = ALT(c[2]); clr[3] = ALT(c[3]); } + template inline TColor::value || !std::is_same::value,T>::type> cast() const { return TColor(T(r), T(g), T(b), T(a)); } + template inline TColor::value && std::is_same::value,T>::type> cast() const { + return TColor( + (uint8_t)CLAMP(ROUND2INT(r), 0, 255), + (uint8_t)CLAMP(ROUND2INT(g), 0, 255), + (uint8_t)CLAMP(ROUND2INT(b), 0, 255), + (uint8_t)CLAMP(ROUND2INT(a), 0, 255) + ); + } // set/get as vector inline const TYPE& operator[](size_t i) const { ASSERT(i<4); return c[i]; } inline TYPE& operator[](size_t i) { ASSERT(i<4); return c[i]; } diff --git a/libs/Common/Types.inl b/libs/Common/Types.inl index a4d0429aa..a5df72c79 100644 --- a/libs/Common/Types.inl +++ b/libs/Common/Types.inl @@ -1557,28 +1557,11 @@ inline TPoint3 Cast(const TPoint3& pt) { return pt; } // Pixel -template -inline TPixel Cast(const TPixel::value, T>::type>& pt) { - return TPixel( - (uint8_t)CLAMP(ROUND2INT(pt.r), 0, 255), - (uint8_t)CLAMP(ROUND2INT(pt.g), 0, 255), - (uint8_t)CLAMP(ROUND2INT(pt.b), 0, 255) - ); -} template inline TPixel Cast(const TPixel& pt) { return pt; } // Color -template -inline TColor Cast(const TColor::value, T>::type>& pt) { - return TColor( - (uint8_t)CLAMP(ROUND2INT(pt.r), 0, 255), - (uint8_t)CLAMP(ROUND2INT(pt.g), 0, 255), - (uint8_t)CLAMP(ROUND2INT(pt.b), 0, 255), - (uint8_t)CLAMP(ROUND2INT(pt.a), 0, 255) - ); -} template inline TColor Cast(const TColor& pt) { return pt; diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index ba1391490..60e0ec805 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -83,7 +83,7 @@ MDEFVAR_OPTDENSE_float(fDescriptorMinMagnitudeThreshold, "Descriptor Min Magnitu MDEFVAR_OPTDENSE_float(fDepthDiffThreshold, "Depth Diff Threshold", "maximum variance allowed for the depths during refinement", "0.01") MDEFVAR_OPTDENSE_float(fNormalDiffThreshold, "Normal Diff Threshold", "maximum variance allowed for the normal during fusion (degrees)", "25") MDEFVAR_OPTDENSE_float(fPairwiseMul, "Pairwise Mul", "pairwise cost scale to match the unary cost", "0.3") -MDEFVAR_OPTDENSE_float(fOptimizerEps, "Optimizer Eps", "MRF optimizer stop epsilon", "0.005") +MDEFVAR_OPTDENSE_float(fOptimizerEps, "Optimizer Eps", "MRF optimizer stop epsilon", "0.001") MDEFVAR_OPTDENSE_int32(nOptimizerMaxIters, "Optimizer Max Iters", "MRF optimizer max number of iterations", "80") MDEFVAR_OPTDENSE_uint32(nSpeckleSize, "Speckle Size", "maximal size of a speckle (small speckles get removed)", "100") MDEFVAR_OPTDENSE_uint32(nIpolGapSize, "Interpolate Gap Size", "interpolate small gaps (left<->right, top<->bottom)", "7") @@ -96,8 +96,8 @@ MDEFVAR_OPTDENSE_uint32(nEstimationIters, "Estimation Iters", "Number of iterati MDEFVAR_OPTDENSE_uint32(nRandomIters, "Random Iters", "Number of iterations for random assignment per pixel", "6") MDEFVAR_OPTDENSE_uint32(nRandomMaxScale, "Random Max Scale", "Maximum number of iterations to skip during random assignment", "2") MDEFVAR_OPTDENSE_float(fRandomDepthRatio, "Random Depth Ratio", "Depth range ratio of the current estimate for random plane assignment", "0.004") -MDEFVAR_OPTDENSE_float(fRandomAngle1Range, "Random Angle1 Range", "Angle 1 range for random plane assignment (in degrees)", "20.0") -MDEFVAR_OPTDENSE_float(fRandomAngle2Range, "Random Angle2 Range", "Angle 2 range for random plane assignment (in degrees)", "12.0") +MDEFVAR_OPTDENSE_float(fRandomAngle1Range, "Random Angle1 Range", "Angle 1 range for random plane assignment (degrees)", "20.0") +MDEFVAR_OPTDENSE_float(fRandomAngle2Range, "Random Angle2 Range", "Angle 2 range for random plane assignment (degrees)", "12.0") MDEFVAR_OPTDENSE_float(fRandomSmoothDepth, "Random Smooth Depth", "Depth variance used during neighbor smoothness assignment (ratio)", "0.02") MDEFVAR_OPTDENSE_float(fRandomSmoothNormal, "Random Smooth Normal", "Normal variance used during neighbor smoothness assignment (degrees)", "13") MDEFVAR_OPTDENSE_float(fRandomSmoothBonus, "Random Smooth Bonus", "Score factor used to encourage smoothness (1 - disabled)", "0.93") @@ -275,7 +275,7 @@ DepthEstimator::DepthEstimator( rnd(SEACAVE::Random::default_seed), #endif idxPixel(_idx), - neighbors(0,nMaxNeighbors), + neighbors(0,2), #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA neighborsClose(0,4), #endif @@ -298,7 +298,7 @@ DepthEstimator::DepthEstimator( idxScore(_depthData0.images.size()<=2 ? 0u : 1u), #endif smoothBonusDepth(1.f-OPTDENSE::fRandomSmoothBonus), smoothBonusNormal((1.f-OPTDENSE::fRandomSmoothBonus)*0.96f), - smoothSigmaDepth(-1.f/(2.f*SQUARE(OPTDENSE::fRandomSmoothDepth))), // used in exp(-x^2 / (2*(0.01^2))) + smoothSigmaDepth(-1.f/(2.f*SQUARE(OPTDENSE::fRandomSmoothDepth))), // used in exp(-x^2 / (2*(0.02^2))) smoothSigmaNormal(-1.f/(2.f*SQUARE(FD2R(OPTDENSE::fRandomSmoothNormal)))), // used in exp(-x^2 / (2*(0.22^2))) thMagnitudeSq(OPTDENSE::fDescriptorMinMagnitudeThreshold>0?SQUARE(OPTDENSE::fDescriptorMinMagnitudeThreshold):-1.f), angle1Range(FD2R(OPTDENSE::fRandomAngle1Range)), @@ -445,6 +445,7 @@ float DepthEstimator::ScorePixelImage(const ViewData& image1, Depth depth, const score *= 1.f - smoothBonusNormal * DENSE_EXP(SQUARE(ACOS(ComputeAngle(normal.ptr(), neighbor.normal.ptr()))) * smoothSigmaNormal); } #endif + ASSERT(ISFINITE(score)); return score; } @@ -480,7 +481,7 @@ float DepthEstimator::ScorePixel(Depth depth, const Normal& normal) if (idxScore == 0) return *std::min_element(scores.cbegin(), scores.cend()); #if 0 - return std::accumulate(scores.begin(), &scores.PartialSort(idxScore), 0.f) / idxScore; + return std::accumulate(scores.cbegin(), &scores.GetNth(idxScore), 0.f) / idxScore; #elif 1 const float* pescore(&scores.PartialSort(idxScore)); const float* pscore(scores.cbegin()); @@ -661,11 +662,9 @@ void DepthEstimator::ProcessPixel(IDX idx) #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA NeighborEstimate& neighbor = neighborsClose[n]; #endif - if (neighbor.normal.dot(viewDir) >= 0) - continue; neighbor.depth = InterpolatePixel(nx, neighbor.depth, neighbor.normal); CorrectNormal(neighbor.normal); - ASSERT(neighbor.depth > 0); + ASSERT(neighbor.depth > 0 && neighbor.normal.dot(viewDir) <= 0); #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE InitPlane(neighbor.depth, neighbor.normal); #endif diff --git a/libs/MVS/DepthMap.h b/libs/MVS/DepthMap.h index e98affe80..98c2794a1 100644 --- a/libs/MVS/DepthMap.h +++ b/libs/MVS/DepthMap.h @@ -223,7 +223,6 @@ struct MVS_API DepthEstimator { enum { nSizeStep = 2 }; enum { TexelChannels = 1 }; enum { nTexels = SQUARE((nSizeHalfWindow*2+nSizeStep)/nSizeStep)*TexelChannels }; - enum { nMaxNeighbors = 8 }; enum ENDIRECTION { LT2RB = 0, @@ -233,7 +232,6 @@ struct MVS_API DepthEstimator { typedef TPoint2 MapRef; typedef CLISTDEF0(MapRef) MapRefArr; - typedef CLISTDEF0IDX(ImageRef,unsigned) DirSamples; typedef Eigen::Matrix TexelVec; struct NeighborData { @@ -351,7 +349,7 @@ struct MVS_API DepthEstimator { const ImageRef p0(p.x-nSizeHalfWindow, p.y-nSizeHalfWindow); const ImageRef p1(p0.x+nSizeWindow, p0.y); const ImageRef p2(p0.x, p0.y+nSizeWindow); - const ImageRef p3(p0.x+nSizeWindow, p0.y+nSizeWindow); + const ImageRef p3(p1.x, p2.y); return (float)(image0Sum(p3) - image0Sum(p2) - image0Sum(p1) + image0Sum(p0)); } #endif diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index 4c349817d..26dcd1fa6 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -1486,12 +1486,13 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b pointcloud.points.RemoveLast(); } else { // this point is valid, store it - point = X*(REAL(1)/confidence); + const REAL nrm(REAL(1)/confidence); + point = X*nrm; ASSERT(ISFINITE(point)); if (bEstimateColor) - pointcloud.colors.AddConstruct(Cast(C*(1.f/confidence))); + pointcloud.colors.AddConstruct((C*(float)nrm).cast()); if (bEstimateNormal) - pointcloud.normals.AddConstruct(normalized(N*(1.f/confidence))); + pointcloud.normals.AddConstruct(normalized(N*(float)nrm)); // invalidate all neighbor depths that do not agree with it for (Depth* pDepth: invalidDepths) *pDepth = 0; From 3e06be846d6e64491fccb025459a666316bd09ec Mon Sep 17 00:00:00 2001 From: cDc Date: Thu, 21 Mar 2019 23:03:11 +0200 Subject: [PATCH 10/70] dense: clean-up --- libs/Common/Util.inl | 134 +++----------------------------------- libs/MVS/DepthMap.cpp | 57 ++++++++-------- libs/MVS/DepthMap.h | 35 ++-------- libs/MVS/SceneDensify.cpp | 4 +- 4 files changed, 44 insertions(+), 186 deletions(-) diff --git a/libs/Common/Util.inl b/libs/Common/Util.inl index 74b484b15..f64a1704e 100644 --- a/libs/Common/Util.inl +++ b/libs/Common/Util.inl @@ -613,152 +613,36 @@ inline TPoint3 CorrectBarycentricCoordinates(TPoint2 b) { template inline void Normal2Dir(const TPoint3& d, TPoint2& p) { ASSERT(ISEQUAL(norm(d), T(1))); - #if 0 - // empirically tested - p.y = TR(atan2(sqrt(d.x*d.x + d.y*d.y), d.z)); p.x = TR(atan2(d.y, d.x)); - #elif 0 - // empirically tested - if (d.x == T(0) && d.y == T(0)) { - ASSERT(ISEQUAL(ABS(d.z), T(1))); - p.y = TR(M_PI_2); - p.x = TR(M_PI_2); - } else { - p.y = TR(atan2(d.z, sqrt(d.x*d.x + d.y*d.y))); - p.x = TR(atan2(d.y, d.x)); - } - #else - // as in PMVS - const T b(asin(CLAMP(d.y, T(-1), T(1)))); - p.y = TR(b); - const T cosb(cos(b)); - if (cosb == T(0)) { - p.x = TR(0); - } else { - const T scl(T(1) / cosb); - const T sina( d.x * scl); - const T cosa(-d.z * scl); - p.x = TR(acos(CLAMP(cosa, T(-1), T(1)))); - if (sina < T(0)) - p.x = -p.x; - } - #endif + p.y = TR(acos(d.z)); } template inline void Dir2Normal(const TPoint2& p, TPoint3& d) { - #if 0 - // empirically tested const T siny(sin(p.y)); d.x = TR(cos(p.x)*siny); d.y = TR(sin(p.x)*siny); d.z = TR(cos(p.y)); - #elif 0 - // empirically tested - const T cosy(cos(p.y)); - d.x = TR(cos(p.x)*cosy); - d.y = TR(sin(p.x)*cosy); - d.z = TR(sin(p.y)); - #else - // as in PMVS - const T cosy(cos(p.y)); - d.x = TR( sin(p.x)*cosy); - d.y = TR( sin(p.y)); - d.z = TR(-cos(p.x)*cosy); - #endif ASSERT(ISEQUAL(norm(d), TR(1))); } // Encodes/decodes a 3D vector in two parameters for the direction and one parameter for the scale -#if 0 -template -inline void Vector2DirScale(const T vect[3], TR dir[2], TR* scale) -{ - const T x2y2(SQUARE(vect[0])+SQUARE(vect[1])); - const T scl(sqrt(x2y2+SQUARE(vect[2]))); - scale[0] = TR(scl); - const T nrm(T(1) / scl); - dir[1] = TR(atan2(sqrt(x2y2)*nrm, vect[2]*nrm)); - dir[0] = TR(atan2(vect[1]*nrm, vect[0]*nrm)); -} -template -inline void DirScale2Vector(const T dir[2], const T* scale, TR vect[3]) -{ - const T cosy(cos(dir[1])*scale[0]); - vect[0] = TR(cos(dir[0])*cosy); - vect[1] = TR(sin(dir[0])*cosy); - vect[2] = TR(sin(dir[1])*scale[0]); -} -#elif 0 -template -inline void Vector2DirScale(const T vect[3], TR dir[2], TR* scale) -{ - const T x2y2(SQUARE(vect[0])+SQUARE(vect[1])); - const T scl(sqrt(x2y2+SQUARE(vect[2]))); - scale[0] = TR(scl); - const T nrm(T(1) / scl); - if (vect[0] == T(0) && vect[1] == T(0)) { - ASSERT(ISEQUAL(ABS(vect[2]*nrm), T(1))); - dir[1] = TR(M_PI_2); - dir[0] = TR(M_PI_2); - } else { - dir[1] = TR(atan2(vect[2]*nrm, sqrt(x2y2)*nrm)); - dir[0] = TR(atan2(vect[1]*nrm, vect[0]*nrm)); - } -} -template -inline void DirScale2Vector(const T dir[2], const T* scale, TR vect[3]) -{ - const T cosy(cos(dir[1])*scale[0]); - vect[0] = TR(cos(dir[0])*cosy); - vect[1] = TR(sin(dir[0])*cosy); - vect[2] = TR(sin(dir[1])*scale[0]); -} -#else template inline void Vector2DirScale(const T vect[3], TR dir[2], TR* scale) { const T scl(sqrt(SQUARE(vect[0])+SQUARE(vect[1])+SQUARE(vect[2]))); + ASSERT(!ISZERO(scl)); scale[0] = TR(scl); - const T b(asin(CLAMP(vect[1]/scale[0], T(-1), T(1)))); - dir[1] = TR(b); - const T cosb(cos(b)); - if (cosb == T(0)) { - dir[0] = TR(0); - } else { - const T invscl(T(1) / (scl*cosb)); - const T sina( vect[0] * invscl); - const T cosa(-vect[2] * invscl); - dir[0] = TR(acos(CLAMP(cosa, T(-1), T(1)))); - if (sina < T(0)) - dir[0] = -dir[0]; - } + dir[0] = TR(atan2(vect[1], vect[0])); + dir[1] = TR(acos(vect[2] / scl)); } template inline void DirScale2Vector(const T dir[2], const T* scale, TR vect[3]) { - const T cos_dir1(cos(dir[1])*scale[0]); - vect[0] = TR( sin(dir[0]) * cos_dir1); - vect[1] = TR( sin(dir[1]) * scale[0]); - vect[2] = TR(-cos(dir[0]) * cos_dir1); -} -#endif -#if TD_VERBOSE == TD_VERBOSE_DEBUG -inline void TestDir() { - typedef REAL Real; - for (int i=0; i<10000; ++i) { - TPoint3 d( - SEACAVE::randomMeanRange(0.0, 1.0), - SEACAVE::randomMeanRange(0.0, 1.0), - SEACAVE::randomMeanRange(0.0, 1.0) - ); - normalize(d); - TPoint2 p; - Normal2Dir(d, p); - TPoint3 _d; - Dir2Normal(p, _d); - ASSERT(ISEQUAL(d, _d)); - } + ASSERT(!ISZERO(*scale)); + const T siny(*scale*sin(dir[1])); + vect[0] = TR(cos(dir[0])*siny); + vect[1] = TR(sin(dir[0])*siny); + vect[2] = TR(*scale*cos(dir[1])); } -#endif /*----------------------------------------------------------------*/ diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index 60e0ec805..37e2ee801 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -230,7 +230,7 @@ unsigned DepthData::DecRef() // 1 2 3 // 1 2 4 7 5 3 6 8 9 --> 4 5 6 // 7 8 9 -void DepthEstimator::MapMatrix2ZigzagIdx(const Image8U::Size& size, DepthEstimator::MapRefArr& coords, BitMatrix& mask, int rawStride) +void DepthEstimator::MapMatrix2ZigzagIdx(const Image8U::Size& size, DepthEstimator::MapRefArr& coords, const BitMatrix& mask, int rawStride) { typedef DepthEstimator::MapRef MapRef; const int w = size.width; @@ -259,7 +259,7 @@ void DepthEstimator::MapMatrix2ZigzagIdx(const Image8U::Size& size, DepthEstimat } } -// replace POWI(0.5f, (int)invScaleRange): 0 1 2 3 4 5 6 7 8 9 10 11 +// replace POWI(0.5f, (int)idxScaleRange): 0 1 2 3 4 5 6 7 8 9 10 11 const float DepthEstimator::scaleRanges[12] = {1.f, 0.5f, 0.25f, 0.125f, 0.0625f, 0.03125f, 0.015625f, 0.0078125f, 0.00390625f, 0.001953125f, 0.0009765625f, 0.00048828125f}; DepthEstimator::DepthEstimator( @@ -363,19 +363,6 @@ bool DepthEstimator::FillPixelPatch() return true; } -#if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE -// compute plane defined by current depth and normal estimate -void DepthEstimator::InitPlane(Depth depth, const Normal& normal) -{ - #if 0 - plane.Set(reinterpret_cast(normal), Vec3f(depth*Cast(X0))); - #else - plane.m_vN = reinterpret_cast(normal); - plane.m_fD = -depth*reinterpret_cast(normal).dot(Cast(X0)); - #endif -} -#endif - // compute pixel's NCC score in the given target image float DepthEstimator::ScorePixelImage(const ViewData& image1, Depth depth, const Normal& normal) { @@ -521,8 +508,8 @@ void DepthEstimator::ProcessPixel(IDX idx) return; float& conf = confMap0(x0); - unsigned invScaleRange(DecodeScoreScale(conf)); - if ((invScaleRange <= 2 || conf > OPTDENSE::fNCCThresholdRefine) && FillPixelPatch()) { + unsigned idxScaleRange(DecodeScoreScale(conf)); + if ((idxScaleRange <= 2 || conf > OPTDENSE::fNCCThresholdRefine) && FillPixelPatch()) { // find neighbors neighbors.Empty(); #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA @@ -656,7 +643,7 @@ void DepthEstimator::ProcessPixel(IDX idx) const ImageRef& nx = neighbor.x; #endif float nconf(confMap0(nx)); - const unsigned ninvScaleRange(DecodeScoreScale(nconf)); + const unsigned nidxScaleRange(DecodeScoreScale(nconf)); if (nconf >= OPTDENSE::fNCCThresholdKeep) continue; #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA @@ -674,24 +661,24 @@ void DepthEstimator::ProcessPixel(IDX idx) conf = nconf; depth = neighbor.depth; normal = neighbor.normal; - invScaleRange = (ninvScaleRange>1 ? ninvScaleRange-1 : ninvScaleRange); + idxScaleRange = (nidxScaleRange>1 ? nidxScaleRange-1 : nidxScaleRange); } } // check few random solutions close to the current estimate in an attempt to find a better estimate float depthRange(MaxDepthDifference(depth, OPTDENSE::fRandomDepthRatio)); - if (invScaleRange > OPTDENSE::nRandomMaxScale) - invScaleRange = OPTDENSE::nRandomMaxScale; - else if (invScaleRange == 0) { + if (idxScaleRange > OPTDENSE::nRandomMaxScale) + idxScaleRange = OPTDENSE::nRandomMaxScale; + else if (idxScaleRange == 0) { if (conf <= thConfSmall) - invScaleRange = 1; + idxScaleRange = 1; else if (conf <= thConfBig) depthRange *= 0.5f; } - float scaleRange(scaleRanges[invScaleRange]); + float scaleRange(scaleRanges[idxScaleRange]); Point2f p; Normal2Dir(normal, p); Normal nnormal; - for (unsigned iter=invScaleRange; iter= 0 && nconf <= 2); if (nconf >= OPTDENSE::fNCCThresholdKeep) continue; @@ -773,7 +759,7 @@ void DepthEstimator::ProcessPixel(IDX idx) } #endif } - conf = EncodeScoreScale(conf, invScaleRange); + conf = EncodeScoreScale(conf, idxScaleRange); } // interpolate given pixel's estimate to the current position @@ -834,6 +820,19 @@ Depth DepthEstimator::InterpolatePixel(const ImageRef& nx, Depth depth, const No return ISINSIDE(depthNew,dMin,dMax) ? depthNew : depth; } +#if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE +// compute plane defined by current depth and normal estimate +void DepthEstimator::InitPlane(Depth depth, const Normal& normal) +{ + #if 0 + plane.Set(reinterpret_cast(normal), Vec3f(depth*Cast(X0))); + #else + plane.m_vN = reinterpret_cast(normal); + plane.m_fD = -depth*reinterpret_cast(normal).dot(Cast(X0)); + #endif +} +#endif + #if DENSE_REFINE == DENSE_REFINE_EXACT DepthEstimator::PixelEstimate DepthEstimator::PerturbEstimate(const PixelEstimate& est, float perturbation) { diff --git a/libs/MVS/DepthMap.h b/libs/MVS/DepthMap.h index 98c2794a1..d07f6b9e5 100644 --- a/libs/MVS/DepthMap.h +++ b/libs/MVS/DepthMap.h @@ -333,13 +333,13 @@ struct MVS_API DepthEstimator { bool PreparePixelPatch(const ImageRef&); bool FillPixelPatch(); - #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE - void InitPlane(Depth, const Normal&); - #endif float ScorePixelImage(const ViewData& image1, Depth, const Normal&); float ScorePixel(Depth, const Normal&); void ProcessPixel(IDX idx); Depth InterpolatePixel(const ImageRef&, Depth, const Normal&) const; + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + void InitPlane(Depth, const Normal&); + #endif #if DENSE_REFINE == DENSE_REFINE_EXACT PixelEstimate PerturbEstimate(const PixelEstimate&, float perturbation); #endif @@ -418,39 +418,14 @@ struct MVS_API DepthEstimator { ASSERT(score >= 0.f && score <= 2.01f); return score*0.1f+(float)invScaleRange; } - static inline unsigned DecodeScale(float score) { - return (unsigned)FLOOR2INT(score); - } static inline unsigned DecodeScoreScale(float& score) { - const unsigned invScaleRange(DecodeScale(score)); + const unsigned invScaleRange((unsigned)FLOOR2INT(score)); score = (score-(float)invScaleRange)*10.f; //ASSERT(score >= 0.f && score <= 2.01f); //problems in multi-threading return invScaleRange; } - static inline float DecodeScore(float score) { - DecodeScoreScale(score); - return score; - } - - // Encodes/decodes a normalized 3D vector in two parameters for the direction - template - static inline void Normal2Dir(const TPoint3& d, TPoint2& p) { - // empirically tested - ASSERT(ISEQUAL(norm(d), T(1))); - p.y = TR(atan2(sqrt(d.x*d.x + d.y*d.y), d.z)); - p.x = TR(atan2(d.y, d.x)); - } - template - static inline void Dir2Normal(const TPoint2& p, TPoint3& d) { - // empirically tested - const T siny(sin(p.y)); - d.x = TR(cos(p.x)*siny); - d.y = TR(sin(p.x)*siny); - d.z = TR(cos(p.y)); - ASSERT(ISEQUAL(norm(d), TR(1))); - } - static void MapMatrix2ZigzagIdx(const Image8U::Size& size, DepthEstimator::MapRefArr& coords, BitMatrix& mask, int rawStride=16); + static void MapMatrix2ZigzagIdx(const Image8U::Size& size, DepthEstimator::MapRefArr& coords, const BitMatrix& mask, int rawStride=16); const float smoothBonusDepth, smoothBonusNormal; const float smoothSigmaDepth, smoothSigmaNormal; diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index 26dcd1fa6..8b55d8701 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -598,7 +598,7 @@ void* STCALL DepthMapsData::EndDepthMapTmp(void* arg) Depth& depth = estimator.depthMap0(x); Normal& normal = estimator.normalMap0(x); float& conf = estimator.confMap0(x); - const unsigned invScaleRange(DepthEstimator::DecodeScoreScale(conf)); + const unsigned idxScaleRange(DepthEstimator::DecodeScoreScale(conf)); ASSERT(depth >= 0); // check if the score is good enough // and that the cross-estimates is close enough to the current estimate @@ -635,7 +635,7 @@ void* STCALL DepthMapsData::EndDepthMapTmp(void* arg) #elif 1 conf = wAngle/(depth*SQUARE(MAXF(conf,1e-2f))); #else - conf = SQRT((float)invScaleRange)*wAngle/(depth*SQUARE(MAXF(conf,1e-2f))); + conf = SQRT((float)idxScaleRange)*wAngle/(depth*SQUARE(MAXF(conf,1e-2f))); #endif #endif } From 4ca6d2913a5893baf6445ccf5587218bc3c2c1e1 Mon Sep 17 00:00:00 2001 From: cDc Date: Tue, 2 Apr 2019 18:44:40 +0300 Subject: [PATCH 11/70] cmake: improve CGAL linking --- build/Modules/FindCGAL.cmake | 11 ++++++++++- libs/MVS/CMakeLists.txt | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/build/Modules/FindCGAL.cmake b/build/Modules/FindCGAL.cmake index 325d36d49..f9dbf2f38 100644 --- a/build/Modules/FindCGAL.cmake +++ b/build/Modules/FindCGAL.cmake @@ -64,7 +64,16 @@ set(CGAL_VERSION "") if(EXISTS "${CGAL_DIR}" AND NOT "${CGAL_DIR}" STREQUAL "") if(EXISTS "${CGAL_DIR}/CGALConfig.cmake") include("${CGAL_DIR}/CGALConfig.cmake") - set(CGAL_LIBS ${CGAL_LIBS} ${CGAL_LIBRARIES} ${CGAL_LIBRARY} ${CGAL_Core_LIBRARY} ${CGAL_ImageIO_LIBRARY} ${CGAL_3RD_PARTY_LIBRARIES} ${CGAL_Core_3RD_PARTY_LIBRARIES} ${CGAL_ImageIO_3RD_PARTY_LIBRARIES} ${MPFR_LIBRARIES} ${GMP_LIBRARIES} ${ZLIB_LIBRARIES}) + set(CGAL_TARGET_LIBS "") + if(TARGET CGAL::CGAL) + get_target_property(CGAL_TARGET_LIBS CGAL::CGAL IMPORTED_LINK_INTERFACE_LIBRARIES_RELEASE) + elseif(TARGET CGAL) + get_target_property(CGAL_TARGET_LIBS CGAL IMPORTED_LINK_INTERFACE_LIBRARIES_RELEASE) + endif() + if(NOT CGAL_TARGET_LIBS) + set(CGAL_TARGET_LIBS "") + endif() + set(CGAL_LIBS ${CGAL_LIBS} ${CGAL_LIBRARIES} ${CGAL_LIBRARY} ${CGAL_TARGET_LIBS} ${CGAL_Core_LIBRARY} ${CGAL_ImageIO_LIBRARY} ${CGAL_3RD_PARTY_LIBRARIES} ${CGAL_Core_3RD_PARTY_LIBRARIES} ${CGAL_ImageIO_3RD_PARTY_LIBRARIES} ${MPFR_LIBRARIES} ${GMP_LIBRARIES} ${ZLIB_LIBRARIES}) set(CGAL_VERSION "${CGAL_MAJOR_VERSION}.${CGAL_MINOR_VERSION}") else() set(CGAL_INCLUDE_DIRS "${CGAL_DIR}/include" "${CGAL_DIR}/auxiliary/gmp/include") diff --git a/libs/MVS/CMakeLists.txt b/libs/MVS/CMakeLists.txt index fa8780f79..14be62072 100644 --- a/libs/MVS/CMakeLists.txt +++ b/libs/MVS/CMakeLists.txt @@ -42,7 +42,7 @@ cxx_library_with_type_no_pch(MVS "Libs" "" "${cxx_default}" set_target_pch(MVS Common.h) # Link its dependencies -TARGET_LINK_LIBRARIES(MVS PRIVATE Common Math IO ${CERES_LIBS} ${CGAL_LIBS} ${CUDA_CUDA_LIBRARY} CGAL::CGAL) +TARGET_LINK_LIBRARIES(MVS PRIVATE Common Math IO ${CERES_LIBS} ${CGAL_LIBS} ${CUDA_CUDA_LIBRARY}) # Install SET_TARGET_PROPERTIES(MVS PROPERTIES From 02732e1b8bd9da70048629b931fb64645f0d3fa5 Mon Sep 17 00:00:00 2001 From: cDc Date: Tue, 30 Apr 2019 16:21:53 +0300 Subject: [PATCH 12/70] dense: fix fuse with missing depth-maps --- libs/MVS/SceneDensify.cpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index 8b55d8701..d98037141 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -1383,12 +1383,14 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b const DepthData& depthData(arrDepthData[idxImage]); ASSERT(!depthData.images.IsEmpty() && !depthData.neighbors.IsEmpty()); for (const ViewScore& neighbor: depthData.neighbors) { - const Image& imageData = scene.images[neighbor.idx.ID]; - DepthIndex& depthIdxs = arrDepthIdx[&imageData-scene.images.Begin()]; - if (depthIdxs.empty()) { - depthIdxs.create(Image8U::Size(imageData.width, imageData.height)); - depthIdxs.memset((uint8_t)NO_ID); - } + DepthIndex& depthIdxs = arrDepthIdx[neighbor.idx.ID]; + if (!depthIdxs.empty()) + continue; + const DepthData& depthDataB(arrDepthData[neighbor.idx.ID]); + if (depthDataB.IsEmpty()) + continue; + depthIdxs.create(depthDataB.depthMap.size()); + depthIdxs.memset((uint8_t)NO_ID); } ASSERT(!depthData.IsEmpty()); const Image8U::Size sizeMap(depthData.depthMap.size()); @@ -1431,12 +1433,14 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b invalidDepths.Empty(); FOREACHPTR(pNeighbor, depthData.neighbors) { const IIndex idxImageB(pNeighbor->idx.ID); + DepthData& depthDataB = arrDepthData[idxImageB]; + if (depthDataB.IsEmpty()) + continue; const Image& imageDataB = scene.images[idxImageB]; const Point3f pt(imageDataB.camera.ProjectPointP3(point)); if (pt.z <= 0) continue; const ImageRef xB(ROUND2INT(pt.x/pt.z), ROUND2INT(pt.y/pt.z)); - DepthData& depthDataB = arrDepthData[idxImageB]; DepthMap& depthMapB = depthDataB.depthMap; if (!depthMapB.isInside(xB)) continue; From d66ecd70dabf10d281c0e4c900e58a4012baf673 Mon Sep 17 00:00:00 2001 From: cDc Date: Wed, 8 May 2019 15:06:50 +0300 Subject: [PATCH 13/70] cmake: fix find CGAL --- build/Modules/FindCGAL.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/Modules/FindCGAL.cmake b/build/Modules/FindCGAL.cmake index f9dbf2f38..fcdc7b4fd 100644 --- a/build/Modules/FindCGAL.cmake +++ b/build/Modules/FindCGAL.cmake @@ -66,9 +66,9 @@ if(EXISTS "${CGAL_DIR}" AND NOT "${CGAL_DIR}" STREQUAL "") include("${CGAL_DIR}/CGALConfig.cmake") set(CGAL_TARGET_LIBS "") if(TARGET CGAL::CGAL) - get_target_property(CGAL_TARGET_LIBS CGAL::CGAL IMPORTED_LINK_INTERFACE_LIBRARIES_RELEASE) + get_target_property(CGAL_TARGET_LIBS CGAL::CGAL INTERFACE_LINK_LIBRARIES) elseif(TARGET CGAL) - get_target_property(CGAL_TARGET_LIBS CGAL IMPORTED_LINK_INTERFACE_LIBRARIES_RELEASE) + get_target_property(CGAL_TARGET_LIBS CGAL INTERFACE_LINK_LIBRARIES) endif() if(NOT CGAL_TARGET_LIBS) set(CGAL_TARGET_LIBS "") From e129d68c09594f23b8b65b2656b7c43fb5709a30 Mon Sep 17 00:00:00 2001 From: cDc Date: Wed, 22 May 2019 11:42:48 +0300 Subject: [PATCH 14/70] interface: import P camera list --- apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp | 12 +- .../InterfaceVisualSFM/InterfaceVisualSFM.cpp | 164 +++++++++++++++--- 2 files changed, 146 insertions(+), 30 deletions(-) diff --git a/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp b/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp index 052004a2b..53e50d8ca 100644 --- a/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp +++ b/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp @@ -85,11 +85,11 @@ bool Initialize(size_t argc, LPCTSTR* argv) ("help,h", "produce this help message") ("working-folder,w", boost::program_options::value(&WORKING_FOLDER), "working directory (default current directory)") ("config-file,c", boost::program_options::value(&OPT::strConfigFileName)->default_value(APPNAME _T(".cfg")), "file name containing program options") - ("archive-type", boost::program_options::value(&OPT::nArchiveType)->default_value(2), "project archive type: 0-text, 1-binary, 2-compressed binary") - ("process-priority", boost::program_options::value(&OPT::nProcessPriority)->default_value(-1), "process priority (below normal by default)") - ("max-threads", boost::program_options::value(&OPT::nMaxThreads)->default_value(0), "maximum number of threads (0 for using all available cores)") + ("archive-type", boost::program_options::value(&OPT::nArchiveType)->default_value(2), "project archive type: 0-text, 1-binary, 2-compressed binary") + ("process-priority", boost::program_options::value(&OPT::nProcessPriority)->default_value(-1), "process priority (below normal by default)") + ("max-threads", boost::program_options::value(&OPT::nMaxThreads)->default_value(0), "maximum number of threads (0 for using all available cores)") #if TD_VERBOSE != TD_VERBOSE_OFF - ("verbosity,v", boost::program_options::value(&g_nVerbosityLevel)->default_value( + ("verbosity,v", boost::program_options::value(&g_nVerbosityLevel)->default_value( #if TD_VERBOSE == TD_VERBOSE_DEBUG 3 #else @@ -105,7 +105,7 @@ bool Initialize(size_t argc, LPCTSTR* argv) ("input-file,i", boost::program_options::value(&OPT::strInputFileName), "input COLMAP folder containing cameras, images and points files OR input MVS project file") ("output-file,o", boost::program_options::value(&OPT::strOutputFileName), "output filename for storing the MVS project") ("image-folder", boost::program_options::value(&OPT::strImageFolder)->default_value(COLMAP_IMAGES_FOLDER), "folder to the undistorted images") - ("normalize,f", boost::program_options::value(&OPT::bNormalizeIntrinsics)->default_value(true), "normalize intrinsics while exporting to MVS format") + ("normalize,f", boost::program_options::value(&OPT::bNormalizeIntrinsics)->default_value(true), "normalize intrinsics while exporting to MVS format") ; boost::program_options::options_description cmdline_options; @@ -434,7 +434,7 @@ bool ImportScene(const String& strFolder, Interface& scene) camera.C = Interface::Pos3d(0,0,0); if (OPT::bNormalizeIntrinsics) { // normalize camera intrinsics - const double fScale(1.0/Camera::GetNormalizationScale(colmapCamera.width, colmapCamera.height)); + const REAL fScale(REAL(1)/Camera::GetNormalizationScale(colmapCamera.width, colmapCamera.height)); camera.K(0,0) *= fScale; camera.K(1,1) *= fScale; camera.K(0,2) *= fScale; diff --git a/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp b/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp index da13a8b1c..368690602 100644 --- a/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp +++ b/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp @@ -42,6 +42,7 @@ #define APPNAME _T("InterfaceVisualSFM") #define MVS_EXT _T(".mvs") #define VSFM_EXT _T(".nvm") +#define CMPMVS_EXT _T(".lst") // S T R U C T S /////////////////////////////////////////////////// @@ -70,11 +71,11 @@ bool Initialize(size_t argc, LPCTSTR* argv) ("help,h", "produce this help message") ("working-folder,w", boost::program_options::value(&WORKING_FOLDER), "working directory (default current directory)") ("config-file,c", boost::program_options::value(&OPT::strConfigFileName)->default_value(APPNAME _T(".cfg")), "file name containing program options") - ("archive-type", boost::program_options::value(&OPT::nArchiveType)->default_value(2), "project archive type: 0-text, 1-binary, 2-compressed binary") - ("process-priority", boost::program_options::value(&OPT::nProcessPriority)->default_value(-1), "process priority (below normal by default)") - ("max-threads", boost::program_options::value(&OPT::nMaxThreads)->default_value(0), "maximum number of threads (0 for using all available cores)") + ("archive-type", boost::program_options::value(&OPT::nArchiveType)->default_value(2), "project archive type: 0-text, 1-binary, 2-compressed binary") + ("process-priority", boost::program_options::value(&OPT::nProcessPriority)->default_value(-1), "process priority (below normal by default)") + ("max-threads", boost::program_options::value(&OPT::nMaxThreads)->default_value(0), "maximum number of threads (0 for using all available cores)") #if TD_VERBOSE != TD_VERBOSE_OFF - ("verbosity,v", boost::program_options::value(&g_nVerbosityLevel)->default_value( + ("verbosity,v", boost::program_options::value(&g_nVerbosityLevel)->default_value( #if TD_VERBOSE == TD_VERBOSE_DEBUG 3 #else @@ -270,16 +271,9 @@ void UndistortImage(const Camera& camera, const REAL& k1, const Image8U3 imgIn, } } // namespace MVS -int main(int argc, LPCTSTR* argv) -{ - #ifdef _DEBUGINFO - // set _crtBreakAlloc index to stop in at allocation - _CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF);// | _CRTDBG_CHECK_ALWAYS_DF); - #endif - - if (!Initialize(argc, argv)) - return EXIT_FAILURE; +int ImportSceneVSFM() +{ TD_TIMER_START(); // read VisualSFM input data @@ -291,7 +285,7 @@ int main(int argc, LPCTSTR* argv) std::vector names; std::vector ptc; if (!PBA::LoadModelFile(MAKE_PATH_SAFE(OPT::strInputFileName), cameras, vertices, measurements, correspondingPoint, correspondingView, names, ptc)) - return EXIT_FAILURE; + return false; // convert data from VisualSFM to OpenMVS MVS::Scene scene(OPT::nMaxThreads); @@ -305,7 +299,7 @@ int main(int argc, LPCTSTR* argv) image.name = MAKE_PATH_FULL(WORKING_FOLDER_FULL, image.name); if (!image.ReloadImage(0, false)) { LOG("error: can not read image %s", image.name.c_str()); - return EXIT_FAILURE; + return false; } // set camera image.platformID = scene.platforms.GetSize(); @@ -318,10 +312,10 @@ int main(int argc, LPCTSTR* argv) camera.C = CMatrix::ZERO; // normalize camera intrinsics const REAL fScale(REAL(1)/MVS::Camera::GetNormalizationScale(image.width, image.height)); - camera.K(0, 0) *= fScale; - camera.K(1, 1) *= fScale; - camera.K(0, 2) *= fScale; - camera.K(1, 2) *= fScale; + camera.K(0,0) *= fScale; + camera.K(1,1) *= fScale; + camera.K(0,2) *= fScale; + camera.K(1,2) *= fScale; // set pose image.poseID = platform.poses.GetSize(); MVS::Platform::Pose& pose = platform.poses.AddEmpty(); @@ -370,7 +364,7 @@ int main(int argc, LPCTSTR* argv) #pragma omp flush (bAbort) continue; #else - return EXIT_FAILURE; + return false; #endif } MVS::UndistortImage(imageData.camera, cameraNVM.GetNormalizedMeasurementDistortion(), imageData.image, imageData.image); @@ -382,21 +376,143 @@ int main(int argc, LPCTSTR* argv) #pragma omp flush (bAbort) continue; #else - return EXIT_FAILURE; + return false; #endif } imageData.ReleaseImage(); } #ifdef _USE_OPENMP if (bAbort) - return EXIT_SUCCESS; + return false; #endif progress.close(); + VERBOSE("Input data imported: %u cameras, %u poses, %u images, %u vertices (%s)", cameras.size(), cameras.size(), cameras.size(), vertices.size(), TD_TIMER_GET_FMT().c_str()); + // write OpenMVS input data - scene.SaveInterface(MAKE_PATH_SAFE(OPT::strOutputFileName)); + return scene.SaveInterface(MAKE_PATH_SAFE(OPT::strOutputFileName)); +} - VERBOSE("Input data imported: %u cameras, %u poses, %u images, %u vertices (%s)", cameras.size(), cameras.size(), cameras.size(), vertices.size(), TD_TIMER_GET_FMT().c_str()); + +template +void _ImageListParseP(const LPSTR* argv, TMatrix& P) +{ + // read projection matrix + P(0,0) = String::FromString(argv[0]); + P(0,1) = String::FromString(argv[1]); + P(0,2) = String::FromString(argv[2]); + P(0,3) = String::FromString(argv[3]); + P(1,0) = String::FromString(argv[4]); + P(1,1) = String::FromString(argv[5]); + P(1,2) = String::FromString(argv[6]); + P(1,3) = String::FromString(argv[7]); + P(2,0) = String::FromString(argv[8]); + P(2,1) = String::FromString(argv[9]); + P(2,2) = String::FromString(argv[10]); + P(2,3) = String::FromString(argv[11]); +} + +int ImportSceneCMPMVS() +{ + TD_TIMER_START(); + + MVS::Scene scene(OPT::nMaxThreads); + + // read CmpMVS input data as a list of images and their projection matrices + std::ifstream iFilein(MAKE_PATH_SAFE(OPT::strInputFileName)); + if (!iFilein.is_open()) + return false; + while (iFilein.good()) { + String strImageName; + std::getline(iFilein, strImageName); + if (strImageName.empty()) + continue; + if (!File::access(MAKE_PATH_SAFE(strImageName))) + return false; + const String strImageNameP(Util::getFileFullName(strImageName)+"_P.txt"); + std::ifstream iFileP(MAKE_PATH_SAFE(strImageNameP)); + if (!iFileP.is_open()) + return false; + String strP; int numLines(0); + while (iFileP.good()) { + String line; + std::getline(iFileP, line); + if (strImageName.empty()) + break; + if (strP.empty()) + strP = line; + else + strP += _T(' ') + line; + ++numLines; + } + if (numLines != 3) + return false; + PMatrix P; + size_t argc; + CAutoPtrArr argv(Util::CommandLineToArgvA(strP, argc)); + if (argc != 12) + return false; + _ImageListParseP(argv, P); + KMatrix K; RMatrix R; CMatrix C; + MVS::DecomposeProjectionMatrix(P, K, R, C); + // set image + MVS::Image& image = scene.images.AddEmpty(); + image.name = strImageName; + Util::ensureUnifySlash(image.name); + image.name = MAKE_PATH_FULL(WORKING_FOLDER_FULL, image.name); + if (!image.ReloadImage(0, false)) { + LOG("error: can not read image %s", image.name.c_str()); + return false; + } + // set camera + image.platformID = scene.platforms.GetSize(); + MVS::Platform& platform = scene.platforms.AddEmpty(); + MVS::Platform::Camera& camera = platform.cameras.AddEmpty(); + image.cameraID = 0; + camera.K = K; + camera.R = RMatrix::IDENTITY; + camera.C = CMatrix::ZERO; + // normalize camera intrinsics + const REAL fScale(REAL(1)/MVS::Camera::GetNormalizationScale(image.width, image.height)); + camera.K(0, 0) *= fScale; + camera.K(1, 1) *= fScale; + camera.K(0, 2) *= fScale; + camera.K(1, 2) *= fScale; + // set pose + image.poseID = platform.poses.GetSize(); + MVS::Platform::Pose& pose = platform.poses.AddEmpty(); + pose.R = R; + pose.C = C; + image.UpdateCamera(scene.platforms); + ++scene.nCalibratedImages; + } + + VERBOSE("Input data imported: %u images (%s)", scene.images.size(), TD_TIMER_GET_FMT().c_str()); + + // write OpenMVS input data + return scene.SaveInterface(MAKE_PATH_SAFE(OPT::strOutputFileName)); +} + + +int main(int argc, LPCTSTR* argv) +{ + #ifdef _DEBUGINFO + // set _crtBreakAlloc index to stop in at allocation + _CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF);// | _CRTDBG_CHECK_ALWAYS_DF); + #endif + + if (!Initialize(argc, argv)) + return EXIT_FAILURE; + + const String strInputFileNameExt(Util::getFileExt(OPT::strInputFileName).ToLower()); + if (strInputFileNameExt == VSFM_EXT) { + if (!ImportSceneVSFM()) + return EXIT_FAILURE; + } else + if (strInputFileNameExt == CMPMVS_EXT) { + if (!ImportSceneCMPMVS()) + return EXIT_FAILURE; + } Finalize(); return EXIT_SUCCESS; From f86c97123c9f1ade2f0112a2c4859b0ebbef527c Mon Sep 17 00:00:00 2001 From: cDc Date: Thu, 23 May 2019 10:23:27 +0300 Subject: [PATCH 15/70] io: use legacy type names if requested --- libs/IO/PLY.cpp | 19 +++++++++++++++---- libs/IO/PLY.h | 7 +++++-- libs/MVS/PointCloud.cpp | 4 +++- libs/MVS/PointCloud.h | 2 +- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/libs/IO/PLY.cpp b/libs/IO/PLY.cpp index 80497f4c1..a1b4d94a6 100644 --- a/libs/IO/PLY.cpp +++ b/libs/IO/PLY.cpp @@ -26,11 +26,11 @@ using namespace SEACAVE; // S T R U C T S /////////////////////////////////////////////////// -const char * PLY::type_names[] = { +const char* const PLY::type_names[] = { "invalid", "int8", "int16", "int32", "uint8", "uint16", "uint32", "float32", "float64", }; -const char * PLY::old_type_names[] = { +const char* const PLY::old_type_names[] = { "invalid", "char", "short", "int", "uchar", "ushort", "uint", "float", "double", }; @@ -66,7 +66,8 @@ Init PLY data as empty. PLY::PLY() : mfp(NULL), f(NULL), ostream(NULL), - which_elem(NULL), other_elems(NULL), current_rules(NULL), rule_list(NULL) + which_elem(NULL), other_elems(NULL), current_rules(NULL), rule_list(NULL), + write_type_names(type_names) { } @@ -998,6 +999,16 @@ void PLY::flush() } +/****************************************************************************** +Use old PLY type names during writing for backward compatibility. +******************************************************************************/ + +void PLY::set_legacy_type_names() +{ + write_type_names = old_type_names; +} + + /****************************************************************************** Get version number and file type of a PlyFile. @@ -1271,7 +1282,7 @@ void PLY::write_scalar_type(OSTREAM* fp, int code) abort_ply("error: write_scalar_type: bad data code = %d", code); /* write the code to a file */ - fp->print("%s", type_names[code]); + fp->print("%s", write_type_names[code]); } diff --git a/libs/IO/PLY.h b/libs/IO/PLY.h index b39ba3720..d6354edef 100644 --- a/libs/IO/PLY.h +++ b/libs/IO/PLY.h @@ -161,6 +161,8 @@ class IO_API PLY void flush(); void release(); + void set_legacy_type_names(); + void get_info(float *, int *); void append_comment(const char *); @@ -298,10 +300,11 @@ class IO_API PLY PlyPropRules *current_rules; /* current propagation rules */ PlyRuleList *rule_list; /* rule list from user */ std::vector vals; /* rule list from user */ + const char* const* write_type_names; /* names of scalar types to be used for writing (new types by default) */ protected: - static const char *type_names[9]; // names of scalar types - static const char *old_type_names[9]; // old names of types for backward compatibility + static const char* const type_names[9]; // names of scalar types + static const char* const old_type_names[9]; // old names of types for backward compatibility static const int ply_type_size[9]; static const RuleName rule_name_list[7]; }; diff --git a/libs/MVS/PointCloud.cpp b/libs/MVS/PointCloud.cpp index c43524d5e..42e61230c 100644 --- a/libs/MVS/PointCloud.cpp +++ b/libs/MVS/PointCloud.cpp @@ -185,7 +185,7 @@ bool PointCloud::Load(const String& fileName) } // Load // save the dense point cloud as PLY file -bool PointCloud::Save(const String& fileName) const +bool PointCloud::Save(const String& fileName, bool bLegacyTypes) const { if (points.IsEmpty()) return false; @@ -195,6 +195,8 @@ bool PointCloud::Save(const String& fileName) const ASSERT(!fileName.IsEmpty()); Util::ensureFolder(fileName); PLY ply; + if (bLegacyTypes) + ply.set_legacy_type_names(); if (!ply.write(fileName, 1, BasicPLY::elem_names, PLY::BINARY_LE, 64*1024)) return false; diff --git a/libs/MVS/PointCloud.h b/libs/MVS/PointCloud.h index 6f351ebd4..ce2a9722d 100644 --- a/libs/MVS/PointCloud.h +++ b/libs/MVS/PointCloud.h @@ -93,7 +93,7 @@ class MVS_API PointCloud Box GetAABB(unsigned minViews) const; bool Load(const String& fileName); - bool Save(const String& fileName) const; + bool Save(const String& fileName, bool bLegacyTypes=false) const; #ifdef _USE_BOOST // implement BOOST serialization From 8711c09fd61468e3ead57f67212e32381f102e5f Mon Sep 17 00:00:00 2001 From: cDc Date: Thu, 23 May 2019 10:24:05 +0300 Subject: [PATCH 16/70] interface: export dense point-cloud to COLMAP --- apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp | 196 ++++++++++++++--------- 1 file changed, 120 insertions(+), 76 deletions(-) diff --git a/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp b/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp index 53e50d8ca..be6a46476 100644 --- a/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp +++ b/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp @@ -318,7 +318,6 @@ struct Image { return true; } bool Write(std::ostream& out) const { - ASSERT(!projs.empty()); out << ID+1 << _T(" ") << q.w() << _T(" ") << q.x() << _T(" ") << q.y() << _T(" ") << q.z() << _T(" ") << t(0) << _T(" ") << t(1) << _T(" ") << t(2) << _T(" ") @@ -555,6 +554,7 @@ bool ExportScene(const String& strFolder, const Interface& scene) // write camera list CLISTDEF0IDX(KMatrix,uint32_t) Ks; + CLISTDEF0IDX(COLMAP::Camera,uint32_t) cams; { const String filenameCameras(strFolder+COLMAP_CAMERAS); LOG_OUT() << "Writing cameras: " << filenameCameras << std::endl; @@ -614,12 +614,17 @@ bool ExportScene(const String& strFolder, const Interface& scene) K(1,1) = cam.params[1]; K(0,2) = cam.params[2]; K(1,2) = cam.params[3]; + cams.emplace_back(cam); } } // create images list COLMAP::Images images; CameraArr cameras; + float maxNumPointsSparse(0); + const float avgViewsPerPoint(3.f); + const uint32_t avgResolutionSmallView(640*480), avgResolutionLargeView(6000*4000); + const uint32_t avgPointsPerSmallView(3000), avgPointsPerLargeView(12000); { images.resize(scene.images.size()); cameras.resize(scene.images.size()); @@ -641,41 +646,125 @@ bool ExportScene(const String& strFolder, const Interface& scene) camera.R = pose.R; camera.C = pose.C; camera.ComposeP(); + const COLMAP::Camera& cam = cams[image.platformID]; + const uint32_t resolutionView(cam.width*cam.height); + const float linearFactor(float(avgResolutionLargeView-resolutionView)/(avgResolutionLargeView-avgResolutionSmallView)); + maxNumPointsSparse += (avgPointsPerSmallView+(avgPointsPerLargeView-avgPointsPerSmallView)*linearFactor)/avgViewsPerPoint; } } - // write points list - { - const String filenamePoints(strFolder+COLMAP_POINTS); - LOG_OUT() << "Writing points: " << filenamePoints << std::endl; - std::ofstream file(filenamePoints); - if (!file.good()) { - VERBOSE("error: unable to open file '%s'", filenamePoints.c_str()); - return false; + // auto-select dense or sparse mode based on number of points + const bool bSparsePointCloud(scene.vertices.size() < (size_t)maxNumPointsSparse); + if (bSparsePointCloud) { + // write points list + { + const String filenamePoints(strFolder+COLMAP_POINTS); + LOG_OUT() << "Writing points: " << filenamePoints << std::endl; + std::ofstream file(filenamePoints); + if (!file.good()) { + VERBOSE("error: unable to open file '%s'", filenamePoints.c_str()); + return false; + } + file << _T("# 3D point list with one line of data per point:") << std::endl; + file << _T("# POINT3D_ID, X, Y, Z, R, G, B, ERROR, TRACK[] as (IMAGE_ID, POINT2D_IDX)") << std::endl; + for (uint32_t ID=0; ID<(uint32_t)scene.vertices.size(); ++ID) { + const Interface::Vertex& vertex = scene.vertices[ID]; + COLMAP::Point point; + point.ID = ID; + point.p = vertex.X; + for (const Interface::Vertex::View& view: vertex.views) { + COLMAP::Image& img = images[view.imageID]; + COLMAP::Point::Track track; + track.idImage = view.imageID; + track.idProj = (uint32_t)img.projs.size(); + point.tracks.push_back(track); + COLMAP::Image::Proj proj; + proj.idPoint = ID; + const Point3 X(vertex.X); + ProjectVertex_3x4_3_2(cameras[view.imageID].P.val, X.ptr(), proj.p.data()); + img.projs.push_back(proj); + } + point.c = scene.verticesColor.empty() ? Interface::Col3(255,255,255) : scene.verticesColor[ID].c; + point.e = 0; + if (!point.Write(file)) + return false; + } } - file << _T("# 3D point list with one line of data per point:") << std::endl; - file << _T("# POINT3D_ID, X, Y, Z, R, G, B, ERROR, TRACK[] as (IMAGE_ID, POINT2D_IDX)") << std::endl; - for (uint32_t ID=0; ID<(uint32_t)scene.vertices.size(); ++ID) { - const Interface::Vertex& vertex = scene.vertices[ID]; - COLMAP::Point point; - point.ID = ID; - point.p = vertex.X; - for (const Interface::Vertex::View& view: vertex.views) { - COLMAP::Image& img = images[view.imageID]; - COLMAP::Point::Track track; - track.idImage = view.imageID; - track.idProj = (uint32_t)img.projs.size(); - point.tracks.push_back(track); - COLMAP::Image::Proj proj; - proj.idPoint = ID; - const Point3 X(vertex.X); - ProjectVertex_3x4_3_2(cameras[view.imageID].P.val, X.ptr(), proj.p.data()); - img.projs.push_back(proj); + + Util::ensureFolder(strFolder+COLMAP_STEREO_FOLDER); + + // write fusion list + { + const String filenameFusion(strFolder+COLMAP_FUSION); + LOG_OUT() << "Writing fusion configuration: " << filenameFusion << std::endl; + std::ofstream file(filenameFusion); + if (!file.good()) { + VERBOSE("error: unable to open file '%s'", filenameFusion.c_str()); + return false; } - point.c = scene.verticesColor.empty() ? Interface::Col3(255,255,255) : scene.verticesColor[ID].c; - point.e = 0; - if (!point.Write(file)) + for (const COLMAP::Image& img: images) { + if (img.projs.empty()) + continue; + file << img.name << std::endl; + if (file.fail()) + return false; + } + } + + // write patch-match list + { + const String filenameFusion(strFolder+COLMAP_PATCHMATCH); + LOG_OUT() << "Writing patch-match configuration: " << filenameFusion << std::endl; + std::ofstream file(filenameFusion); + if (!file.good()) { + VERBOSE("error: unable to open file '%s'", filenameFusion.c_str()); return false; + } + for (const COLMAP::Image& img: images) { + if (img.projs.empty()) + continue; + file << img.name << std::endl; + if (file.fail()) + return false; + file << _T("__auto__, 20") << std::endl; + if (file.fail()) + return false; + } + } + + Util::ensureFolder(strFolder+COLMAP_STEREO_CONSISTENCYGRAPHS_FOLDER); + Util::ensureFolder(strFolder+COLMAP_STEREO_DEPTHMAPS_FOLDER); + Util::ensureFolder(strFolder+COLMAP_STEREO_NORMALMAPS_FOLDER); + } else { + // export dense point-cloud + const String filenameDensePoints(strFolder+COLMAP_DENSE_POINTS); + const String filenameDenseVisPoints(strFolder+COLMAP_DENSE_POINTS_VISIBILITY); + LOG_OUT() << "Writing points: " << filenameDensePoints << " and " << filenameDenseVisPoints << std::endl; + File file(filenameDenseVisPoints, File::WRITE, File::CREATE | File::TRUNCATE); + if (!file.isOpen()) { + VERBOSE("error: unable to write file '%s'", filenameDenseVisPoints.c_str()); + return false; + } + const uint64_t numPoints(scene.vertices.size()); + file.write(&numPoints, sizeof(uint64_t)); + PointCloud pointcloud; + for (size_t i=0; i Date: Tue, 2 Apr 2019 18:50:02 +0300 Subject: [PATCH 17/70] dense: improve refine stage (cherry picked from commit d42ec8d4f04d69a2e41257bf391ccb29b1a147bb) --- libs/MVS/DepthMap.cpp | 470 +++++++++++++++++++------------------- libs/MVS/DepthMap.h | 15 +- libs/MVS/SceneDensify.cpp | 9 +- 3 files changed, 243 insertions(+), 251 deletions(-) diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index 37e2ee801..2de201fdb 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -91,7 +91,6 @@ MDEFVAR_OPTDENSE_uint32(nOptimize, "Optimize", "should we filter the extracted d MDEFVAR_OPTDENSE_uint32(nEstimateColors, "Estimate Colors", "should we estimate the colors for the dense point-cloud?", "2", "0", "1") MDEFVAR_OPTDENSE_uint32(nEstimateNormals, "Estimate Normals", "should we estimate the normals for the dense point-cloud?", "0", "1", "2") MDEFVAR_OPTDENSE_float(fNCCThresholdKeep, "NCC Threshold Keep", "Maximum 1-NCC score accepted for a match", "0.5", "0.3") -MDEFVAR_OPTDENSE_float(fNCCThresholdRefine, "NCC Threshold Refine", "1-NCC score under which a match is not refined anymore", "0.03") MDEFVAR_OPTDENSE_uint32(nEstimationIters, "Estimation Iters", "Number of iterations for depth-map refinement", "4") MDEFVAR_OPTDENSE_uint32(nRandomIters, "Random Iters", "Number of iterations for random assignment per pixel", "6") MDEFVAR_OPTDENSE_uint32(nRandomMaxScale, "Random Max Scale", "Maximum number of iterations to skip during random assignment", "2") @@ -305,6 +304,7 @@ DepthEstimator::DepthEstimator( angle2Range(FD2R(OPTDENSE::fRandomAngle2Range)), thConfSmall(OPTDENSE::fNCCThresholdKeep*0.25f), thConfBig(OPTDENSE::fNCCThresholdKeep*0.5f), + thConfRand(OPTDENSE::fNCCThresholdKeep*1.08f), thRobust(OPTDENSE::fNCCThresholdKeep*1.2f) #if DENSE_REFINE == DENSE_REFINE_EXACT , thPerturbation(1.f/POW(2.f,float(nIter+1))) @@ -504,262 +504,270 @@ void DepthEstimator::ProcessPixel(IDX idx) { // compute pixel coordinates from pixel index and its neighbors ASSERT(dir == LT2RB || dir == RB2LT); - if (!PreparePixelPatch(dir == LT2RB ? coords[idx] : coords[coords.GetSize()-1-idx])) + if (!PreparePixelPatch(dir == LT2RB ? coords[idx] : coords[coords.GetSize()-1-idx]) || !FillPixelPatch()) return; - - float& conf = confMap0(x0); - unsigned idxScaleRange(DecodeScoreScale(conf)); - if ((idxScaleRange <= 2 || conf > OPTDENSE::fNCCThresholdRefine) && FillPixelPatch()) { - // find neighbors - neighbors.Empty(); - #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA - neighborsClose.Empty(); - #endif - if (dir == LT2RB) { - // direction from left-top to right-bottom corner - if (x0.x > nSizeHalfWindow) { - const ImageRef nx(x0.x-1, x0.y); - const Depth ndepth(depthMap0(nx)); - if (ndepth > 0) { - #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA - neighbors.emplace_back(nx); - neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) - #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE - , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) - #endif - }); - #else - neighbors.emplace_back(NeighborData{nx,ndepth,normalMap0(nx)}); + // find neighbors + neighbors.Empty(); + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + neighborsClose.Empty(); + #endif + if (dir == LT2RB) { + // direction from left-top to right-bottom corner + if (x0.x > nSizeHalfWindow) { + const ImageRef nx(x0.x-1, x0.y); + const Depth ndepth(depthMap0(nx)); + if (ndepth > 0) { + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + neighbors.emplace_back(nx); + neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) #endif - } + }); + #else + neighbors.emplace_back(NeighborData{nx,ndepth,normalMap0(nx)}); + #endif } - if (x0.y > nSizeHalfWindow) { - const ImageRef nx(x0.x, x0.y-1); - const Depth ndepth(depthMap0(nx)); - if (ndepth > 0) { - #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA - neighbors.emplace_back(nx); - neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) - #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE - , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) - #endif - }); - #else - neighbors.emplace_back(NeighborData{nx,ndepth,normalMap0(nx)}); + } + if (x0.y > nSizeHalfWindow) { + const ImageRef nx(x0.x, x0.y-1); + const Depth ndepth(depthMap0(nx)); + if (ndepth > 0) { + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + neighbors.emplace_back(nx); + neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) #endif - } - } - #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA - if (x0.x < size.width-nSizeHalfWindow) { - const ImageRef nx(x0.x+1, x0.y); - const Depth ndepth(depthMap0(nx)); - if (ndepth > 0) - neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) - #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE - , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) - #endif - }); - } - if (x0.y < size.height-nSizeHalfWindow) { - const ImageRef nx(x0.x, x0.y+1); - const Depth ndepth(depthMap0(nx)); - if (ndepth > 0) - neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) - #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE - , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) - #endif - }); + }); + #else + neighbors.emplace_back(NeighborData{nx,ndepth,normalMap0(nx)}); + #endif } - #endif - } else { - ASSERT(dir == RB2LT); - // direction from right-bottom to left-top corner - if (x0.x < size.width-nSizeHalfWindow) { - const ImageRef nx(x0.x+1, x0.y); - const Depth ndepth(depthMap0(nx)); - if (ndepth > 0) { - #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA - neighbors.emplace_back(nx); - neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) - #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE - , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) - #endif - }); - #else - neighbors.emplace_back(NeighborData{nx,ndepth,normalMap0(nx)}); + } + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + if (x0.x < size.width-nSizeHalfWindow) { + const ImageRef nx(x0.x+1, x0.y); + const Depth ndepth(depthMap0(nx)); + if (ndepth > 0) + neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) #endif - } - } - if (x0.y < size.height-nSizeHalfWindow) { - const ImageRef nx(x0.x, x0.y+1); - const Depth ndepth(depthMap0(nx)); - if (ndepth > 0) { - #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA - neighbors.emplace_back(nx); - neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) - #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE - , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) - #endif - }); - #else - neighbors.emplace_back(NeighborData{nx,ndepth,normalMap0(nx)}); + }); + } + if (x0.y < size.height-nSizeHalfWindow) { + const ImageRef nx(x0.x, x0.y+1); + const Depth ndepth(depthMap0(nx)); + if (ndepth > 0) + neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) #endif - } - } - #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA - if (x0.x > nSizeHalfWindow) { - const ImageRef nx(x0.x-1, x0.y); - const Depth ndepth(depthMap0(nx)); - if (ndepth > 0) - neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) - #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE - , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) - #endif - }); - } - if (x0.y > nSizeHalfWindow) { - const ImageRef nx(x0.x, x0.y-1); - const Depth ndepth(depthMap0(nx)); - if (ndepth > 0) - neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) - #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE - , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) - #endif - }); - } - #endif + }); } - Depth& depth = depthMap0(x0); - Normal& normal = normalMap0(x0); - const Normal viewDir(Cast(static_cast(X0))); - ASSERT(depth > 0 && normal.dot(viewDir) < 0); - #if DENSE_REFINE == DENSE_REFINE_ITER - // check if any of the neighbor estimates are better then the current estimate - #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA - FOREACH(n, neighbors) { - const ImageRef& nx = neighbors[n]; - #else - for (NeighborData& neighbor: neighbors) { - const ImageRef& nx = neighbor.x; #endif - float nconf(confMap0(nx)); - const unsigned nidxScaleRange(DecodeScoreScale(nconf)); - if (nconf >= OPTDENSE::fNCCThresholdKeep) - continue; - #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA - NeighborEstimate& neighbor = neighborsClose[n]; - #endif - neighbor.depth = InterpolatePixel(nx, neighbor.depth, neighbor.normal); - CorrectNormal(neighbor.normal); - ASSERT(neighbor.depth > 0 && neighbor.normal.dot(viewDir) <= 0); - #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE - InitPlane(neighbor.depth, neighbor.normal); - #endif - nconf = ScorePixel(neighbor.depth, neighbor.normal); - ASSERT(nconf >= 0 && nconf <= 2); - if (conf > nconf) { - conf = nconf; - depth = neighbor.depth; - normal = neighbor.normal; - idxScaleRange = (nidxScaleRange>1 ? nidxScaleRange-1 : nidxScaleRange); + } else { + ASSERT(dir == RB2LT); + // direction from right-bottom to left-top corner + if (x0.x < size.width-nSizeHalfWindow) { + const ImageRef nx(x0.x+1, x0.y); + const Depth ndepth(depthMap0(nx)); + if (ndepth > 0) { + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + neighbors.emplace_back(nx); + neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) + #endif + }); + #else + neighbors.emplace_back(NeighborData{nx,ndepth,normalMap0(nx)}); + #endif } } - // check few random solutions close to the current estimate in an attempt to find a better estimate - float depthRange(MaxDepthDifference(depth, OPTDENSE::fRandomDepthRatio)); - if (idxScaleRange > OPTDENSE::nRandomMaxScale) - idxScaleRange = OPTDENSE::nRandomMaxScale; - else if (idxScaleRange == 0) { - if (conf <= thConfSmall) - idxScaleRange = 1; - else if (conf <= thConfBig) - depthRange *= 0.5f; - } - float scaleRange(scaleRanges[idxScaleRange]); - Point2f p; - Normal2Dir(normal, p); - Normal nnormal; - for (unsigned iter=idxScaleRange; iter= 0) - continue; - #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE - InitPlane(ndepth, nnormal); - #endif - const float nconf(ScorePixel(ndepth, nnormal)); - ASSERT(nconf >= 0); - if (conf > nconf) { - conf = nconf; - depth = ndepth; - normal = nnormal; - p = np; - scaleRange = scaleRanges[++idxScaleRange]; + if (x0.y < size.height-nSizeHalfWindow) { + const ImageRef nx(x0.x, x0.y+1); + const Depth ndepth(depthMap0(nx)); + if (ndepth > 0) { + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + neighbors.emplace_back(nx); + neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) + #endif + }); + #else + neighbors.emplace_back(NeighborData{nx,ndepth,normalMap0(nx)}); + #endif } } - #else - // current pixel estimate - PixelEstimate currEstimate{depth, normal}; - // propagate depth estimate from the best neighbor estimate - PixelEstimate prevEstimate; float prevCost(FLT_MAX); #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA - FOREACH(n, neighbors) { - const ImageRef& nx = neighbors[n]; - #else - for (const NeighborData& neighbor: neighbors) { - const ImageRef& nx = neighbor.x; + if (x0.x > nSizeHalfWindow) { + const ImageRef nx(x0.x-1, x0.y); + const Depth ndepth(depthMap0(nx)); + if (ndepth > 0) + neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) + #endif + }); + } + if (x0.y > nSizeHalfWindow) { + const ImageRef nx(x0.x, x0.y-1); + const Depth ndepth(depthMap0(nx)); + if (ndepth > 0) + neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) + #endif + }); + } #endif - float nconf(confMap0(nx)); - const unsigned nidxScaleRange(DecodeScoreScale(nconf)); - ASSERT(nconf >= 0 && nconf <= 2); - if (nconf >= OPTDENSE::fNCCThresholdKeep) - continue; - if (prevCost <= nconf) - continue; - #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA - const NeighborEstimate& neighbor = neighborsClose[n]; - #endif - if (neighbor.normal.dot(viewDir) >= 0) - continue; - prevEstimate.depth = InterpolatePixel(nx, neighbor.depth, neighbor.normal); - prevEstimate.normal = neighbor.normal; - CorrectNormal(prevEstimate.normal); - prevCost = nconf; + } + float& conf = confMap0(x0); + Depth& depth = depthMap0(x0); + Normal& normal = normalMap0(x0); + const Normal viewDir(Cast(static_cast(X0))); + ASSERT(depth > 0 && normal.dot(viewDir) < 0); + #if DENSE_REFINE == DENSE_REFINE_ITER + // check if any of the neighbor estimates are better then the current estimate + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + FOREACH(n, neighbors) { + const ImageRef& nx = neighbors[n]; + #else + for (NeighborData& neighbor: neighbors) { + const ImageRef& nx = neighbor.x; + #endif + if (confMap0(nx) >= OPTDENSE::fNCCThresholdKeep) + continue; + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + NeighborEstimate& neighbor = neighborsClose[n]; + #endif + neighbor.depth = InterpolatePixel(nx, neighbor.depth, neighbor.normal); + CorrectNormal(neighbor.normal); + ASSERT(neighbor.depth > 0 && neighbor.normal.dot(viewDir) <= 0); + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + InitPlane(neighbor.depth, neighbor.normal); + #endif + const float nconf(ScorePixel(neighbor.depth, neighbor.normal)); + ASSERT(nconf >= 0 && nconf <= 2); + if (conf > nconf) { + conf = nconf; + depth = neighbor.depth; + normal = neighbor.normal; } - if (prevCost == FLT_MAX) - prevEstimate = PerturbEstimate(currEstimate, thPerturbation); - // randomly sampled estimate - PixelEstimate randEstimate(PerturbEstimate(currEstimate, thPerturbation)); - // select best pixel estimate - const int numCosts = 5; - float costs[numCosts] = {0,0,0,0,0}; - const Depth depths[numCosts] = { - currEstimate.depth, prevEstimate.depth, randEstimate.depth, - currEstimate.depth, randEstimate.depth}; - const Normal normals[numCosts] = { - currEstimate.normal, prevEstimate.normal, - randEstimate.normal, randEstimate.normal, - currEstimate.normal}; - conf = FLT_MAX; - for (int idxCost=0; idxCost= thConfRand) { + // try completely random values in order to find an initial estimate + for (unsigned iter=0; iter= 0); if (conf > nconf) { conf = nconf; depth = ndepth; normal = nnormal; + if (conf < thConfRand) + goto RefineIters; } } + return; + } + RefineIters: + // try random values around the current estimate in order to refine it + const float depthRange(MaxDepthDifference(depth, OPTDENSE::fRandomDepthRatio)); + float scaleRange(scaleRanges[idxScaleRange]); + Point2f p; + Normal2Dir(normal, p); + Normal nnormal; + for (unsigned iter=0; iter= 0) + continue; + #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE + InitPlane(ndepth, nnormal); + #endif + const float nconf(ScorePixel(ndepth, nnormal)); + ASSERT(nconf >= 0); + if (conf > nconf) { + conf = nconf; + depth = ndepth; + normal = nnormal; + p = np; + scaleRange = scaleRanges[++idxScaleRange]; + } + } + #else + // current pixel estimate + PixelEstimate currEstimate{depth, normal}; + // propagate depth estimate from the best neighbor estimate + PixelEstimate prevEstimate; float prevCost(FLT_MAX); + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + FOREACH(n, neighbors) { + const ImageRef& nx = neighbors[n]; + #else + for (const NeighborData& neighbor: neighbors) { + const ImageRef& nx = neighbor.x; + #endif + float nconf(confMap0(nx)); + const unsigned nidxScaleRange(DecodeScoreScale(nconf)); + ASSERT(nconf >= 0 && nconf <= 2); + if (nconf >= OPTDENSE::fNCCThresholdKeep) + continue; + if (prevCost <= nconf) + continue; + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + const NeighborEstimate& neighbor = neighborsClose[n]; #endif + if (neighbor.normal.dot(viewDir) >= 0) + continue; + prevEstimate.depth = InterpolatePixel(nx, neighbor.depth, neighbor.normal); + prevEstimate.normal = neighbor.normal; + CorrectNormal(prevEstimate.normal); + prevCost = nconf; } - conf = EncodeScoreScale(conf, idxScaleRange); + if (prevCost == FLT_MAX) + prevEstimate = PerturbEstimate(currEstimate, thPerturbation); + // randomly sampled estimate + PixelEstimate randEstimate(PerturbEstimate(currEstimate, thPerturbation)); + // select best pixel estimate + const int numCosts = 5; + float costs[numCosts] = {0,0,0,0,0}; + const Depth depths[numCosts] = { + currEstimate.depth, prevEstimate.depth, randEstimate.depth, + currEstimate.depth, randEstimate.depth}; + const Normal normals[numCosts] = { + currEstimate.normal, prevEstimate.normal, + randEstimate.normal, randEstimate.normal, + currEstimate.normal}; + conf = FLT_MAX; + for (int idxCost=0; idxCost= 0); + if (conf > nconf) { + conf = nconf; + depth = ndepth; + normal = nnormal; + } + } + #endif } // interpolate given pixel's estimate to the current position diff --git a/libs/MVS/DepthMap.h b/libs/MVS/DepthMap.h index d07f6b9e5..6f1cf106c 100644 --- a/libs/MVS/DepthMap.h +++ b/libs/MVS/DepthMap.h @@ -117,7 +117,6 @@ extern unsigned nOptimize; extern unsigned nEstimateColors; extern unsigned nEstimateNormals; extern float fNCCThresholdKeep; -extern float fNCCThresholdRefine; extern unsigned nEstimationIters; extern unsigned nRandomIters; extern unsigned nRandomMaxScale; @@ -413,25 +412,13 @@ struct MVS_API DepthEstimator { normal = RMatrixBaseF(normal.cross(viewDir), MINF((ACOS(cosAngLen/norm(viewDir))-FD2R(90.f))*1.01f, -0.001f)) * normal; } - // encode/decode NCC score and refinement level in one float - static inline float EncodeScoreScale(float score, unsigned invScaleRange=0) { - ASSERT(score >= 0.f && score <= 2.01f); - return score*0.1f+(float)invScaleRange; - } - static inline unsigned DecodeScoreScale(float& score) { - const unsigned invScaleRange((unsigned)FLOOR2INT(score)); - score = (score-(float)invScaleRange)*10.f; - //ASSERT(score >= 0.f && score <= 2.01f); //problems in multi-threading - return invScaleRange; - } - static void MapMatrix2ZigzagIdx(const Image8U::Size& size, DepthEstimator::MapRefArr& coords, const BitMatrix& mask, int rawStride=16); const float smoothBonusDepth, smoothBonusNormal; const float smoothSigmaDepth, smoothSigmaNormal; const float thMagnitudeSq; const float angle1Range, angle2Range; - const float thConfSmall, thConfBig; + const float thConfSmall, thConfBig, thConfRand; const float thRobust; #if DENSE_REFINE == DENSE_REFINE_EXACT const float thPerturbation; diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index d98037141..d14d8e3f2 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -560,7 +560,7 @@ void* STCALL DepthMapsData::ScoreDepthMapTmp(void* arg) if (!estimator.PreparePixelPatch(x) || !estimator.FillPixelPatch()) { estimator.depthMap0(x) = 0; estimator.normalMap0(x) = Normal::ZERO; - estimator.confMap0(x) = DepthEstimator::EncodeScoreScale(2.f); + estimator.confMap0(x) = 2.f; continue; } Depth& depth = estimator.depthMap0(x); @@ -574,7 +574,7 @@ void* STCALL DepthMapsData::ScoreDepthMapTmp(void* arg) // replace invalid normal with random values normal = estimator.RandomNormal(viewDir); } - estimator.confMap0(x) = DepthEstimator::EncodeScoreScale(estimator.ScorePixel(depth, normal)); + estimator.confMap0(x) = estimator.ScorePixel(depth, normal); } return NULL; } @@ -598,7 +598,6 @@ void* STCALL DepthMapsData::EndDepthMapTmp(void* arg) Depth& depth = estimator.depthMap0(x); Normal& normal = estimator.normalMap0(x); float& conf = estimator.confMap0(x); - const unsigned idxScaleRange(DepthEstimator::DecodeScoreScale(conf)); ASSERT(depth >= 0); // check if the score is good enough // and that the cross-estimates is close enough to the current estimate @@ -632,10 +631,8 @@ void* STCALL DepthMapsData::EndDepthMapTmp(void* arg) #endif #if 1 conf = wAngle/MAXF(conf,1e-2f); - #elif 1 - conf = wAngle/(depth*SQUARE(MAXF(conf,1e-2f))); #else - conf = SQRT((float)idxScaleRange)*wAngle/(depth*SQUARE(MAXF(conf,1e-2f))); + conf = wAngle/(depth*SQUARE(MAXF(conf,1e-2f))); #endif #endif } From 51af2fb289abb5a75c63ca616a15540fa79bea1b Mon Sep 17 00:00:00 2001 From: cDc Date: Tue, 2 Apr 2019 18:51:50 +0300 Subject: [PATCH 18/70] dense: fine-tune patch-match --- apps/DensifyPointCloud/DensifyPointCloud.cpp | 2 + apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp | 2 + .../InterfaceVisualSFM/InterfaceVisualSFM.cpp | 2 + apps/ReconstructMesh/ReconstructMesh.cpp | 2 + apps/RefineMesh/RefineMesh.cpp | 2 + apps/TextureMesh/TextureMesh.cpp | 2 + apps/Viewer/Viewer.cpp | 2 + libs/Common/Types.h | 26 +++-- libs/Common/Types.inl | 103 ++++++++++++++++-- libs/Common/Util.cpp | 15 +++ libs/Common/Util.h | 2 + libs/MVS/DepthMap.cpp | 31 +++--- libs/MVS/DepthMap.h | 9 +- libs/MVS/Scene.cpp | 37 +++---- libs/MVS/SceneDensify.cpp | 86 ++++++++------- 15 files changed, 227 insertions(+), 96 deletions(-) diff --git a/apps/DensifyPointCloud/DensifyPointCloud.cpp b/apps/DensifyPointCloud/DensifyPointCloud.cpp index f3101f2e9..b96cfae46 100644 --- a/apps/DensifyPointCloud/DensifyPointCloud.cpp +++ b/apps/DensifyPointCloud/DensifyPointCloud.cpp @@ -194,6 +194,8 @@ bool Initialize(size_t argc, LPCTSTR* argv) // start memory dumper MiniDumper::Create(APPNAME, WORKING_FOLDER); #endif + + Util::Init(); return true; } diff --git a/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp b/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp index be6a46476..651b44c98 100644 --- a/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp +++ b/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp @@ -185,6 +185,8 @@ bool Initialize(size_t argc, LPCTSTR* argv) // start memory dumper MiniDumper::Create(APPNAME, WORKING_FOLDER); #endif + + Util::Init(); return true; } diff --git a/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp b/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp index 368690602..75bce304b 100644 --- a/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp +++ b/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp @@ -158,6 +158,8 @@ bool Initialize(size_t argc, LPCTSTR* argv) // start memory dumper MiniDumper::Create(APPNAME, WORKING_FOLDER); #endif + + Util::Init(); return true; } diff --git a/apps/ReconstructMesh/ReconstructMesh.cpp b/apps/ReconstructMesh/ReconstructMesh.cpp index 14d325e01..8651715e0 100644 --- a/apps/ReconstructMesh/ReconstructMesh.cpp +++ b/apps/ReconstructMesh/ReconstructMesh.cpp @@ -189,6 +189,8 @@ bool Initialize(size_t argc, LPCTSTR* argv) // start memory dumper MiniDumper::Create(APPNAME, WORKING_FOLDER); #endif + + Util::Init(); return true; } diff --git a/apps/RefineMesh/RefineMesh.cpp b/apps/RefineMesh/RefineMesh.cpp index a8c65136d..1cbd6e227 100644 --- a/apps/RefineMesh/RefineMesh.cpp +++ b/apps/RefineMesh/RefineMesh.cpp @@ -195,6 +195,8 @@ bool Initialize(size_t argc, LPCTSTR* argv) // start memory dumper MiniDumper::Create(APPNAME, WORKING_FOLDER); #endif + + Util::Init(); return true; } diff --git a/apps/TextureMesh/TextureMesh.cpp b/apps/TextureMesh/TextureMesh.cpp index ee99b89b0..9e610a8ba 100644 --- a/apps/TextureMesh/TextureMesh.cpp +++ b/apps/TextureMesh/TextureMesh.cpp @@ -179,6 +179,8 @@ bool Initialize(size_t argc, LPCTSTR* argv) // start memory dumper MiniDumper::Create(APPNAME, WORKING_FOLDER); #endif + + Util::Init(); return true; } diff --git a/apps/Viewer/Viewer.cpp b/apps/Viewer/Viewer.cpp index 028675a6a..2470e3abf 100644 --- a/apps/Viewer/Viewer.cpp +++ b/apps/Viewer/Viewer.cpp @@ -181,6 +181,8 @@ bool Initialize(size_t argc, LPCTSTR* argv) // start memory dumper MiniDumper::Create(APPNAME, WORKING_FOLDER); #endif + + Util::Init(); return true; } diff --git a/libs/Common/Types.h b/libs/Common/Types.h index 2355c8fd3..39e95aaa1 100644 --- a/libs/Common/Types.h +++ b/libs/Common/Types.h @@ -1480,8 +1480,12 @@ class TDMatrix : public cv::Mat_ return pt.width>=0 && pt.height>=0 && pt.width - inline bool isInside(const cv::Point_& pt) const { - return int(pt.x)>=0 && int(pt.y)>=0 && int(pt.x)::value,bool>::type isInside(const cv::Point_& pt) const { + return pt.x>=0 && pt.y>=0 && pt.x + inline typename std::enable_if::value,bool>::type isInside(const cv::Point_& pt) const { + return pt.x>=T(0) && pt.y>=T(0) && pt.x<=T(Base::size().width) && pt.y<=T(Base::size().height); } /// Is this coordinate inside the 2D matrix, and not too close to the edges? @@ -1490,12 +1494,20 @@ class TDMatrix : public cv::Mat_ return pt.width>=border && pt.height>=border && pt.width - inline bool isInsideWithBorder(const cv::Point_& pt, int border) const { - return int(pt.x)>=border && int(pt.y)>=border && int(pt.x)::value,bool>::type isInsideWithBorder(const cv::Point_& pt, int border) const { + return pt.x>=border && pt.y>=border && pt.x + inline typename std::enable_if::value,bool>::type isInsideWithBorder(const cv::Point_& pt, int border) const { + return pt.x>=T(border) && pt.y>=T(border) && pt.x<=T(Base::size().width-(border+1)) && pt.y<=T(Base::size().height-(border+1)); } template - inline bool isInsideWithBorder(const cv::Point_& pt) const { - return int(pt.x)>=border && int(pt.y)>=border && int(pt.x)::value,bool>::type isInsideWithBorder(const cv::Point_& pt) const { + return pt.x>=border && pt.y>=border && pt.x + inline typename std::enable_if::value,bool>::type isInsideWithBorder(const cv::Point_& pt) const { + return pt.x>=T(border) && pt.y>=T(border) && pt.x<=T(Base::size().width-(border+1)) && pt.y<=T(Base::size().height-(border+1)); } /// Remove the given element from the vector @@ -2003,7 +2015,7 @@ class TImage : public TDMatrix INTERTYPE sample(const SAMPLER& sampler, const TPoint2& pt) const; template - void toGray(TImage& out, int code, bool bNormalize=false) const; + void toGray(TImage& out, int code, bool bNormalize=false, bool bSRGB=false) const; unsigned computeMaxResolution(unsigned& level, unsigned minImageSize=320, unsigned maxImageSize=INT_MAX) const; static unsigned computeMaxResolution(unsigned width, unsigned height, unsigned& level, unsigned minImageSize=320, unsigned maxImageSize=INT_MAX); diff --git a/libs/Common/Types.inl b/libs/Common/Types.inl index a5df72c79..eb1582583 100644 --- a/libs/Common/Types.inl +++ b/libs/Common/Types.inl @@ -1524,6 +1524,79 @@ DEFINE_GENERIC_CVDATATYPE(SEACAVE::Image32F4, uint8_t) namespace SEACAVE { +namespace CONVERT { + +// convert sRGB to/from linear value +// (see http://en.wikipedia.org/wiki/SRGB) +template +constexpr T sRGB2RGB(T x) { + return x <= T(0.04045) ? x * (T(1)/T(12.92)) : POW((x + T(0.055)) * (T(1)/T(1.055)), T(2.4)); +} +template +constexpr T RGB2sRGB(T x) { + return x <= T(0.0031308) ? T(12.92) * x : T(1.055) * POW(x, T(1)/T(2.4)) - T(0.055); +} +static const CAutoPtrArr g_ptrsRGB82RGBf([]() { + float* const buffer = new float[256]; + for (int i=0; i<256; ++i) + buffer[i] = sRGB2RGB(float(i)/255.f); + return buffer; +}()); + +// color conversion helper structures +template +struct NormRGB_t { + const TO v; + inline NormRGB_t(TI _v) : v(TO(_v)*(TO(1)/TO(255))) {} + inline operator TO () const { return v; } +}; +template +struct RGBUnNorm_t { + const TO v; + inline RGBUnNorm_t(TI _v) : v(TO(_v)*TO(255)) {} + inline operator TO () const { return v; } +}; +template +struct RoundF2U_t { + const TO v; + inline RoundF2U_t(TI _v) : v(ROUND2INT(_v)) {} + inline operator TO () const { return v; } +}; +template +struct sRGB2RGB_t { + const TO v; + inline sRGB2RGB_t(TI _v) : v(sRGB2RGB(TO(_v))) {} + inline operator TO () const { return v; } +}; +template +struct NormsRGB2RGB_t { + const TO v; + inline NormsRGB2RGB_t(TI _v) : v(sRGB2RGB(TO(_v)*(TO(1)/TO(255)))) {} + inline operator TO () const { return v; } +}; +template <> +struct NormsRGB2RGB_t { + const float v; + inline NormsRGB2RGB_t(uint8_t _v) : v(g_ptrsRGB82RGBf[_v]) {} + inline operator float () const { return v; } +}; +template +struct NormsRGB2RGBUnNorm_t { + const TO v; + inline NormsRGB2RGBUnNorm_t(TI _v) : v(sRGB2RGB(TO(_v)*(TO(1)/TO(255)))*TO(255)) {} + inline operator TO () const { return v; } +}; +template <> +struct NormsRGB2RGBUnNorm_t { + const float v; + inline NormsRGB2RGBUnNorm_t(uint8_t _v) : v(g_ptrsRGB82RGBf[_v]*255.f) {} + inline operator float () const { return v; } +}; + +} // namespace CONVERT +/*----------------------------------------------------------------*/ + + // C L A S S ////////////////////////////////////////////////////// template @@ -2050,23 +2123,21 @@ INTERTYPE TImage::sample(const SAMPLER& sampler, const TPoint2 template -void TImage::toGray(TImage& out, int code, bool bNormalize) const +void TImage::toGray(TImage& out, int code, bool bNormalize, bool bSRGB) const { #if 1 ASSERT(code==cv::COLOR_RGB2GRAY || code==cv::COLOR_RGBA2GRAY || code==cv::COLOR_BGR2GRAY || code==cv::COLOR_BGRA2GRAY); static const T coeffsRGB[] = {T(0.299), T(0.587), T(0.114)}; static const T coeffsBGR[] = {T(0.114), T(0.587), T(0.299)}; - static const T coeffsRGBn[] = {T(0.299/255), T(0.587/255), T(0.114/255)}; - static const T coeffsBGRn[] = {T(0.114/255), T(0.587/255), T(0.299/255)}; const float* coeffs; switch (code) { case cv::COLOR_BGR2GRAY: case cv::COLOR_BGRA2GRAY: - coeffs = (bNormalize ? coeffsBGRn : coeffsBGR); + coeffs = coeffsBGR; break; case cv::COLOR_RGB2GRAY: case cv::COLOR_RGBA2GRAY: - coeffs = (bNormalize ? coeffsRGBn : coeffsRGB); + coeffs = coeffsRGB; break; default: ASSERT("Unsupported image format" == NULL); @@ -2080,8 +2151,26 @@ void TImage::toGray(TImage& out, int code, bool bNormalize) const T* dst = out.cv::Mat::template ptr(); T* const dstEnd = dst + out.area(); typedef typename cv::DataType::channel_type ST; - for (const ST* src=cv::Mat::template ptr(); dst!=dstEnd; src+=scn) - *dst++ = cb*T(src[0]) + cg*T(src[1]) + cr*T(src[2]); + if (bSRGB) { + if (bNormalize) { + typedef typename CONVERT::NormsRGB2RGB_t ColConv; + for (const ST* src=cv::Mat::template ptr(); dst!=dstEnd; src+=scn) + *dst++ = T(cb*ColConv(src[0]) + cg*ColConv(src[1]) + cr*ColConv(src[2])); + } else { + typedef typename CONVERT::NormsRGB2RGBUnNorm_t ColConv; + for (const ST* src=cv::Mat::template ptr(); dst!=dstEnd; src+=scn) + *dst++ = T(cb*ColConv(src[0]) + cg*ColConv(src[1]) + cr*ColConv(src[2])); + } + } else { + if (bNormalize) { + typedef typename CONVERT::NormRGB_t ColConv; + for (const ST* src=cv::Mat::template ptr(); dst!=dstEnd; src+=scn) + *dst++ = T(cb*ColConv(src[0]) + cg*ColConv(src[1]) + cr*ColConv(src[2])); + } else { + for (const ST* src=cv::Mat::template ptr(); dst!=dstEnd; src+=scn) + *dst++ = T(cb*src[0] + cg*src[1] + cr*src[2]); + } + } #else cv::Mat cimg; convertTo(cimg, cv::DataType::type); diff --git a/libs/Common/Util.cpp b/libs/Common/Util.cpp index 91c08797f..10c685aae 100644 --- a/libs/Common/Util.cpp +++ b/libs/Common/Util.cpp @@ -433,6 +433,21 @@ String Util::GetOSInfo() /*----------------------------------------------------------------*/ +// Initialize various global variables (ex: random-number-generator state). +void Util::Init() +{ + #ifdef _RELEASE + const time_t t(Util::getTime()); + std::srand((unsigned)t); + cv::setRNGSeed((int)t); + #else + std::srand((unsigned)0); + cv::setRNGSeed((int)0); + #endif +} +/*----------------------------------------------------------------*/ + + /** * Set global variable for availability of SSE instructions. */ diff --git a/libs/Common/Util.h b/libs/Common/Util.h index d41b829ed..0e10540ec 100644 --- a/libs/Common/Util.h +++ b/libs/Common/Util.h @@ -749,6 +749,8 @@ class GENERAL_API Util } + static void Init(); + static String GetCPUInfo(); static String GetRAMInfo(); static String GetOSInfo(); diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index 2de201fdb..9c4d77c8a 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -90,13 +90,13 @@ MDEFVAR_OPTDENSE_uint32(nIpolGapSize, "Interpolate Gap Size", "interpolate small MDEFVAR_OPTDENSE_uint32(nOptimize, "Optimize", "should we filter the extracted depth-maps?", "7") // see DepthFlags MDEFVAR_OPTDENSE_uint32(nEstimateColors, "Estimate Colors", "should we estimate the colors for the dense point-cloud?", "2", "0", "1") MDEFVAR_OPTDENSE_uint32(nEstimateNormals, "Estimate Normals", "should we estimate the normals for the dense point-cloud?", "0", "1", "2") -MDEFVAR_OPTDENSE_float(fNCCThresholdKeep, "NCC Threshold Keep", "Maximum 1-NCC score accepted for a match", "0.5", "0.3") +MDEFVAR_OPTDENSE_float(fNCCThresholdKeep, "NCC Threshold Keep", "Maximum 1-NCC score accepted for a match", "0.55", "0.3") MDEFVAR_OPTDENSE_uint32(nEstimationIters, "Estimation Iters", "Number of iterations for depth-map refinement", "4") MDEFVAR_OPTDENSE_uint32(nRandomIters, "Random Iters", "Number of iterations for random assignment per pixel", "6") MDEFVAR_OPTDENSE_uint32(nRandomMaxScale, "Random Max Scale", "Maximum number of iterations to skip during random assignment", "2") -MDEFVAR_OPTDENSE_float(fRandomDepthRatio, "Random Depth Ratio", "Depth range ratio of the current estimate for random plane assignment", "0.004") -MDEFVAR_OPTDENSE_float(fRandomAngle1Range, "Random Angle1 Range", "Angle 1 range for random plane assignment (degrees)", "20.0") -MDEFVAR_OPTDENSE_float(fRandomAngle2Range, "Random Angle2 Range", "Angle 2 range for random plane assignment (degrees)", "12.0") +MDEFVAR_OPTDENSE_float(fRandomDepthRatio, "Random Depth Ratio", "Depth range ratio of the current estimate for random plane assignment", "0.003", "0.004") +MDEFVAR_OPTDENSE_float(fRandomAngle1Range, "Random Angle1 Range", "Angle 1 range for random plane assignment (degrees)", "16.0", "20.0") +MDEFVAR_OPTDENSE_float(fRandomAngle2Range, "Random Angle2 Range", "Angle 2 range for random plane assignment (degrees)", "10.0", "12.0") MDEFVAR_OPTDENSE_float(fRandomSmoothDepth, "Random Smooth Depth", "Depth variance used during neighbor smoothness assignment (ratio)", "0.02") MDEFVAR_OPTDENSE_float(fRandomSmoothNormal, "Random Smooth Normal", "Normal variance used during neighbor smoothness assignment (degrees)", "13") MDEFVAR_OPTDENSE_float(fRandomSmoothBonus, "Random Smooth Bonus", "Score factor used to encourage smoothness (1 - disabled)", "0.93") @@ -290,6 +290,7 @@ DepthEstimator::DepthEstimator( #endif coords(_coords), size(_depthData0.images.First().image.size()), dMin(_depthData0.dMin), dMax(_depthData0.dMax), + dMinSqr(SQRT(_depthData0.dMin)), dMaxSqr(SQRT(_depthData0.dMax)), dir(nIter%2 ? RB2LT : LT2RB), #if DENSE_AGGNCC == DENSE_AGGNCC_NTH idxScore((_depthData0.images.size()-1)/3), @@ -302,9 +303,9 @@ DepthEstimator::DepthEstimator( thMagnitudeSq(OPTDENSE::fDescriptorMinMagnitudeThreshold>0?SQUARE(OPTDENSE::fDescriptorMinMagnitudeThreshold):-1.f), angle1Range(FD2R(OPTDENSE::fRandomAngle1Range)), angle2Range(FD2R(OPTDENSE::fRandomAngle2Range)), - thConfSmall(OPTDENSE::fNCCThresholdKeep*0.25f), - thConfBig(OPTDENSE::fNCCThresholdKeep*0.5f), - thConfRand(OPTDENSE::fNCCThresholdKeep*1.08f), + thConfSmall(OPTDENSE::fNCCThresholdKeep*0.2f), + thConfBig(OPTDENSE::fNCCThresholdKeep*0.4f), + thConfRand(OPTDENSE::fNCCThresholdKeep*0.9f), thRobust(OPTDENSE::fNCCThresholdKeep*1.2f) #if DENSE_REFINE == DENSE_REFINE_EXACT , thPerturbation(1.f/POW(2.f,float(nIter+1))) @@ -425,11 +426,12 @@ float DepthEstimator::ScorePixelImage(const ViewData& image1, Depth depth, const // encourage smoothness for (const NeighborEstimate& neighbor: neighborsClose) { #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE - score *= 1.f - smoothBonusDepth * DENSE_EXP(SQUARE(plane.Distance(neighbor.X)/depth) * smoothSigmaDepth); + const float factorDepth(DENSE_EXP(SQUARE(plane.Distance(neighbor.X)/depth) * smoothSigmaDepth)); #else - score *= 1.f - smoothBonusDepth * DENSE_EXP(SQUARE((depth-neighbor.depth)/depth) * smoothSigmaDepth); + const float factorDepth(DENSE_EXP(SQUARE((depth-neighbor.depth)/depth) * smoothSigmaDepth)); #endif - score *= 1.f - smoothBonusNormal * DENSE_EXP(SQUARE(ACOS(ComputeAngle(normal.ptr(), neighbor.normal.ptr()))) * smoothSigmaNormal); + const float factorNormal(DENSE_EXP(SQUARE(ACOS(ComputeAngle(normal.ptr(), neighbor.normal.ptr()))) * smoothSigmaNormal)); + score *= (1.f - smoothBonusDepth * factorDepth) * (1.f - smoothBonusNormal * factorNormal); } #endif ASSERT(ISFINITE(score)); @@ -658,8 +660,9 @@ void DepthEstimator::ProcessPixel(IDX idx) normal = neighbor.normal; } } - // check few random solutions close to the current estimate in an attempt to find a better estimate + // try random values around the current estimate in order to refine it unsigned idxScaleRange(0); + RefineIters: if (conf <= thConfSmall) idxScaleRange = 2; else if (conf <= thConfBig) @@ -667,7 +670,7 @@ void DepthEstimator::ProcessPixel(IDX idx) else if (conf >= thConfRand) { // try completely random values in order to find an initial estimate for (unsigned iter=0; iter= 0); @@ -681,10 +684,8 @@ void DepthEstimator::ProcessPixel(IDX idx) } return; } - RefineIters: - // try random values around the current estimate in order to refine it - const float depthRange(MaxDepthDifference(depth, OPTDENSE::fRandomDepthRatio)); float scaleRange(scaleRanges[idxScaleRange]); + const float depthRange(MaxDepthDifference(depth, OPTDENSE::fRandomDepthRatio)); Point2f p; Normal2Dir(normal, p); Normal nnormal; diff --git a/libs/MVS/DepthMap.h b/libs/MVS/DepthMap.h index 6f1cf106c..426b8b6d1 100644 --- a/libs/MVS/DepthMap.h +++ b/libs/MVS/DepthMap.h @@ -316,6 +316,7 @@ struct MVS_API DepthEstimator { const MapRefArr& coords; const Image8U::Size size; const Depth dMin, dMax; + const Depth dMinSqr, dMaxSqr; const ENDIRECTION dir; #if DENSE_AGGNCC == DENSE_AGGNCC_NTH || DENSE_AGGNCC == DENSE_AGGNCC_MINMEAN const IDX idxScore; @@ -394,13 +395,13 @@ struct MVS_API DepthEstimator { } // generate random depth and normal - inline Depth RandomDepth(Depth dMin, Depth dMax) { - ASSERT(dMin > 0); - return rnd.randomRange(dMin, dMax); + inline Depth RandomDepth(Depth dMinSqr, Depth dMaxSqr) { + ASSERT(dMinSqr > 0 && dMinSqr < dMaxSqr); + return SQUARE(rnd.randomRange(dMinSqr, dMaxSqr)); } inline Normal RandomNormal(const Point3f& viewRay) { Normal normal; - Dir2Normal(Point2f(rnd.randomRange(FD2R(0.f),FD2R(360.f)), rnd.randomRange(FD2R(120.f),FD2R(180.f))), normal); + Dir2Normal(Point2f(rnd.randomRange(FD2R(0.f),FD2R(180.f)), rnd.randomRange(FD2R(90.f),FD2R(180.f))), normal); return normal.dot(viewRay) > 0 ? -normal : normal; } diff --git a/libs/MVS/Scene.cpp b/libs/MVS/Scene.cpp index 23c48e7a5..7932ae348 100644 --- a/libs/MVS/Scene.cpp +++ b/libs/MVS/Scene.cpp @@ -427,7 +427,7 @@ inline float Footprint(const Camera& camera, const Point3f& X) { const Point3 cX(camera.TransformPointW2C(Cast(X))); return (float)norm(camera.TransformPointC2I(Point3(cX.x+fSphereRadius,cX.y,cX.z))-camera.TransformPointC2I(cX))+std::numeric_limits::epsilon(); #else - return (float)(camera.GetFocalLength()/camera.PointDepth(Cast(X))); + return (float)(camera.GetFocalLength()/camera.PointDepth(X)); #endif } @@ -470,8 +470,7 @@ bool Scene::SelectNeighborViews(uint32_t ID, IndexArr& points, unsigned nMinView // score shared views const Point3f V1(imageData.camera.C - Cast(point)); const float footprint1(Footprint(imageData.camera, point)); - FOREACHPTR(pView, views) { - const PointCloud::View& view = *pView; + for (const PointCloud::View& view: views) { if (view == ID) continue; const Image& imageData2 = images[view]; @@ -498,40 +497,36 @@ bool Scene::SelectNeighborViews(uint32_t ID, IndexArr& points, unsigned nMinView ASSERT(nPoints > 3); // select best neighborViews - Point2fArr pointsA(0, points.GetSize()), pointsB(0, points.GetSize()); + Point2fArr projs(0, points.GetSize()); FOREACH(IDB, images) { const Image& imageDataB = images[IDB]; if (!imageDataB.IsValid()) continue; const Score& score = scores[IDB]; - if (score.points == 0) + if (score.points < 3) continue; ASSERT(ID != IDB); // compute how well the matched features are spread out (image covered area) const Point2f boundsA(imageData.GetSize()); const Point2f boundsB(imageDataB.GetSize()); - ASSERT(pointsA.IsEmpty() && pointsB.IsEmpty()); - FOREACHPTR(pIdx, points) { - const PointCloud::ViewArr& views = pointcloud.pointViews[*pIdx]; + ASSERT(projs.IsEmpty()); + for (uint32_t idx: points) { + const PointCloud::ViewArr& views = pointcloud.pointViews[idx]; ASSERT(views.IsSorted()); ASSERT(views.FindFirst(ID) != PointCloud::ViewArr::NO_INDEX); if (views.FindFirst(IDB) == PointCloud::ViewArr::NO_INDEX) continue; - const PointCloud::Point& point = pointcloud.points[*pIdx]; - Point2f& ptA = pointsA.AddConstruct(imageData.camera.ProjectPointP(point)); - Point2f& ptB = pointsB.AddConstruct(imageDataB.camera.ProjectPointP(point)); - if (!imageData.camera.IsInside(ptA, boundsA) || !imageDataB.camera.IsInside(ptB, boundsB)) { - pointsA.RemoveLast(); - pointsB.RemoveLast(); - } + const PointCloud::Point& point = pointcloud.points[idx]; + Point2f& ptA = projs.AddConstruct(imageData.camera.ProjectPointP(point)); + Point2f ptB = imageDataB.camera.ProjectPointP(point); + if (!imageData.camera.IsInside(ptA, boundsA) || !imageDataB.camera.IsInside(ptB, boundsB)) + projs.RemoveLast(); } - ASSERT(pointsA.GetSize() == pointsB.GetSize() && pointsA.GetSize() <= score.points); - if (pointsA.IsEmpty()) + ASSERT(projs.GetSize() <= score.points); + if (projs.IsEmpty()) continue; - const float areaA(ComputeCoveredArea((const float*)pointsA.Begin(), pointsA.GetSize(), boundsA.ptr())); - const float areaB(ComputeCoveredArea((const float*)pointsB.Begin(), pointsB.GetSize(), boundsB.ptr())); - const float area(MINF(areaA, areaB)); - pointsA.Empty(); pointsB.Empty(); + const float area(ComputeCoveredArea((const float*)projs.Begin(), projs.GetSize(), boundsA.ptr())); + projs.Empty(); // store image score ViewScore& neighbor = neighbors.AddEmpty(); neighbor.idx.ID = IDB; diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index d14d8e3f2..a5177b863 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -413,16 +413,13 @@ std::pair TriangulatePointsDelaunay(CGAL::Delaunay& delaunay, const ASSERT(sizeof(Point3) == sizeof(X3D)); ASSERT(sizeof(Point3) == sizeof(CGAL::Point)); std::pair depthBounds(FLT_MAX, 0.f); - FOREACH(p, points) { - const PointCloud::Point& point = scene.pointcloud.points[points[p]]; - const Point3 ptCam(image.camera.TransformPointW2C(Cast(point))); - const Point2 ptImg(image.camera.TransformPointC2I(ptCam)); - delaunay.insert(CGAL::Point(ptImg.x, ptImg.y, ptCam.z)); - const Depth depth((float)ptCam.z); - if (depthBounds.first > depth) - depthBounds.first = depth; - if (depthBounds.second < depth) - depthBounds.second = depth; + for (uint32_t idx: points) { + const Point3f pt(image.camera.ProjectPointP3(scene.pointcloud.points[idx])); + delaunay.insert(CGAL::Point(pt.x/pt.z, pt.y/pt.z, pt.z)); + if (depthBounds.first > pt.z) + depthBounds.first = pt.z; + if (depthBounds.second < pt.z) + depthBounds.second = pt.z; } // if full size depth-map requested if (OPTDENSE::bAddCorners) { @@ -445,7 +442,7 @@ std::pair TriangulatePointsDelaunay(CGAL::Delaunay& delaunay, const if (cfc == 0) continue; // normally this should never happen const CGAL::FaceCirculator done(cfc); - Point3d& poszA = (Point3d&)vcorner->point(); + Point3d& poszA = reinterpret_cast(vcorner->point()); const Point2d& posA = reinterpret_cast(poszA); const Ray3d rayA(Point3d::ZERO, normalized(image.camera.TransformPointI2C(poszA))); DepthDistArr depths(0, numPoints); @@ -459,9 +456,9 @@ std::pair TriangulatePointsDelaunay(CGAL::Delaunay& delaunay, const // compute the depth as the intersection of the corner ray with // the plane defined by the face's vertices { - const Point3d& poszB0 = (const Point3d&)fc->vertex(0)->point(); - const Point3d& poszB1 = (const Point3d&)fc->vertex(1)->point(); - const Point3d& poszB2 = (const Point3d&)fc->vertex(2)->point(); + const Point3d& poszB0 = reinterpret_cast(fc->vertex(0)->point()); + const Point3d& poszB1 = reinterpret_cast(fc->vertex(1)->point()); + const Point3d& poszB2 = reinterpret_cast(fc->vertex(2)->point()); const Planed planeB( image.camera.TransformPointI2C(poszB0), image.camera.TransformPointI2C(poszB1), @@ -470,9 +467,13 @@ std::pair TriangulatePointsDelaunay(CGAL::Delaunay& delaunay, const const Point3d poszB(rayA.Intersects(planeB)); if (poszB.z <= 0) continue; - const Point2d posB((reinterpret_cast(poszB0)+reinterpret_cast(poszB1)+reinterpret_cast(poszB2))/3.f); - const REAL dist(norm(posB-posA)); - depths.StoreTop(DepthDist((float)poszB.z, 1.f/(float)dist)); + const Point2d posB(( + reinterpret_cast(poszB0)+ + reinterpret_cast(poszB1)+ + reinterpret_cast(poszB2))/3.f + ); + const double dist(norm(posB-posA)); + depths.StoreTop(DepthDist(CLAMP((float)poszB.z,depthBounds.first,depthBounds.second), INVERT((float)dist))); } Continue:; } while (++cfc != done); @@ -520,8 +521,9 @@ bool DepthMapsData::InitDepthMap(DepthData& depthData) inline void operator()(const ImageRef& pt) { if (!depthMap.isInside(pt)) return; - const float z(INVERT(normalPlane.dot(P.TransformPointI2C(Point2f(pt))))); - ASSERT(z > 0); + const Depth z(INVERT(normalPlane.dot(P.TransformPointI2C(Point2f(pt))))); + if (z <= 0) // due to numerical instability + return; depthMap(pt) = z; normalMap(pt) = normal; } @@ -529,9 +531,9 @@ bool DepthMapsData::InitDepthMap(DepthData& depthData) RasterDepthDataPlaneData data = {camera, depthData.depthMap, depthData.normalMap}; for (CGAL::Delaunay::Face_iterator it=delaunay.faces_begin(); it!=delaunay.faces_end(); ++it) { const CGAL::Delaunay::Face& face = *it; - const Point3f i0((const Point3&)face.vertex(0)->point()); - const Point3f i1((const Point3&)face.vertex(1)->point()); - const Point3f i2((const Point3&)face.vertex(2)->point()); + const Point3f i0(reinterpret_cast(face.vertex(0)->point())); + const Point3f i1(reinterpret_cast(face.vertex(1)->point())); + const Point3f i2(reinterpret_cast(face.vertex(2)->point())); // compute the plane defined by the 3 points const Point3f c0(camera.TransformPointI2C(i0)); const Point3f c1(camera.TransformPointI2C(i1)); @@ -566,9 +568,9 @@ void* STCALL DepthMapsData::ScoreDepthMapTmp(void* arg) Depth& depth = estimator.depthMap0(x); Normal& normal = estimator.normalMap0(x); const Normal viewDir(Cast(static_cast(estimator.X0))); - if (depth <= 0) { + if (!ISINSIDE(depth, estimator.dMin, estimator.dMax)) { // init with random values - depth = estimator.RandomDepth(estimator.dMin, estimator.dMax); + depth = estimator.RandomDepth(estimator.dMinSqr, estimator.dMaxSqr); normal = estimator.RandomNormal(viewDir); } else if (normal.dot(viewDir) >= 0) { // replace invalid normal with random values @@ -595,16 +597,15 @@ void* STCALL DepthMapsData::EndDepthMapTmp(void* arg) const float fOptimAngle(FD2R(OPTDENSE::fOptimAngle)); while ((idx=(IDX)Thread::safeInc(estimator.idxPixel)) < estimator.coords.GetSize()) { const ImageRef& x = estimator.coords[idx]; + ASSERT(estimator.depthMap0(x) >= 0); Depth& depth = estimator.depthMap0(x); - Normal& normal = estimator.normalMap0(x); float& conf = estimator.confMap0(x); - ASSERT(depth >= 0); // check if the score is good enough // and that the cross-estimates is close enough to the current estimate - if (conf > OPTDENSE::fNCCThresholdKeep) { + if (conf >= OPTDENSE::fNCCThresholdKeep) { #if 1 // used if gap-interpolation is active conf = 0; - normal = Normal::ZERO; + estimator.normalMap0(x) = Normal::ZERO; #endif depth = 0; } else { @@ -1321,24 +1322,25 @@ bool DepthMapsData::FilterDepthMap(DepthData& depthDataRef, const IIndexArr& idx // fuse all valid depth-maps in the same 3D point cloud; // join points very likely to represent the same 3D point and // filter out points blocking the view -struct Proj { - union { - uint32_t idxPixel; - struct { - uint16_t x, y; // image pixel coordinates - }; - }; - inline Proj() {} - inline Proj(uint32_t _idxPixel) : idxPixel(_idxPixel) {} - inline Proj(const ImageRef& ir) : x(ir.x), y(ir.y) {} - inline ImageRef GetCoord() const { return ImageRef(x,y); } -}; -typedef SEACAVE::cList ProjArr; -typedef SEACAVE::cList ProjsArr; void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, bool bEstimateNormal) { TD_TIMER_STARTD(); + struct Proj { + union { + uint32_t idxPixel; + struct { + uint16_t x, y; // image pixel coordinates + }; + }; + inline Proj() {} + inline Proj(uint32_t _idxPixel) : idxPixel(_idxPixel) {} + inline Proj(const ImageRef& ir) : x(ir.x), y(ir.y) {} + inline ImageRef GetCoord() const { return ImageRef(x,y); } + }; + typedef SEACAVE::cList ProjArr; + typedef SEACAVE::cList ProjsArr; + // find best connected images IndexScoreArr connections(0, scene.images.GetSize()); size_t nPointsEstimate(0); From 7eb794ca14c1af95a0df88308af4cc8e3609a139 Mon Sep 17 00:00:00 2001 From: cDc Date: Sat, 25 May 2019 23:15:40 +0300 Subject: [PATCH 19/70] common: fix VC2019 warnings --- .appveyor.yml | 2 +- build/Utils.cmake | 35 ++++++++++++++++++++++++++++------- libs/Common/Types.h | 1 + libs/Common/Util.h | 9 ++------- 4 files changed, 32 insertions(+), 15 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index f0c62c275..aeb6dee08 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -108,5 +108,5 @@ test_script: #- sh: ctest -j4 on_success: - - cmd: 7z a OpenMVS_x64.7z "C:\projects\openmvs\bin\bin\x64\%Configuration%\*.exe" "C:\projects\openmvs\bin\bin\x64\%Configuration%\*.dll" + - cmd: 7z a OpenMVS_x64.7z "C:\projects\openmvs\bin\bin\vc15\x64\%Configuration%\*.exe" "C:\projects\openmvs\bin\bin\vc15\x64\%Configuration%\*.dll" - cmd: appveyor PushArtifact OpenMVS_x64.7z diff --git a/build/Utils.cmake b/build/Utils.cmake index fc713bdf1..d24b8453b 100644 --- a/build/Utils.cmake +++ b/build/Utils.cmake @@ -174,7 +174,11 @@ macro(ComposePackageLibSuffix) set(PACKAGE_LIB_SUFFIX_DBG "") set(PACKAGE_LIB_SUFFIX_REL "") if(MSVC) - if("${MSVC_VERSION}" STREQUAL "1900") + if("${MSVC_VERSION}" STREQUAL "1921") + set(PACKAGE_LIB_SUFFIX "/vc16") + elseif("${MSVC_VERSION}" STREQUAL "1916") + set(PACKAGE_LIB_SUFFIX "/vc15") + elseif("${MSVC_VERSION}" STREQUAL "1900") set(PACKAGE_LIB_SUFFIX "/vc14") elseif("${MSVC_VERSION}" STREQUAL "1800") set(PACKAGE_LIB_SUFFIX "/vc12") @@ -422,13 +426,30 @@ macro(optimize_default_compiler_settings) set(BUILD_EXTRA_EXE_LINKER_FLAGS_DEBUG "") # try to enable C++14/C++11 support - check_cxx_compiler_flag(--std=c++14 SUPPORTS_STD_CXX14) - check_cxx_compiler_flag(--std=c++11 SUPPORTS_STD_CXX11) - if(SUPPORTS_STD_CXX14) - set(CMAKE_CXX_STANDARD 14) - elseif(SUPPORTS_STD_CXX11) - set(CMAKE_CXX_STANDARD 11) + if(${CMAKE_VERSION} VERSION_LESS "3.8.2") + check_cxx_compiler_flag(--std=c++14 SUPPORTS_STD_CXX14) + check_cxx_compiler_flag(--std=c++11 SUPPORTS_STD_CXX11) + if(SUPPORTS_STD_CXX14) + set(CMAKE_CXX_STANDARD 14) + elseif(SUPPORTS_STD_CXX11) + set(CMAKE_CXX_STANDARD 11) + endif() + else() + foreach(i ${CMAKE_CXX_COMPILE_FEATURES}) + if("${i}" STREQUAL "cxx_std_14") + set(CMAKE_CXX_STANDARD 14) + break() + elseif("${i}" STREQUAL "cxx_std_11") + set(CMAKE_CXX_STANDARD 11) + endif() + endforeach() + endif() + if(CLANG AND (CMAKE_CXX_STANDARD EQUAL 11 OR CMAKE_CXX_STANDARD EQUAL 14)) + set(CMAKE_EXE_LINKER_FLAGS "-stdlib=libc++") + add_extra_compiler_option(-stdlib=libc++) endif() + set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_CXX_EXTENSIONS OFF) message("Compiling with C++${CMAKE_CXX_STANDARD}") if(MINGW) diff --git a/libs/Common/Types.h b/libs/Common/Types.h index 39e95aaa1..aad57d11a 100644 --- a/libs/Common/Types.h +++ b/libs/Common/Types.h @@ -41,6 +41,7 @@ #endif #include #include +#include #include #include #include diff --git a/libs/Common/Util.h b/libs/Common/Util.h index 0e10540ec..dee6aab24 100644 --- a/libs/Common/Util.h +++ b/libs/Common/Util.h @@ -631,8 +631,7 @@ class GENERAL_API Util if (wsz == NULL) return String(); #if 1 - const std::wstring ws(wsz); - return std::string(ws.cbegin(), ws.cend()); + return std::wstring_convert, wchar_t>().to_bytes(wsz); #elif 1 std::mbstate_t state = std::mbstate_t(); const size_t len(std::wcsrtombs(NULL, &wsz, 0, &state)); @@ -642,7 +641,7 @@ class GENERAL_API Util if (std::wcsrtombs(&mbstr[0], &wsz, mbstr.size(), &state) == static_cast(-1)) return String(); return String(&mbstr[0]); - #elif 1 + #else const std::wstring ws(wsz); const std::locale locale(""); typedef std::codecvt converter_type; @@ -654,10 +653,6 @@ class GENERAL_API Util if (converter.out(state, ws.data(), ws.data() + ws.length(), from_next, &to[0], &to[0] + to.size(), to_next) != converter_type::ok) return String(); return std::string(&to[0], to_next); - #else - typedef std::codecvt_utf8 convert_typeX; - std::wstring_convert converterX; - return converterX.to_bytes(wstr); #endif } From d06d0a0cca0f5e86add6907717e1a36147619b6d Mon Sep 17 00:00:00 2001 From: cDc Date: Sun, 26 May 2019 00:13:31 +0300 Subject: [PATCH 20/70] common: fix build with opencv older than 3.4 --- libs/Common/Util.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libs/Common/Util.cpp b/libs/Common/Util.cpp index 10c685aae..a32268cf7 100644 --- a/libs/Common/Util.cpp +++ b/libs/Common/Util.cpp @@ -439,11 +439,15 @@ void Util::Init() #ifdef _RELEASE const time_t t(Util::getTime()); std::srand((unsigned)t); + #if CV_MAJOR_VERSION > 3 || (CV_MAJOR_VERSION == 3 && CV_MINOR_VERSION >= 4) cv::setRNGSeed((int)t); + #endif #else std::srand((unsigned)0); + #if CV_MAJOR_VERSION > 3 || (CV_MAJOR_VERSION == 3 && CV_MINOR_VERSION >= 4) cv::setRNGSeed((int)0); #endif + #endif } /*----------------------------------------------------------------*/ From 4a302c5e412b72feb8354dd2bd029dae7f7f0311 Mon Sep 17 00:00:00 2001 From: cDc Date: Sun, 26 May 2019 10:55:37 +0300 Subject: [PATCH 21/70] io: remove unused code --- CMakeLists.txt | 1 - build/Templates/ConfigLocal.h.in | 3 - libs/IO/CMakeLists.txt | 10 - libs/IO/Common.h | 8 - libs/IO/EXIF.cpp | 656 --------- libs/IO/EXIF.h | 157 --- libs/IO/ImageEXIF.cpp | 224 --- libs/IO/ImageEXIF.h | 54 - libs/IO/TinyXML2.cpp | 2257 ------------------------------ libs/IO/TinyXML2.h | 2075 --------------------------- 10 files changed, 5445 deletions(-) delete mode 100644 libs/IO/EXIF.cpp delete mode 100644 libs/IO/EXIF.h delete mode 100644 libs/IO/ImageEXIF.cpp delete mode 100644 libs/IO/ImageEXIF.h delete mode 100644 libs/IO/TinyXML2.cpp delete mode 100644 libs/IO/TinyXML2.h diff --git a/CMakeLists.txt b/CMakeLists.txt index ad6583a14..6d9677e10 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,7 +41,6 @@ ConfigLibrary() # List configuration options SET(OpenMVS_USE_NONFREE ON CACHE BOOL "Build non-free (patented) functionality") -SET(OpenMVS_USE_EXIV2 OFF CACHE BOOL "Link and use EXIV2 library") SET(OpenMVS_USE_CERES OFF CACHE BOOL "Enable CERES optimization library") SET(OpenMVS_USE_FAST_FLOAT2INT ON CACHE BOOL "Use an optimized code to convert real numbers to int") SET(OpenMVS_USE_FAST_INVSQRT OFF CACHE BOOL "Use an optimized code to compute the inverse square root (slower in fact on modern compilers)") diff --git a/build/Templates/ConfigLocal.h.in b/build/Templates/ConfigLocal.h.in index d81e950ab..5fbc77b78 100644 --- a/build/Templates/ConfigLocal.h.in +++ b/build/Templates/ConfigLocal.h.in @@ -34,9 +34,6 @@ // TIFF codec #cmakedefine _USE_TIFF -// EXIV2 parser -#cmakedefine _USE_EXIV2 - // OpenGL support #cmakedefine _USE_OPENGL diff --git a/libs/IO/CMakeLists.txt b/libs/IO/CMakeLists.txt index a069fc3d2..a354376be 100644 --- a/libs/IO/CMakeLists.txt +++ b/libs/IO/CMakeLists.txt @@ -23,16 +23,6 @@ if(TIFF_FOUND) else() SET(TIFF_LIBRARIES "") endif() -if(OpenMVS_USE_EXIV2) - FIND_PACKAGE(EXIV2 QUIET) - if(EXIV2_FOUND) - INCLUDE_DIRECTORIES(${EXIV2_INCLUDE_DIRS}) - LIST(APPEND OpenMVS_DEFINITIONS -D_USE_EXIV2) - SET(_USE_EXIV2 TRUE CACHE INTERNAL "") - else() - SET(EXIV2_LIBS "") - endif() -endif() # List sources files FILE(GLOB PCH_C "Common.cpp") diff --git a/libs/IO/Common.h b/libs/IO/Common.h index dee7db38b..e43aaac44 100644 --- a/libs/IO/Common.h +++ b/libs/IO/Common.h @@ -36,9 +36,6 @@ #ifdef _USE_TIFF #define _IMAGE_TIFF // add TIFF support #endif -#ifdef _USE_EXIV2 -#define _IMAGE_EXIF // complete EXIF info support based on Exiv2 -#endif #include "ImageSCI.h" #ifdef _IMAGE_BMP @@ -59,13 +56,8 @@ #ifdef _IMAGE_TIFF #include "ImageTIFF.h" #endif -#ifdef _IMAGE_EXIF -#include "ImageEXIF.h" -#endif -#include "EXIF.h" #include "PLY.h" #include "OBJ.h" -#include "TinyXML2.h" /*----------------------------------------------------------------*/ #endif // __IO_COMMON_H__ diff --git a/libs/IO/EXIF.cpp b/libs/IO/EXIF.cpp deleted file mode 100644 index a17af248b..000000000 --- a/libs/IO/EXIF.cpp +++ /dev/null @@ -1,656 +0,0 @@ -/************************************************************************** - exif.cpp -- A simple ISO C++ library to parse basic EXIF - information from a JPEG file. - - Copyright (c) 2010-2013 Mayank Lahiri - mlahiri@gmail.com - All rights reserved (BSD License). - - See exif.h for version history. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - -- Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - -- Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESS - OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN - NO EVENT SHALL THE FREEBSD PROJECT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY - OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, - EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ -#include "Common.h" -#include "EXIF.h" - -using std::string; - -namespace { - // IF Entry - struct IFEntry { - // Raw fields - unsigned short tag; - unsigned short format; - unsigned data; - unsigned length; - - // Parsed fields - string val_string; - unsigned short val_16; - unsigned val_32; - double val_rational; - unsigned char val_byte; - }; - - // Helper functions - unsigned parse32(const unsigned char *buf, bool intel) { - if (intel) - return ((unsigned)buf[3]<<24) | - ((unsigned)buf[2]<<16) | - ((unsigned)buf[1]<<8) | - buf[0]; - - return ((unsigned)buf[0]<<24) | - ((unsigned)buf[1]<<16) | - ((unsigned)buf[2]<<8) | - buf[3]; - } - - unsigned short parse16(const unsigned char *buf, bool intel) { - if (intel) - return ((unsigned) buf[1]<<8) | buf[0]; - return ((unsigned) buf[0]<<8) | buf[1]; - } - - string parseEXIFString(const unsigned char *buf, - const unsigned num_components, - const unsigned data, - const unsigned base, - const unsigned len, - bool intel) { - string value; - if (num_components <= 4){ - value.resize(num_components); - char j = intel ? 0 : 24; - char j_m = intel ? -8 : 8; - for(unsigned i=0; i> j & 0xff; - if (value[num_components-1] == '\0') - value.resize(num_components-1); - } else { - if (base+data+num_components <= len) - value.assign( (const char*)(buf+base+data), num_components-1 ); - } - return value; - } - - double parseEXIFRational(const unsigned char *buf, bool intel) { - double numerator = 0; - double denominator = 1; - - numerator = (double) parse32(buf, intel); - denominator= (double) parse32(buf+4, intel); - if(denominator < 1e-20) - return 0; - return numerator/denominator; - } - - IFEntry parseIFEntry(const unsigned char *buf, - const unsigned offs, - const bool alignIntel, - const unsigned base, - const unsigned len) { - IFEntry result; - - // Each directory entry is composed of: - // 2 bytes: tag number (data field) - // 2 bytes: data format - // 4 bytes: number of components - // 4 bytes: data value or offset to data value - result.tag = parse16(buf + offs, alignIntel); - result.format = parse16(buf + offs + 2, alignIntel); - result.length = parse32(buf + offs + 4, alignIntel); - result.data = parse32(buf + offs + 8, alignIntel); - - // Parse value in specified format - switch (result.format) { - case 1: - result.val_byte = (unsigned char) *(buf + offs + 8); - break; - case 2: - result.val_string = parseEXIFString(buf, result.length, result.data, base, len, alignIntel); - break; - case 3: - result.val_16 = parse16((const unsigned char *) buf + offs + 8, alignIntel); - break; - case 4: - result.val_32 = result.data; - break; - case 5: - if (base + result.data + 8 <= len) - result.val_rational = parseEXIFRational(buf + base + result.data, alignIntel); - break; - case 7: - case 9: - case 10: - break; - default: - result.tag = 0xFF; - } - return result; - } -} - -// -// Locates the EXIF segment and parses it using parseFromEXIFSegment -// -int EXIFInfo::parseFrom(const unsigned char *buf, unsigned len) { - enum JPEG_MARKERS { - START = 0xFF, - SOI = 0xD8, - EOI = 0xD9, - SOS = 0xDA, - COM = 0xFE, - JFIF = 0xE0, - EXIF = 0xE1, - IPTC = 0xED, - }; - // Sanity check: all JPEG files start with 0xFFD8 and end with 0xFFD9 - // This check also ensures that the user has supplied a correct value for len. - if (!buf || len < 16) - return PARSE_EXIF_ERROR_NO_EXIF; - if (buf[0] != START || buf[1] != SOI) - return PARSE_EXIF_ERROR_NO_JPEG; - // not always valid, sometimes 0xFF is added for padding - //if (buf[len-2] != 0xFF || buf[len-1] != 0xD9) - // return PARSE_EXIF_ERROR_NO_JPEG; - - // Scan for EXIF header (bytes 0xFF 0xE1) and do a sanity check by - // looking for bytes "Exif\0\0". The marker length data is in Motorola - // byte order, which results in the 'false' parameter to parse16(). - // The marker has to contain at least the TIFF header, otherwise the - // EXIF data is corrupt. So the minimum length specified here has to be: - // 2 bytes: section size - // 6 bytes: "Exif\0\0" string - // 2 bytes: TIFF header (either "II" or "MM" string) - // 2 bytes: TIFF magic (short 0x2a00 in Motorola byte order) - // 4 bytes: Offset to first IFD - // ========= - // 16 bytes - for (unsigned pos=2; pos len) - return PARSE_EXIF_ERROR_NO_JPEG; - // skip the section - pos += section_length; - } - } - } - return PARSE_EXIF_ERROR_NO_EXIF; -} - -int EXIFInfo::parseFrom(const string &data) { - return parseFrom((const unsigned char *)data.data(), (unsigned)data.length()); -} - -// -// Main parsing function for an EXIF segment. -// -// PARAM: 'buf' start of the EXIF TIFF, which must be the bytes "Exif\0\0". -// PARAM: 'len' length of buffer -// -int EXIFInfo::parseFromEXIFSegment(const unsigned char *buf, unsigned len) { - clear(); - bool alignIntel = true; // byte alignment (defined in EXIF header) - unsigned offs = 0; // current offset into buffer - if (!buf || len < 6) - return PARSE_EXIF_ERROR_NO_EXIF; - - if (!std::equal(buf, buf+6, "Exif\0\0")) - return PARSE_EXIF_ERROR_NO_EXIF; - offs += 6; - - // Now parsing the TIFF header. The first two bytes are either "II" or - // "MM" for Intel or Motorola byte alignment. Sanity check by parsing - // the unsigned short that follows, making sure it equals 0x2a. The - // last 4 bytes are an offset into the first IFD, which are added to - // the global offset counter. For this block, we expect the following - // minimum size: - // 2 bytes: 'II' or 'MM' - // 2 bytes: 0x002a - // 4 bytes: offset to first IDF - // ----------------------------- - // 8 bytes - if (offs + 8 > len) - return PARSE_EXIF_ERROR_CORRUPT; - unsigned tiff_header_start = offs; - if (buf[offs] == 'I' && buf[offs+1] == 'I') - alignIntel = true; - else { - if(buf[offs] == 'M' && buf[offs+1] == 'M') - alignIntel = false; - else - return PARSE_EXIF_ERROR_UNKNOWN_BYTEALIGN; - } - this->ByteAlign = alignIntel; - offs += 2; - if (0x2a != parse16(buf+offs, alignIntel)) - return PARSE_EXIF_ERROR_CORRUPT; - offs += 2; - unsigned first_ifd_offset = parse32(buf + offs, alignIntel); - offs += first_ifd_offset - 4; - if (offs >= len) - return PARSE_EXIF_ERROR_CORRUPT; - - // Now parsing the first Image File Directory (IFD0, for the main image). - // An IFD consists of a variable number of 12-byte directory entries. The - // first two bytes of the IFD section contain the number of directory - // entries in the section. The last 4 bytes of the IFD contain an offset - // to the next IFD, which means this IFD must contain exactly 6 + 12 * num - // bytes of data. - if (offs + 2 > len) - return PARSE_EXIF_ERROR_CORRUPT; - int num_entries = parse16(buf + offs, alignIntel); - if (offs + 6 + 12 * num_entries > len) - return PARSE_EXIF_ERROR_CORRUPT; - offs += 2; - unsigned exif_sub_ifd_offset = len; - unsigned gps_sub_ifd_offset = len; - while (--num_entries >= 0) { - IFEntry result = parseIFEntry(buf, offs, alignIntel, tiff_header_start, len); - offs += 12; - switch(result.tag) { - case 0x102: - // Bits per sample - if (result.format == 3) - this->BitsPerSample = result.val_16; - break; - - case 0x10E: - // Image description - if (result.format == 2) - this->ImageDescription = result.val_string; - break; - - case 0x10F: - // Digicam make - if (result.format == 2) - this->Make = result.val_string; - break; - - case 0x110: - // Digicam model - if (result.format == 2) - this->Model = result.val_string; - break; - - case 0x112: - // Orientation of image - if (result.format == 3) - this->Orientation = result.val_16; - break; - - case 0x131: - // Software used for image - if (result.format == 2) - this->Software = result.val_string; - break; - - case 0x132: - // EXIF/TIFF date/time of image modification - if (result.format == 2) - this->DateTime = result.val_string; - break; - - case 0x8298: - // Copyright information - if (result.format == 2) - this->Copyright = result.val_string; - break; - - case 0x8825: - // GPS IFS offset - gps_sub_ifd_offset = tiff_header_start + result.data; - break; - - case 0x8769: - // EXIF SubIFD offset - exif_sub_ifd_offset = tiff_header_start + result.data; - break; - } - } - - // Jump to the EXIF SubIFD if it exists and parse all the information - // there. Note that it's possible that the EXIF SubIFD doesn't exist. - // The EXIF SubIFD contains most of the interesting information that a - // typical user might want. - if (exif_sub_ifd_offset + 4 <= len) { - offs = exif_sub_ifd_offset; - int num_entries = parse16(buf + offs, alignIntel); - if (offs + 6 + 12 * num_entries > len) - return PARSE_EXIF_ERROR_CORRUPT; - offs += 2; - while (--num_entries >= 0) { - IFEntry result = parseIFEntry(buf, offs, alignIntel, tiff_header_start, len); - switch(result.tag) { - case 0x829a: - // Exposure time in seconds - if (result.format == 5) - this->ExposureTime = result.val_rational; - break; - - case 0x829d: - // FNumber - if (result.format == 5) - this->FNumber = result.val_rational; - break; - - case 0x8827: - // ISO Speed Rating - if (result.format == 3) - this->ISOSpeedRatings = result.val_16; - break; - - case 0x9003: - // Original date and time - if (result.format == 2) - this->DateTimeOriginal = result.val_string; - break; - - case 0x9004: - // Digitization date and time - if (result.format == 2) - this->DateTimeDigitized = result.val_string; - break; - - case 0x9201: - // Shutter speed value - if (result.format == 5) - this->ShutterSpeedValue = result.val_rational; - break; - - case 0x9204: - // Exposure bias value - if (result.format == 5) - this->ExposureBiasValue = result.val_rational; - break; - - case 0x9206: - // Subject distance - if (result.format == 5) - this->SubjectDistance = result.val_rational; - break; - - case 0x9209: - // Flash used - if (result.format == 3) - this->Flash = result.data ? 1 : 0; - break; - - case 0x920a: - // Focal length - if (result.format == 5) - this->FocalLength = result.val_rational; - break; - - case 0x9207: - // Metering mode - if (result.format == 3) - this->MeteringMode = result.val_16; - break; - - case 0x9291: - // Subsecond original time - if (result.format == 2) - this->SubSecTimeOriginal = result.val_string; - break; - - case 0xa002: - // EXIF Image width - if (result.format == 4) - this->ImageWidth = result.val_32; - if (result.format == 3) - this->ImageWidth = result.val_16; - break; - - case 0xa003: - // EXIF Image height - if (result.format == 4) - this->ImageHeight = result.val_32; - if (result.format == 3) - this->ImageHeight = result.val_16; - break; - - case 0xa20e: - // Focal plane X resolution - if (result.format == 5) - this->FocalPlaneXResolution = result.val_rational; - break; - - case 0xa20f: - // Focal plane Y resolution - if (result.format == 5) - this->FocalPlaneYResolution = result.val_rational; - break; - - case 0xa210: - // Focal plane resolution unit - if (result.format == 3) - this->FocalPlaneResolutionUnit = result.val_16; - break; - - case 0xa405: - // Focal length in 35mm film - if (result.format == 3) - this->FocalLengthIn35mm = result.val_16; - break; - } - offs += 12; - } - } - - // Jump to the GPS SubIFD if it exists and parse all the information - // there. Note that it's possible that the GPS SubIFD doesn't exist. - if (gps_sub_ifd_offset + 4 <= len) { - offs = gps_sub_ifd_offset; - int num_entries = parse16(buf + offs, alignIntel); - if (offs + 6 + 12 * num_entries > len) - return PARSE_EXIF_ERROR_CORRUPT; - offs += 2; - while (--num_entries >= 0) { - unsigned short tag = parse16(buf + offs, alignIntel); - unsigned short format = parse16(buf + offs + 2, alignIntel); - unsigned length = parse32(buf + offs + 4, alignIntel); - unsigned data = parse32(buf + offs + 8, alignIntel); - switch(tag) { - case 1: - // GPS north or south - this->GeoLocation.LatComponents.direction = *(buf + offs + 8); - if ('S' == this->GeoLocation.LatComponents.direction) - this->GeoLocation.Latitude = -this->GeoLocation.Latitude; - break; - - case 2: - // GPS latitude - if (format == 5 && length == 3) { - this->GeoLocation.LatComponents.degrees = - parseEXIFRational(buf + data + tiff_header_start, alignIntel); - this->GeoLocation.LatComponents.minutes = - parseEXIFRational(buf + data + tiff_header_start + 8, alignIntel); - this->GeoLocation.LatComponents.seconds = - parseEXIFRational(buf + data + tiff_header_start + 16, alignIntel); - this->GeoLocation.Latitude = - this->GeoLocation.LatComponents.degrees + - this->GeoLocation.LatComponents.minutes / 60 + - this->GeoLocation.LatComponents.seconds / 3600; - if ('S' == this->GeoLocation.LatComponents.direction) - this->GeoLocation.Latitude = -this->GeoLocation.Latitude; - } - break; - - case 3: - // GPS east or west - this->GeoLocation.LonComponents.direction = *(buf + offs + 8); - if ('W' == this->GeoLocation.LonComponents.direction) - this->GeoLocation.Longitude = -this->GeoLocation.Longitude; - break; - - case 4: - // GPS longitude - if (format == 5 && length == 3) { - this->GeoLocation.LonComponents.degrees = - parseEXIFRational(buf + data + tiff_header_start, alignIntel); - this->GeoLocation.LonComponents.minutes = - parseEXIFRational(buf + data + tiff_header_start + 8, alignIntel); - this->GeoLocation.LonComponents.seconds = - parseEXIFRational(buf + data + tiff_header_start + 16, alignIntel); - this->GeoLocation.Longitude = - this->GeoLocation.LonComponents.degrees + - this->GeoLocation.LonComponents.minutes / 60 + - this->GeoLocation.LonComponents.seconds / 3600; - if ('W' == this->GeoLocation.LonComponents.direction) - this->GeoLocation.Longitude = -this->GeoLocation.Longitude; - } - break; - - case 5: - // GPS altitude reference (below or above sea level) - this->GeoLocation.AltitudeRef = *(buf + offs + 8); - if (1 == this->GeoLocation.AltitudeRef) - this->GeoLocation.Altitude = -this->GeoLocation.Altitude; - break; - - case 6: - // GPS altitude reference - if (format == 5) { - this->GeoLocation.Altitude = - parseEXIFRational(buf + data + tiff_header_start, alignIntel); - if (1 == this->GeoLocation.AltitudeRef) - this->GeoLocation.Altitude = -this->GeoLocation.Altitude; - } - break; - - case 7: - // GPS timestamp - if (format == 5 && length == 3) { - std::ostringstream os; - os.setf(std::ios_base::scientific, std::ios_base::floatfield); - os.precision(8); - os << parseEXIFRational(buf + data + tiff_header_start, alignIntel); - os << parseEXIFRational(buf + data + tiff_header_start + 8, alignIntel); - os << parseEXIFRational(buf + data + tiff_header_start + 16, alignIntel); - this->GeoLocation.GPSTimeStamp = os.str(); - } - break; - - case 29: - // GPS datestamp - if (format == 2) - this->GeoLocation.GPSDateStamp = parseEXIFString(buf, length, data, tiff_header_start, len, alignIntel); - break; - - case 30: - // GPS differential - if (format == 3) - this->GeoLocation.GPSDifferential = parse16(buf + data + tiff_header_start, alignIntel); - break; - } - offs += 12; - } - } - - return PARSE_EXIF_SUCCESS; -} - -void EXIFInfo::clear() { - // Strings - ImageDescription = ""; - Make = ""; - Model = ""; - Software = ""; - DateTime = ""; - DateTimeOriginal = ""; - DateTimeDigitized = ""; - SubSecTimeOriginal= ""; - Copyright = ""; - - // Shorts / unsigned / double - ByteAlign = 0; - Orientation = 0; - - BitsPerSample = 0; - ExposureTime = 0; - FNumber = 0; - ISOSpeedRatings = 0; - ShutterSpeedValue = 0; - ExposureBiasValue = 0; - SubjectDistance = 0; - FocalLength = 0; - FocalLengthIn35mm = 0; - FocalPlaneXResolution = 0; - FocalPlaneYResolution = 0; - FocalPlaneResolutionUnit = 0; - Flash = 0; - MeteringMode = 0; - ImageWidth = 0; - ImageHeight = 0; - - // Geolocation - GeoLocation.Latitude = 0; - GeoLocation.Longitude = 0; - GeoLocation.Altitude = 0; - GeoLocation.AltitudeRef = 0; - GeoLocation.GPSDifferential = 0; - GeoLocation.GPSTimeStamp = ""; - GeoLocation.GPSDateStamp = ""; - GeoLocation.LatComponents.degrees = 0; - GeoLocation.LatComponents.minutes = 0; - GeoLocation.LatComponents.seconds = 0; - GeoLocation.LatComponents.direction = 0; - GeoLocation.LonComponents.degrees = 0; - GeoLocation.LonComponents.minutes = 0; - GeoLocation.LonComponents.seconds = 0; - GeoLocation.LonComponents.direction = 0; -} diff --git a/libs/IO/EXIF.h b/libs/IO/EXIF.h deleted file mode 100644 index d39395c5d..000000000 --- a/libs/IO/EXIF.h +++ /dev/null @@ -1,157 +0,0 @@ -/************************************************************************** - exif.h -- A simple ISO C++ library to parse basic EXIF - information from a JPEG file. - - Based on the description of the EXIF file format at: - -- http://park2.wakwak.com/~tsuruzoh/Computer/Digicams/exif-e.html - -- http://www.media.mit.edu/pia/Research/deepview/exif.html - -- http://www.exif.org/Exif2-2.PDF - - Copyright (c) 2010-2013 Mayank Lahiri - mlahiri@gmail.com - All rights reserved. - - VERSION HISTORY: - ================ - - current: modified by cDc@seacave - - 2.1: Released July 2013 - -- fixed a bug where JPEGs without an EXIF SubIFD would not be parsed - -- fixed a bug in parsing GPS coordinate seconds - -- fixed makefile bug - -- added two pathological test images from Matt Galloway - http://www.galloway.me.uk/2012/01/uiimageorientation-exif-orientation-sample-images/ - -- split main parsing routine for easier integration into Firefox - - 2.0: Released February 2013 - -- complete rewrite - -- no new/delete - -- added GPS support - - 1.0: Released 2010 - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - -- Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - -- Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESS - OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN - NO EVENT SHALL THE FREEBSD PROJECT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY - OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, - EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ -#ifndef __EXIF_H -#define __EXIF_H - -#include - -// -// Class responsible for storing and parsing EXIF information from a JPEG blob -// -class IO_API EXIFInfo { - public: - // Parsing function for an entire JPEG image buffer. - // - // PARAM 'data': A pointer to a JPEG image. - // PARAM 'length': The length of the JPEG image. - // RETURN: PARSE_EXIF_SUCCESS (0) on succes with 'result' filled out - // error code otherwise, as defined by the PARSE_EXIF_ERROR_* macros - int parseFrom(const unsigned char *data, unsigned length); - int parseFrom(const std::string &data); - - // Parsing function for an EXIF segment. This is used internally by parseFrom() - // but can be called for special cases where only the EXIF section is - // available (i.e., a blob starting with the bytes "Exif\0\0"). - int parseFromEXIFSegment(const unsigned char *buf, unsigned len); - - // Set all data members to default values. - void clear(); - - // Data fields filled out by parseFrom() - char ByteAlign; // 0 = Motorola byte alignment, 1 = Intel - std::string ImageDescription; // Image description - std::string Make; // Camera manufacturer's name - std::string Model; // Camera model - unsigned short Orientation; // Image orientation, start of data corresponds to - // 0: unspecified in EXIF data - // 1: upper left of image - // 3: lower right of image - // 6: upper right of image - // 8: lower left of image - // 9: undefined - unsigned short BitsPerSample; // Number of bits per component - std::string Software; // Software used - std::string DateTime; // File change date and time - std::string DateTimeOriginal; // Original file date and time (may not exist) - std::string DateTimeDigitized; // Digitization date and time (may not exist) - std::string SubSecTimeOriginal; // Sub-second time that original picture was taken - std::string Copyright; // File copyright information - double ExposureTime; // Exposure time in seconds - double FNumber; // F/stop - unsigned short ISOSpeedRatings; // ISO speed - double ShutterSpeedValue; // Shutter speed (reciprocal of exposure time) - double ExposureBiasValue; // Exposure bias value in EV - double SubjectDistance; // Distance to focus point in meters - double FocalLength; // Focal length of lens in millimeters - unsigned short FocalLengthIn35mm; // Focal length in 35mm film - double FocalPlaneXResolution; // Indicates the number of pixels in the image width (X) direction per FocalPlaneResolutionUnit on the camera focal plane (may not exist) - double FocalPlaneYResolution; // Indicates the number of pixels in the image width (Y) direction per FocalPlaneResolutionUnit on the camera focal plane (may not exist) - unsigned short FocalPlaneResolutionUnit;// Indicates the unit for measuring FocalPlaneXResolution and FocalPlaneYResolution (may not exist) - // 0: unspecified in EXIF data - // 1: no absolute unit of measurement - // 2: inch - // 3: centimeter - char Flash; // 0 = no flash, 1 = flash used - unsigned short MeteringMode; // Metering mode - // 1: average - // 2: center weighted average - // 3: spot - // 4: multi-spot - // 5: multi-segment - unsigned ImageWidth; // Image width reported in EXIF data - unsigned ImageHeight; // Image height reported in EXIF data - struct Geolocation_t { // GPS information embedded in file - double Latitude; // Image latitude expressed as decimal - double Longitude; // Image longitude expressed as decimal - double Altitude; // Altitude in meters, relative to sea level - char AltitudeRef; // 0 = above sea level, -1 = below sea level - unsigned short GPSDifferential; // Indicates whether differential correction is applied to the GPS receiver (may not exist) - // 0: measurement without differential correction - // 1: differential correction applied - std::string GPSTimeStamp; // Indicates the time as UTC (Coordinated Universal Time) (may not exist) - std::string GPSDateStamp; // A character string recording date and time information relative to UTC (Coordinated Universal Time) YYYY:MM:DD (may not exist) - struct Coord_t { - double degrees; - double minutes; - double seconds; - char direction; - } LatComponents, LonComponents; // Latitude, Longitude expressed in deg/min/sec - } GeoLocation; - EXIFInfo() { - clear(); - } -}; - -// Parse was successful -#define PARSE_EXIF_SUCCESS 0 -// No JPEG markers found in buffer, possibly invalid JPEG file -#define PARSE_EXIF_ERROR_NO_JPEG 1982 -// No EXIF header found in JPEG file. -#define PARSE_EXIF_ERROR_NO_EXIF 1983 -// Byte alignment specified in EXIF file was unknown (not Motorola or Intel). -#define PARSE_EXIF_ERROR_UNKNOWN_BYTEALIGN 1984 -// EXIF header was found, but data was corrupted. -#define PARSE_EXIF_ERROR_CORRUPT 1985 - -#endif diff --git a/libs/IO/ImageEXIF.cpp b/libs/IO/ImageEXIF.cpp deleted file mode 100644 index 3e7c507a9..000000000 --- a/libs/IO/ImageEXIF.cpp +++ /dev/null @@ -1,224 +0,0 @@ -//////////////////////////////////////////////////////////////////// -// ImageEXIF.cpp -// -// Copyright 2007 cDc@seacave -// Distributed under the Boost Software License, Version 1.0 -// (See http://www.boost.org/LICENSE_1_0.txt) - -#include "Common.h" - -#ifdef _IMAGE_EXIF -#include "ImageEXIF.h" - -#if !defined(EXV_HAVE_STDINT_H) && _MSC_VER < 1600 -#define EXV_HAVE_STDINT_H -#endif -#include -#include - -using namespace SEACAVE; - - -// D E F I N E S /////////////////////////////////////////////////// - - -// S T R U C T S /////////////////////////////////////////////////// - -class Exiv2IO : public Exiv2::BasicIo { -public: - Exiv2IO(ISTREAM& f) : m_stream(f) {} - virtual ~Exiv2IO() {} - virtual int open() { return 0; } - virtual int close() { return 0; } - virtual long write(const Exiv2::byte* data, long wcount) { return 0; /*m_stream.write(data, wcount);*/ } - virtual long write(Exiv2::BasicIo& src) { return 0; } - virtual int putb(Exiv2::byte data) { return 0; } - virtual Exiv2::DataBuf read(long rcount) - { - Exiv2::DataBuf buf(rcount); - buf.size_ = (long)m_stream.read(buf.pData_, buf.size_); - if (buf.size_ == STREAM_ERROR) - buf.size_ = 0; - return buf; - } - virtual long read(Exiv2::byte* buf, long rcount) - { - const size_t ret = m_stream.read(buf, rcount); - if (ret == STREAM_ERROR) - return 0; - return (long)ret; - } - virtual int getb() - { - int ret = 0; - if (STREAM_ERROR == m_stream.read(&ret, 1)) - return EOF; - return ret; - } - virtual void transfer(Exiv2::BasicIo& src) {} - #ifdef _MSC_VER - virtual int seek(uint64_t offset, Position pos) - #else - virtual int seek(long offset, Position pos) - #endif - { - bool ret; - switch (pos) { - case beg: - ret = m_stream.setPos(offset); - break; - case cur: - ret = m_stream.setPos(m_stream.getPos()+offset); - break; - case end: - ret = m_stream.setPos(m_stream.getSize()-offset); - break; - } - if (ret == true) - return 0; - return -1; - } - virtual Exiv2::byte* mmap(bool isWriteable) { return NULL; } - virtual int munmap() { return 0; } - virtual long tell() const { return (long)m_stream.getPos(); } - virtual long size() const { return (long)m_stream.getSize(); } - virtual bool isopen() const { return true; } - virtual int error() const { return 0; } - virtual bool eof() const { return (m_stream.getPos() == m_stream.getSize()); } - virtual std::string path() const { return std::string(); } - #ifdef EXV_UNICODE_PATH - virtual std::wstring wpath() const { return std::wstring(); } - #endif - virtual Exiv2IO::AutoPtr temporary() const { return Exiv2IO::AutoPtr(); } - -protected: - ISTREAM& m_stream; -}; // class Exiv2IO - -struct Exiv2Struct { - Exiv2::Image::AutoPtr pImage; -}; - - - -// S T R U C T S /////////////////////////////////////////////////// - -CImageEXIF::CImageEXIF() -{ -} // Constructor - -CImageEXIF::~CImageEXIF() -{ - //clean up - Close(); -} // Destructor -/*----------------------------------------------------------------*/ - -void CImageEXIF::Close() -{ -} -/*----------------------------------------------------------------*/ - -HRESULT CImageEXIF::ReadHeader() -{ - m_state = new Exiv2Struct; - Exiv2Struct& state = *m_state; - state.pImage = Exiv2::ImageFactory::open(Exiv2::BasicIo::AutoPtr(new Exiv2IO(*((ISTREAM*)m_pStream)))); - if (state.pImage.get() == NULL) { - m_state.Release(); - return _FAIL; - } - state.pImage->readMetadata(); - return _OK; -} // ReadHeader -/*----------------------------------------------------------------*/ - -HRESULT CImageEXIF::ReadData(void* pData, PIXELFORMAT dataFormat, Size nStride, Size lineWidth) -{ - return _FAIL; -} // ReadData -/*----------------------------------------------------------------*/ - -HRESULT CImageEXIF::WriteHeader(PIXELFORMAT imageFormat, Size width, Size height, BYTE numLevels) -{ - //TODO: to implement the EXIF encoder - return _OK; -} // WriteHeader -/*----------------------------------------------------------------*/ - -HRESULT CImageEXIF::WriteData(void* pData, PIXELFORMAT dataFormat, Size nStride, Size lineWidth) -{ - //TODO: to implement the EXIF encoder - return _OK; -} // WriteData -/*----------------------------------------------------------------*/ - -bool CImageEXIF::HasEXIF() const -{ - if (m_state == NULL) - return false; - return !m_state->pImage->exifData().empty(); -} // HasEXIF -bool CImageEXIF::HasIPTC() const -{ - if (m_state == NULL) - return false; - return !m_state->pImage->iptcData().empty(); -} // HasIPTC -bool CImageEXIF::HasXMP() const -{ - if (m_state == NULL) - return false; - return !m_state->pImage->xmpData().empty(); -} // HasXMP -/*----------------------------------------------------------------*/ - -String CImageEXIF::ReadKeyEXIF(const String& name, bool bInterpret) const -{ - const Exiv2Struct& state = *m_state; - const Exiv2::ExifData& exifData = state.pImage->exifData(); - ASSERT(!exifData.empty()); - const Exiv2::ExifKey key("Exif."+name); - Exiv2::ExifData::const_iterator it = exifData.findKey(key); - if (it == exifData.end()) - return String(); - if (bInterpret) - return it->print(); - return it->value().toString(); -} // ReadKeyEXIF -/*----------------------------------------------------------------*/ - -void CImageEXIF::DumpAll() -{ - const Exiv2Struct& state = *m_state; - // print EXIF info - const Exiv2::ExifData& exifData = state.pImage->exifData(); - if (exifData.empty()) { - LOG(LT_IMAGE, _T("Info: Image constains no EXIF data")); - } else { - Exiv2::ExifData::const_iterator end = exifData.end(); - for (Exiv2::ExifData::const_iterator i = exifData.begin(); i != end; ++i) - LOG(LT_IMAGE, _T("%s %d %s %d %s"), i->key().c_str(), i->tag(), i->typeName(), i->count(), i->print().c_str()); - } - // print IPTC info - const Exiv2::IptcData& iptcData = state.pImage->iptcData(); - if (iptcData.empty()) { - LOG(LT_IMAGE, _T("Info: Image constains no IPTC data")); - } else { - Exiv2::IptcData::const_iterator end = iptcData.end(); - for (Exiv2::IptcData::const_iterator i = iptcData.begin(); i != end; ++i) - LOG(LT_IMAGE, _T("%s %d %s %d %s"), i->key().c_str(), i->tag(), i->typeName(), i->count(), i->print().c_str()); - } - // print XMP info - const Exiv2::XmpData& xmpData = state.pImage->xmpData(); - if (xmpData.empty()) { - LOG(LT_IMAGE, _T("Info: Image constains no XMP data")); - } else { - Exiv2::XmpData::const_iterator end = xmpData.end(); - for (Exiv2::XmpData::const_iterator i = xmpData.begin(); i != end; ++i) - LOG(LT_IMAGE, _T("%s %d %s %d %s"), i->key().c_str(), i->tag(), i->typeName(), i->count(), i->print().c_str()); - } -} // DumpAll -/*----------------------------------------------------------------*/ - -#endif // _IMAGE_EXIF diff --git a/libs/IO/ImageEXIF.h b/libs/IO/ImageEXIF.h deleted file mode 100644 index a4703ca6e..000000000 --- a/libs/IO/ImageEXIF.h +++ /dev/null @@ -1,54 +0,0 @@ -//////////////////////////////////////////////////////////////////// -// ImageEXIF.h -// -// Copyright 2007 cDc@seacave -// Distributed under the Boost Software License, Version 1.0 -// (See http://www.boost.org/LICENSE_1_0.txt) - -#ifndef __SEACAVE_IMAGEEXIV_H__ -#define __SEACAVE_IMAGEEXIV_H__ - - -// D E F I N E S /////////////////////////////////////////////////// - -struct Exiv2Struct; - - -// I N C L U D E S ///////////////////////////////////////////////// - -#include "Image.h" - - -namespace SEACAVE { - -// S T R U C T S /////////////////////////////////////////////////// - -class IO_API CImageEXIF : public CImage -{ -public: - CImageEXIF(); - virtual ~CImageEXIF(); - - void Close(); - - HRESULT ReadHeader(); - HRESULT ReadData(void*, PIXELFORMAT, Size nStride, Size lineWidth); - HRESULT WriteHeader(PIXELFORMAT, Size width, Size height, BYTE numLevels); - HRESULT WriteData(void*, PIXELFORMAT, Size nStride, Size lineWidth); - - bool HasEXIF() const; - bool HasIPTC() const; - bool HasXMP() const; - - String ReadKeyEXIF(const String& name, bool bInterpret=true) const; - - void DumpAll(); - -protected: - CAutoPtr m_state; -}; // class CImageEXIF -/*----------------------------------------------------------------*/ - -} // namespace SEACAVE - -#endif // __SEACAVE_IMAGEEXIV_H__ diff --git a/libs/IO/TinyXML2.cpp b/libs/IO/TinyXML2.cpp deleted file mode 100644 index 272decb9a..000000000 --- a/libs/IO/TinyXML2.cpp +++ /dev/null @@ -1,2257 +0,0 @@ -/* -Original code by Lee Thomason (www.grinninglizard.com) - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any -damages arising from the use of this software. - -Permission is granted to anyone to use this software for any -purpose, including commercial applications, and to alter it and -redistribute it freely, subject to the following restrictions: - -1. The origin of this software must not be misrepresented; you must -not claim that you wrote the original software. If you use this -software in a product, an acknowledgment in the product documentation -would be appreciated but is not required. - -2. Altered source versions must be plainly marked as such, and -must not be misrepresented as being the original software. - -3. This notice may not be removed or altered from any source -distribution. -*/ - -#include "Common.h" -#include "TinyXML2.h" - -#ifdef ANDROID_NDK -# include -#else -# include -#endif - -static const char LINE_FEED = (char)0x0a; // all line endings are normalized to LF -static const char LF = LINE_FEED; -static const char CARRIAGE_RETURN = (char)0x0d; // CR gets filtered out -static const char CR = CARRIAGE_RETURN; -static const char SINGLE_QUOTE = '\''; -static const char DOUBLE_QUOTE = '\"'; - -// Bunch of unicode info at: -// http://www.unicode.org/faq/utf_bom.html -// ef bb bf (Microsoft "lead bytes") - designates UTF-8 - -static const unsigned char TIXML_UTF_LEAD_0 = 0xefU; -static const unsigned char TIXML_UTF_LEAD_1 = 0xbbU; -static const unsigned char TIXML_UTF_LEAD_2 = 0xbfU; - -namespace tinyxml2 -{ - -struct Entity { - const char* pattern; - int length; - char value; -}; - -static const int NUM_ENTITIES = 5; -static const Entity entities[NUM_ENTITIES] = { - { "quot", 4, DOUBLE_QUOTE }, - { "amp", 3, '&' }, - { "apos", 4, SINGLE_QUOTE }, - { "lt", 2, '<' }, - { "gt", 2, '>' } -}; - - -StrPair::~StrPair() -{ - Reset(); -} - - -void StrPair::Reset() -{ - if ( _flags & NEEDS_DELETE ) { - delete [] _start; - } - _flags = 0; - _start = 0; - _end = 0; -} - - -void StrPair::SetStr( const char* str, int flags ) -{ - Reset(); - size_t len = strlen( str ); - _start = new char[ len+1 ]; - memcpy( _start, str, len+1 ); - _end = _start + len; - _flags = flags | NEEDS_DELETE; -} - - -char* StrPair::ParseText( char* p, const char* endTag, int strFlags ) -{ - TIXMLASSERT( endTag && *endTag ); - - char* start = p; - char endChar = *endTag; - size_t length = strlen( endTag ); - - // Inner loop of text parsing. - while ( *p ) { - if ( *p == endChar && strncmp( p, endTag, length ) == 0 ) { - Set( start, p, strFlags ); - return p + length; - } - ++p; - } - return 0; -} - - -char* StrPair::ParseName( char* p ) -{ - if ( !p || !(*p) ) { - return 0; - } - - char* const start = p; - - while( *p && ( p == start ? XMLUtil::IsNameStartChar( *p ) : XMLUtil::IsNameChar( *p ) )) { - ++p; - } - - if ( p > start ) { - Set( start, p, 0 ); - return p; - } - return 0; -} - - -void StrPair::CollapseWhitespace() -{ - // Adjusting _start would cause undefined behavior on delete[] - TIXMLASSERT( ( _flags & NEEDS_DELETE ) == 0 ); - // Trim leading space. - _start = XMLUtil::SkipWhiteSpace( _start ); - - if ( _start && *_start ) { - char* p = _start; // the read pointer - char* q = _start; // the write pointer - - while( *p ) { - if ( XMLUtil::IsWhiteSpace( *p )) { - p = XMLUtil::SkipWhiteSpace( p ); - if ( *p == 0 ) { - break; // don't write to q; this trims the trailing space. - } - *q = ' '; - ++q; - } - *q = *p; - ++q; - ++p; - } - *q = 0; - } -} - - -const char* StrPair::GetStr() -{ - if ( _flags & NEEDS_FLUSH ) { - *_end = 0; - _flags ^= NEEDS_FLUSH; - - if ( _flags ) { - char* p = _start; // the read pointer - char* q = _start; // the write pointer - - while( p < _end ) { - if ( (_flags & NEEDS_NEWLINE_NORMALIZATION) && *p == CR ) { - // CR-LF pair becomes LF - // CR alone becomes LF - // LF-CR becomes LF - if ( *(p+1) == LF ) { - p += 2; - } - else { - ++p; - } - *q++ = LF; - } - else if ( (_flags & NEEDS_NEWLINE_NORMALIZATION) && *p == LF ) { - if ( *(p+1) == CR ) { - p += 2; - } - else { - ++p; - } - *q++ = LF; - } - else if ( (_flags & NEEDS_ENTITY_PROCESSING) && *p == '&' ) { - // Entities handled by tinyXML2: - // - special entities in the entity table [in/out] - // - numeric character reference [in] - // 中 or 中 - - if ( *(p+1) == '#' ) { - const int buflen = 10; - char buf[buflen] = { 0 }; - int len = 0; - p = const_cast( XMLUtil::GetCharacterRef( p, buf, &len ) ); - TIXMLASSERT( 0 <= len && len <= buflen ); - TIXMLASSERT( q + len <= p ); - memcpy( q, buf, len ); - q += len; - } - else { - int i=0; - for(; i(p); - // Check for BOM: - if ( *(pu+0) == TIXML_UTF_LEAD_0 - && *(pu+1) == TIXML_UTF_LEAD_1 - && *(pu+2) == TIXML_UTF_LEAD_2 ) { - *bom = true; - p += 3; - } - return p; -} - - -void XMLUtil::ConvertUTF32ToUTF8( unsigned long input, char* output, int* length ) -{ - const unsigned long BYTE_MASK = 0xBF; - const unsigned long BYTE_MARK = 0x80; - const unsigned long FIRST_BYTE_MARK[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; - - if (input < 0x80) { - *length = 1; - } - else if ( input < 0x800 ) { - *length = 2; - } - else if ( input < 0x10000 ) { - *length = 3; - } - else if ( input < 0x200000 ) { - *length = 4; - } - else { - *length = 0; // This code won't covert this correctly anyway. - return; - } - - output += *length; - - // Scary scary fall throughs. - switch (*length) { - case 4: - --output; - *output = (char)((input | BYTE_MARK) & BYTE_MASK); - input >>= 6; - case 3: - --output; - *output = (char)((input | BYTE_MARK) & BYTE_MASK); - input >>= 6; - case 2: - --output; - *output = (char)((input | BYTE_MARK) & BYTE_MASK); - input >>= 6; - case 1: - --output; - *output = (char)(input | FIRST_BYTE_MARK[*length]); - default: - break; - } -} - - -const char* XMLUtil::GetCharacterRef( const char* p, char* value, int* length ) -{ - // Presume an entity, and pull it out. - *length = 0; - - if ( *(p+1) == '#' && *(p+2) ) { - unsigned long ucs = 0; - ptrdiff_t delta = 0; - unsigned mult = 1; - - if ( *(p+2) == 'x' ) { - // Hexadecimal. - if ( !*(p+3) ) { - return 0; - } - - const char* q = p+3; - q = strchr( q, ';' ); - - if ( !q || !*q ) { - return 0; - } - - delta = q-p; - --q; - - while ( *q != 'x' ) { - if ( *q >= '0' && *q <= '9' ) { - ucs += mult * (*q - '0'); - } - else if ( *q >= 'a' && *q <= 'f' ) { - ucs += mult * (*q - 'a' + 10); - } - else if ( *q >= 'A' && *q <= 'F' ) { - ucs += mult * (*q - 'A' + 10 ); - } - else { - return 0; - } - mult *= 16; - --q; - } - } - else { - // Decimal. - if ( !*(p+2) ) { - return 0; - } - - const char* q = p+2; - q = strchr( q, ';' ); - - if ( !q || !*q ) { - return 0; - } - - delta = q-p; - --q; - - while ( *q != '#' ) { - if ( *q >= '0' && *q <= '9' ) { - ucs += mult * (*q - '0'); - } - else { - return 0; - } - mult *= 10; - --q; - } - } - // convert the UCS to UTF-8 - ConvertUTF32ToUTF8( ucs, value, length ); - return p + delta + 1; - } - return p+1; -} - - -void XMLUtil::ToStr( int v, char* buffer, int bufferSize ) -{ - TIXML_SNPRINTF( buffer, bufferSize, "%d", v ); -} - - -void XMLUtil::ToStr( unsigned v, char* buffer, int bufferSize ) -{ - TIXML_SNPRINTF( buffer, bufferSize, "%u", v ); -} - - -void XMLUtil::ToStr( bool v, char* buffer, int bufferSize ) -{ - TIXML_SNPRINTF( buffer, bufferSize, "%d", v ? 1 : 0 ); -} - -/* - ToStr() of a number is a very tricky topic. - https://github.com/leethomason/tinyxml2/issues/106 -*/ -void XMLUtil::ToStr( float v, char* buffer, int bufferSize ) -{ - TIXML_SNPRINTF( buffer, bufferSize, "%.8g", v ); -} - - -void XMLUtil::ToStr( double v, char* buffer, int bufferSize ) -{ - TIXML_SNPRINTF( buffer, bufferSize, "%.17g", v ); -} - - -bool XMLUtil::ToInt( const char* str, int* value ) -{ - if ( TIXML_SSCANF( str, "%d", value ) == 1 ) { - return true; - } - return false; -} - -bool XMLUtil::ToUnsigned( const char* str, unsigned *value ) -{ - if ( TIXML_SSCANF( str, "%u", value ) == 1 ) { - return true; - } - return false; -} - -bool XMLUtil::ToBool( const char* str, bool* value ) -{ - int ival = 0; - if ( ToInt( str, &ival )) { - *value = (ival==0) ? false : true; - return true; - } - if ( StringEqual( str, "true" ) ) { - *value = true; - return true; - } - else if ( StringEqual( str, "false" ) ) { - *value = false; - return true; - } - return false; -} - - -bool XMLUtil::ToFloat( const char* str, float* value ) -{ - if ( TIXML_SSCANF( str, "%f", value ) == 1 ) { - return true; - } - return false; -} - -bool XMLUtil::ToDouble( const char* str, double* value ) -{ - if ( TIXML_SSCANF( str, "%lf", value ) == 1 ) { - return true; - } - return false; -} - - -char* XMLDocument::Identify( char* p, XMLNode** node ) -{ - char* const start = p; - p = XMLUtil::SkipWhiteSpace( p ); - if( !p || !*p ) { - return p; - } - - // What is this thing? - // These strings define the matching patters: - static const char* xmlHeader = { "_memPool = &_commentPool; - p += xmlHeaderLen; - } - else if ( XMLUtil::StringEqual( p, commentHeader, commentHeaderLen ) ) { - returnNode = new (_commentPool.Alloc()) XMLComment( this ); - returnNode->_memPool = &_commentPool; - p += commentHeaderLen; - } - else if ( XMLUtil::StringEqual( p, cdataHeader, cdataHeaderLen ) ) { - XMLText* text = new (_textPool.Alloc()) XMLText( this ); - returnNode = text; - returnNode->_memPool = &_textPool; - p += cdataHeaderLen; - text->SetCData( true ); - } - else if ( XMLUtil::StringEqual( p, dtdHeader, dtdHeaderLen ) ) { - returnNode = new (_commentPool.Alloc()) XMLUnknown( this ); - returnNode->_memPool = &_commentPool; - p += dtdHeaderLen; - } - else if ( XMLUtil::StringEqual( p, elementHeader, elementHeaderLen ) ) { - returnNode = new (_elementPool.Alloc()) XMLElement( this ); - returnNode->_memPool = &_elementPool; - p += elementHeaderLen; - } - else { - returnNode = new (_textPool.Alloc()) XMLText( this ); - returnNode->_memPool = &_textPool; - p = start; // Back it up, all the text counts. - } - - *node = returnNode; - return p; -} - - -bool XMLDocument::Accept( XMLVisitor* visitor ) const -{ - if ( visitor->VisitEnter( *this ) ) { - for ( const XMLNode* node=FirstChild(); node; node=node->NextSibling() ) { - if ( !node->Accept( visitor ) ) { - break; - } - } - } - return visitor->VisitExit( *this ); -} - - -// --------- XMLNode ----------- // - -XMLNode::XMLNode( XMLDocument* doc ) : - _document( doc ), - _parent( 0 ), - _firstChild( 0 ), _lastChild( 0 ), - _prev( 0 ), _next( 0 ), - _memPool( 0 ) -{ -} - - -XMLNode::~XMLNode() -{ - DeleteChildren(); - if ( _parent ) { - _parent->Unlink( this ); - } -} - -const char* XMLNode::Value() const -{ - return _value.GetStr(); -} - -void XMLNode::SetValue( const char* str, bool staticMem ) -{ - if ( staticMem ) { - _value.SetInternedStr( str ); - } - else { - _value.SetStr( str ); - } -} - - -void XMLNode::DeleteChildren() -{ - while( _firstChild ) { - XMLNode* node = _firstChild; - Unlink( node ); - - DeleteNode( node ); - } - _firstChild = _lastChild = 0; -} - - -void XMLNode::Unlink( XMLNode* child ) -{ - if ( child == _firstChild ) { - _firstChild = _firstChild->_next; - } - if ( child == _lastChild ) { - _lastChild = _lastChild->_prev; - } - - if ( child->_prev ) { - child->_prev->_next = child->_next; - } - if ( child->_next ) { - child->_next->_prev = child->_prev; - } - child->_parent = 0; -} - - -void XMLNode::DeleteChild( XMLNode* node ) -{ - TIXMLASSERT( node->_parent == this ); - DeleteNode( node ); -} - - -XMLNode* XMLNode::InsertEndChild( XMLNode* addThis ) -{ - if (addThis->_document != _document) - return 0; - - if (addThis->_parent) - addThis->_parent->Unlink( addThis ); - else - addThis->_memPool->SetTracked(); - - if ( _lastChild ) { - TIXMLASSERT( _firstChild ); - TIXMLASSERT( _lastChild->_next == 0 ); - _lastChild->_next = addThis; - addThis->_prev = _lastChild; - _lastChild = addThis; - - addThis->_next = 0; - } - else { - TIXMLASSERT( _firstChild == 0 ); - _firstChild = _lastChild = addThis; - - addThis->_prev = 0; - addThis->_next = 0; - } - addThis->_parent = this; - return addThis; -} - - -XMLNode* XMLNode::InsertFirstChild( XMLNode* addThis ) -{ - if (addThis->_document != _document) - return 0; - - if (addThis->_parent) - addThis->_parent->Unlink( addThis ); - else - addThis->_memPool->SetTracked(); - - if ( _firstChild ) { - TIXMLASSERT( _lastChild ); - TIXMLASSERT( _firstChild->_prev == 0 ); - - _firstChild->_prev = addThis; - addThis->_next = _firstChild; - _firstChild = addThis; - - addThis->_prev = 0; - } - else { - TIXMLASSERT( _lastChild == 0 ); - _firstChild = _lastChild = addThis; - - addThis->_prev = 0; - addThis->_next = 0; - } - addThis->_parent = this; - return addThis; -} - - -XMLNode* XMLNode::InsertAfterChild( XMLNode* afterThis, XMLNode* addThis ) -{ - if (addThis->_document != _document) - return 0; - - TIXMLASSERT( afterThis->_parent == this ); - - if ( afterThis->_parent != this ) { - return 0; - } - - if ( afterThis->_next == 0 ) { - // The last node or the only node. - return InsertEndChild( addThis ); - } - if (addThis->_parent) - addThis->_parent->Unlink( addThis ); - else - addThis->_memPool->SetTracked(); - addThis->_prev = afterThis; - addThis->_next = afterThis->_next; - afterThis->_next->_prev = addThis; - afterThis->_next = addThis; - addThis->_parent = this; - return addThis; -} - - - - -const XMLElement* XMLNode::FirstChildElement( const char* value ) const -{ - for( XMLNode* node=_firstChild; node; node=node->_next ) { - XMLElement* element = node->ToElement(); - if ( element ) { - if ( !value || XMLUtil::StringEqual( element->Name(), value ) ) { - return element; - } - } - } - return 0; -} - - -const XMLElement* XMLNode::LastChildElement( const char* value ) const -{ - for( XMLNode* node=_lastChild; node; node=node->_prev ) { - XMLElement* element = node->ToElement(); - if ( element ) { - if ( !value || XMLUtil::StringEqual( element->Name(), value ) ) { - return element; - } - } - } - return 0; -} - - -const XMLElement* XMLNode::NextSiblingElement( const char* value ) const -{ - for( XMLNode* node=this->_next; node; node = node->_next ) { - const XMLElement* element = node->ToElement(); - if ( element - && (!value || XMLUtil::StringEqual( value, node->Value() ))) { - return element; - } - } - return 0; -} - - -const XMLElement* XMLNode::PreviousSiblingElement( const char* value ) const -{ - for( XMLNode* node=_prev; node; node = node->_prev ) { - const XMLElement* element = node->ToElement(); - if ( element - && (!value || XMLUtil::StringEqual( value, node->Value() ))) { - return element; - } - } - return 0; -} - - -char* XMLNode::ParseDeep( char* p, StrPair* parentEnd ) -{ - // This is a recursive method, but thinking about it "at the current level" - // it is a pretty simple flat list: - // - // - // - // With a special case: - // - // - // - // - // Where the closing element (/foo) *must* be the next thing after the opening - // element, and the names must match. BUT the tricky bit is that the closing - // element will be read by the child. - // - // 'endTag' is the end tag for this node, it is returned by a call to a child. - // 'parentEnd' is the end tag for the parent, which is filled in and returned. - - while( p && *p ) { - XMLNode* node = 0; - - p = _document->Identify( p, &node ); - if ( p == 0 || node == 0 ) { - break; - } - - StrPair endTag; - p = node->ParseDeep( p, &endTag ); - if ( !p ) { - DeleteNode( node ); - node = 0; - if ( !_document->Error() ) { - _document->SetError( XML_ERROR_PARSING, 0, 0 ); - } - break; - } - - XMLElement* ele = node->ToElement(); - // We read the end tag. Return it to the parent. - if ( ele && ele->ClosingType() == XMLElement::CLOSING ) { - if ( parentEnd ) { - *parentEnd = ele->_value; - } - node->_memPool->SetTracked(); // created and then immediately deleted. - DeleteNode( node ); - return p; - } - - // Handle an end tag returned to this level. - // And handle a bunch of annoying errors. - if ( ele ) { - bool mismatch = false; - if ( endTag.Empty() && ele->ClosingType() == XMLElement::OPEN ) { - mismatch = true; - } - else if ( !endTag.Empty() && ele->ClosingType() != XMLElement::OPEN ) { - mismatch = true; - } - else if ( !endTag.Empty() ) { - if ( !XMLUtil::StringEqual( endTag.GetStr(), node->Value() )) { - mismatch = true; - } - } - if ( mismatch ) { - _document->SetError( XML_ERROR_MISMATCHED_ELEMENT, node->Value(), 0 ); - p = 0; - } - } - if ( p == 0 ) { - DeleteNode( node ); - node = 0; - } - if ( node ) { - this->InsertEndChild( node ); - } - } - return 0; -} - -void XMLNode::DeleteNode( XMLNode* node ) -{ - if ( node == 0 ) { - return; - } - MemPool* pool = node->_memPool; - node->~XMLNode(); - pool->Free( node ); -} - -// --------- XMLText ---------- // -char* XMLText::ParseDeep( char* p, StrPair* ) -{ - const char* start = p; - if ( this->CData() ) { - p = _value.ParseText( p, "]]>", StrPair::NEEDS_NEWLINE_NORMALIZATION ); - if ( !p ) { - _document->SetError( XML_ERROR_PARSING_CDATA, start, 0 ); - } - return p; - } - else { - int flags = _document->ProcessEntities() ? StrPair::TEXT_ELEMENT : StrPair::TEXT_ELEMENT_LEAVE_ENTITIES; - if ( _document->WhitespaceMode() == COLLAPSE_WHITESPACE ) { - flags |= StrPair::COLLAPSE_WHITESPACE; - } - - p = _value.ParseText( p, "<", flags ); - if ( !p ) { - _document->SetError( XML_ERROR_PARSING_TEXT, start, 0 ); - } - if ( p && *p ) { - return p-1; - } - } - return 0; -} - - -XMLNode* XMLText::ShallowClone( XMLDocument* doc ) const -{ - if ( !doc ) { - doc = _document; - } - XMLText* text = doc->NewText( Value() ); // fixme: this will always allocate memory. Intern? - text->SetCData( this->CData() ); - return text; -} - - -bool XMLText::ShallowEqual( const XMLNode* compare ) const -{ - const XMLText* text = compare->ToText(); - return ( text && XMLUtil::StringEqual( text->Value(), Value() ) ); -} - - -bool XMLText::Accept( XMLVisitor* visitor ) const -{ - return visitor->Visit( *this ); -} - - -// --------- XMLComment ---------- // - -XMLComment::XMLComment( XMLDocument* doc ) : XMLNode( doc ) -{ -} - - -XMLComment::~XMLComment() -{ -} - - -char* XMLComment::ParseDeep( char* p, StrPair* ) -{ - // Comment parses as text. - const char* start = p; - p = _value.ParseText( p, "-->", StrPair::COMMENT ); - if ( p == 0 ) { - _document->SetError( XML_ERROR_PARSING_COMMENT, start, 0 ); - } - return p; -} - - -XMLNode* XMLComment::ShallowClone( XMLDocument* doc ) const -{ - if ( !doc ) { - doc = _document; - } - XMLComment* comment = doc->NewComment( Value() ); // fixme: this will always allocate memory. Intern? - return comment; -} - - -bool XMLComment::ShallowEqual( const XMLNode* compare ) const -{ - const XMLComment* comment = compare->ToComment(); - return ( comment && XMLUtil::StringEqual( comment->Value(), Value() )); -} - - -bool XMLComment::Accept( XMLVisitor* visitor ) const -{ - return visitor->Visit( *this ); -} - - -// --------- XMLDeclaration ---------- // - -XMLDeclaration::XMLDeclaration( XMLDocument* doc ) : XMLNode( doc ) -{ -} - - -XMLDeclaration::~XMLDeclaration() -{ - //printf( "~XMLDeclaration\n" ); -} - - -char* XMLDeclaration::ParseDeep( char* p, StrPair* ) -{ - // Declaration parses as text. - const char* start = p; - p = _value.ParseText( p, "?>", StrPair::NEEDS_NEWLINE_NORMALIZATION ); - if ( p == 0 ) { - _document->SetError( XML_ERROR_PARSING_DECLARATION, start, 0 ); - } - return p; -} - - -XMLNode* XMLDeclaration::ShallowClone( XMLDocument* doc ) const -{ - if ( !doc ) { - doc = _document; - } - XMLDeclaration* dec = doc->NewDeclaration( Value() ); // fixme: this will always allocate memory. Intern? - return dec; -} - - -bool XMLDeclaration::ShallowEqual( const XMLNode* compare ) const -{ - const XMLDeclaration* declaration = compare->ToDeclaration(); - return ( declaration && XMLUtil::StringEqual( declaration->Value(), Value() )); -} - - - -bool XMLDeclaration::Accept( XMLVisitor* visitor ) const -{ - return visitor->Visit( *this ); -} - -// --------- XMLUnknown ---------- // - -XMLUnknown::XMLUnknown( XMLDocument* doc ) : XMLNode( doc ) -{ -} - - -XMLUnknown::~XMLUnknown() -{ -} - - -char* XMLUnknown::ParseDeep( char* p, StrPair* ) -{ - // Unknown parses as text. - const char* start = p; - - p = _value.ParseText( p, ">", StrPair::NEEDS_NEWLINE_NORMALIZATION ); - if ( !p ) { - _document->SetError( XML_ERROR_PARSING_UNKNOWN, start, 0 ); - } - return p; -} - - -XMLNode* XMLUnknown::ShallowClone( XMLDocument* doc ) const -{ - if ( !doc ) { - doc = _document; - } - XMLUnknown* text = doc->NewUnknown( Value() ); // fixme: this will always allocate memory. Intern? - return text; -} - - -bool XMLUnknown::ShallowEqual( const XMLNode* compare ) const -{ - const XMLUnknown* unknown = compare->ToUnknown(); - return ( unknown && XMLUtil::StringEqual( unknown->Value(), Value() )); -} - - -bool XMLUnknown::Accept( XMLVisitor* visitor ) const -{ - return visitor->Visit( *this ); -} - -// --------- XMLAttribute ---------- // - -const char* XMLAttribute::Name() const -{ - return _name.GetStr(); -} - -const char* XMLAttribute::Value() const -{ - return _value.GetStr(); -} - -char* XMLAttribute::ParseDeep( char* p, bool processEntities ) -{ - // Parse using the name rules: bug fix, was using ParseText before - p = _name.ParseName( p ); - if ( !p || !*p ) { - return 0; - } - - // Skip white space before = - p = XMLUtil::SkipWhiteSpace( p ); - if ( !p || *p != '=' ) { - return 0; - } - - ++p; // move up to opening quote - p = XMLUtil::SkipWhiteSpace( p ); - if ( *p != '\"' && *p != '\'' ) { - return 0; - } - - char endTag[2] = { *p, 0 }; - ++p; // move past opening quote - - p = _value.ParseText( p, endTag, processEntities ? StrPair::ATTRIBUTE_VALUE : StrPair::ATTRIBUTE_VALUE_LEAVE_ENTITIES ); - return p; -} - - -void XMLAttribute::SetName( const char* n ) -{ - _name.SetStr( n ); -} - - -XMLError XMLAttribute::QueryIntValue( int* value ) const -{ - if ( XMLUtil::ToInt( Value(), value )) { - return XML_NO_ERROR; - } - return XML_WRONG_ATTRIBUTE_TYPE; -} - - -XMLError XMLAttribute::QueryUnsignedValue( unsigned int* value ) const -{ - if ( XMLUtil::ToUnsigned( Value(), value )) { - return XML_NO_ERROR; - } - return XML_WRONG_ATTRIBUTE_TYPE; -} - - -XMLError XMLAttribute::QueryBoolValue( bool* value ) const -{ - if ( XMLUtil::ToBool( Value(), value )) { - return XML_NO_ERROR; - } - return XML_WRONG_ATTRIBUTE_TYPE; -} - - -XMLError XMLAttribute::QueryFloatValue( float* value ) const -{ - if ( XMLUtil::ToFloat( Value(), value )) { - return XML_NO_ERROR; - } - return XML_WRONG_ATTRIBUTE_TYPE; -} - - -XMLError XMLAttribute::QueryDoubleValue( double* value ) const -{ - if ( XMLUtil::ToDouble( Value(), value )) { - return XML_NO_ERROR; - } - return XML_WRONG_ATTRIBUTE_TYPE; -} - - -void XMLAttribute::SetAttribute( const char* v ) -{ - _value.SetStr( v ); -} - - -void XMLAttribute::SetAttribute( int v ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( v, buf, BUF_SIZE ); - _value.SetStr( buf ); -} - - -void XMLAttribute::SetAttribute( unsigned v ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( v, buf, BUF_SIZE ); - _value.SetStr( buf ); -} - - -void XMLAttribute::SetAttribute( bool v ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( v, buf, BUF_SIZE ); - _value.SetStr( buf ); -} - -void XMLAttribute::SetAttribute( double v ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( v, buf, BUF_SIZE ); - _value.SetStr( buf ); -} - -void XMLAttribute::SetAttribute( float v ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( v, buf, BUF_SIZE ); - _value.SetStr( buf ); -} - - -// --------- XMLElement ---------- // -XMLElement::XMLElement( XMLDocument* doc ) : XMLNode( doc ), - _closingType( 0 ), - _rootAttribute( 0 ) -{ -} - - -XMLElement::~XMLElement() -{ - while( _rootAttribute ) { - XMLAttribute* next = _rootAttribute->_next; - DeleteAttribute( _rootAttribute ); - _rootAttribute = next; - } -} - - -XMLAttribute* XMLElement::FindAttribute( const char* name ) -{ - for( XMLAttribute* a = _rootAttribute; a; a = a->_next ) { - if ( XMLUtil::StringEqual( a->Name(), name ) ) { - return a; - } - } - return 0; -} - - -const XMLAttribute* XMLElement::FindAttribute( const char* name ) const -{ - for( XMLAttribute* a = _rootAttribute; a; a = a->_next ) { - if ( XMLUtil::StringEqual( a->Name(), name ) ) { - return a; - } - } - return 0; -} - - -const char* XMLElement::Attribute( const char* name, const char* value ) const -{ - const XMLAttribute* a = FindAttribute( name ); - if ( !a ) { - return 0; - } - if ( !value || XMLUtil::StringEqual( a->Value(), value )) { - return a->Value(); - } - return 0; -} - - -const char* XMLElement::GetText() const -{ - if ( FirstChild() && FirstChild()->ToText() ) { - return FirstChild()->Value(); - } - return 0; -} - - -void XMLElement::SetText( const char* inText ) -{ - if ( FirstChild() && FirstChild()->ToText() ) - FirstChild()->SetValue( inText ); - else { - XMLText* theText = GetDocument()->NewText( inText ); - InsertFirstChild( theText ); - } -} - - -void XMLElement::SetText( int v ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( v, buf, BUF_SIZE ); - SetText( buf ); -} - - -void XMLElement::SetText( unsigned v ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( v, buf, BUF_SIZE ); - SetText( buf ); -} - - -void XMLElement::SetText( bool v ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( v, buf, BUF_SIZE ); - SetText( buf ); -} - - -void XMLElement::SetText( float v ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( v, buf, BUF_SIZE ); - SetText( buf ); -} - - -void XMLElement::SetText( double v ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( v, buf, BUF_SIZE ); - SetText( buf ); -} - - -XMLError XMLElement::QueryIntText( int* ival ) const -{ - if ( FirstChild() && FirstChild()->ToText() ) { - const char* t = FirstChild()->Value(); - if ( XMLUtil::ToInt( t, ival ) ) { - return XML_SUCCESS; - } - return XML_CAN_NOT_CONVERT_TEXT; - } - return XML_NO_TEXT_NODE; -} - - -XMLError XMLElement::QueryUnsignedText( unsigned* uval ) const -{ - if ( FirstChild() && FirstChild()->ToText() ) { - const char* t = FirstChild()->Value(); - if ( XMLUtil::ToUnsigned( t, uval ) ) { - return XML_SUCCESS; - } - return XML_CAN_NOT_CONVERT_TEXT; - } - return XML_NO_TEXT_NODE; -} - - -XMLError XMLElement::QueryBoolText( bool* bval ) const -{ - if ( FirstChild() && FirstChild()->ToText() ) { - const char* t = FirstChild()->Value(); - if ( XMLUtil::ToBool( t, bval ) ) { - return XML_SUCCESS; - } - return XML_CAN_NOT_CONVERT_TEXT; - } - return XML_NO_TEXT_NODE; -} - - -XMLError XMLElement::QueryDoubleText( double* dval ) const -{ - if ( FirstChild() && FirstChild()->ToText() ) { - const char* t = FirstChild()->Value(); - if ( XMLUtil::ToDouble( t, dval ) ) { - return XML_SUCCESS; - } - return XML_CAN_NOT_CONVERT_TEXT; - } - return XML_NO_TEXT_NODE; -} - - -XMLError XMLElement::QueryFloatText( float* fval ) const -{ - if ( FirstChild() && FirstChild()->ToText() ) { - const char* t = FirstChild()->Value(); - if ( XMLUtil::ToFloat( t, fval ) ) { - return XML_SUCCESS; - } - return XML_CAN_NOT_CONVERT_TEXT; - } - return XML_NO_TEXT_NODE; -} - - - -XMLAttribute* XMLElement::FindOrCreateAttribute( const char* name ) -{ - XMLAttribute* last = 0; - XMLAttribute* attrib = 0; - for( attrib = _rootAttribute; - attrib; - last = attrib, attrib = attrib->_next ) { - if ( XMLUtil::StringEqual( attrib->Name(), name ) ) { - break; - } - } - if ( !attrib ) { - attrib = new (_document->_attributePool.Alloc() ) XMLAttribute(); - attrib->_memPool = &_document->_attributePool; - if ( last ) { - last->_next = attrib; - } - else { - _rootAttribute = attrib; - } - attrib->SetName( name ); - attrib->_memPool->SetTracked(); // always created and linked. - } - return attrib; -} - - -void XMLElement::DeleteAttribute( const char* name ) -{ - XMLAttribute* prev = 0; - for( XMLAttribute* a=_rootAttribute; a; a=a->_next ) { - if ( XMLUtil::StringEqual( name, a->Name() ) ) { - if ( prev ) { - prev->_next = a->_next; - } - else { - _rootAttribute = a->_next; - } - DeleteAttribute( a ); - break; - } - prev = a; - } -} - - -char* XMLElement::ParseAttributes( char* p ) -{ - const char* start = p; - XMLAttribute* prevAttribute = 0; - - // Read the attributes. - while( p ) { - p = XMLUtil::SkipWhiteSpace( p ); - if ( !p || !(*p) ) { - _document->SetError( XML_ERROR_PARSING_ELEMENT, start, Name() ); - return 0; - } - - // attribute. - if (XMLUtil::IsNameStartChar( *p ) ) { - XMLAttribute* attrib = new (_document->_attributePool.Alloc() ) XMLAttribute(); - attrib->_memPool = &_document->_attributePool; - attrib->_memPool->SetTracked(); - - p = attrib->ParseDeep( p, _document->ProcessEntities() ); - if ( !p || Attribute( attrib->Name() ) ) { - DeleteAttribute( attrib ); - _document->SetError( XML_ERROR_PARSING_ATTRIBUTE, start, p ); - return 0; - } - // There is a minor bug here: if the attribute in the source xml - // document is duplicated, it will not be detected and the - // attribute will be doubly added. However, tracking the 'prevAttribute' - // avoids re-scanning the attribute list. Preferring performance for - // now, may reconsider in the future. - if ( prevAttribute ) { - prevAttribute->_next = attrib; - } - else { - _rootAttribute = attrib; - } - prevAttribute = attrib; - } - // end of the tag - else if ( *p == '/' && *(p+1) == '>' ) { - _closingType = CLOSED; - return p+2; // done; sealed element. - } - // end of the tag - else if ( *p == '>' ) { - ++p; - break; - } - else { - _document->SetError( XML_ERROR_PARSING_ELEMENT, start, p ); - return 0; - } - } - return p; -} - -void XMLElement::DeleteAttribute( XMLAttribute* attribute ) -{ - if ( attribute == 0 ) { - return; - } - MemPool* pool = attribute->_memPool; - attribute->~XMLAttribute(); - pool->Free( attribute ); -} - -// -// -// foobar -// -char* XMLElement::ParseDeep( char* p, StrPair* strPair ) -{ - // Read the element name. - p = XMLUtil::SkipWhiteSpace( p ); - if ( !p ) { - return 0; - } - - // The closing element is the form. It is - // parsed just like a regular element then deleted from - // the DOM. - if ( *p == '/' ) { - _closingType = CLOSING; - ++p; - } - - p = _value.ParseName( p ); - if ( _value.Empty() ) { - return 0; - } - - p = ParseAttributes( p ); - if ( !p || !*p || _closingType ) { - return p; - } - - p = XMLNode::ParseDeep( p, strPair ); - return p; -} - - - -XMLNode* XMLElement::ShallowClone( XMLDocument* doc ) const -{ - if ( !doc ) { - doc = _document; - } - XMLElement* element = doc->NewElement( Value() ); // fixme: this will always allocate memory. Intern? - for( const XMLAttribute* a=FirstAttribute(); a; a=a->Next() ) { - element->SetAttribute( a->Name(), a->Value() ); // fixme: this will always allocate memory. Intern? - } - return element; -} - - -bool XMLElement::ShallowEqual( const XMLNode* compare ) const -{ - const XMLElement* other = compare->ToElement(); - if ( other && XMLUtil::StringEqual( other->Value(), Value() )) { - - const XMLAttribute* a=FirstAttribute(); - const XMLAttribute* b=other->FirstAttribute(); - - while ( a && b ) { - if ( !XMLUtil::StringEqual( a->Value(), b->Value() ) ) { - return false; - } - a = a->Next(); - b = b->Next(); - } - if ( a || b ) { - // different count - return false; - } - return true; - } - return false; -} - - -bool XMLElement::Accept( XMLVisitor* visitor ) const -{ - if ( visitor->VisitEnter( *this, _rootAttribute ) ) { - for ( const XMLNode* node=FirstChild(); node; node=node->NextSibling() ) { - if ( !node->Accept( visitor ) ) { - break; - } - } - } - return visitor->VisitExit( *this ); -} - - -// --------- XMLDocument ----------- // - -// Warning: List must match 'enum XMLError' -const char* XMLDocument::_errorNames[XML_ERROR_COUNT] = { - "XML_SUCCESS", - "XML_NO_ATTRIBUTE", - "XML_WRONG_ATTRIBUTE_TYPE", - "XML_ERROR_FILE_NOT_FOUND", - "XML_ERROR_FILE_COULD_NOT_BE_OPENED", - "XML_ERROR_FILE_READ_ERROR", - "XML_ERROR_ELEMENT_MISMATCH", - "XML_ERROR_PARSING_ELEMENT", - "XML_ERROR_PARSING_ATTRIBUTE", - "XML_ERROR_IDENTIFYING_TAG", - "XML_ERROR_PARSING_TEXT", - "XML_ERROR_PARSING_CDATA", - "XML_ERROR_PARSING_COMMENT", - "XML_ERROR_PARSING_DECLARATION", - "XML_ERROR_PARSING_UNKNOWN", - "XML_ERROR_EMPTY_DOCUMENT", - "XML_ERROR_MISMATCHED_ELEMENT", - "XML_ERROR_PARSING", - "XML_CAN_NOT_CONVERT_TEXT", - "XML_NO_TEXT_NODE" -}; - - -XMLDocument::XMLDocument( bool processEntities, Whitespace whitespace ) : - XMLNode( 0 ), - _writeBOM( false ), - _processEntities( processEntities ), - _errorID( XML_NO_ERROR ), - _whitespace( whitespace ), - _errorStr1( 0 ), - _errorStr2( 0 ), - _charBuffer( 0 ) -{ - _document = this; // avoid warning about 'this' in initializer list -} - - -XMLDocument::~XMLDocument() -{ - Clear(); -} - - -void XMLDocument::Clear() -{ - DeleteChildren(); - - _errorID = XML_NO_ERROR; - _errorStr1 = 0; - _errorStr2 = 0; - - delete [] _charBuffer; - _charBuffer = 0; - -#if 0 - _textPool.Trace( "text" ); - _elementPool.Trace( "element" ); - _commentPool.Trace( "comment" ); - _attributePool.Trace( "attribute" ); -#endif - -#ifndef _RELEASE - if ( Error() == false ) { - TIXMLASSERT( _elementPool.CurrentAllocs() == _elementPool.Untracked() ); - TIXMLASSERT( _attributePool.CurrentAllocs() == _attributePool.Untracked() ); - TIXMLASSERT( _textPool.CurrentAllocs() == _textPool.Untracked() ); - TIXMLASSERT( _commentPool.CurrentAllocs() == _commentPool.Untracked() ); - } -#endif -} - - -XMLElement* XMLDocument::NewElement( const char* name ) -{ - XMLElement* ele = new (_elementPool.Alloc()) XMLElement( this ); - ele->_memPool = &_elementPool; - ele->SetName( name ); - return ele; -} - - -XMLComment* XMLDocument::NewComment( const char* str ) -{ - XMLComment* comment = new (_commentPool.Alloc()) XMLComment( this ); - comment->_memPool = &_commentPool; - comment->SetValue( str ); - return comment; -} - - -XMLText* XMLDocument::NewText( const char* str ) -{ - XMLText* text = new (_textPool.Alloc()) XMLText( this ); - text->_memPool = &_textPool; - text->SetValue( str ); - return text; -} - - -XMLDeclaration* XMLDocument::NewDeclaration( const char* str ) -{ - XMLDeclaration* dec = new (_commentPool.Alloc()) XMLDeclaration( this ); - dec->_memPool = &_commentPool; - dec->SetValue( str ? str : "xml version=\"1.0\" encoding=\"UTF-8\"" ); - return dec; -} - - -XMLUnknown* XMLDocument::NewUnknown( const char* str ) -{ - XMLUnknown* unk = new (_commentPool.Alloc()) XMLUnknown( this ); - unk->_memPool = &_commentPool; - unk->SetValue( str ); - return unk; -} - -static FILE* callfopen( const char* filepath, const char* mode ) -{ -#if defined(_MSC_VER) && (_MSC_VER >= 1400 ) && (!defined WINCE) - FILE* fp = 0; - errno_t err = fopen_s( &fp, filepath, mode ); - if ( err ) { - return 0; - } -#else - FILE* fp = fopen( filepath, mode ); -#endif - return fp; -} - -XMLError XMLDocument::LoadFile( const char* filename ) -{ - Clear(); - FILE* fp = callfopen( filename, "rb" ); - if ( !fp ) { - SetError( XML_ERROR_FILE_NOT_FOUND, filename, 0 ); - return _errorID; - } - LoadFile( fp ); - fclose( fp ); - return _errorID; -} - - -XMLError XMLDocument::LoadFile( FILE* fp ) -{ - Clear(); - - fseek( fp, 0, SEEK_SET ); - if ( fgetc( fp ) == EOF && ferror( fp ) != 0 ) { - SetError( XML_ERROR_FILE_READ_ERROR, 0, 0 ); - return _errorID; - } - - fseek( fp, 0, SEEK_END ); - const long filelength = ftell( fp ); - fseek( fp, 0, SEEK_SET ); - if ( filelength == -1L ) { - SetError( XML_ERROR_FILE_READ_ERROR, 0, 0 ); - return _errorID; - } - - const size_t size = filelength; - if ( size == 0 ) { - SetError( XML_ERROR_EMPTY_DOCUMENT, 0, 0 ); - return _errorID; - } - - _charBuffer = new char[size+1]; - size_t read = fread( _charBuffer, 1, size, fp ); - if ( read != size ) { - SetError( XML_ERROR_FILE_READ_ERROR, 0, 0 ); - return _errorID; - } - - _charBuffer[size] = 0; - - const char* p = _charBuffer; - p = XMLUtil::SkipWhiteSpace( p ); - p = XMLUtil::ReadBOM( p, &_writeBOM ); - if ( !p || !*p ) { - SetError( XML_ERROR_EMPTY_DOCUMENT, 0, 0 ); - return _errorID; - } - - ParseDeep( _charBuffer + (p-_charBuffer), 0 ); - return _errorID; -} - - -XMLError XMLDocument::SaveFile( const char* filename, bool compact ) -{ - FILE* fp = callfopen( filename, "w" ); - if ( !fp ) { - SetError( XML_ERROR_FILE_COULD_NOT_BE_OPENED, filename, 0 ); - return _errorID; - } - SaveFile(fp, compact); - fclose( fp ); - return _errorID; -} - - -XMLError XMLDocument::SaveFile( FILE* fp, bool compact ) -{ - XMLPrinter stream( fp, compact ); - Print( &stream ); - return _errorID; -} - - -XMLError XMLDocument::Parse( const char* p, size_t len ) -{ - const char* start = p; - Clear(); - - if ( len == 0 || !p || !*p ) { - SetError( XML_ERROR_EMPTY_DOCUMENT, 0, 0 ); - return _errorID; - } - if ( len == (size_t)(-1) ) { - len = strlen( p ); - } - _charBuffer = new char[ len+1 ]; - memcpy( _charBuffer, p, len ); - _charBuffer[len] = 0; - - p = XMLUtil::SkipWhiteSpace( p ); - p = XMLUtil::ReadBOM( p, &_writeBOM ); - if ( !p || !*p ) { - SetError( XML_ERROR_EMPTY_DOCUMENT, 0, 0 ); - return _errorID; - } - - ptrdiff_t delta = p - start; // skip initial whitespace, BOM, etc. - ParseDeep( _charBuffer+delta, 0 ); - if (_errorID) { - // clean up now essentially dangling memory. - // and the parse fail can put objects in the - // pools that are dead and inaccessible. - DeleteChildren(); - _elementPool.Clear(); - _attributePool.Clear(); - _textPool.Clear(); - _commentPool.Clear(); - } - return _errorID; -} - - -void XMLDocument::Print( XMLPrinter* streamer ) const -{ - XMLPrinter stdStreamer( stdout ); - if ( !streamer ) { - streamer = &stdStreamer; - } - Accept( streamer ); -} - - -void XMLDocument::SetError( XMLError error, const char* str1, const char* str2 ) -{ - _errorID = error; - _errorStr1 = str1; - _errorStr2 = str2; -} - -const char* XMLDocument::ErrorName() const -{ - TIXMLASSERT(_errorID >= 0 && _errorID < XML_ERROR_COUNT ); - return _errorNames[_errorID]; -} - -void XMLDocument::PrintError() const -{ - if ( _errorID ) { - static const int LEN = 20; - char buf1[LEN] = { 0 }; - char buf2[LEN] = { 0 }; - - if ( _errorStr1 ) { - TIXML_SNPRINTF( buf1, LEN, "%s", _errorStr1 ); - } - if ( _errorStr2 ) { - TIXML_SNPRINTF( buf2, LEN, "%s", _errorStr2 ); - } - - printf( "XMLDocument error id=%d '%s' str1=%s str2=%s\n", - _errorID, ErrorName(), buf1, buf2 ); - } -} - - -XMLPrinter::XMLPrinter( FILE* file, bool compact, int depth ) : - _elementJustOpened( false ), - _firstElement( true ), - _fp( file ), - _depth( depth ), - _textDepth( -1 ), - _processEntities( true ), - _compactMode( compact ) -{ - for( int i=0; i'] = true; // not required, but consistency is nice - _buffer.Push( 0 ); -} - - -void XMLPrinter::Print( const char* format, ... ) -{ - va_list va; - va_start( va, format ); - - if ( _fp ) { - vfprintf( _fp, format, va ); - } - else { -#if defined(_MSC_VER) && (_MSC_VER >= 1400 ) - #if defined(WINCE) - int len = 512; - do { - len = len*2; - char* str = new char[len](); - len = _vsnprintf(str, len, format, va); - delete[] str; - }while (len < 0); - #else - int len = _vscprintf( format, va ); - #endif -#else - int len = vsnprintf( 0, 0, format, va ); -#endif - // Close out and re-start the va-args - va_end( va ); - va_start( va, format ); - char* p = _buffer.PushArr( len ) - 1; // back up over the null terminator. -#if defined(_MSC_VER) && (_MSC_VER >= 1400 ) - #if defined(WINCE) - _vsnprintf( p, len+1, format, va ); - #else - vsnprintf_s( p, len+1, _TRUNCATE, format, va ); - #endif -#else - vsnprintf( p, len+1, format, va ); -#endif - } - va_end( va ); -} - - -void XMLPrinter::PrintSpace( int depth ) -{ - for( int i=0; i 0 && *q < ENTITY_RANGE ) { - // Check for entities. If one is found, flush - // the stream up until the entity, write the - // entity, and keep looking. - if ( flag[(unsigned)(*q)] ) { - while ( p < q ) { - Print( "%c", *p ); - ++p; - } - for( int i=0; i 0) ) { - Print( "%s", p ); - } -} - - -void XMLPrinter::PushHeader( bool writeBOM, bool writeDec ) -{ - if ( writeBOM ) { - static const unsigned char bom[] = { TIXML_UTF_LEAD_0, TIXML_UTF_LEAD_1, TIXML_UTF_LEAD_2, 0 }; - Print( "%s", bom ); - } - if ( writeDec ) { - PushDeclaration( "xml version=\"1.0\"" ); - } -} - - -void XMLPrinter::OpenElement( const char* name, bool compactMode ) -{ - if ( _elementJustOpened ) { - SealElement(); - } - _stack.Push( name ); - - if ( _textDepth < 0 && !_firstElement && !compactMode ) { - Print( "\n" ); - } - if ( !compactMode ) { - PrintSpace( _depth ); - } - - Print( "<%s", name ); - _elementJustOpened = true; - _firstElement = false; - ++_depth; -} - - -void XMLPrinter::PushAttribute( const char* name, const char* value ) -{ - TIXMLASSERT( _elementJustOpened ); - Print( " %s=\"", name ); - PrintString( value, false ); - Print( "\"" ); -} - - -void XMLPrinter::PushAttribute( const char* name, int v ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( v, buf, BUF_SIZE ); - PushAttribute( name, buf ); -} - - -void XMLPrinter::PushAttribute( const char* name, unsigned v ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( v, buf, BUF_SIZE ); - PushAttribute( name, buf ); -} - - -void XMLPrinter::PushAttribute( const char* name, bool v ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( v, buf, BUF_SIZE ); - PushAttribute( name, buf ); -} - - -void XMLPrinter::PushAttribute( const char* name, double v ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( v, buf, BUF_SIZE ); - PushAttribute( name, buf ); -} - - -void XMLPrinter::CloseElement( bool compactMode ) -{ - --_depth; - const char* name = _stack.Pop(); - - if ( _elementJustOpened ) { - Print( "/>" ); - } - else { - if ( _textDepth < 0 && !compactMode) { - Print( "\n" ); - PrintSpace( _depth ); - } - Print( "", name ); - } - - if ( _textDepth == _depth ) { - _textDepth = -1; - } - if ( _depth == 0 && !compactMode) { - Print( "\n" ); - } - _elementJustOpened = false; -} - - -void XMLPrinter::SealElement() -{ - _elementJustOpened = false; - Print( ">" ); -} - - -void XMLPrinter::PushText( const char* text, bool cdata ) -{ - _textDepth = _depth-1; - - if ( _elementJustOpened ) { - SealElement(); - } - if ( cdata ) { - Print( "" ); - } - else { - PrintString( text, true ); - } -} - -void XMLPrinter::PushText( int value ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( value, buf, BUF_SIZE ); - PushText( buf, false ); -} - - -void XMLPrinter::PushText( unsigned value ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( value, buf, BUF_SIZE ); - PushText( buf, false ); -} - - -void XMLPrinter::PushText( bool value ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( value, buf, BUF_SIZE ); - PushText( buf, false ); -} - - -void XMLPrinter::PushText( float value ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( value, buf, BUF_SIZE ); - PushText( buf, false ); -} - - -void XMLPrinter::PushText( double value ) -{ - char buf[BUF_SIZE]; - XMLUtil::ToStr( value, buf, BUF_SIZE ); - PushText( buf, false ); -} - - -void XMLPrinter::PushComment( const char* comment ) -{ - if ( _elementJustOpened ) { - SealElement(); - } - if ( _textDepth < 0 && !_firstElement && !_compactMode) { - Print( "\n" ); - PrintSpace( _depth ); - } - _firstElement = false; - Print( "", comment ); -} - - -void XMLPrinter::PushDeclaration( const char* value ) -{ - if ( _elementJustOpened ) { - SealElement(); - } - if ( _textDepth < 0 && !_firstElement && !_compactMode) { - Print( "\n" ); - PrintSpace( _depth ); - } - _firstElement = false; - Print( "", value ); -} - - -void XMLPrinter::PushUnknown( const char* value ) -{ - if ( _elementJustOpened ) { - SealElement(); - } - if ( _textDepth < 0 && !_firstElement && !_compactMode) { - Print( "\n" ); - PrintSpace( _depth ); - } - _firstElement = false; - Print( "", value ); -} - - -bool XMLPrinter::VisitEnter( const XMLDocument& doc ) -{ - _processEntities = doc.ProcessEntities(); - if ( doc.HasBOM() ) { - PushHeader( true, false ); - } - return true; -} - - -bool XMLPrinter::VisitEnter( const XMLElement& element, const XMLAttribute* attribute ) -{ - const XMLElement* parentElem = element.Parent()->ToElement(); - bool compactMode = parentElem ? CompactMode(*parentElem) : _compactMode; - OpenElement( element.Name(), compactMode ); - while ( attribute ) { - PushAttribute( attribute->Name(), attribute->Value() ); - attribute = attribute->Next(); - } - return true; -} - - -bool XMLPrinter::VisitExit( const XMLElement& element ) -{ - CloseElement( CompactMode(element) ); - return true; -} - - -bool XMLPrinter::Visit( const XMLText& text ) -{ - PushText( text.Value(), text.CData() ); - return true; -} - - -bool XMLPrinter::Visit( const XMLComment& comment ) -{ - PushComment( comment.Value() ); - return true; -} - -bool XMLPrinter::Visit( const XMLDeclaration& declaration ) -{ - PushDeclaration( declaration.Value() ); - return true; -} - - -bool XMLPrinter::Visit( const XMLUnknown& unknown ) -{ - PushUnknown( unknown.Value() ); - return true; -} - -} // namespace tinyxml2 - diff --git a/libs/IO/TinyXML2.h b/libs/IO/TinyXML2.h deleted file mode 100644 index 3e95a402f..000000000 --- a/libs/IO/TinyXML2.h +++ /dev/null @@ -1,2075 +0,0 @@ -/* -Original code by Lee Thomason (www.grinninglizard.com) - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any -damages arising from the use of this software. - -Permission is granted to anyone to use this software for any -purpose, including commercial applications, and to alter it and -redistribute it freely, subject to the following restrictions: - -1. The origin of this software must not be misrepresented; you must -not claim that you wrote the original software. If you use this -software in a product, an acknowledgment in the product documentation -would be appreciated but is not required. - -2. Altered source versions must be plainly marked as such, and -must not be misrepresented as being the original software. - -3. This notice may not be removed or altered from any source -distribution. -*/ - -#ifndef TINYXML2_INCLUDED -#define TINYXML2_INCLUDED - -#if defined(ANDROID_NDK) || defined(__BORLANDC__) -# include -# include -# include -# include -# include -# include -#else -# include -# include -# include -# include -# include -# include -#endif - -/* - TODO: intern strings instead of allocation. -*/ -/* - gcc: - g++ -Wall -DDEBUG tinyxml2.cpp xmltest.cpp -o gccxmltest.exe - - Formatting, Artistic Style: - AStyle.exe --style=1tbs --indent-switches --break-closing-brackets --indent-preprocessor tinyxml2.cpp tinyxml2.h -*/ - -#ifdef _MSC_VER -# pragma warning(push) -# pragma warning(disable: 4251) -#endif - -#define TINYXML2_LIB IO_API - - -#ifndef _RELEASE -# if defined(_MSC_VER) -# define TIXMLASSERT( x ) ASSERT(x) //if ( !(x)) { __debugbreak(); } //if ( !(x)) WinDebugBreak() -# elif defined (ANDROID_NDK) -# include -# define TIXMLASSERT( x ) if ( !(x)) { __android_log_assert( "assert", "grinliz", "ASSERT in '%s' at %d.", __FILE__, __LINE__ ); } -# else -# include -# define TIXMLASSERT ASSERT -# endif -# else -# define TIXMLASSERT( x ) {} -#endif - - -#if defined(_MSC_VER) && (_MSC_VER >= 1400 ) && (!defined WINCE) -// Microsoft visual studio, version 2005 and higher. -/*int _snprintf_s( - char *buffer, - size_t sizeOfBuffer, - size_t count, - const char *format [, - argument] ... -);*/ -inline int TIXML_SNPRINTF( char* buffer, size_t size, const char* format, ... ) -{ - va_list va; - va_start( va, format ); - int result = vsnprintf_s( buffer, size, _TRUNCATE, format, va ); - va_end( va ); - return result; -} -#define TIXML_SSCANF sscanf_s -#elif defined WINCE -#define TIXML_SNPRINTF _snprintf -#define TIXML_SSCANF sscanf -#else -// GCC version 3 and higher -//#warning( "Using sn* functions." ) -#define TIXML_SNPRINTF snprintf -#define TIXML_SSCANF sscanf -#endif - -/* Versioning, past 1.0.14: - http://semver.org/ -*/ -static const int TIXML2_MAJOR_VERSION = 2; -static const int TIXML2_MINOR_VERSION = 2; -static const int TIXML2_PATCH_VERSION = 0; - -namespace tinyxml2 -{ -class XMLDocument; -class XMLElement; -class XMLAttribute; -class XMLComment; -class XMLText; -class XMLDeclaration; -class XMLUnknown; -class XMLPrinter; - -/* - A class that wraps strings. Normally stores the start and end - pointers into the XML file itself, and will apply normalization - and entity translation if actually read. Can also store (and memory - manage) a traditional char[] -*/ -class StrPair -{ -public: - enum { - NEEDS_ENTITY_PROCESSING = 0x01, - NEEDS_NEWLINE_NORMALIZATION = 0x02, - COLLAPSE_WHITESPACE = 0x04, - - TEXT_ELEMENT = NEEDS_ENTITY_PROCESSING | NEEDS_NEWLINE_NORMALIZATION, - TEXT_ELEMENT_LEAVE_ENTITIES = NEEDS_NEWLINE_NORMALIZATION, - ATTRIBUTE_NAME = 0, - ATTRIBUTE_VALUE = NEEDS_ENTITY_PROCESSING | NEEDS_NEWLINE_NORMALIZATION, - ATTRIBUTE_VALUE_LEAVE_ENTITIES = NEEDS_NEWLINE_NORMALIZATION, - COMMENT = NEEDS_NEWLINE_NORMALIZATION - }; - - StrPair() : _flags( 0 ), _start( 0 ), _end( 0 ) {} - ~StrPair(); - - void Set( char* start, char* end, int flags ) { - Reset(); - _start = start; - _end = end; - _flags = flags | NEEDS_FLUSH; - } - - const char* GetStr(); - - bool Empty() const { - return _start == _end; - } - - void SetInternedStr( const char* str ) { - Reset(); - _start = const_cast(str); - } - - void SetStr( const char* str, int flags=0 ); - - char* ParseText( char* in, const char* endTag, int strFlags ); - char* ParseName( char* in ); - -private: - void Reset(); - void CollapseWhitespace(); - - enum { - NEEDS_FLUSH = 0x100, - NEEDS_DELETE = 0x200 - }; - - // After parsing, if *_end != 0, it can be set to zero. - int _flags; - char* _start; - char* _end; -}; - - -/* - A dynamic array of Plain Old Data. Doesn't support constructors, etc. - Has a small initial memory pool, so that low or no usage will not - cause a call to new/delete -*/ -template -class DynArray -{ -public: - DynArray< T, INIT >() { - _mem = _pool; - _allocated = INIT; - _size = 0; - } - - ~DynArray() { - if ( _mem != _pool ) { - delete [] _mem; - } - } - - void Clear() { - _size = 0; - } - - void Push( T t ) { - EnsureCapacity( _size+1 ); - _mem[_size++] = t; - } - - T* PushArr( int count ) { - EnsureCapacity( _size+count ); - T* ret = &_mem[_size]; - _size += count; - return ret; - } - - T Pop() { - return _mem[--_size]; - } - - void PopArr( int count ) { - TIXMLASSERT( _size >= count ); - _size -= count; - } - - bool Empty() const { - return _size == 0; - } - - T& operator[](int i) { - TIXMLASSERT( i>= 0 && i < _size ); - return _mem[i]; - } - - const T& operator[](int i) const { - TIXMLASSERT( i>= 0 && i < _size ); - return _mem[i]; - } - - const T& PeekTop() const { - TIXMLASSERT( _size > 0 ); - return _mem[ _size - 1]; - } - - int Size() const { - return _size; - } - - int Capacity() const { - return _allocated; - } - - const T* Mem() const { - return _mem; - } - - T* Mem() { - return _mem; - } - -private: - void EnsureCapacity( int cap ) { - if ( cap > _allocated ) { - int newAllocated = cap * 2; - T* newMem = new T[newAllocated]; - memcpy( newMem, _mem, sizeof(T)*_size ); // warning: not using constructors, only works for PODs - if ( _mem != _pool ) { - delete [] _mem; - } - _mem = newMem; - _allocated = newAllocated; - } - } - - T* _mem; - T _pool[INIT]; - int _allocated; // objects allocated - int _size; // number objects in use -}; - - -/* - Parent virtual class of a pool for fast allocation - and deallocation of objects. -*/ -class MemPool -{ -public: - MemPool() {} - virtual ~MemPool() {} - - virtual int ItemSize() const = 0; - virtual void* Alloc() = 0; - virtual void Free( void* ) = 0; - virtual void SetTracked() = 0; - virtual void Clear() = 0; -}; - - -/* - Template child class to create pools of the correct type. -*/ -template< int SIZE > -class MemPoolT : public MemPool -{ -public: - MemPoolT() : _root(0), _currentAllocs(0), _nAllocs(0), _maxAllocs(0), _nUntracked(0) {} - ~MemPoolT() { - Clear(); - } - - void Clear() { - // Delete the blocks. - while( !_blockPtrs.Empty()) { - Block* b = _blockPtrs.Pop(); - delete b; - } - _root = 0; - _currentAllocs = 0; - _nAllocs = 0; - _maxAllocs = 0; - _nUntracked = 0; - } - - virtual int ItemSize() const { - return SIZE; - } - int CurrentAllocs() const { - return _currentAllocs; - } - - virtual void* Alloc() { - if ( !_root ) { - // Need a new block. - Block* block = new Block(); - _blockPtrs.Push( block ); - - for( int i=0; ichunk[i].next = &block->chunk[i+1]; - } - block->chunk[COUNT-1].next = 0; - _root = block->chunk; - } - void* result = _root; - _root = _root->next; - - ++_currentAllocs; - if ( _currentAllocs > _maxAllocs ) { - _maxAllocs = _currentAllocs; - } - _nAllocs++; - _nUntracked++; - return result; - } - - virtual void Free( void* mem ) { - if ( !mem ) { - return; - } - --_currentAllocs; - Chunk* chunk = static_cast( mem ); -#ifndef _RELEASE - memset( chunk, 0xfe, sizeof(Chunk) ); -#endif - chunk->next = _root; - _root = chunk; - } - void Trace( const char* name ) { - printf( "Mempool %s watermark=%d [%dk] current=%d size=%d nAlloc=%d blocks=%d\n", - name, _maxAllocs, _maxAllocs*SIZE/1024, _currentAllocs, SIZE, _nAllocs, _blockPtrs.Size() ); - } - - void SetTracked() { - _nUntracked--; - } - - int Untracked() const { - return _nUntracked; - } - - // This number is perf sensitive. 4k seems like a good tradeoff on my machine. - // The test file is large, 170k. - // Release: VS2010 gcc(no opt) - // 1k: 4000 - // 2k: 4000 - // 4k: 3900 21000 - // 16k: 5200 - // 32k: 4300 - // 64k: 4000 21000 - enum { COUNT = (4*1024)/SIZE }; // Some compilers do not accept to use COUNT in private part if COUNT is private - -private: - union Chunk { - Chunk* next; - char mem[SIZE]; - }; - struct Block { - Chunk chunk[COUNT]; - }; - DynArray< Block*, 10 > _blockPtrs; - Chunk* _root; - - int _currentAllocs; - int _nAllocs; - int _maxAllocs; - int _nUntracked; -}; - - - -/** - Implements the interface to the "Visitor pattern" (see the Accept() method.) - If you call the Accept() method, it requires being passed a XMLVisitor - class to handle callbacks. For nodes that contain other nodes (Document, Element) - you will get called with a VisitEnter/VisitExit pair. Nodes that are always leafs - are simply called with Visit(). - - If you return 'true' from a Visit method, recursive parsing will continue. If you return - false, no children of this node or its siblings will be visited. - - All flavors of Visit methods have a default implementation that returns 'true' (continue - visiting). You need to only override methods that are interesting to you. - - Generally Accept() is called on the XMLDocument, although all nodes support visiting. - - You should never change the document from a callback. - - @sa XMLNode::Accept() -*/ -class TINYXML2_LIB XMLVisitor -{ -public: - virtual ~XMLVisitor() {} - - /// Visit a document. - virtual bool VisitEnter( const XMLDocument& /*doc*/ ) { - return true; - } - /// Visit a document. - virtual bool VisitExit( const XMLDocument& /*doc*/ ) { - return true; - } - - /// Visit an element. - virtual bool VisitEnter( const XMLElement& /*element*/, const XMLAttribute* /*firstAttribute*/ ) { - return true; - } - /// Visit an element. - virtual bool VisitExit( const XMLElement& /*element*/ ) { - return true; - } - - /// Visit a declaration. - virtual bool Visit( const XMLDeclaration& /*declaration*/ ) { - return true; - } - /// Visit a text node. - virtual bool Visit( const XMLText& /*text*/ ) { - return true; - } - /// Visit a comment node. - virtual bool Visit( const XMLComment& /*comment*/ ) { - return true; - } - /// Visit an unknown node. - virtual bool Visit( const XMLUnknown& /*unknown*/ ) { - return true; - } -}; - -// WARNING: must match XMLErrorNames[] -enum XMLError { - XML_SUCCESS = 0, - XML_NO_ERROR = 0, - XML_NO_ATTRIBUTE, - XML_WRONG_ATTRIBUTE_TYPE, - XML_ERROR_FILE_NOT_FOUND, - XML_ERROR_FILE_COULD_NOT_BE_OPENED, - XML_ERROR_FILE_READ_ERROR, - XML_ERROR_ELEMENT_MISMATCH, - XML_ERROR_PARSING_ELEMENT, - XML_ERROR_PARSING_ATTRIBUTE, - XML_ERROR_IDENTIFYING_TAG, - XML_ERROR_PARSING_TEXT, - XML_ERROR_PARSING_CDATA, - XML_ERROR_PARSING_COMMENT, - XML_ERROR_PARSING_DECLARATION, - XML_ERROR_PARSING_UNKNOWN, - XML_ERROR_EMPTY_DOCUMENT, - XML_ERROR_MISMATCHED_ELEMENT, - XML_ERROR_PARSING, - XML_CAN_NOT_CONVERT_TEXT, - XML_NO_TEXT_NODE, - - XML_ERROR_COUNT -}; - - -/* - Utility functionality. -*/ -class XMLUtil -{ -public: - static const char* SkipWhiteSpace( const char* p ) { - while( IsWhiteSpace(*p) ) { - ++p; - } - return p; - } - static char* SkipWhiteSpace( char* p ) { - return const_cast( SkipWhiteSpace( const_cast(p) ) ); - } - - // Anything in the high order range of UTF-8 is assumed to not be whitespace. This isn't - // correct, but simple, and usually works. - static bool IsWhiteSpace( char p ) { - return !IsUTF8Continuation(p) && isspace( static_cast(p) ); - } - - inline static bool IsNameStartChar( unsigned char ch ) { - return ( ( ch < 128 ) ? isalpha( ch ) : 1 ) - || ch == ':' - || ch == '_'; - } - - inline static bool IsNameChar( unsigned char ch ) { - return IsNameStartChar( ch ) - || isdigit( ch ) - || ch == '.' - || ch == '-'; - } - - inline static bool StringEqual( const char* p, const char* q, int nChar=INT_MAX ) { - int n = 0; - if ( p == q ) { - return true; - } - while( *p && *q && *p == *q && n(const_cast(this)->FirstChildElement( value )); - } - - /// Get the last child node, or null if none exists. - const XMLNode* LastChild() const { - return _lastChild; - } - - XMLNode* LastChild() { - return const_cast(const_cast(this)->LastChild() ); - } - - /** Get the last child element or optionally the last child - element with the specified name. - */ - const XMLElement* LastChildElement( const char* value=0 ) const; - - XMLElement* LastChildElement( const char* value=0 ) { - return const_cast(const_cast(this)->LastChildElement(value) ); - } - - /// Get the previous (left) sibling node of this node. - const XMLNode* PreviousSibling() const { - return _prev; - } - - XMLNode* PreviousSibling() { - return _prev; - } - - /// Get the previous (left) sibling element of this node, with an optionally supplied name. - const XMLElement* PreviousSiblingElement( const char* value=0 ) const ; - - XMLElement* PreviousSiblingElement( const char* value=0 ) { - return const_cast(const_cast(this)->PreviousSiblingElement( value ) ); - } - - /// Get the next (right) sibling node of this node. - const XMLNode* NextSibling() const { - return _next; - } - - XMLNode* NextSibling() { - return _next; - } - - /// Get the next (right) sibling element of this node, with an optionally supplied name. - const XMLElement* NextSiblingElement( const char* value=0 ) const; - - XMLElement* NextSiblingElement( const char* value=0 ) { - return const_cast(const_cast(this)->NextSiblingElement( value ) ); - } - - /** - Add a child node as the last (right) child. - If the child node is already part of the document, - it is moved from its old location to the new location. - Returns the addThis argument or 0 if the node does not - belong to the same document. - */ - XMLNode* InsertEndChild( XMLNode* addThis ); - - XMLNode* LinkEndChild( XMLNode* addThis ) { - return InsertEndChild( addThis ); - } - /** - Add a child node as the first (left) child. - If the child node is already part of the document, - it is moved from its old location to the new location. - Returns the addThis argument or 0 if the node does not - belong to the same document. - */ - XMLNode* InsertFirstChild( XMLNode* addThis ); - /** - Add a node after the specified child node. - If the child node is already part of the document, - it is moved from its old location to the new location. - Returns the addThis argument or 0 if the afterThis node - is not a child of this node, or if the node does not - belong to the same document. - */ - XMLNode* InsertAfterChild( XMLNode* afterThis, XMLNode* addThis ); - - /** - Delete all the children of this node. - */ - void DeleteChildren(); - - /** - Delete a child of this node. - */ - void DeleteChild( XMLNode* node ); - - /** - Make a copy of this node, but not its children. - You may pass in a Document pointer that will be - the owner of the new Node. If the 'document' is - null, then the node returned will be allocated - from the current Document. (this->GetDocument()) - - Note: if called on a XMLDocument, this will return null. - */ - virtual XMLNode* ShallowClone( XMLDocument* document ) const = 0; - - /** - Test if 2 nodes are the same, but don't test children. - The 2 nodes do not need to be in the same Document. - - Note: if called on a XMLDocument, this will return false. - */ - virtual bool ShallowEqual( const XMLNode* compare ) const = 0; - - /** Accept a hierarchical visit of the nodes in the TinyXML-2 DOM. Every node in the - XML tree will be conditionally visited and the host will be called back - via the XMLVisitor interface. - - This is essentially a SAX interface for TinyXML-2. (Note however it doesn't re-parse - the XML for the callbacks, so the performance of TinyXML-2 is unchanged by using this - interface versus any other.) - - The interface has been based on ideas from: - - - http://www.saxproject.org/ - - http://c2.com/cgi/wiki?HierarchicalVisitorPattern - - Which are both good references for "visiting". - - An example of using Accept(): - @verbatim - XMLPrinter printer; - tinyxmlDoc.Accept( &printer ); - const char* xmlcstr = printer.CStr(); - @endverbatim - */ - virtual bool Accept( XMLVisitor* visitor ) const = 0; - - // internal - virtual char* ParseDeep( char*, StrPair* ); - -protected: - XMLNode( XMLDocument* ); - virtual ~XMLNode(); - XMLNode( const XMLNode& ); // not supported - XMLNode& operator=( const XMLNode& ); // not supported - - XMLDocument* _document; - XMLNode* _parent; - mutable StrPair _value; - - XMLNode* _firstChild; - XMLNode* _lastChild; - - XMLNode* _prev; - XMLNode* _next; - -private: - MemPool* _memPool; - void Unlink( XMLNode* child ); - static void DeleteNode( XMLNode* node ); -}; - - -/** XML text. - - Note that a text node can have child element nodes, for example: - @verbatim - This is bold - @endverbatim - - A text node can have 2 ways to output the next. "normal" output - and CDATA. It will default to the mode it was parsed from the XML file and - you generally want to leave it alone, but you can change the output mode with - SetCData() and query it with CData(). -*/ -class TINYXML2_LIB XMLText : public XMLNode -{ - friend class XMLBase; - friend class XMLDocument; -public: - virtual bool Accept( XMLVisitor* visitor ) const; - - virtual XMLText* ToText() { - return this; - } - virtual const XMLText* ToText() const { - return this; - } - - /// Declare whether this should be CDATA or standard text. - void SetCData( bool isCData ) { - _isCData = isCData; - } - /// Returns true if this is a CDATA text element. - bool CData() const { - return _isCData; - } - - char* ParseDeep( char*, StrPair* endTag ); - virtual XMLNode* ShallowClone( XMLDocument* document ) const; - virtual bool ShallowEqual( const XMLNode* compare ) const; - -protected: - XMLText( XMLDocument* doc ) : XMLNode( doc ), _isCData( false ) {} - virtual ~XMLText() {} - XMLText( const XMLText& ); // not supported - XMLText& operator=( const XMLText& ); // not supported - -private: - bool _isCData; -}; - - -/** An XML Comment. */ -class TINYXML2_LIB XMLComment : public XMLNode -{ - friend class XMLDocument; -public: - virtual XMLComment* ToComment() { - return this; - } - virtual const XMLComment* ToComment() const { - return this; - } - - virtual bool Accept( XMLVisitor* visitor ) const; - - char* ParseDeep( char*, StrPair* endTag ); - virtual XMLNode* ShallowClone( XMLDocument* document ) const; - virtual bool ShallowEqual( const XMLNode* compare ) const; - -protected: - XMLComment( XMLDocument* doc ); - virtual ~XMLComment(); - XMLComment( const XMLComment& ); // not supported - XMLComment& operator=( const XMLComment& ); // not supported - -private: -}; - - -/** In correct XML the declaration is the first entry in the file. - @verbatim - - @endverbatim - - TinyXML-2 will happily read or write files without a declaration, - however. - - The text of the declaration isn't interpreted. It is parsed - and written as a string. -*/ -class TINYXML2_LIB XMLDeclaration : public XMLNode -{ - friend class XMLDocument; -public: - virtual XMLDeclaration* ToDeclaration() { - return this; - } - virtual const XMLDeclaration* ToDeclaration() const { - return this; - } - - virtual bool Accept( XMLVisitor* visitor ) const; - - char* ParseDeep( char*, StrPair* endTag ); - virtual XMLNode* ShallowClone( XMLDocument* document ) const; - virtual bool ShallowEqual( const XMLNode* compare ) const; - -protected: - XMLDeclaration( XMLDocument* doc ); - virtual ~XMLDeclaration(); - XMLDeclaration( const XMLDeclaration& ); // not supported - XMLDeclaration& operator=( const XMLDeclaration& ); // not supported -}; - - -/** Any tag that TinyXML-2 doesn't recognize is saved as an - unknown. It is a tag of text, but should not be modified. - It will be written back to the XML, unchanged, when the file - is saved. - - DTD tags get thrown into XMLUnknowns. -*/ -class TINYXML2_LIB XMLUnknown : public XMLNode -{ - friend class XMLDocument; -public: - virtual XMLUnknown* ToUnknown() { - return this; - } - virtual const XMLUnknown* ToUnknown() const { - return this; - } - - virtual bool Accept( XMLVisitor* visitor ) const; - - char* ParseDeep( char*, StrPair* endTag ); - virtual XMLNode* ShallowClone( XMLDocument* document ) const; - virtual bool ShallowEqual( const XMLNode* compare ) const; - -protected: - XMLUnknown( XMLDocument* doc ); - virtual ~XMLUnknown(); - XMLUnknown( const XMLUnknown& ); // not supported - XMLUnknown& operator=( const XMLUnknown& ); // not supported -}; - - - -/** An attribute is a name-value pair. Elements have an arbitrary - number of attributes, each with a unique name. - - @note The attributes are not XMLNodes. You may only query the - Next() attribute in a list. -*/ -class TINYXML2_LIB XMLAttribute -{ - friend class XMLElement; -public: - /// The name of the attribute. - const char* Name() const; - - /// The value of the attribute. - const char* Value() const; - - /// The next attribute in the list. - const XMLAttribute* Next() const { - return _next; - } - - /** IntValue interprets the attribute as an integer, and returns the value. - If the value isn't an integer, 0 will be returned. There is no error checking; - use QueryIntValue() if you need error checking. - */ - int IntValue() const { - int i=0; - QueryIntValue( &i ); - return i; - } - /// Query as an unsigned integer. See IntValue() - unsigned UnsignedValue() const { - unsigned i=0; - QueryUnsignedValue( &i ); - return i; - } - /// Query as a boolean. See IntValue() - bool BoolValue() const { - bool b=false; - QueryBoolValue( &b ); - return b; - } - /// Query as a double. See IntValue() - double DoubleValue() const { - double d=0; - QueryDoubleValue( &d ); - return d; - } - /// Query as a float. See IntValue() - float FloatValue() const { - float f=0; - QueryFloatValue( &f ); - return f; - } - - /** QueryIntValue interprets the attribute as an integer, and returns the value - in the provided parameter. The function will return XML_NO_ERROR on success, - and XML_WRONG_ATTRIBUTE_TYPE if the conversion is not successful. - */ - XMLError QueryIntValue( int* value ) const; - /// See QueryIntValue - XMLError QueryUnsignedValue( unsigned int* value ) const; - /// See QueryIntValue - XMLError QueryBoolValue( bool* value ) const; - /// See QueryIntValue - XMLError QueryDoubleValue( double* value ) const; - /// See QueryIntValue - XMLError QueryFloatValue( float* value ) const; - - /// Set the attribute to a string value. - void SetAttribute( const char* value ); - /// Set the attribute to value. - void SetAttribute( int value ); - /// Set the attribute to value. - void SetAttribute( unsigned value ); - /// Set the attribute to value. - void SetAttribute( bool value ); - /// Set the attribute to value. - void SetAttribute( double value ); - /// Set the attribute to value. - void SetAttribute( float value ); - -private: - enum { BUF_SIZE = 200 }; - - XMLAttribute() : _next( 0 ), _memPool( 0 ) {} - virtual ~XMLAttribute() {} - - XMLAttribute( const XMLAttribute& ); // not supported - void operator=( const XMLAttribute& ); // not supported - void SetName( const char* name ); - - char* ParseDeep( char* p, bool processEntities ); - - mutable StrPair _name; - mutable StrPair _value; - XMLAttribute* _next; - MemPool* _memPool; -}; - - -/** The element is a container class. It has a value, the element name, - and can contain other elements, text, comments, and unknowns. - Elements also contain an arbitrary number of attributes. -*/ -class TINYXML2_LIB XMLElement : public XMLNode -{ - friend class XMLBase; - friend class XMLDocument; -public: - /// Get the name of an element (which is the Value() of the node.) - const char* Name() const { - return Value(); - } - /// Set the name of the element. - void SetName( const char* str, bool staticMem=false ) { - SetValue( str, staticMem ); - } - - virtual XMLElement* ToElement() { - return this; - } - virtual const XMLElement* ToElement() const { - return this; - } - virtual bool Accept( XMLVisitor* visitor ) const; - - /** Given an attribute name, Attribute() returns the value - for the attribute of that name, or null if none - exists. For example: - - @verbatim - const char* value = ele->Attribute( "foo" ); - @endverbatim - - The 'value' parameter is normally null. However, if specified, - the attribute will only be returned if the 'name' and 'value' - match. This allow you to write code: - - @verbatim - if ( ele->Attribute( "foo", "bar" ) ) callFooIsBar(); - @endverbatim - - rather than: - @verbatim - if ( ele->Attribute( "foo" ) ) { - if ( strcmp( ele->Attribute( "foo" ), "bar" ) == 0 ) callFooIsBar(); - } - @endverbatim - */ - const char* Attribute( const char* name, const char* value=0 ) const; - - /** Given an attribute name, IntAttribute() returns the value - of the attribute interpreted as an integer. 0 will be - returned if there is an error. For a method with error - checking, see QueryIntAttribute() - */ - int IntAttribute( const char* name ) const { - int i=0; - QueryIntAttribute( name, &i ); - return i; - } - /// See IntAttribute() - unsigned UnsignedAttribute( const char* name ) const { - unsigned i=0; - QueryUnsignedAttribute( name, &i ); - return i; - } - /// See IntAttribute() - bool BoolAttribute( const char* name ) const { - bool b=false; - QueryBoolAttribute( name, &b ); - return b; - } - /// See IntAttribute() - double DoubleAttribute( const char* name ) const { - double d=0; - QueryDoubleAttribute( name, &d ); - return d; - } - /// See IntAttribute() - float FloatAttribute( const char* name ) const { - float f=0; - QueryFloatAttribute( name, &f ); - return f; - } - - /** Given an attribute name, QueryIntAttribute() returns - XML_NO_ERROR, XML_WRONG_ATTRIBUTE_TYPE if the conversion - can't be performed, or XML_NO_ATTRIBUTE if the attribute - doesn't exist. If successful, the result of the conversion - will be written to 'value'. If not successful, nothing will - be written to 'value'. This allows you to provide default - value: - - @verbatim - int value = 10; - QueryIntAttribute( "foo", &value ); // if "foo" isn't found, value will still be 10 - @endverbatim - */ - XMLError QueryIntAttribute( const char* name, int* value ) const { - const XMLAttribute* a = FindAttribute( name ); - if ( !a ) { - return XML_NO_ATTRIBUTE; - } - return a->QueryIntValue( value ); - } - /// See QueryIntAttribute() - XMLError QueryUnsignedAttribute( const char* name, unsigned int* value ) const { - const XMLAttribute* a = FindAttribute( name ); - if ( !a ) { - return XML_NO_ATTRIBUTE; - } - return a->QueryUnsignedValue( value ); - } - /// See QueryIntAttribute() - XMLError QueryBoolAttribute( const char* name, bool* value ) const { - const XMLAttribute* a = FindAttribute( name ); - if ( !a ) { - return XML_NO_ATTRIBUTE; - } - return a->QueryBoolValue( value ); - } - /// See QueryIntAttribute() - XMLError QueryDoubleAttribute( const char* name, double* value ) const { - const XMLAttribute* a = FindAttribute( name ); - if ( !a ) { - return XML_NO_ATTRIBUTE; - } - return a->QueryDoubleValue( value ); - } - /// See QueryIntAttribute() - XMLError QueryFloatAttribute( const char* name, float* value ) const { - const XMLAttribute* a = FindAttribute( name ); - if ( !a ) { - return XML_NO_ATTRIBUTE; - } - return a->QueryFloatValue( value ); - } - - - /** Given an attribute name, QueryAttribute() returns - XML_NO_ERROR, XML_WRONG_ATTRIBUTE_TYPE if the conversion - can't be performed, or XML_NO_ATTRIBUTE if the attribute - doesn't exist. It is overloaded for the primitive types, - and is a generally more convenient replacement of - QueryIntAttribute() and related functions. - - If successful, the result of the conversion - will be written to 'value'. If not successful, nothing will - be written to 'value'. This allows you to provide default - value: - - @verbatim - int value = 10; - QueryAttribute( "foo", &value ); // if "foo" isn't found, value will still be 10 - @endverbatim - */ - int QueryAttribute( const char* name, int* value ) const { - return QueryIntAttribute( name, value ); - } - - int QueryAttribute( const char* name, unsigned int* value ) const { - return QueryUnsignedAttribute( name, value ); - } - - int QueryAttribute( const char* name, bool* value ) const { - return QueryBoolAttribute( name, value ); - } - - int QueryAttribute( const char* name, double* value ) const { - return QueryDoubleAttribute( name, value ); - } - - int QueryAttribute( const char* name, float* value ) const { - return QueryFloatAttribute( name, value ); - } - - /// Sets the named attribute to value. - void SetAttribute( const char* name, const char* value ) { - XMLAttribute* a = FindOrCreateAttribute( name ); - a->SetAttribute( value ); - } - /// Sets the named attribute to value. - void SetAttribute( const char* name, int value ) { - XMLAttribute* a = FindOrCreateAttribute( name ); - a->SetAttribute( value ); - } - /// Sets the named attribute to value. - void SetAttribute( const char* name, unsigned value ) { - XMLAttribute* a = FindOrCreateAttribute( name ); - a->SetAttribute( value ); - } - /// Sets the named attribute to value. - void SetAttribute( const char* name, bool value ) { - XMLAttribute* a = FindOrCreateAttribute( name ); - a->SetAttribute( value ); - } - /// Sets the named attribute to value. - void SetAttribute( const char* name, double value ) { - XMLAttribute* a = FindOrCreateAttribute( name ); - a->SetAttribute( value ); - } - /// Sets the named attribute to value. - void SetAttribute( const char* name, float value ) { - XMLAttribute* a = FindOrCreateAttribute( name ); - a->SetAttribute( value ); - } - - /** - Delete an attribute. - */ - void DeleteAttribute( const char* name ); - - /// Return the first attribute in the list. - const XMLAttribute* FirstAttribute() const { - return _rootAttribute; - } - /// Query a specific attribute in the list. - const XMLAttribute* FindAttribute( const char* name ) const; - - /** Convenience function for easy access to the text inside an element. Although easy - and concise, GetText() is limited compared to getting the XMLText child - and accessing it directly. - - If the first child of 'this' is a XMLText, the GetText() - returns the character string of the Text node, else null is returned. - - This is a convenient method for getting the text of simple contained text: - @verbatim - This is text - const char* str = fooElement->GetText(); - @endverbatim - - 'str' will be a pointer to "This is text". - - Note that this function can be misleading. If the element foo was created from - this XML: - @verbatim - This is text - @endverbatim - - then the value of str would be null. The first child node isn't a text node, it is - another element. From this XML: - @verbatim - This is text - @endverbatim - GetText() will return "This is ". - */ - const char* GetText() const; - - /** Convenience function for easy access to the text inside an element. Although easy - and concise, SetText() is limited compared to creating an XMLText child - and mutating it directly. - - If the first child of 'this' is a XMLText, SetText() sets its value to - the given string, otherwise it will create a first child that is an XMLText. - - This is a convenient method for setting the text of simple contained text: - @verbatim - This is text - fooElement->SetText( "Hullaballoo!" ); - Hullaballoo! - @endverbatim - - Note that this function can be misleading. If the element foo was created from - this XML: - @verbatim - This is text - @endverbatim - - then it will not change "This is text", but rather prefix it with a text element: - @verbatim - Hullaballoo!This is text - @endverbatim - - For this XML: - @verbatim - - @endverbatim - SetText() will generate - @verbatim - Hullaballoo! - @endverbatim - */ - void SetText( const char* inText ); - /// Convenience method for setting text inside and element. See SetText() for important limitations. - void SetText( int value ); - /// Convenience method for setting text inside and element. See SetText() for important limitations. - void SetText( unsigned value ); - /// Convenience method for setting text inside and element. See SetText() for important limitations. - void SetText( bool value ); - /// Convenience method for setting text inside and element. See SetText() for important limitations. - void SetText( double value ); - /// Convenience method for setting text inside and element. See SetText() for important limitations. - void SetText( float value ); - - /** - Convenience method to query the value of a child text node. This is probably best - shown by example. Given you have a document is this form: - @verbatim - - 1 - 1.4 - - @endverbatim - - The QueryIntText() and similar functions provide a safe and easier way to get to the - "value" of x and y. - - @verbatim - int x = 0; - float y = 0; // types of x and y are contrived for example - const XMLElement* xElement = pointElement->FirstChildElement( "x" ); - const XMLElement* yElement = pointElement->FirstChildElement( "y" ); - xElement->QueryIntText( &x ); - yElement->QueryFloatText( &y ); - @endverbatim - - @returns XML_SUCCESS (0) on success, XML_CAN_NOT_CONVERT_TEXT if the text cannot be converted - to the requested type, and XML_NO_TEXT_NODE if there is no child text to query. - - */ - XMLError QueryIntText( int* ival ) const; - /// See QueryIntText() - XMLError QueryUnsignedText( unsigned* uval ) const; - /// See QueryIntText() - XMLError QueryBoolText( bool* bval ) const; - /// See QueryIntText() - XMLError QueryDoubleText( double* dval ) const; - /// See QueryIntText() - XMLError QueryFloatText( float* fval ) const; - - // internal: - enum { - OPEN, // - CLOSED, // - CLOSING // - }; - int ClosingType() const { - return _closingType; - } - char* ParseDeep( char* p, StrPair* endTag ); - virtual XMLNode* ShallowClone( XMLDocument* document ) const; - virtual bool ShallowEqual( const XMLNode* compare ) const; - -private: - XMLElement( XMLDocument* doc ); - virtual ~XMLElement(); - XMLElement( const XMLElement& ); // not supported - void operator=( const XMLElement& ); // not supported - - XMLAttribute* FindAttribute( const char* name ); - XMLAttribute* FindOrCreateAttribute( const char* name ); - //void LinkAttribute( XMLAttribute* attrib ); - char* ParseAttributes( char* p ); - static void DeleteAttribute( XMLAttribute* attribute ); - - enum { BUF_SIZE = 200 }; - int _closingType; - // The attribute list is ordered; there is no 'lastAttribute' - // because the list needs to be scanned for dupes before adding - // a new attribute. - XMLAttribute* _rootAttribute; -}; - - -enum Whitespace { - PRESERVE_WHITESPACE, - COLLAPSE_WHITESPACE -}; - - -/** A Document binds together all the functionality. - It can be saved, loaded, and printed to the screen. - All Nodes are connected and allocated to a Document. - If the Document is deleted, all its Nodes are also deleted. -*/ -class TINYXML2_LIB XMLDocument : public XMLNode -{ - friend class XMLElement; -public: - /// constructor - XMLDocument( bool processEntities = true, Whitespace = PRESERVE_WHITESPACE ); - ~XMLDocument(); - - virtual XMLDocument* ToDocument() { - return this; - } - virtual const XMLDocument* ToDocument() const { - return this; - } - - /** - Parse an XML file from a character string. - Returns XML_NO_ERROR (0) on success, or - an errorID. - - You may optionally pass in the 'nBytes', which is - the number of bytes which will be parsed. If not - specified, TinyXML-2 will assume 'xml' points to a - null terminated string. - */ - XMLError Parse( const char* xml, size_t nBytes=(size_t)(-1) ); - - /** - Load an XML file from disk. - Returns XML_NO_ERROR (0) on success, or - an errorID. - */ - XMLError LoadFile( const char* filename ); - - /** - Load an XML file from disk. You are responsible - for providing and closing the FILE*. - - Returns XML_NO_ERROR (0) on success, or - an errorID. - */ - XMLError LoadFile( FILE* ); - - /** - Save the XML file to disk. - Returns XML_NO_ERROR (0) on success, or - an errorID. - */ - XMLError SaveFile( const char* filename, bool compact = false ); - - /** - Save the XML file to disk. You are responsible - for providing and closing the FILE*. - - Returns XML_NO_ERROR (0) on success, or - an errorID. - */ - XMLError SaveFile( FILE* fp, bool compact = false ); - - bool ProcessEntities() const { - return _processEntities; - } - Whitespace WhitespaceMode() const { - return _whitespace; - } - - /** - Returns true if this document has a leading Byte Order Mark of UTF8. - */ - bool HasBOM() const { - return _writeBOM; - } - /** Sets whether to write the BOM when writing the file. - */ - void SetBOM( bool useBOM ) { - _writeBOM = useBOM; - } - - /** Return the root element of DOM. Equivalent to FirstChildElement(). - To get the first node, use FirstChild(). - */ - XMLElement* RootElement() { - return FirstChildElement(); - } - const XMLElement* RootElement() const { - return FirstChildElement(); - } - - /** Print the Document. If the Printer is not provided, it will - print to stdout. If you provide Printer, this can print to a file: - @verbatim - XMLPrinter printer( fp ); - doc.Print( &printer ); - @endverbatim - - Or you can use a printer to print to memory: - @verbatim - XMLPrinter printer; - doc.Print( &printer ); - // printer.CStr() has a const char* to the XML - @endverbatim - */ - void Print( XMLPrinter* streamer=0 ) const; - virtual bool Accept( XMLVisitor* visitor ) const; - - /** - Create a new Element associated with - this Document. The memory for the Element - is managed by the Document. - */ - XMLElement* NewElement( const char* name ); - /** - Create a new Comment associated with - this Document. The memory for the Comment - is managed by the Document. - */ - XMLComment* NewComment( const char* comment ); - /** - Create a new Text associated with - this Document. The memory for the Text - is managed by the Document. - */ - XMLText* NewText( const char* text ); - /** - Create a new Declaration associated with - this Document. The memory for the object - is managed by the Document. - - If the 'text' param is null, the standard - declaration is used.: - @verbatim - - @endverbatim - */ - XMLDeclaration* NewDeclaration( const char* text=0 ); - /** - Create a new Unknown associated with - this Document. The memory for the object - is managed by the Document. - */ - XMLUnknown* NewUnknown( const char* text ); - - /** - Delete a node associated with this document. - It will be unlinked from the DOM. - */ - void DeleteNode( XMLNode* node ) { - node->_parent->DeleteChild( node ); - } - - void SetError( XMLError error, const char* str1, const char* str2 ); - - /// Return true if there was an error parsing the document. - bool Error() const { - return _errorID != XML_NO_ERROR; - } - /// Return the errorID. - XMLError ErrorID() const { - return _errorID; - } - const char* ErrorName() const; - - /// Return a possibly helpful diagnostic location or string. - const char* GetErrorStr1() const { - return _errorStr1; - } - /// Return a possibly helpful secondary diagnostic location or string. - const char* GetErrorStr2() const { - return _errorStr2; - } - /// If there is an error, print it to stdout. - void PrintError() const; - - /// Clear the document, resetting it to the initial state. - void Clear(); - - // internal - char* Identify( char* p, XMLNode** node ); - - virtual XMLNode* ShallowClone( XMLDocument* /*document*/ ) const { - return 0; - } - virtual bool ShallowEqual( const XMLNode* /*compare*/ ) const { - return false; - } - -private: - XMLDocument( const XMLDocument& ); // not supported - void operator=( const XMLDocument& ); // not supported - - bool _writeBOM; - bool _processEntities; - XMLError _errorID; - Whitespace _whitespace; - const char* _errorStr1; - const char* _errorStr2; - char* _charBuffer; - - MemPoolT< sizeof(XMLElement) > _elementPool; - MemPoolT< sizeof(XMLAttribute) > _attributePool; - MemPoolT< sizeof(XMLText) > _textPool; - MemPoolT< sizeof(XMLComment) > _commentPool; - - static const char* _errorNames[XML_ERROR_COUNT]; -}; - - -/** - A XMLHandle is a class that wraps a node pointer with null checks; this is - an incredibly useful thing. Note that XMLHandle is not part of the TinyXML-2 - DOM structure. It is a separate utility class. - - Take an example: - @verbatim - - - - - - - @endverbatim - - Assuming you want the value of "attributeB" in the 2nd "Child" element, it's very - easy to write a *lot* of code that looks like: - - @verbatim - XMLElement* root = document.FirstChildElement( "Document" ); - if ( root ) - { - XMLElement* element = root->FirstChildElement( "Element" ); - if ( element ) - { - XMLElement* child = element->FirstChildElement( "Child" ); - if ( child ) - { - XMLElement* child2 = child->NextSiblingElement( "Child" ); - if ( child2 ) - { - // Finally do something useful. - @endverbatim - - And that doesn't even cover "else" cases. XMLHandle addresses the verbosity - of such code. A XMLHandle checks for null pointers so it is perfectly safe - and correct to use: - - @verbatim - XMLHandle docHandle( &document ); - XMLElement* child2 = docHandle.FirstChild( "Document" ).FirstChild( "Element" ).FirstChild().NextSibling().ToElement(); - if ( child2 ) - { - // do something useful - @endverbatim - - Which is MUCH more concise and useful. - - It is also safe to copy handles - internally they are nothing more than node pointers. - @verbatim - XMLHandle handleCopy = handle; - @endverbatim - - See also XMLConstHandle, which is the same as XMLHandle, but operates on const objects. -*/ -class TINYXML2_LIB XMLHandle -{ -public: - /// Create a handle from any node (at any depth of the tree.) This can be a null pointer. - XMLHandle( XMLNode* node ) { - _node = node; - } - /// Create a handle from a node. - XMLHandle( XMLNode& node ) { - _node = &node; - } - /// Copy constructor - XMLHandle( const XMLHandle& ref ) { - _node = ref._node; - } - /// Assignment - XMLHandle& operator=( const XMLHandle& ref ) { - _node = ref._node; - return *this; - } - - /// Get the first child of this handle. - XMLHandle FirstChild() { - return XMLHandle( _node ? _node->FirstChild() : 0 ); - } - /// Get the first child element of this handle. - XMLHandle FirstChildElement( const char* value=0 ) { - return XMLHandle( _node ? _node->FirstChildElement( value ) : 0 ); - } - /// Get the last child of this handle. - XMLHandle LastChild() { - return XMLHandle( _node ? _node->LastChild() : 0 ); - } - /// Get the last child element of this handle. - XMLHandle LastChildElement( const char* _value=0 ) { - return XMLHandle( _node ? _node->LastChildElement( _value ) : 0 ); - } - /// Get the previous sibling of this handle. - XMLHandle PreviousSibling() { - return XMLHandle( _node ? _node->PreviousSibling() : 0 ); - } - /// Get the previous sibling element of this handle. - XMLHandle PreviousSiblingElement( const char* _value=0 ) { - return XMLHandle( _node ? _node->PreviousSiblingElement( _value ) : 0 ); - } - /// Get the next sibling of this handle. - XMLHandle NextSibling() { - return XMLHandle( _node ? _node->NextSibling() : 0 ); - } - /// Get the next sibling element of this handle. - XMLHandle NextSiblingElement( const char* _value=0 ) { - return XMLHandle( _node ? _node->NextSiblingElement( _value ) : 0 ); - } - - /// Safe cast to XMLNode. This can return null. - XMLNode* ToNode() { - return _node; - } - /// Safe cast to XMLElement. This can return null. - XMLElement* ToElement() { - return ( ( _node == 0 ) ? 0 : _node->ToElement() ); - } - /// Safe cast to XMLText. This can return null. - XMLText* ToText() { - return ( ( _node == 0 ) ? 0 : _node->ToText() ); - } - /// Safe cast to XMLUnknown. This can return null. - XMLUnknown* ToUnknown() { - return ( ( _node == 0 ) ? 0 : _node->ToUnknown() ); - } - /// Safe cast to XMLDeclaration. This can return null. - XMLDeclaration* ToDeclaration() { - return ( ( _node == 0 ) ? 0 : _node->ToDeclaration() ); - } - -private: - XMLNode* _node; -}; - - -/** - A variant of the XMLHandle class for working with const XMLNodes and Documents. It is the - same in all regards, except for the 'const' qualifiers. See XMLHandle for API. -*/ -class TINYXML2_LIB XMLConstHandle -{ -public: - XMLConstHandle( const XMLNode* node ) { - _node = node; - } - XMLConstHandle( const XMLNode& node ) { - _node = &node; - } - XMLConstHandle( const XMLConstHandle& ref ) { - _node = ref._node; - } - - XMLConstHandle& operator=( const XMLConstHandle& ref ) { - _node = ref._node; - return *this; - } - - const XMLConstHandle FirstChild() const { - return XMLConstHandle( _node ? _node->FirstChild() : 0 ); - } - const XMLConstHandle FirstChildElement( const char* value=0 ) const { - return XMLConstHandle( _node ? _node->FirstChildElement( value ) : 0 ); - } - const XMLConstHandle LastChild() const { - return XMLConstHandle( _node ? _node->LastChild() : 0 ); - } - const XMLConstHandle LastChildElement( const char* _value=0 ) const { - return XMLConstHandle( _node ? _node->LastChildElement( _value ) : 0 ); - } - const XMLConstHandle PreviousSibling() const { - return XMLConstHandle( _node ? _node->PreviousSibling() : 0 ); - } - const XMLConstHandle PreviousSiblingElement( const char* _value=0 ) const { - return XMLConstHandle( _node ? _node->PreviousSiblingElement( _value ) : 0 ); - } - const XMLConstHandle NextSibling() const { - return XMLConstHandle( _node ? _node->NextSibling() : 0 ); - } - const XMLConstHandle NextSiblingElement( const char* _value=0 ) const { - return XMLConstHandle( _node ? _node->NextSiblingElement( _value ) : 0 ); - } - - - const XMLNode* ToNode() const { - return _node; - } - const XMLElement* ToElement() const { - return ( ( _node == 0 ) ? 0 : _node->ToElement() ); - } - const XMLText* ToText() const { - return ( ( _node == 0 ) ? 0 : _node->ToText() ); - } - const XMLUnknown* ToUnknown() const { - return ( ( _node == 0 ) ? 0 : _node->ToUnknown() ); - } - const XMLDeclaration* ToDeclaration() const { - return ( ( _node == 0 ) ? 0 : _node->ToDeclaration() ); - } - -private: - const XMLNode* _node; -}; - - -/** - Printing functionality. The XMLPrinter gives you more - options than the XMLDocument::Print() method. - - It can: - -# Print to memory. - -# Print to a file you provide. - -# Print XML without a XMLDocument. - - Print to Memory - - @verbatim - XMLPrinter printer; - doc.Print( &printer ); - SomeFunction( printer.CStr() ); - @endverbatim - - Print to a File - - You provide the file pointer. - @verbatim - XMLPrinter printer( fp ); - doc.Print( &printer ); - @endverbatim - - Print without a XMLDocument - - When loading, an XML parser is very useful. However, sometimes - when saving, it just gets in the way. The code is often set up - for streaming, and constructing the DOM is just overhead. - - The Printer supports the streaming case. The following code - prints out a trivially simple XML file without ever creating - an XML document. - - @verbatim - XMLPrinter printer( fp ); - printer.OpenElement( "foo" ); - printer.PushAttribute( "foo", "bar" ); - printer.CloseElement(); - @endverbatim -*/ -class TINYXML2_LIB XMLPrinter : public XMLVisitor -{ -public: - /** Construct the printer. If the FILE* is specified, - this will print to the FILE. Else it will print - to memory, and the result is available in CStr(). - If 'compact' is set to true, then output is created - with only required whitespace and newlines. - */ - XMLPrinter( FILE* file=0, bool compact = false, int depth = 0 ); - virtual ~XMLPrinter() {} - - /** If streaming, write the BOM and declaration. */ - void PushHeader( bool writeBOM, bool writeDeclaration ); - /** If streaming, start writing an element. - The element must be closed with CloseElement() - */ - void OpenElement( const char* name, bool compactMode=false ); - /// If streaming, add an attribute to an open element. - void PushAttribute( const char* name, const char* value ); - void PushAttribute( const char* name, int value ); - void PushAttribute( const char* name, unsigned value ); - void PushAttribute( const char* name, bool value ); - void PushAttribute( const char* name, double value ); - /// If streaming, close the Element. - virtual void CloseElement( bool compactMode=false ); - - /// Add a text node. - void PushText( const char* text, bool cdata=false ); - /// Add a text node from an integer. - void PushText( int value ); - /// Add a text node from an unsigned. - void PushText( unsigned value ); - /// Add a text node from a bool. - void PushText( bool value ); - /// Add a text node from a float. - void PushText( float value ); - /// Add a text node from a double. - void PushText( double value ); - - /// Add a comment - void PushComment( const char* comment ); - - void PushDeclaration( const char* value ); - void PushUnknown( const char* value ); - - virtual bool VisitEnter( const XMLDocument& /*doc*/ ); - virtual bool VisitExit( const XMLDocument& /*doc*/ ) { - return true; - } - - virtual bool VisitEnter( const XMLElement& element, const XMLAttribute* attribute ); - virtual bool VisitExit( const XMLElement& element ); - - virtual bool Visit( const XMLText& text ); - virtual bool Visit( const XMLComment& comment ); - virtual bool Visit( const XMLDeclaration& declaration ); - virtual bool Visit( const XMLUnknown& unknown ); - - /** - If in print to memory mode, return a pointer to - the XML file in memory. - */ - const char* CStr() const { - return _buffer.Mem(); - } - /** - If in print to memory mode, return the size - of the XML file in memory. (Note the size returned - includes the terminating null.) - */ - int CStrSize() const { - return _buffer.Size(); - } - /** - If in print to memory mode, reset the buffer to the - beginning. - */ - void ClearBuffer() { - _buffer.Clear(); - _buffer.Push(0); - } - -protected: - virtual bool CompactMode( const XMLElement& ) { return _compactMode; } - - /** Prints out the space before an element. You may override to change - the space and tabs used. A PrintSpace() override should call Print(). - */ - virtual void PrintSpace( int depth ); - void Print( const char* format, ... ); - - void SealElement(); - bool _elementJustOpened; - DynArray< const char*, 10 > _stack; - -private: - void PrintString( const char*, bool restrictedEntitySet ); // prints out, after detecting entities. - - bool _firstElement; - FILE* _fp; - int _depth; - int _textDepth; - bool _processEntities; - bool _compactMode; - - enum { - ENTITY_RANGE = 64, - BUF_SIZE = 200 - }; - bool _entityFlag[ENTITY_RANGE]; - bool _restrictedEntityFlag[ENTITY_RANGE]; - - DynArray< char, 20 > _buffer; -}; - - -} // tinyxml2 - -#if defined(_MSC_VER) -# pragma warning(pop) -#endif - -#endif // TINYXML2_INCLUDED From 1f1cf6f7d02c739fe891a6d4ca42df63891937e8 Mon Sep 17 00:00:00 2001 From: cDc Date: Sun, 26 May 2019 12:47:20 +0300 Subject: [PATCH 22/70] dense: improve confidence computation --- libs/MVS/SceneDensify.cpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index a5177b863..8424f7cfa 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -118,6 +118,13 @@ class EVTAdjustDepthMap : public Event /*----------------------------------------------------------------*/ +// convert the ZNCC score to a weight used to average the fused points +inline float Conf2Weight(float conf, Depth depth) { + return 1.f/(MAXF(1.f-conf,0.03f)*depth*depth); +} +/*----------------------------------------------------------------*/ + + // S T R U C T S /////////////////////////////////////////////////// @@ -602,7 +609,7 @@ void* STCALL DepthMapsData::EndDepthMapTmp(void* arg) float& conf = estimator.confMap0(x); // check if the score is good enough // and that the cross-estimates is close enough to the current estimate - if (conf >= OPTDENSE::fNCCThresholdKeep) { + if (depth <= 0 || conf >= OPTDENSE::fNCCThresholdKeep) { #if 1 // used if gap-interpolation is active conf = 0; estimator.normalMap0(x) = Normal::ZERO; @@ -610,7 +617,8 @@ void* STCALL DepthMapsData::EndDepthMapTmp(void* arg) depth = 0; } else { #if 1 - conf = 1.f/(MAXF(conf,1e-2f)*depth); + // converted ZNCC [0-2] score, where 0 is best, to [0-1] confidence, where 1 is best + conf = conf>=1.f ? 0.f : 1.f-conf; #else #if 1 FOREACH(i, estimator.images) @@ -1419,13 +1427,12 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b PointCloud::ViewArr& views = pointcloud.pointViews.AddEmpty(); views.Insert(idxImage); PointCloud::WeightArr& weights = pointcloud.pointWeights.AddEmpty(); - weights.Insert(depthData.confMap(x)); + REAL confidence(weights.emplace_back(Conf2Weight(depthData.confMap(x),depth))); ProjArr& pointProjs = projs.AddEmpty(); pointProjs.Insert(Proj(x)); const PointCloud::Normal normal(imageData.camera.R.t()*Cast(depthData.normalMap(x))); ASSERT(ISEQUAL(norm(normal), 1.f)); // check the projection in the neighbor depth-maps - REAL confidence(weights.First()); Point3 X(point*confidence); Pixel32F C(Cast(imageData.image(x))*confidence); PointCloud::Normal N(normal*confidence); @@ -1456,7 +1463,7 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b if (normal.dot(normalB) > normalError) { // add view to the 3D point ASSERT(views.FindFirst(idxImageB) == PointCloud::ViewArr::NO_INDEX); - const float confidenceB(depthDataB.confMap(xB)); + const float confidenceB(Conf2Weight(depthDataB.confMap(xB),depthB)); const IIndex idx(views.InsertSort(idxImageB)); weights.InsertAt(idx, confidenceB); pointProjs.InsertAt(idx, Proj(xB)); From 3af59f77c9b0d7b0f7ecedf06a0197bcb606f322 Mon Sep 17 00:00:00 2001 From: cDc Date: Sat, 8 Jun 2019 10:38:11 +0300 Subject: [PATCH 23/70] mesh: small bug fix --- libs/MVS/SceneReconstruct.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/MVS/SceneReconstruct.cpp b/libs/MVS/SceneReconstruct.cpp index 88b8c73f0..81005bcd8 100644 --- a/libs/MVS/SceneReconstruct.cpp +++ b/libs/MVS/SceneReconstruct.cpp @@ -399,7 +399,7 @@ void fetchCellFacets(const delaunay_t& Tr, const std::vector& hullFacet // create the 4 frustum planes ASSERT(facets.empty()); typedef TFrustum Frustum; - Frustum frustum(imageData.camera.P, imageData.width, imageData.width, 0, 1); + Frustum frustum(imageData.camera.P, imageData.width, imageData.height, 0, 1); // loop over all cells const point_t ptOrigin(MVS2CGAL(imageData.camera.C)); for (const facet_t& face: hullFacets) { From 3e9c369a21c54aa0f947fee89f297713a9b90b06 Mon Sep 17 00:00:00 2001 From: shuaidu Date: Mon, 24 Jun 2019 18:18:12 +0800 Subject: [PATCH 24/70] mesh: intersect with infinite tetrahedron bug fix (#450) (cherry picked from commit c70f51f2c87b4cb6548084067571746d16358d2a) --- libs/MVS/SceneReconstruct.cpp | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/libs/MVS/SceneReconstruct.cpp b/libs/MVS/SceneReconstruct.cpp index 81005bcd8..5a87e74f4 100644 --- a/libs/MVS/SceneReconstruct.cpp +++ b/libs/MVS/SceneReconstruct.cpp @@ -536,11 +536,12 @@ bool intersect(const delaunay_t& Tr, const segment_t& seg, const std::vector= 0) { // skip this cell if the intersection is not in the desired direction - inter.dist = inter.ray.IntersectsDist(getFacetPlane(in_facet)); - if ((inter.dist > prevDist) != inter.bigger) + const REAL interDist(inter.ray.IntersectsDist(getFacetPlane(in_facet))); + if ((interDist > prevDist) != inter.bigger) continue; // vertices of facet i: j = 4 * i, vertices = facet_vertex_order[j,j+1,j+2] negative orientation inter.facet = in_facet; + inter.dist = interDist; switch (nb_coplanar) { case 0: { // face intersection @@ -570,8 +571,12 @@ bool intersect(const delaunay_t& Tr, const segment_t& seg, const std::vectorindex(inter.v1))); - out_facets.push_back(facet_t(c, c->index(inter.v2))); + const facet_t f1(c, c->index(inter.v1)); + if (!Tr.is_infinite(f1)) + out_facets.push_back(f1); + const facet_t f2(c, c->index(inter.v2)); + if (!Tr.is_infinite(f2)) + out_facets.push_back(f2); } while (++ifc != efc); return true; } case 2: { @@ -601,19 +606,24 @@ bool intersect(const delaunay_t& Tr, const segment_t& seg, const std::vector& out_facets; - inline cell_back_inserter_t(const intersection_t& inter, std::vector& _out_facets) - : v(inter.v1), current_cell(inter.facet.first), out_facets(_out_facets) {} + inline cell_back_inserter_t(const delaunay_t& _Tr, const intersection_t& inter, std::vector& _out_facets) + : Tr(_Tr), v(inter.v1), current_cell(inter.facet.first), out_facets(_out_facets) {} inline cell_back_inserter_t& operator*() { return *this; } inline cell_back_inserter_t& operator++(int) { return *this; } inline void operator=(cell_handle_t c) { - if (c != current_cell) - out_facets.push_back(facet_t(c, c->index(v))); + if (c == current_cell) + return; + const facet_t f(c, c->index(v)); + if (Tr.is_infinite(f)) + return; + out_facets.push_back(f); } }; - Tr.finite_incident_cells(inter.v1, cell_back_inserter_t(inter, out_facets)); + Tr.finite_incident_cells(inter.v1, cell_back_inserter_t(Tr, inter, out_facets)); return true; } } // coplanar with 3 edges = tangent = impossible? From 0b5bfea4232cc197b885e96a167293bb34fa5c7f Mon Sep 17 00:00:00 2001 From: cDc Date: Fri, 6 Sep 2019 15:31:21 +0300 Subject: [PATCH 25/70] interface: fix COLMAP import (cherry picked from commit eeaed441b67e2c8bc894a2779cb238973591a80f) --- apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp | 37 +++++++++++------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp b/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp index 651b44c98..27eabf0e8 100644 --- a/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp +++ b/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp @@ -412,16 +412,17 @@ bool ImportScene(const String& strFolder, Interface& scene) VERBOSE("error: unable to open file '%s'", filenameCameras.c_str()); return false; } - typedef std::unordered_set CamerasSet; + typedef std::unordered_map CamerasSet; CamerasSet setCameras; - { - COLMAP::Camera camera; - while (file.good() && camera.Read(file)) - setCameras.emplace(camera); - } - mapCameras.reserve(setCameras.size()); - for (const COLMAP::Camera& colmapCamera: setCameras) { - mapCameras[colmapCamera.ID] = (uint32_t)scene.platforms.size(); + COLMAP::Camera colmapCamera; + while (file.good() && colmapCamera.Read(file)) { + const auto setIt(setCameras.emplace(colmapCamera, (uint32_t)scene.platforms.size())); + mapCameras.emplace(colmapCamera.ID, setIt.first->second); + if (!setIt.second) { + // reuse existing platform + continue; + } + // create new platform Interface::Platform platform; platform.name = String::FormatString(_T("platform%03u"), colmapCamera.ID); // only one camera per platform supported Interface::Platform::Camera camera; @@ -460,20 +461,16 @@ bool ImportScene(const String& strFolder, Interface& scene) VERBOSE("error: unable to open file '%s'", filenameImages.c_str()); return false; } - { - COLMAP::Image image; - while (file.good() && image.Read(file)) - mapImages[image]; - } - for (ImagesMap::value_type& it: mapImages) { - it.second = (uint32_t)scene.images.size(); + COLMAP::Image imageColmap; + while (file.good() && imageColmap.Read(file)) { + mapImages.emplace(imageColmap, (uint32_t)scene.images.size()); Interface::Platform::Pose pose; - Eigen::Map(pose.R.val) = it.first.q.toRotationMatrix(); + Eigen::Map(pose.R.val) = imageColmap.q.toRotationMatrix(); EnsureRotationMatrix((Matrix3x3d&)pose.R); - Eigen::Map(&pose.C.x) = -(it.first.q.inverse() * it.first.t); + Eigen::Map(&pose.C.x) = -(imageColmap.q.inverse() * imageColmap.t); Interface::Image image; - image.name = OPT::strImageFolder+it.first.name; - image.platformID = mapCameras.at(it.first.idCamera); + image.name = OPT::strImageFolder+imageColmap.name; + image.platformID = mapCameras.at(imageColmap.idCamera); image.cameraID = 0; Interface::Platform& platform = scene.platforms[image.platformID]; image.poseID = (uint32_t)platform.poses.size(); From eb8dbc810dd4a7aca55a8ca63ab7d538e705f123 Mon Sep 17 00:00:00 2001 From: cDc Date: Thu, 12 Sep 2019 10:47:26 +0300 Subject: [PATCH 26/70] dense: fix debug asserts (cherry picked from commit ea15e142c5068c57a3d83ae80051e80a8e4532eb) --- libs/MVS/DepthMap.cpp | 6 ++-- libs/MVS/SceneDensify.cpp | 68 +++++++++++++++++++++------------------ 2 files changed, 39 insertions(+), 35 deletions(-) diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index 9c4d77c8a..12973ac3a 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -441,7 +441,7 @@ float DepthEstimator::ScorePixelImage(const ViewData& image1, Depth depth, const // compute pixel's NCC score float DepthEstimator::ScorePixel(Depth depth, const Normal& normal) { - ASSERT(depth > 0 && normal.dot(Cast(static_cast(X0))) < 0); + ASSERT(depth > 0 && normal.dot(Cast(static_cast(X0))) <= 0); // compute score for this pixel as seen in each view ASSERT(scores.size() == images.size()); FOREACH(idxView, images) @@ -631,7 +631,7 @@ void DepthEstimator::ProcessPixel(IDX idx) Depth& depth = depthMap0(x0); Normal& normal = normalMap0(x0); const Normal viewDir(Cast(static_cast(X0))); - ASSERT(depth > 0 && normal.dot(viewDir) < 0); + ASSERT(depth > 0 && normal.dot(viewDir) <= 0); #if DENSE_REFINE == DENSE_REFINE_ITER // check if any of the neighbor estimates are better then the current estimate #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA @@ -774,7 +774,7 @@ void DepthEstimator::ProcessPixel(IDX idx) // interpolate given pixel's estimate to the current position Depth DepthEstimator::InterpolatePixel(const ImageRef& nx, Depth depth, const Normal& normal) const { - ASSERT(depth > 0 && normal.dot(image0.camera.TransformPointI2C(Cast(nx))) < 0); + ASSERT(depth > 0 && normal.dot(image0.camera.TransformPointI2C(Cast(nx))) <= 0); Depth depthNew; #if 1 // compute as intersection of the lines diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index 8424f7cfa..5ec91fc73 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -843,7 +843,7 @@ bool DepthMapsData::RemoveSmallSegments(DepthData& depthData) NormalMap& normalMap = depthData.normalMap; ConfidenceMap& confMap = depthData.confMap; ASSERT(!depthMap.empty()); - ImageRef size(depthMap.size()); + const ImageRef size(depthMap.size()); // allocate memory on heap for dynamic programming arrays TImage done_map(size, false); @@ -871,33 +871,35 @@ bool DepthMapsData::RemoveSmallSegments(DepthData& depthData) while (seg_list_curr < seg_list_count) { // get address of current pixel in this segment const ImageRef addr_curr(seg_list[seg_list_curr]); - - // fill list with neighbor positions - neighbor[0] = ImageRef(addr_curr.x-1, addr_curr.y ); - neighbor[1] = ImageRef(addr_curr.x+1, addr_curr.y ); - neighbor[2] = ImageRef(addr_curr.x , addr_curr.y-1); - neighbor[3] = ImageRef(addr_curr.x , addr_curr.y+1); - - // for all neighbors do const Depth& depth_curr = depthMap(addr_curr); - for (int i=0; i<4; ++i) { - // get neighbor pixel address - const ImageRef& addr_neighbor(neighbor[i]); - // check if neighbor is inside image - if (addr_neighbor.x>=0 && addr_neighbor.y>=0 && addr_neighbor.x0 && IsDepthSimilar(depth_curr, depth_neighbor, fDepthDiffThreshold)) { - // add neighbor coordinates to segment list - seg_list[seg_list_count++] = addr_neighbor; - // set neighbor pixel in done_map to "done" - // (otherwise a pixel may be added 2 times to the list, as - // neighbor of one pixel and as neighbor of another pixel) - done = true; + + if (depth_curr>0) { + // fill list with neighbor positions + neighbor[0] = ImageRef(addr_curr.x-1, addr_curr.y ); + neighbor[1] = ImageRef(addr_curr.x+1, addr_curr.y ); + neighbor[2] = ImageRef(addr_curr.x , addr_curr.y-1); + neighbor[3] = ImageRef(addr_curr.x , addr_curr.y+1); + + // for all neighbors do + for (int i=0; i<4; ++i) { + // get neighbor pixel address + const ImageRef& addr_neighbor(neighbor[i]); + // check if neighbor is inside image + if (addr_neighbor.x>=0 && addr_neighbor.y>=0 && addr_neighbor.x0 && IsDepthSimilar(depth_curr, depth_neighbor, fDepthDiffThreshold)) { + // add neighbor coordinates to segment list + seg_list[seg_list_count++] = addr_neighbor; + // set neighbor pixel in done_map to "done" + // (otherwise a pixel may be added 2 times to the list, as + // neighbor of one pixel and as neighbor of another pixel) + done = true; + } } } } @@ -935,7 +937,7 @@ bool DepthMapsData::GapInterpolation(DepthData& depthData) NormalMap& normalMap = depthData.normalMap; ConfidenceMap& confMap = depthData.confMap; ASSERT(!depthMap.empty()); - ImageRef size(depthMap.size()); + const ImageRef size(depthMap.size()); // 1. Row-wise: // for each row do @@ -1851,13 +1853,15 @@ void Scene::DenseReconstructionEstimate(void* pData) // apply filters if (OPTDENSE::nOptimize & (OPTDENSE::REMOVE_SPECKLES)) { TD_TIMER_START(); - data.detphMaps.RemoveSmallSegments(depthData); - DEBUG_ULTIMATE("Depth-map %3u filtered: remove small segments (%s)", idx, TD_TIMER_GET_FMT().c_str()); + if (data.detphMaps.RemoveSmallSegments(depthData)) { + DEBUG_ULTIMATE("Depth-map %3u filtered: remove small segments (%s)", idx, TD_TIMER_GET_FMT().c_str()); + } } if (OPTDENSE::nOptimize & (OPTDENSE::FILL_GAPS)) { TD_TIMER_START(); - data.detphMaps.GapInterpolation(depthData); - DEBUG_ULTIMATE("Depth-map %3u filtered: gap interpolation (%s)", idx, TD_TIMER_GET_FMT().c_str()); + if (data.detphMaps.GapInterpolation(depthData)) { + DEBUG_ULTIMATE("Depth-map %3u filtered: gap interpolation (%s)", idx, TD_TIMER_GET_FMT().c_str()); + } } // save depth-map data.events.AddEventFirst(new EVTSaveDepthMap(evtImage.idxImage)); From 564e4c5ea44fe646b0345777648c551291699d0c Mon Sep 17 00:00:00 2001 From: indianajohn Date: Fri, 20 Sep 2019 01:52:32 -0700 Subject: [PATCH 27/70] dense: expose SceneDensify data structures and functions (#476) --- libs/Common/Types.inl | 2 +- libs/MVS.h | 1 + libs/MVS/Scene.h | 4 + libs/MVS/SceneDensify.cpp | 172 ++++++++++++++------------------------ libs/MVS/SceneDensify.h | 106 +++++++++++++++++++++++ 5 files changed, 174 insertions(+), 111 deletions(-) create mode 100644 libs/MVS/SceneDensify.h diff --git a/libs/Common/Types.inl b/libs/Common/Types.inl index eb1582583..6a23b7c59 100644 --- a/libs/Common/Types.inl +++ b/libs/Common/Types.inl @@ -2147,7 +2147,7 @@ void TImage::toGray(TImage& out, int code, bool bNormalize, bool bSRGB) out.create(rows, cols); ASSERT(cv::Mat::isContinuous()); ASSERT(out.cv::Mat::isContinuous()); - const int scn(cv::Mat::channels()); + const int scn(this->cv::Mat::channels()); T* dst = out.cv::Mat::template ptr(); T* const dstEnd = dst + out.area(); typedef typename cv::DataType::channel_type ST; diff --git a/libs/MVS.h b/libs/MVS.h index a8937c08f..d4ec4937d 100644 --- a/libs/MVS.h +++ b/libs/MVS.h @@ -41,6 +41,7 @@ #include "Math/Common.h" #include "MVS/Common.h" #include "MVS/Scene.h" +#include "MVS/SceneDensify.h" #endif // _MVS_MVS_H_ diff --git a/libs/MVS/Scene.h b/libs/MVS/Scene.h index 581ad120e..782e9a315 100644 --- a/libs/MVS/Scene.h +++ b/libs/MVS/Scene.h @@ -45,6 +45,9 @@ // S T R U C T S /////////////////////////////////////////////////// namespace MVS { + +// Forward declarations +struct MVS_API DenseDepthMapData; class MVS_API Scene { @@ -79,6 +82,7 @@ class MVS_API Scene // Dense reconstruction bool DenseReconstruction(); + bool ComputeDepthMaps(DenseDepthMapData& data); void DenseReconstructionEstimate(void*); void DenseReconstructionFilter(void*); void PointCloudFilter(int thRemove=-1); diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index 5ec91fc73..0f6b0cc4f 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -31,6 +31,7 @@ #include "Common.h" #include "Scene.h" +#include "SceneDensify.h" // MRF: view selection #include "../Math/TRWS/MRFEnergy.h" // CGAL: depth-map initialization @@ -125,46 +126,8 @@ inline float Conf2Weight(float conf, Depth depth) { /*----------------------------------------------------------------*/ - // S T R U C T S /////////////////////////////////////////////////// -// structure used to compute all depth-maps -class DepthMapsData -{ -public: - DepthMapsData(Scene& _scene); - ~DepthMapsData(); - - bool SelectViews(IIndexArr& images, IIndexArr& imagesMap, IIndexArr& neighborsMap); - bool SelectViews(DepthData& depthData); - bool InitViews(DepthData& depthData, IIndex idxNeighbor, IIndex numNeighbors); - bool InitDepthMap(DepthData& depthData); - bool EstimateDepthMap(IIndex idxImage); - - bool RemoveSmallSegments(DepthData& depthData); - bool GapInterpolation(DepthData& depthData); - - bool FilterDepthMap(DepthData& depthData, const IIndexArr& idxNeighbors, bool bAdjust=true); - void FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, bool bEstimateNormal); - -protected: - static void* STCALL ScoreDepthMapTmp(void*); - static void* STCALL EstimateDepthMapTmp(void*); - static void* STCALL EndDepthMapTmp(void*); - -public: - Scene& scene; - - DepthDataArr arrDepthData; - - // used internally to estimate the depth-maps - Image8U::Size prevDepthMapSize; // remember the size of the last estimated depth-map - Image8U::Size prevDepthMapSizeTrg; // ... same for target image - DepthEstimator::MapRefArr coords; // map pixel index to zigzag matrix coordinates - DepthEstimator::MapRefArr coordsTrg; // ... same for target image -}; -/*----------------------------------------------------------------*/ - DepthMapsData::DepthMapsData(Scene& _scene) : @@ -1560,32 +1523,54 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b // S T R U C T S /////////////////////////////////////////////////// -struct DenseDepthMapData { - Scene& scene; - IIndexArr images; - IIndexArr neighborsMap; - DepthMapsData detphMaps; - volatile Thread::safe_t idxImage; - SEACAVE::EventQueue events; // internal events queue (processed by the working threads) - Semaphore sem; - CAutoPtr progress; - - DenseDepthMapData(Scene& _scene) - : scene(_scene), detphMaps(_scene), idxImage(0), sem(1) {} - - void SignalCompleteDepthmapFilter() { - ASSERT(idxImage > 0); - if (Thread::safeDec(idxImage) == 0) - sem.Signal((unsigned)images.GetSize()*2); - } -}; - static void* DenseReconstructionEstimateTmp(void*); static void* DenseReconstructionFilterTmp(void*); -bool Scene::DenseReconstruction() -{ + +bool Scene::DenseReconstruction() { DenseDepthMapData data(*this); + + // estimate depth-maps + if (!ComputeDepthMaps(data)) + return false; + + // fuse all depth-maps + pointcloud.Release(); + data.depthMaps.FuseDepthMaps(pointcloud, OPTDENSE::nEstimateColors == 2, OPTDENSE::nEstimateNormals == 2); + #if TD_VERBOSE != TD_VERBOSE_OFF + if (g_nVerbosityLevel > 2) { + // print number of points with 3+ views + size_t nPoints1m(0), nPoints2(0), nPoints3p(0); + FOREACHPTR(pViews, pointcloud.pointViews) { + switch (pViews->GetSize()) + { + case 0: + case 1: + ++nPoints1m; + break; + case 2: + ++nPoints2; + break; + default: + ++nPoints3p; + } + } + VERBOSE("Dense point-cloud composed of:\n\t%u points with 1- views\n\t%u points with 2 views\n\t%u points with 3+ views", nPoints1m, nPoints2, nPoints3p); + } + #endif + if (!pointcloud.IsEmpty()) { + if (pointcloud.colors.IsEmpty() && OPTDENSE::nEstimateColors == 1) + EstimatePointColors(images, pointcloud); + if (pointcloud.normals.IsEmpty() && OPTDENSE::nEstimateNormals == 1) + EstimatePointNormals(images, pointcloud); + } + return true; +} // DenseReconstruction +/*----------------------------------------------------------------*/ + +// do first half of dense reconstruction: depth map computation +// results are saved to "data" +bool Scene::ComputeDepthMaps(DenseDepthMapData& data) { { // maps global view indices to our list of views to be processed IIndexArr imagesMap; @@ -1665,8 +1650,8 @@ bool Scene::DenseReconstruction() #endif const IIndex idxImage(data.images[idx]); ASSERT(imagesMap[idxImage] != NO_ID); - DepthData& depthData(data.detphMaps.arrDepthData[idxImage]); - if (!data.detphMaps.SelectViews(depthData)) { + DepthData& depthData(data.depthMaps.arrDepthData[idxImage]); + if (!data.depthMaps.SelectViews(depthData)) { #ifdef DENSE_USE_OPENMP #pragma omp critical #endif @@ -1680,7 +1665,7 @@ bool Scene::DenseReconstruction() data.images.RemoveAt(idx); } // globally select a target view for each reference image - if (OPTDENSE::nNumViews == 1 && !data.detphMaps.SelectViews(data.images, imagesMap, data.neighborsMap)) { + if (OPTDENSE::nNumViews == 1 && !data.depthMaps.SelectViews(data.images, imagesMap, data.neighborsMap)) { VERBOSE("error: no valid images to be dense reconstructed"); return false; } @@ -1738,40 +1723,8 @@ bool Scene::DenseReconstruction() return false; data.progress.Release(); } - - // fuse all depth-maps - pointcloud.Release(); - data.detphMaps.FuseDepthMaps(pointcloud, OPTDENSE::nEstimateColors == 2, OPTDENSE::nEstimateNormals == 2); - #if TD_VERBOSE != TD_VERBOSE_OFF - if (g_nVerbosityLevel > 2) { - // print number of points with 3+ views - size_t nPoints1m(0), nPoints2(0), nPoints3p(0); - FOREACHPTR(pViews, pointcloud.pointViews) { - switch (pViews->GetSize()) - { - case 0: - case 1: - ++nPoints1m; - break; - case 2: - ++nPoints2; - break; - default: - ++nPoints3p; - } - } - VERBOSE("Dense point-cloud composed of:\n\t%u points with 1- views\n\t%u points with 2 views\n\t%u points with 3+ views", nPoints1m, nPoints2, nPoints3p); - } - #endif - - if (!pointcloud.IsEmpty()) { - if (pointcloud.colors.IsEmpty() && OPTDENSE::nEstimateColors == 1) - EstimatePointColors(images, pointcloud); - if (pointcloud.normals.IsEmpty() && OPTDENSE::nEstimateNormals == 1) - EstimatePointNormals(images, pointcloud); - } return true; -} // DenseReconstruction +} // ComputeDepthMaps /*----------------------------------------------------------------*/ void* DenseReconstructionEstimateTmp(void* arg) { @@ -1781,8 +1734,7 @@ void* DenseReconstructionEstimateTmp(void* arg) { } // initialize the dense reconstruction with the sparse point cloud -void Scene::DenseReconstructionEstimate(void* pData) -{ +void Scene::DenseReconstructionEstimate(void* pData) { DenseDepthMapData& data = *((DenseDepthMapData*)pData); while (true) { CAutoPtr evt(data.events.GetEvent()); @@ -1798,10 +1750,10 @@ void Scene::DenseReconstructionEstimate(void* pData) } // select views to reconstruct the depth-map for this image const IIndex idx = data.images[evtImage.idxImage]; - DepthData& depthData(data.detphMaps.arrDepthData[idx]); + DepthData& depthData(data.depthMaps.arrDepthData[idx]); // init images pair: reference image and the best neighbor view ASSERT(data.neighborsMap.IsEmpty() || data.neighborsMap[evtImage.idxImage] != NO_ID); - if (!data.detphMaps.InitViews(depthData, data.neighborsMap.IsEmpty()?NO_ID:data.neighborsMap[evtImage.idxImage], OPTDENSE::nNumViews)) { + if (!data.depthMaps.InitViews(depthData, data.neighborsMap.IsEmpty()?NO_ID:data.neighborsMap[evtImage.idxImage], OPTDENSE::nNumViews)) { // process next image data.events.AddEvent(new EVTProcessImage((IIndex)Thread::safeInc(data.idxImage))); break; @@ -1830,7 +1782,7 @@ void Scene::DenseReconstructionEstimate(void* pData) data.events.AddEvent(new EVTProcessImage((uint32_t)Thread::safeInc(data.idxImage))); // extract depth map data.sem.Wait(); - data.detphMaps.EstimateDepthMap(data.images[evtImage.idxImage]); + data.depthMaps.EstimateDepthMap(data.images[evtImage.idxImage]); data.sem.Signal(); if (OPTDENSE::nOptimize & OPTDENSE::OPTIMIZE) { // optimize depth-map @@ -1844,7 +1796,7 @@ void Scene::DenseReconstructionEstimate(void* pData) case EVT_OPTIMIZEDEPTHMAP: { const EVTOptimizeDepthMap& evtImage = *((EVTOptimizeDepthMap*)(Event*)evt); const IIndex idx = data.images[evtImage.idxImage]; - DepthData& depthData(data.detphMaps.arrDepthData[idx]); + DepthData& depthData(data.depthMaps.arrDepthData[idx]); #if TD_VERBOSE != TD_VERBOSE_OFF // save depth map as image if (g_nVerbosityLevel > 3) @@ -1853,13 +1805,13 @@ void Scene::DenseReconstructionEstimate(void* pData) // apply filters if (OPTDENSE::nOptimize & (OPTDENSE::REMOVE_SPECKLES)) { TD_TIMER_START(); - if (data.detphMaps.RemoveSmallSegments(depthData)) { + if (data.depthMaps.RemoveSmallSegments(depthData)) { DEBUG_ULTIMATE("Depth-map %3u filtered: remove small segments (%s)", idx, TD_TIMER_GET_FMT().c_str()); } } if (OPTDENSE::nOptimize & (OPTDENSE::FILL_GAPS)) { TD_TIMER_START(); - if (data.detphMaps.GapInterpolation(depthData)) { + if (data.depthMaps.GapInterpolation(depthData)) { DEBUG_ULTIMATE("Depth-map %3u filtered: gap interpolation (%s)", idx, TD_TIMER_GET_FMT().c_str()); } } @@ -1870,7 +1822,7 @@ void Scene::DenseReconstructionEstimate(void* pData) case EVT_SAVEDEPTHMAP: { const EVTSaveDepthMap& evtImage = *((EVTSaveDepthMap*)(Event*)evt); const IIndex idx = data.images[evtImage.idxImage]; - DepthData& depthData(data.detphMaps.arrDepthData[idx]); + DepthData& depthData(data.depthMaps.arrDepthData[idx]); #if TD_VERBOSE != TD_VERBOSE_OFF // save depth map as image if (g_nVerbosityLevel > 2) { @@ -1916,7 +1868,7 @@ void Scene::DenseReconstructionFilter(void* pData) case EVT_FILTERDEPTHMAP: { const EVTFilterDepthMap& evtImage = *((EVTFilterDepthMap*)(Event*)evt); const IIndex idx = data.images[evtImage.idxImage]; - DepthData& depthData(data.detphMaps.arrDepthData[idx]); + DepthData& depthData(data.depthMaps.arrDepthData[idx]); if (!depthData.IsValid()) { data.SignalCompleteDepthmapFilter(); break; @@ -1927,7 +1879,7 @@ void Scene::DenseReconstructionFilter(void* pData) IIndexArr idxNeighbors(0, depthData.neighbors.GetSize()); FOREACH(n, depthData.neighbors) { const IIndex idxView = depthData.neighbors[n].idx.ID; - DepthData& depthDataPair = data.detphMaps.arrDepthData[idxView]; + DepthData& depthDataPair = data.depthMaps.arrDepthData[idxView]; if (!depthDataPair.IsValid()) continue; if (depthDataPair.IncRef(ComposeDepthFilePath(idxView, "dmap")) == 0) { @@ -1940,14 +1892,14 @@ void Scene::DenseReconstructionFilter(void* pData) break; } // filter the depth-map for this image - if (data.detphMaps.FilterDepthMap(depthData, idxNeighbors, OPTDENSE::bFilterAdjust)) { + if (data.depthMaps.FilterDepthMap(depthData, idxNeighbors, OPTDENSE::bFilterAdjust)) { // load the filtered maps after all depth-maps were filtered data.events.AddEvent(new EVTAdjustDepthMap(evtImage.idxImage)); } // unload referenced depth-maps FOREACHPTR(pIdxNeighbor, idxNeighbors) { const IIndex idxView = depthData.neighbors[*pIdxNeighbor].idx.ID; - DepthData& depthDataPair = data.detphMaps.arrDepthData[idxView]; + DepthData& depthDataPair = data.depthMaps.arrDepthData[idxView]; depthDataPair.DecRef(); } depthData.DecRef(); @@ -1957,7 +1909,7 @@ void Scene::DenseReconstructionFilter(void* pData) case EVT_ADJUSTDEPTHMAP: { const EVTAdjustDepthMap& evtImage = *((EVTAdjustDepthMap*)(Event*)evt); const IIndex idx = data.images[evtImage.idxImage]; - DepthData& depthData(data.detphMaps.arrDepthData[idx]); + DepthData& depthData(data.depthMaps.arrDepthData[idx]); ASSERT(depthData.IsValid()); data.sem.Wait(); // load filtered maps diff --git a/libs/MVS/SceneDensify.h b/libs/MVS/SceneDensify.h new file mode 100644 index 000000000..6d7d0efe0 --- /dev/null +++ b/libs/MVS/SceneDensify.h @@ -0,0 +1,106 @@ +/* +* SceneDensify.h +* +* Copyright (c) 2014-2015 SEACAVE +* +* Author(s): +* +* cDc +* +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* +* Additional Terms: +* +* You are required to preserve legal notices and author attributions in +* that material or in the Appropriate Legal Notices displayed by works +* containing it. +*/ + +#ifndef _MVS_SCENEDENSIFY_H_ +#define _MVS_SCENEDENSIFY_H_ + + +// I N C L U D E S ///////////////////////////////////////////////// + + +// S T R U C T S /////////////////////////////////////////////////// + +namespace MVS { + +// Forward declarations +class MVS_API Scene; + +// structure used to compute all depth-maps +class MVS_API DepthMapsData +{ +public: + DepthMapsData(Scene& _scene); + ~DepthMapsData(); + + bool SelectViews(IIndexArr& images, IIndexArr& imagesMap, IIndexArr& neighborsMap); + bool SelectViews(DepthData& depthData); + bool InitViews(DepthData& depthData, IIndex idxNeighbor, IIndex numNeighbors); + bool InitDepthMap(DepthData& depthData); + bool EstimateDepthMap(IIndex idxImage); + + bool RemoveSmallSegments(DepthData& depthData); + bool GapInterpolation(DepthData& depthData); + + bool FilterDepthMap(DepthData& depthData, const IIndexArr& idxNeighbors, bool bAdjust=true); + void FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, bool bEstimateNormal); + +protected: + static void* STCALL ScoreDepthMapTmp(void*); + static void* STCALL EstimateDepthMapTmp(void*); + static void* STCALL EndDepthMapTmp(void*); + +public: + Scene& scene; + + DepthDataArr arrDepthData; + + // used internally to estimate the depth-maps + Image8U::Size prevDepthMapSize; // remember the size of the last estimated depth-map + Image8U::Size prevDepthMapSizeTrg; // ... same for target image + DepthEstimator::MapRefArr coords; // map pixel index to zigzag matrix coordinates + DepthEstimator::MapRefArr coordsTrg; // ... same for target image +}; +/*----------------------------------------------------------------*/ + +struct MVS_API DenseDepthMapData { + Scene& scene; + IIndexArr images; + IIndexArr neighborsMap; + DepthMapsData depthMaps; + volatile Thread::safe_t idxImage; + SEACAVE::EventQueue events; // internal events queue (processed by the working threads) + Semaphore sem; + CAutoPtr progress; + + DenseDepthMapData(Scene& _scene) + : scene(_scene), depthMaps(_scene), idxImage(0), sem(1) {} + + void SignalCompleteDepthmapFilter() { + ASSERT(idxImage > 0); + if (Thread::safeDec(idxImage) == 0) + sem.Signal((unsigned)images.GetSize()*2); + } +}; +/*----------------------------------------------------------------*/ + +} // namespace MVS + +#endif From dae77e7ae9cfba9818d1c8b46ce0c64c69408e96 Mon Sep 17 00:00:00 2001 From: cDc Date: Wed, 2 Oct 2019 18:50:16 +0300 Subject: [PATCH 28/70] common: use boost throw_exception declaration --- libs/Common/Types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/Common/Types.h b/libs/Common/Types.h index aad57d11a..ef2138046 100644 --- a/libs/Common/Types.h +++ b/libs/Common/Types.h @@ -88,7 +88,7 @@ #define BOOST_NO_EXCEPTIONS #endif #ifdef BOOST_NO_EXCEPTIONS -namespace boost { void throw_exception(std::exception const&); } +#include #endif #define BOOST_NO_UNREACHABLE_RETURN_DETECTION // include headers that implement serialization support From bb13d4a67bf5b3d3d54a30238e2c0f23a2fa5cf0 Mon Sep 17 00:00:00 2001 From: cDc Date: Thu, 3 Oct 2019 13:50:09 +0300 Subject: [PATCH 29/70] interface: add support for sensor band name --- libs/MVS/Interface.h | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/libs/MVS/Interface.h b/libs/MVS/Interface.h index 18c586897..97d96240e 100644 --- a/libs/MVS/Interface.h +++ b/libs/MVS/Interface.h @@ -5,12 +5,14 @@ // I N C L U D E S ///////////////////////////////////////////////// #include +#include +#include // D E F I N E S /////////////////////////////////////////////////// #define MVSI_PROJECT_ID "MVSI" // identifies the project stream -#define MVSI_PROJECT_VER ((uint32_t)3) // identifies the version of a project stream +#define MVSI_PROJECT_VER ((uint32_t)4) // identifies the version of a project stream // set a default namespace name if none given #ifndef _INTERFACE_NAMESPACE @@ -55,8 +57,17 @@ class Point3_ inline operator EVecMap () { return EVecMap((Type*)this); } #endif + const Type* ptr() const { return &x; } + Type* ptr() { return &x; } Type operator()(int r) const { return (&x)[r]; } Type& operator()(int r) { return (&x)[r]; } + Point3_ operator - () const { + return Point3_( + -x, + -y, + -z + ); + } Point3_ operator + (const Point3_& X) const { return Point3_( x+X.x, @@ -363,6 +374,7 @@ struct Interface // structure describing a camera mounted on a platform struct Camera { std::string name; // camera's name + std::string bandName; // camera's band name, ex: RGB, BLUE, GREEN, RED, NIR, THERMAL, etc (optional) uint32_t width, height; // image resolution in pixels for all images sharing this camera (optional) Mat33d K; // camera's intrinsics matrix (normalized if image resolution not specified) Mat33d R; // camera's rotation matrix relative to the platform @@ -375,6 +387,9 @@ struct Interface template void serialize(Archive& ar, const unsigned int version) { ar & name; + if (version > 3) { + ar & bandName; + } if (version > 0) { ar & width; ar & height; @@ -391,6 +406,14 @@ struct Interface Mat33d R; // platform's rotation matrix Pos3d C; // platform's translation vector in the global coordinate system + Pose() {} + template + Pose(const MAT& _R, const POS& _C) : R(_R), C(_C) {} + + // translation vector t = -RC + inline Pos3d GetTranslation() const { return R*(-C); } + inline void SetTranslation(const Pos3d& T) { C = R.t()*(-T); } + template void serialize(Archive& ar, const unsigned int /*version*/) { ar & R; @@ -435,8 +458,8 @@ struct Interface uint32_t poseID; // ID of the pose of the associated platform uint32_t ID; // ID of this image in the global space (optional) - Image() : poseID(NO_ID), ID(NO_ID) {} - + Image() : platformID(NO_ID), cameraID(NO_ID), poseID(NO_ID), ID(NO_ID) {} + bool IsValid() const { return poseID != NO_ID; } template @@ -538,7 +561,7 @@ struct Interface VertexArr vertices; // array of reconstructed 3D points NormalArr verticesNormal; // array of reconstructed 3D points' normal (optional) ColorArr verticesColor; // array of reconstructed 3D points' color (optional) - LineArr lines; // array of reconstructed 3D lines + LineArr lines; // array of reconstructed 3D lines (optional) NormalArr linesNormal; // array of reconstructed 3D lines' normal (optional) ColorArr linesColor; // array of reconstructed 3D lines' color (optional) Mat44d transform; // transformation used to convert from absolute to relative coordinate system (optional) From 2674d3b8c77a500cf2c581690d4297c74d1938c3 Mon Sep 17 00:00:00 2001 From: cDc Date: Thu, 3 Oct 2019 14:41:08 +0300 Subject: [PATCH 30/70] mvs: integrate global ID to Image --- apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp | 1 + apps/InterfaceOpenMVG/InterfaceOpenMVG.cpp | 2 + .../InterfacePhotoScan/InterfacePhotoScan.cpp | 1 + .../InterfaceVisualSFM/InterfaceVisualSFM.cpp | 3 +- libs/MVS/DepthMap.h | 10 +++ libs/MVS/Image.h | 9 ++- libs/MVS/Scene.cpp | 2 + libs/MVS/SceneDensify.cpp | 76 +++++++++---------- 8 files changed, 62 insertions(+), 42 deletions(-) diff --git a/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp b/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp index 27eabf0e8..04406eeee 100644 --- a/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp +++ b/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp @@ -472,6 +472,7 @@ bool ImportScene(const String& strFolder, Interface& scene) image.name = OPT::strImageFolder+imageColmap.name; image.platformID = mapCameras.at(imageColmap.idCamera); image.cameraID = 0; + image.ID = imageColmap.ID; Interface::Platform& platform = scene.platforms[image.platformID]; image.poseID = (uint32_t)platform.poses.size(); platform.poses.push_back(pose); diff --git a/apps/InterfaceOpenMVG/InterfaceOpenMVG.cpp b/apps/InterfaceOpenMVG/InterfaceOpenMVG.cpp index ce3c4d0ae..c410d0d75 100644 --- a/apps/InterfaceOpenMVG/InterfaceOpenMVG.cpp +++ b/apps/InterfaceOpenMVG/InterfaceOpenMVG.cpp @@ -602,6 +602,7 @@ int main(int argc, LPCTSTR* argv) const String srcImage(MAKE_PATH_FULL(WORKING_FOLDER_FULL, pathRoot+image.name)); image.name = MAKE_PATH_FULL(WORKING_FOLDER_FULL, OPT::strOutputImageFolder+image.name); Util::ensureDirectory(image.name); + image.ID = static_cast(view.first); image.platformID = map_intrinsic.at(view.second->id_intrinsic); MVS::Platform& platform = scene.platforms[image.platformID]; image.cameraID = 0; @@ -688,6 +689,7 @@ int main(int argc, LPCTSTR* argv) image.name = imageBAF.name; Util::ensureUnifySlash(image.name); image.name = MAKE_PATH_FULL(WORKING_FOLDER_FULL, image.name); + image.ID = imageBAF.id_camera; image.platformID = imageBAF.id_camera; MVS::Platform& platform = scene.platforms[image.platformID]; image.cameraID = 0; diff --git a/apps/InterfacePhotoScan/InterfacePhotoScan.cpp b/apps/InterfacePhotoScan/InterfacePhotoScan.cpp index 25d9db5b4..0b5f1effa 100644 --- a/apps/InterfacePhotoScan/InterfacePhotoScan.cpp +++ b/apps/InterfacePhotoScan/InterfacePhotoScan.cpp @@ -344,6 +344,7 @@ bool ParseImageListXML(MVS::Scene& scene, PlatformDistCoeffs& pltDistCoeffs, siz imageData.name = MAKE_PATH_FULL(strPath, imageData.name); imageData.platformID = camera->UnsignedAttribute(_T("sensor_id")); imageData.cameraID = 0; // only one camera per platform supported by this format + imageData.ID = ID; if (!camera->BoolAttribute(_T("enabled"))) { imageData.poseID = NO_ID; DEBUG_EXTRA("warning: uncalibrated image '%s'", name); diff --git a/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp b/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp index 75bce304b..6230be3ce 100644 --- a/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp +++ b/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp @@ -292,7 +292,7 @@ int ImportSceneVSFM() // convert data from VisualSFM to OpenMVS MVS::Scene scene(OPT::nMaxThreads); scene.platforms.Reserve((uint32_t)cameras.size()); - scene.images.Reserve((uint32_t)cameras.size()); + scene.images.Reserve((MVS::IIndex)cameras.size()); scene.nCalibratedImages = 0; for (size_t idx=0; idx(idx); const PBA::Camera& cameraNVM = cameras[idx]; camera.K = MVS::Platform::Camera::ComposeK(cameraNVM.GetFocalLength(), cameraNVM.GetFocalLength(), image.width, image.height); camera.R = RMatrix::IDENTITY; diff --git a/libs/MVS/DepthMap.h b/libs/MVS/DepthMap.h index 426b8b6d1..d24d7948c 100644 --- a/libs/MVS/DepthMap.h +++ b/libs/MVS/DepthMap.h @@ -149,6 +149,13 @@ struct MVS_API DepthData { Image32F image; // image float intensities Image* pImageData; // image data + inline IIndex GetID() const { + return pImageData->ID; + } + inline IIndex GetLocalID(const ImageArr& images) const { + return (IIndex)(pImageData - images.begin()); + } + template static bool ScaleImage(const IMAGE& image, IMAGE& imageScaled, float scale) { if (ABS(scale-1.f) < 0.15f) @@ -189,6 +196,9 @@ struct MVS_API DepthData { return depthMap.empty(); } + const ViewData& GetView() const { return images.front(); } + const Camera& GetCamera() const { return GetView().camera; } + void GetNormal(const ImageRef& ir, Point3f& N, const TImage* pPointMap=NULL) const; void GetNormal(const Point2f& x, Point3f& N, const TImage* pPointMap=NULL) const; diff --git a/libs/MVS/Image.h b/libs/MVS/Image.h index 20d68065e..48fc17230 100644 --- a/libs/MVS/Image.h +++ b/libs/MVS/Image.h @@ -78,11 +78,12 @@ class MVS_API Image uint32_t platformID; // ID of the associated platform uint32_t cameraID; // ID of the associated camera on the associated platform uint32_t poseID; // ID of the pose of the associated platform + uint32_t ID; // global ID of the image String name; // image file name (relative path) Camera camera; // view's pose uint32_t width, height; // image size Image8U3 image; // image color pixels - ViewScoreArr neighbors; // score&store the neighbor images + ViewScoreArr neighbors; // scored neighbor images float scale; // image scale relative to the original size float avgDepth; // average depth of the points seen by this camera @@ -119,10 +120,11 @@ class MVS_API Image #ifdef _USE_BOOST // implement BOOST serialization template - void save(Archive& ar, const unsigned int /*version*/) const { + void save(Archive& ar, const unsigned int version) const { ar & platformID; ar & cameraID; ar & poseID; + ar & ID; const String relName(MAKE_PATH_REL(WORKING_FOLDER_FULL, name)); ar & relName; ar & width & height; @@ -130,10 +132,11 @@ class MVS_API Image ar & avgDepth; } template - void load(Archive& ar, const unsigned int /*version*/) { + void load(Archive& ar, const unsigned int version) { ar & platformID; ar & cameraID; ar & poseID; + ar & ID; ar & name; name = MAKE_PATH_FULL(WORKING_FOLDER_FULL, name); ar & width & height; diff --git a/libs/MVS/Scene.cpp b/libs/MVS/Scene.cpp index 7932ae348..d3d85eda8 100644 --- a/libs/MVS/Scene.cpp +++ b/libs/MVS/Scene.cpp @@ -123,6 +123,7 @@ bool Scene::LoadInterface(const String & fileName) } imageData.platformID = image.platformID; imageData.cameraID = image.cameraID; + imageData.ID = image.ID; // init camera const Interface::Platform::Camera& camera = obj.platforms[image.platformID].cameras[image.cameraID]; if (camera.HasResolution()) { @@ -231,6 +232,7 @@ bool Scene::SaveInterface(const String & fileName) const image.poseID = imageData.poseID; image.platformID = imageData.platformID; image.cameraID = imageData.cameraID; + image.ID = imageData.ID; } // export 3D points diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index 0f6b0cc4f..bf6a34852 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -344,7 +344,7 @@ bool DepthMapsData::InitViews(DepthData& depthData, IIndex idxNeighbor, IIndex n if (g_nVerbosityLevel > 2) { String msg; for (IIndex i=1; i 4) { - ExportDepthMap(ComposeDepthFilePath(idxImage, "init.png"), depthData.depthMap); - ExportNormalMap(ComposeDepthFilePath(idxImage, "init.normal.png"), depthData.normalMap); - ExportPointCloud(ComposeDepthFilePath(idxImage, "init.ply"), *depthData.images.First().pImageData, depthData.depthMap, depthData.normalMap); + ExportDepthMap(ComposeDepthFilePath(image.GetID(), "init.png"), depthData.depthMap); + ExportNormalMap(ComposeDepthFilePath(image.GetID(), "init.normal.png"), depthData.normalMap); + ExportPointCloud(ComposeDepthFilePath(image.GetID(), "init.ply"), *depthData.images.First().pImageData, depthData.depthMap, depthData.normalMap); } #endif } @@ -725,9 +725,9 @@ bool DepthMapsData::EstimateDepthMap(IIndex idxImage) #if TD_VERBOSE != TD_VERBOSE_OFF // save rough depth map as image if (g_nVerbosityLevel > 4) { - ExportDepthMap(ComposeDepthFilePath(idxImage, "rough.png"), depthData.depthMap); - ExportNormalMap(ComposeDepthFilePath(idxImage, "rough.normal.png"), depthData.normalMap); - ExportPointCloud(ComposeDepthFilePath(idxImage, "rough.ply"), *depthData.images.First().pImageData, depthData.depthMap, depthData.normalMap); + ExportDepthMap(ComposeDepthFilePath(image.GetID(), "rough.png"), depthData.depthMap); + ExportNormalMap(ComposeDepthFilePath(image.GetID(), "rough.normal.png"), depthData.normalMap); + ExportPointCloud(ComposeDepthFilePath(image.GetID(), "rough.ply"), *depthData.images.First().pImageData, depthData.depthMap, depthData.normalMap); } #endif } @@ -756,7 +756,7 @@ bool DepthMapsData::EstimateDepthMap(IIndex idxImage) #if 1 && TD_VERBOSE != TD_VERBOSE_OFF // save intermediate depth map as image if (g_nVerbosityLevel > 4) { - const String path(ComposeDepthFilePath(image.pImageData-scene.images.Begin(), "iter")+String::ToString(iter)); + const String path(ComposeDepthFilePath(image.GetID(), "iter")+String::ToString(iter)); ExportDepthMap(path+".png", depthData.depthMap); ExportNormalMap(path+".normal.png", depthData.normalMap); ExportPointCloud(path+".ply", *depthData.images.First().pImageData, depthData.depthMap, depthData.normalMap); @@ -787,10 +787,10 @@ bool DepthMapsData::EstimateDepthMap(IIndex idxImage) estimators.Release(); } - DEBUG_EXTRA("Depth-map for image %3u %s: %dx%d (%s)", image.pImageData-scene.images.Begin(), + DEBUG_EXTRA("Depth-map for image %3u %s: %dx%d (%s)", image.GetID(), depthData.images.GetSize() > 2 ? String::FormatString("estimated using %2u images", depthData.images.GetSize()-1).c_str() : - String::FormatString("with image %3u estimated", depthData.images[1].pImageData-scene.images.Begin()).c_str(), + String::FormatString("with image %3u estimated", depthData.images[1].GetID()).c_str(), size.width, size.height, TD_TIMER_GET_FMT().c_str()); return true; } // EstimateDepthMap @@ -1049,7 +1049,7 @@ bool DepthMapsData::FilterDepthMap(DepthData& depthDataRef, const IIndexArr& idx const IIndex nMinViews(MINF(OPTDENSE::nMinViewsFilter,scene.nCalibratedImages-1)); const IIndex nMinViewsAdjust(MINF(OPTDENSE::nMinViewsFilterAdjust,scene.nCalibratedImages-1)); if (N < nMinViews || N < nMinViewsAdjust) { - DEBUG("error: depth map %3u can not be filtered", depthDataRef.images.First().pImageData-scene.images.Begin()); + DEBUG("error: depth map %3u can not be filtered", depthDataRef.GetView().GetID()); return false; } @@ -1119,7 +1119,7 @@ bool DepthMapsData::FilterDepthMap(DepthData& depthDataRef, const IIndexArr& idx } #if TD_VERBOSE != TD_VERBOSE_OFF if (g_nVerbosityLevel > 3) - ExportDepthMap(MAKE_PATH(String::FormatString("depthRender%04u.%04u.png", depthDataRef.images.First().pImageData-scene.images.Begin(), idxView)), depthMap); + ExportDepthMap(MAKE_PATH(String::FormatString("depthRender%04u.%04u.png", depthDataRef.GetView().GetID(), idxView)), depthMap); #endif } @@ -1280,12 +1280,12 @@ bool DepthMapsData::FilterDepthMap(DepthData& depthDataRef, const IIndexArr& idx } } } - if (!SaveDepthMap(ComposeDepthFilePath(imageRef.pImageData-scene.images.Begin(), "filtered.dmap"), newDepthMap) || - !SaveConfidenceMap(ComposeDepthFilePath(imageRef.pImageData-scene.images.Begin(), "filtered.cmap"), newConfMap)) + if (!SaveDepthMap(ComposeDepthFilePath(imageRef.GetID(), "filtered.dmap"), newDepthMap) || + !SaveConfidenceMap(ComposeDepthFilePath(imageRef.GetID(), "filtered.cmap"), newConfMap)) return false; #if TD_VERBOSE != TD_VERBOSE_OFF - DEBUG("Depth map %3u filtered using %u other images: %u/%u depths discarded (%s)", imageRef.pImageData-scene.images.Begin(), N, nDiscarded, nProcessed, TD_TIMER_GET_FMT().c_str()); + DEBUG("Depth map %3u filtered using %u other images: %u/%u depths discarded (%s)", imageRef.GetID(), N, nDiscarded, nProcessed, TD_TIMER_GET_FMT().c_str()); #endif return true; @@ -1321,7 +1321,7 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b DepthData& depthData = arrDepthData[i]; if (!depthData.IsValid()) continue; - if (depthData.IncRef(ComposeDepthFilePath(i, "dmap")) == 0) + if (depthData.IncRef(ComposeDepthFilePath(depthData.GetView().GetID(), "dmap")) == 0) return; ASSERT(!depthData.IsEmpty()); IndexScore& connection = connections.AddEmpty(); @@ -1759,10 +1759,10 @@ void Scene::DenseReconstructionEstimate(void* pData) { break; } // try to load already compute depth-map for this image - if (File::access(ComposeDepthFilePath(idx, "dmap"))) { + if (File::access(ComposeDepthFilePath(depthData.GetView().GetID(), "dmap"))) { if (OPTDENSE::nOptimize & OPTDENSE::OPTIMIZE) { - if (!depthData.Load(ComposeDepthFilePath(idx, "dmap"))) { - VERBOSE("error: invalid depth-map '%s'", ComposeDepthFilePath(idx, "dmap").c_str()); + if (!depthData.Load(ComposeDepthFilePath(depthData.GetView().GetID(), "dmap"))) { + VERBOSE("error: invalid depth-map '%s'", ComposeDepthFilePath(depthData.GetView().GetID(), "dmap").c_str()); exit(EXIT_FAILURE); } // optimize depth-map @@ -1800,19 +1800,19 @@ void Scene::DenseReconstructionEstimate(void* pData) { #if TD_VERBOSE != TD_VERBOSE_OFF // save depth map as image if (g_nVerbosityLevel > 3) - ExportDepthMap(ComposeDepthFilePath(idx, "raw.png"), depthData.depthMap); + ExportDepthMap(ComposeDepthFilePath(depthData.GetView().GetID(), "raw.png"), depthData.depthMap); #endif // apply filters if (OPTDENSE::nOptimize & (OPTDENSE::REMOVE_SPECKLES)) { TD_TIMER_START(); if (data.depthMaps.RemoveSmallSegments(depthData)) { - DEBUG_ULTIMATE("Depth-map %3u filtered: remove small segments (%s)", idx, TD_TIMER_GET_FMT().c_str()); + DEBUG_ULTIMATE("Depth-map %3u filtered: remove small segments (%s)", depthData.GetView().GetID(), TD_TIMER_GET_FMT().c_str()); } } if (OPTDENSE::nOptimize & (OPTDENSE::FILL_GAPS)) { TD_TIMER_START(); if (data.depthMaps.GapInterpolation(depthData)) { - DEBUG_ULTIMATE("Depth-map %3u filtered: gap interpolation (%s)", idx, TD_TIMER_GET_FMT().c_str()); + DEBUG_ULTIMATE("Depth-map %3u filtered: gap interpolation (%s)", depthData.GetView().GetID(), TD_TIMER_GET_FMT().c_str()); } } // save depth-map @@ -1826,17 +1826,17 @@ void Scene::DenseReconstructionEstimate(void* pData) { #if TD_VERBOSE != TD_VERBOSE_OFF // save depth map as image if (g_nVerbosityLevel > 2) { - ExportDepthMap(ComposeDepthFilePath(idx, "png"), depthData.depthMap); - ExportConfidenceMap(ComposeDepthFilePath(idx, "conf.png"), depthData.confMap); - ExportPointCloud(ComposeDepthFilePath(idx, "ply"), *depthData.images.First().pImageData, depthData.depthMap, depthData.normalMap); + ExportDepthMap(ComposeDepthFilePath(depthData.GetView().GetID(), "png"), depthData.depthMap); + ExportConfidenceMap(ComposeDepthFilePath(depthData.GetView().GetID(), "conf.png"), depthData.confMap); + ExportPointCloud(ComposeDepthFilePath(depthData.GetView().GetID(), "ply"), *depthData.images.First().pImageData, depthData.depthMap, depthData.normalMap); if (g_nVerbosityLevel > 4) { - ExportNormalMap(ComposeDepthFilePath(idx, "normal.png"), depthData.normalMap); - depthData.confMap.Save(ComposeDepthFilePath(idx, "conf.pfm")); + ExportNormalMap(ComposeDepthFilePath(depthData.GetView().GetID(), "normal.png"), depthData.normalMap); + depthData.confMap.Save(ComposeDepthFilePath(depthData.GetView().GetID(), "conf.pfm")); } } #endif // save compute depth-map for this image - depthData.Save(ComposeDepthFilePath(idx, "dmap")); + depthData.Save(ComposeDepthFilePath(depthData.GetView().GetID(), "dmap")); depthData.ReleaseImages(); depthData.Release(); data.progress->operator++(); @@ -1874,7 +1874,7 @@ void Scene::DenseReconstructionFilter(void* pData) break; } // make sure all depth-maps are loaded - depthData.IncRef(ComposeDepthFilePath(idx, "dmap")); + depthData.IncRef(ComposeDepthFilePath(depthData.GetView().GetID(), "dmap")); const unsigned numMaxNeighbors(8); IIndexArr idxNeighbors(0, depthData.neighbors.GetSize()); FOREACH(n, depthData.neighbors) { @@ -1882,7 +1882,7 @@ void Scene::DenseReconstructionFilter(void* pData) DepthData& depthDataPair = data.depthMaps.arrDepthData[idxView]; if (!depthDataPair.IsValid()) continue; - if (depthDataPair.IncRef(ComposeDepthFilePath(idxView, "dmap")) == 0) { + if (depthDataPair.IncRef(ComposeDepthFilePath(depthDataPair.GetView().GetID(), "dmap")) == 0) { // signal error and terminate data.events.AddEventFirst(new EVTFail); return; @@ -1913,26 +1913,26 @@ void Scene::DenseReconstructionFilter(void* pData) ASSERT(depthData.IsValid()); data.sem.Wait(); // load filtered maps - if (depthData.IncRef(ComposeDepthFilePath(idx, "dmap")) == 0 || - !LoadDepthMap(ComposeDepthFilePath(idx, "filtered.dmap"), depthData.depthMap) || - !LoadConfidenceMap(ComposeDepthFilePath(idx, "filtered.cmap"), depthData.confMap)) + if (depthData.IncRef(ComposeDepthFilePath(depthData.GetView().GetID(), "dmap")) == 0 || + !LoadDepthMap(ComposeDepthFilePath(depthData.GetView().GetID(), "filtered.dmap"), depthData.depthMap) || + !LoadConfidenceMap(ComposeDepthFilePath(depthData.GetView().GetID(), "filtered.cmap"), depthData.confMap)) { // signal error and terminate data.events.AddEventFirst(new EVTFail); return; } ASSERT(depthData.GetRef() == 1); - File::deleteFile(ComposeDepthFilePath(idx, "filtered.dmap").c_str()); - File::deleteFile(ComposeDepthFilePath(idx, "filtered.cmap").c_str()); + File::deleteFile(ComposeDepthFilePath(depthData.GetView().GetID(), "filtered.dmap").c_str()); + File::deleteFile(ComposeDepthFilePath(depthData.GetView().GetID(), "filtered.cmap").c_str()); #if TD_VERBOSE != TD_VERBOSE_OFF // save depth map as image if (g_nVerbosityLevel > 2) { - ExportDepthMap(ComposeDepthFilePath(idx, "filtered.png"), depthData.depthMap); - ExportPointCloud(ComposeDepthFilePath(idx, "filtered.ply"), *depthData.images.First().pImageData, depthData.depthMap, depthData.normalMap); + ExportDepthMap(ComposeDepthFilePath(depthData.GetView().GetID(), "filtered.png"), depthData.depthMap); + ExportPointCloud(ComposeDepthFilePath(depthData.GetView().GetID(), "filtered.ply"), *depthData.images.First().pImageData, depthData.depthMap, depthData.normalMap); } #endif // save filtered depth-map for this image - depthData.Save(ComposeDepthFilePath(idx, "dmap")); + depthData.Save(ComposeDepthFilePath(depthData.GetView().GetID(), "dmap")); depthData.DecRef(); data.progress->operator++(); break; } From c77535edd49d996518570f937c6939c5741fcf98 Mon Sep 17 00:00:00 2001 From: cDc Date: Fri, 4 Oct 2019 12:11:30 +0300 Subject: [PATCH 31/70] dense: unify DMAP format --- libs/MVS/DepthMap.cpp | 175 +++++++++++++++++++++++++++++++++++++++++- libs/MVS/DepthMap.h | 11 +++ 2 files changed, 184 insertions(+), 2 deletions(-) diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index 12973ac3a..952506565 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -188,12 +188,35 @@ void DepthData::GetNormal(const Point2f& pt, Point3f& N, const TImage* bool DepthData::Save(const String& fileName) const { ASSERT(IsValid() && !depthMap.empty() && !confMap.empty()); - return SerializeSave(*this, fileName, ARCHIVE_BINARY_ZIP); + const String fileNameTmp(fileName+".tmp"); { + // serialize out the current state + IIndexArr IDs(0, images.size()); + for (const ViewData& image: images) + IDs.push_back(image.GetID()); + const ViewData& image0 = GetView(); + if (!ExportDepthDataRaw(fileNameTmp, IDs, depthMap.size(), image0.camera.K, image0.camera.R, image0.camera.C, dMin, dMax, depthMap, normalMap, confMap)) + return false; + } + if (!File::renameFile(fileNameTmp, fileName)) { + DEBUG_EXTRA("error: can not access dmap file '%s'", fileName.c_str()); + File::deleteFile(fileNameTmp); + return false; + } + return true; } bool DepthData::Load(const String& fileName) { ASSERT(IsValid()); - return SerializeLoad(*this, fileName, ARCHIVE_BINARY_ZIP); + // serialize in the saved state + IIndexArr IDs; + cv::Size imageSize; + Camera camera; + if (!ImportDepthDataRaw(fileName, IDs, imageSize, camera.K, camera.R, camera.C, dMin, dMax, depthMap, normalMap, confMap)) + return false; + ASSERT(IDs.size() == images.size()); + ASSERT(IDs.front() == GetView().GetID()); + ASSERT(depthMap.size() == imageSize); + return true; } /*----------------------------------------------------------------*/ @@ -1352,6 +1375,154 @@ bool MVS::ExportPointCloud(const String& fileName, const Image& imageData, const /*----------------------------------------------------------------*/ +struct HeaderDepthDataRaw { + enum { + HAS_DEPTH = (1<<0), + HAS_NORMAL = (1<<1), + HAS_CONF = (1<<2), + }; + uint16_t name; // file type + uint8_t type; // content type + uint8_t padding; // reserve + uint32_t imageWidth, imageHeight; // image resolution + uint32_t depthWidth, depthHeight; // depth-map resolution + float dMin, dMax; // depth range for this view + uint32_t nIDs; // number of view IDs + // view ID followed by neighbor view IDs: uint32_t* IDs + // camera, rotation and translation matrices (row-major): double K[3][3], R[3][3], C[3] + // depth, normal, confidence maps + inline HeaderDepthDataRaw() : name(0), type(0), padding(0) {} + static uint16_t HeaderDepthDataRawName() { return *reinterpret_cast("DR"); } + int GetStep() const { return ROUND2INT((float)imageWidth/depthWidth); } +}; + +bool MVS::ExportDepthDataRaw(const String& fileName, + const IIndexArr& IDs, const cv::Size& imageSize, + const KMatrix& K, const RMatrix& R, const CMatrix& C, + Depth dMin, Depth dMax, + const DepthMap& depthMap, const NormalMap& normalMap, const ConfidenceMap& confMap) +{ + ASSERT(!depthMap.empty()); + ASSERT(confMap.empty() || depthMap.size() == confMap.size()); + ASSERT(depthMap.width() <= imageSize.width && depthMap.height() <= imageSize.height); + + FILE *f = fopen(fileName, "wb"); + if (f == NULL) { + DEBUG("error: opening file '%s' for writing depth-data", fileName.c_str()); + return false; + } + + // write header + HeaderDepthDataRaw header; + header.name = HeaderDepthDataRaw::HeaderDepthDataRawName(); + header.type = HeaderDepthDataRaw::HAS_DEPTH; + header.imageWidth = (uint32_t)imageSize.width; + header.imageHeight = (uint32_t)imageSize.height; + header.depthWidth = (uint32_t)depthMap.cols; + header.depthHeight = (uint32_t)depthMap.rows; + header.dMin = dMin; + header.dMax = dMax; + header.nIDs = IDs.size(); + if (!normalMap.empty()) + header.type |= HeaderDepthDataRaw::HAS_NORMAL; + if (!confMap.empty()) + header.type |= HeaderDepthDataRaw::HAS_CONF; + fwrite(&header, sizeof(HeaderDepthDataRaw), 1, f); + + // write neighbor IDs + STATIC_ASSERT(sizeof(uint32_t) == sizeof(IIndex)); + fwrite(IDs.data(), sizeof(IIndex), IDs.size(), f); + + // write pose + STATIC_ASSERT(sizeof(double) == sizeof(REAL)); + fwrite(K.val, sizeof(REAL), 9, f); + fwrite(R.val, sizeof(REAL), 9, f); + fwrite(C.ptr(), sizeof(REAL), 3, f); + + // write depth-map + fwrite(depthMap.getData(), sizeof(float), depthMap.area(), f); + + // write normal-map + if ((header.type & HeaderDepthDataRaw::HAS_NORMAL) != 0) + fwrite(normalMap.getData(), sizeof(float)*3, normalMap.area(), f); + + // write confidence-map + if ((header.type & HeaderDepthDataRaw::HAS_CONF) != 0) + fwrite(confMap.getData(), sizeof(float), confMap.area(), f); + + const bool bRet(ferror(f) == 0); + fclose(f); + return bRet; +} // ExportDepthDataRaw + +bool MVS::ImportDepthDataRaw(const String& fileName, + IIndexArr& IDs, cv::Size& imageSize, + KMatrix& K, RMatrix& R, CMatrix& C, + Depth& dMin, Depth& dMax, + DepthMap& depthMap, NormalMap& normalMap, ConfidenceMap& confMap, unsigned flags) +{ + FILE *f = fopen(fileName, "rb"); + if (f == NULL) { + DEBUG("error: opening file '%s' for reading depth-data", fileName.c_str()); + return false; + } + + // read header + HeaderDepthDataRaw header; + if (fread(&header, sizeof(HeaderDepthDataRaw), 1, f) != 1 || + header.name != HeaderDepthDataRaw::HeaderDepthDataRawName() || + (header.type & HeaderDepthDataRaw::HAS_DEPTH) == 0 || + header.depthWidth <= 0 || header.depthHeight <= 0 || + header.imageWidth < header.depthWidth || header.imageHeight < header.depthHeight) + { + DEBUG("error: invalid depth-data file '%s'", fileName.c_str()); + return false; + } + + // read neighbor IDs + STATIC_ASSERT(sizeof(uint32_t) == sizeof(IIndex)); + IDs.resize(header.nIDs); + fread(IDs.data(), sizeof(IIndex), IDs.size(), f); + + // read pose + STATIC_ASSERT(sizeof(double) == sizeof(REAL)); + fread(K.val, sizeof(REAL), 9, f); + fread(R.val, sizeof(REAL), 9, f); + fread(C.ptr(), sizeof(REAL), 3, f); + + // read depth-map + dMin = header.dMin; + dMax = header.dMax; + imageSize.width = header.imageWidth; + imageSize.height = header.imageHeight; + depthMap.create(header.depthHeight, header.depthWidth); + fread(depthMap.getData(), sizeof(float), depthMap.area(), f); + + // read normal-map + if ((header.type & HeaderDepthDataRaw::HAS_NORMAL) != 0) { + if ((flags & HeaderDepthDataRaw::HAS_NORMAL) != 0) { + normalMap.create(header.depthHeight, header.depthWidth); + fread(normalMap.getData(), sizeof(float)*3, normalMap.area(), f); + } else { + fseek(f, sizeof(float)*3*header.depthWidth*header.depthHeight, SEEK_CUR); + } + } + + // read confidence-map + if ((header.type & HeaderDepthDataRaw::HAS_CONF) != 0) { + if ((flags & HeaderDepthDataRaw::HAS_CONF) != 0) { + confMap.create(header.depthHeight, header.depthWidth); + fread(confMap.getData(), sizeof(float), confMap.area(), f); + } + } + + const bool bRet(ferror(f) == 0); + fclose(f); + return bRet; +} // ImportDepthDataRaw +/*----------------------------------------------------------------*/ + + // compare the estimated and ground-truth depth-maps void MVS::CompareDepthMaps(const DepthMap& depthMap, const DepthMap& depthMapGT, uint32_t idxImage, float threshold) { diff --git a/libs/MVS/DepthMap.h b/libs/MVS/DepthMap.h index d24d7948c..4ba7219e5 100644 --- a/libs/MVS/DepthMap.h +++ b/libs/MVS/DepthMap.h @@ -460,6 +460,17 @@ MVS_API bool ExportNormalMap(const String& fileName, const NormalMap& normalMap) MVS_API bool ExportConfidenceMap(const String& fileName, const ConfidenceMap& confMap); MVS_API bool ExportPointCloud(const String& fileName, const Image&, const DepthMap&, const NormalMap&); +MVS_API bool ExportDepthDataRaw(const String&, + const IIndexArr&, const cv::Size& imageSize, + const KMatrix&, const RMatrix&, const CMatrix&, + Depth dMin, Depth dMax, + const DepthMap&, const NormalMap&, const ConfidenceMap&); +MVS_API bool ImportDepthDataRaw(const String&, + IIndexArr&, cv::Size& imageSize, + KMatrix&, RMatrix&, CMatrix&, + Depth& dMin, Depth& dMax, + DepthMap&, NormalMap&, ConfidenceMap&, unsigned flags=7); + MVS_API void CompareDepthMaps(const DepthMap& depthMap, const DepthMap& depthMapGT, uint32_t idxImage, float threshold=0.01f); MVS_API void CompareNormalMaps(const NormalMap& normalMap, const NormalMap& normalMapGT, uint32_t idxImage); /*----------------------------------------------------------------*/ From 5c460687b47c2d7bc00a7f51e0cf5513fb7a2ead Mon Sep 17 00:00:00 2001 From: cDc Date: Fri, 4 Oct 2019 14:06:42 +0300 Subject: [PATCH 32/70] mvs: small fixes --- apps/DensifyPointCloud/DensifyPointCloud.cpp | 6 +++--- libs/Common/Types.h | 1 + libs/MVS.h | 1 - libs/MVS/Image.h | 4 ++-- libs/MVS/Scene.h | 1 + libs/MVS/SceneDensify.cpp | 10 ++++++---- 6 files changed, 13 insertions(+), 10 deletions(-) diff --git a/apps/DensifyPointCloud/DensifyPointCloud.cpp b/apps/DensifyPointCloud/DensifyPointCloud.cpp index b96cfae46..aeaa18cc7 100644 --- a/apps/DensifyPointCloud/DensifyPointCloud.cpp +++ b/apps/DensifyPointCloud/DensifyPointCloud.cpp @@ -167,8 +167,8 @@ bool Initialize(size_t argc, LPCTSTR* argv) OPT::strOutputFileName = Util::getFileFullName(OPT::strInputFileName) + _T("_dense.mvs"); // init dense options - if (!Util::isFullPath(OPT::strDenseConfigFileName)) - OPT::strDenseConfigFileName = MAKE_PATH(OPT::strDenseConfigFileName); + if (!OPT::strDenseConfigFileName.IsEmpty()) + OPT::strDenseConfigFileName = MAKE_PATH_SAFE(OPT::strDenseConfigFileName); OPTDENSE::init(); const bool bValidConfig(OPTDENSE::oConfig.Load(OPT::strDenseConfigFileName)); OPTDENSE::update(); @@ -180,7 +180,7 @@ bool Initialize(size_t argc, LPCTSTR* argv) OPTDENSE::nOptimize = nOptimize; OPTDENSE::nEstimateColors = nEstimateColors; OPTDENSE::nEstimateNormals = nEstimateNormals; - if (!bValidConfig) + if (!bValidConfig && !OPT::strDenseConfigFileName.IsEmpty()) OPTDENSE::oConfig.Save(OPT::strDenseConfigFileName); // initialize global options diff --git a/libs/Common/Types.h b/libs/Common/Types.h index ef2138046..3e6507fa9 100644 --- a/libs/Common/Types.h +++ b/libs/Common/Types.h @@ -84,6 +84,7 @@ #ifdef _USE_BOOST #if 1 // disable exception support +#define BOOST_NO_UNREACHABLE_RETURN_DETECTION #define BOOST_EXCEPTION_DISABLE #define BOOST_NO_EXCEPTIONS #endif diff --git a/libs/MVS.h b/libs/MVS.h index d4ec4937d..a8937c08f 100644 --- a/libs/MVS.h +++ b/libs/MVS.h @@ -41,7 +41,6 @@ #include "Math/Common.h" #include "MVS/Common.h" #include "MVS/Scene.h" -#include "MVS/SceneDensify.h" #endif // _MVS_MVS_H_ diff --git a/libs/MVS/Image.h b/libs/MVS/Image.h index 48fc17230..627ec5cee 100644 --- a/libs/MVS/Image.h +++ b/libs/MVS/Image.h @@ -120,7 +120,7 @@ class MVS_API Image #ifdef _USE_BOOST // implement BOOST serialization template - void save(Archive& ar, const unsigned int version) const { + void save(Archive& ar, const unsigned int /*version*/) const { ar & platformID; ar & cameraID; ar & poseID; @@ -132,7 +132,7 @@ class MVS_API Image ar & avgDepth; } template - void load(Archive& ar, const unsigned int version) { + void load(Archive& ar, const unsigned int /*version*/) { ar & platformID; ar & cameraID; ar & poseID; diff --git a/libs/MVS/Scene.h b/libs/MVS/Scene.h index 782e9a315..7956525b5 100644 --- a/libs/MVS/Scene.h +++ b/libs/MVS/Scene.h @@ -37,6 +37,7 @@ #include "DepthMap.h" #include "Mesh.h" +#include "SceneDensify.h" // D E F I N E S /////////////////////////////////////////////////// diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index bf6a34852..fd5b518c0 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -31,7 +31,6 @@ #include "Common.h" #include "Scene.h" -#include "SceneDensify.h" // MRF: view selection #include "../Math/TRWS/MRFEnergy.h" // CGAL: depth-map initialization @@ -1526,7 +1525,8 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b static void* DenseReconstructionEstimateTmp(void*); static void* DenseReconstructionFilterTmp(void*); -bool Scene::DenseReconstruction() { +bool Scene::DenseReconstruction() +{ DenseDepthMapData data(*this); // estimate depth-maps @@ -1570,7 +1570,8 @@ bool Scene::DenseReconstruction() { // do first half of dense reconstruction: depth map computation // results are saved to "data" -bool Scene::ComputeDepthMaps(DenseDepthMapData& data) { +bool Scene::ComputeDepthMaps(DenseDepthMapData& data) +{ { // maps global view indices to our list of views to be processed IIndexArr imagesMap; @@ -1734,7 +1735,8 @@ void* DenseReconstructionEstimateTmp(void* arg) { } // initialize the dense reconstruction with the sparse point cloud -void Scene::DenseReconstructionEstimate(void* pData) { +void Scene::DenseReconstructionEstimate(void* pData) +{ DenseDepthMapData& data = *((DenseDepthMapData*)pData); while (true) { CAutoPtr evt(data.events.GetEvent()); From 268f0a6a5349bdc5f2ae92fdeb69dbb5c398fc45 Mon Sep 17 00:00:00 2001 From: cDc Date: Tue, 8 Oct 2019 11:21:14 +0300 Subject: [PATCH 33/70] interface: add support for old interface format --- libs/MVS/Scene.cpp | 50 +++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/libs/MVS/Scene.cpp b/libs/MVS/Scene.cpp index d3d85eda8..dd94c21d8 100644 --- a/libs/MVS/Scene.cpp +++ b/libs/MVS/Scene.cpp @@ -71,34 +71,34 @@ bool Scene::LoadInterface(const String & fileName) // import platforms and cameras ASSERT(!obj.platforms.empty()); platforms.Reserve((uint32_t)obj.platforms.size()); - for (Interface::PlatformArr::const_iterator itPlatform=obj.platforms.begin(); itPlatform!=obj.platforms.end(); ++itPlatform) { + for (const Interface::Platform& itPlatform: obj.platforms) { Platform& platform = platforms.AddEmpty(); - platform.name = itPlatform->name; - platform.cameras.Reserve((uint32_t)itPlatform->cameras.size()); - for (Interface::Platform::CameraArr::const_iterator itCamera=itPlatform->cameras.begin(); itCamera!=itPlatform->cameras.end(); ++itCamera) { + platform.name = itPlatform.name; + platform.cameras.Reserve((uint32_t)itPlatform.cameras.size()); + for (const Interface::Platform::Camera& itCamera: itPlatform.cameras) { Platform::Camera& camera = platform.cameras.AddEmpty(); - camera.K = itCamera->K; - camera.R = itCamera->R; - camera.C = itCamera->C; - if (!itCamera->IsNormalized()) { + camera.K = itCamera.K; + camera.R = itCamera.R; + camera.C = itCamera.C; + if (!itCamera.IsNormalized()) { // normalize K - ASSERT(itCamera->HasResolution()); - const REAL scale(REAL(1)/camera.GetNormalizationScale(itCamera->width,itCamera->height)); + ASSERT(itCamera.HasResolution()); + const REAL scale(REAL(1)/camera.GetNormalizationScale(itCamera.width,itCamera.height)); camera.K(0,0) *= scale; camera.K(1,1) *= scale; camera.K(0,2) *= scale; camera.K(1,2) *= scale; } - DEBUG_EXTRA("Camera model loaded: platform %u; camera %2u; f %.3fx%.3f; poses %u", platforms.GetSize()-1, platform.cameras.GetSize()-1, camera.K(0,0), camera.K(1,1), itPlatform->poses.size()); + DEBUG_EXTRA("Camera model loaded: platform %u; camera %2u; f %.3fx%.3f; poses %u", platforms.size()-1, platform.cameras.size()-1, camera.K(0,0), camera.K(1,1), itPlatform.poses.size()); } - ASSERT(platform.cameras.GetSize() == itPlatform->cameras.size()); - platform.poses.Reserve((uint32_t)itPlatform->poses.size()); - for (Interface::Platform::PoseArr::const_iterator itPose=itPlatform->poses.begin(); itPose!=itPlatform->poses.end(); ++itPose) { + ASSERT(platform.cameras.GetSize() == itPlatform.cameras.size()); + platform.poses.Reserve((uint32_t)itPlatform.poses.size()); + for (const Interface::Platform::Pose& itPose: itPlatform.poses) { Platform::Pose& pose = platform.poses.AddEmpty(); - pose.R = itPose->R; - pose.C = itPose->C; + pose.R = itPose.R; + pose.C = itPose.C; } - ASSERT(platform.poses.GetSize() == itPlatform->poses.size()); + ASSERT(platform.poses.GetSize() == itPlatform.poses.size()); } ASSERT(platforms.GetSize() == obj.platforms.size()); if (platforms.IsEmpty()) @@ -109,9 +109,8 @@ bool Scene::LoadInterface(const String & fileName) size_t nTotalPixels(0); ASSERT(!obj.images.empty()); images.Reserve((uint32_t)obj.images.size()); - for (Interface::ImageArr::const_iterator it=obj.images.begin(); it!=obj.images.end(); ++it) { - const Interface::Image& image = *it; - const uint32_t ID(images.GetSize()); + for (const Interface::Image& image: obj.images) { + const uint32_t ID(images.size()); Image& imageData = images.AddEmpty(); imageData.name = image.name; Util::ensureUnifySlash(imageData.name); @@ -123,7 +122,7 @@ bool Scene::LoadInterface(const String & fileName) } imageData.platformID = image.platformID; imageData.cameraID = image.cameraID; - imageData.ID = image.ID; + imageData.ID = (image.ID == NO_ID ? ID : image.ID); // init camera const Interface::Platform::Camera& camera = obj.platforms[image.platformID].cameras[image.cameraID]; if (camera.HasResolution()) { @@ -200,12 +199,10 @@ bool Scene::SaveInterface(const String & fileName) const // export platforms obj.platforms.reserve(platforms.GetSize()); - FOREACH(i, platforms) { - const Platform& platform = platforms[i]; + for (const Platform& platform: platforms) { Interface::Platform plat; plat.cameras.reserve(platform.cameras.GetSize()); - FOREACH(j, platform.cameras) { - const Platform::Camera& camera = platform.cameras[j]; + for (const Platform::Camera& camera: platform.cameras) { Interface::Platform::Camera cam; cam.K = camera.K; cam.R = camera.R; @@ -213,8 +210,7 @@ bool Scene::SaveInterface(const String & fileName) const plat.cameras.push_back(cam); } plat.poses.reserve(platform.poses.GetSize()); - FOREACH(j, platform.poses) { - const Platform::Pose& pose = platform.poses[j]; + for (const Platform::Pose& pose: platform.poses) { Interface::Platform::Pose p; p.R = pose.R; p.C = pose.C; From cbf172eac467c644e75615cc771fd69f5792a883 Mon Sep 17 00:00:00 2001 From: cDc Date: Wed, 16 Oct 2019 10:51:49 +0300 Subject: [PATCH 34/70] common: fix gcc --- libs/Common/Plane.inl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libs/Common/Plane.inl b/libs/Common/Plane.inl index 59ea42b04..2fcbb76f1 100644 --- a/libs/Common/Plane.inl +++ b/libs/Common/Plane.inl @@ -390,7 +390,11 @@ template void TFrustum::Set(const MATRIX3x4& m, TYPE w, TYPE h, TYPE n, TYPE f) { MATRIX4x4 M(MATRIX4x4::Identity()); + #ifdef __GNUC__ + M.topLeftCorner(3,4) = m; + #else M.template topLeftCorner<3,4>() = m; + #endif Set(M, w, h, n, f); } // Set /*----------------------------------------------------------------*/ From 061668119471eb86cff6f2e4b06b7c296c737b54 Mon Sep 17 00:00:00 2001 From: cDc Date: Wed, 16 Oct 2019 18:35:58 +0300 Subject: [PATCH 35/70] dense: add image file name to DMAP --- libs/MVS/DepthMap.cpp | 38 ++++++++++++++++++++++++++++---------- libs/MVS/DepthMap.h | 4 ++-- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index 952506565..15fddf7c8 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -194,7 +194,7 @@ bool DepthData::Save(const String& fileName) const for (const ViewData& image: images) IDs.push_back(image.GetID()); const ViewData& image0 = GetView(); - if (!ExportDepthDataRaw(fileNameTmp, IDs, depthMap.size(), image0.camera.K, image0.camera.R, image0.camera.C, dMin, dMax, depthMap, normalMap, confMap)) + if (!ExportDepthDataRaw(fileNameTmp, image0.pImageData->name, IDs, depthMap.size(), image0.camera.K, image0.camera.R, image0.camera.C, dMin, dMax, depthMap, normalMap, confMap)) return false; } if (!File::renameFile(fileNameTmp, fileName)) { @@ -208,10 +208,11 @@ bool DepthData::Load(const String& fileName) { ASSERT(IsValid()); // serialize in the saved state + String imageFileName; IIndexArr IDs; cv::Size imageSize; Camera camera; - if (!ImportDepthDataRaw(fileName, IDs, imageSize, camera.K, camera.R, camera.C, dMin, dMax, depthMap, normalMap, confMap)) + if (!ImportDepthDataRaw(fileName, imageFileName, IDs, imageSize, camera.K, camera.R, camera.C, dMin, dMax, depthMap, normalMap, confMap)) return false; ASSERT(IDs.size() == images.size()); ASSERT(IDs.front() == GetView().GetID()); @@ -1387,8 +1388,8 @@ struct HeaderDepthDataRaw { uint32_t imageWidth, imageHeight; // image resolution uint32_t depthWidth, depthHeight; // depth-map resolution float dMin, dMax; // depth range for this view - uint32_t nIDs; // number of view IDs - // view ID followed by neighbor view IDs: uint32_t* IDs + // image file name length followed by the characters: uint16_t nFileNameSize; char* FileName + // number of view IDs followed by view ID and neighbor view IDs: uint32_t nIDs; uint32_t* IDs // camera, rotation and translation matrices (row-major): double K[3][3], R[3][3], C[3] // depth, normal, confidence maps inline HeaderDepthDataRaw() : name(0), type(0), padding(0) {} @@ -1396,7 +1397,7 @@ struct HeaderDepthDataRaw { int GetStep() const { return ROUND2INT((float)imageWidth/depthWidth); } }; -bool MVS::ExportDepthDataRaw(const String& fileName, +bool MVS::ExportDepthDataRaw(const String& fileName, const String& imageFileName, const IIndexArr& IDs, const cv::Size& imageSize, const KMatrix& K, const RMatrix& R, const CMatrix& C, Depth dMin, Depth dMax, @@ -1422,16 +1423,24 @@ bool MVS::ExportDepthDataRaw(const String& fileName, header.depthHeight = (uint32_t)depthMap.rows; header.dMin = dMin; header.dMax = dMax; - header.nIDs = IDs.size(); if (!normalMap.empty()) header.type |= HeaderDepthDataRaw::HAS_NORMAL; if (!confMap.empty()) header.type |= HeaderDepthDataRaw::HAS_CONF; fwrite(&header, sizeof(HeaderDepthDataRaw), 1, f); + // write image file name + STATIC_ASSERT(sizeof(String::value_type) == sizeof(char)); + const String FileName(MAKE_PATH_REL(Util::getFullPath(Util::getFilePath(fileName)), Util::getFullPath(imageFileName))); + const uint16_t nFileNameSize((uint16_t)FileName.length()); + fwrite(&nFileNameSize, sizeof(uint16_t), 1, f); + fwrite(FileName.c_str(), sizeof(char), nFileNameSize, f); + // write neighbor IDs STATIC_ASSERT(sizeof(uint32_t) == sizeof(IIndex)); - fwrite(IDs.data(), sizeof(IIndex), IDs.size(), f); + const uint32_t nIDs(IDs.size()); + fwrite(&nIDs, sizeof(IIndex), 1, f); + fwrite(IDs.data(), sizeof(IIndex), nIDs, f); // write pose STATIC_ASSERT(sizeof(double) == sizeof(REAL)); @@ -1455,7 +1464,7 @@ bool MVS::ExportDepthDataRaw(const String& fileName, return bRet; } // ExportDepthDataRaw -bool MVS::ImportDepthDataRaw(const String& fileName, +bool MVS::ImportDepthDataRaw(const String& fileName, String& imageFileName, IIndexArr& IDs, cv::Size& imageSize, KMatrix& K, RMatrix& R, CMatrix& C, Depth& dMin, Depth& dMax, @@ -1479,10 +1488,19 @@ bool MVS::ImportDepthDataRaw(const String& fileName, return false; } + // read image file name + STATIC_ASSERT(sizeof(String::value_type) == sizeof(char)); + uint16_t nFileNameSize; + fread(&nFileNameSize, sizeof(uint16_t), 1, f); + imageFileName.resize(nFileNameSize); + fread(&imageFileName[0], sizeof(char), nFileNameSize, f); + // read neighbor IDs STATIC_ASSERT(sizeof(uint32_t) == sizeof(IIndex)); - IDs.resize(header.nIDs); - fread(IDs.data(), sizeof(IIndex), IDs.size(), f); + uint32_t nIDs; + fread(&nIDs, sizeof(IIndex), 1, f); + IDs.resize(nIDs); + fread(IDs.data(), sizeof(IIndex), nIDs, f); // read pose STATIC_ASSERT(sizeof(double) == sizeof(REAL)); diff --git a/libs/MVS/DepthMap.h b/libs/MVS/DepthMap.h index 4ba7219e5..d8f00409b 100644 --- a/libs/MVS/DepthMap.h +++ b/libs/MVS/DepthMap.h @@ -460,12 +460,12 @@ MVS_API bool ExportNormalMap(const String& fileName, const NormalMap& normalMap) MVS_API bool ExportConfidenceMap(const String& fileName, const ConfidenceMap& confMap); MVS_API bool ExportPointCloud(const String& fileName, const Image&, const DepthMap&, const NormalMap&); -MVS_API bool ExportDepthDataRaw(const String&, +MVS_API bool ExportDepthDataRaw(const String&, const String& imageFileName, const IIndexArr&, const cv::Size& imageSize, const KMatrix&, const RMatrix&, const CMatrix&, Depth dMin, Depth dMax, const DepthMap&, const NormalMap&, const ConfidenceMap&); -MVS_API bool ImportDepthDataRaw(const String&, +MVS_API bool ImportDepthDataRaw(const String&, String& imageFileName, IIndexArr&, cv::Size& imageSize, KMatrix&, RMatrix&, CMatrix&, Depth& dMin, Depth& dMax, From 6c9ed8d41d6bc44b72ade24bf2c0ee935945461c Mon Sep 17 00:00:00 2001 From: cDc Date: Tue, 22 Oct 2019 12:15:25 +0300 Subject: [PATCH 36/70] Merge branch 'master' into 'develop' --- .appveyor.yml | 18 ++++++++++++++---- libs/Common/CUDA.cpp | 5 +++++ libs/MVS/DepthMap.cpp | 8 ++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index aeb6dee08..30592ae3e 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -3,7 +3,7 @@ version: "{build}" image: - Visual Studio 2017 - - Ubuntu + - ubuntu1804 platform: - x64 @@ -19,9 +19,19 @@ configuration: # scripts that are called at very beginning, before repo cloning init: + #------------------ + # Windows 10 + #------------------ + - cmd: ver - cmd: cmake --version - cmd: msbuild /version + #------------------ + # Ubuntu 18.04 LTS + #------------------ + - sh: lsb_release -a + - sh: cmake --version + # branches to build branches: # blacklist @@ -54,7 +64,7 @@ install: - cmd: cd "%APPVEYOR_BUILD_FOLDER%" #------------------ - # Ubuntu 16.04 LTS + # Ubuntu 18.04 LTS #------------------ - sh: sudo apt-get update -qq && sudo apt-get install -qq - sh: sudo apt-get -y install build-essential git mercurial cmake libpng-dev libjpeg-dev libtiff-dev libglu1-mesa-dev libxmu-dev libxi-dev @@ -74,7 +84,7 @@ for: - matrix: only: - - image: Ubuntu + - image: ubuntu1804 cache: - '/usr/lib/x86_64-linux-gnu/' @@ -92,7 +102,7 @@ build_script: - cmd: cmake --build . --target ALL_BUILD --config %Configuration% -- /maxcpucount:4 #------------------ - # Ubuntu 16.04 LTS + # Ubuntu 18.04 LTS #------------------ - sh: hg clone https://bitbucket.org/eigen/eigen#3.2 - sh: mkdir eigen_build && cd eigen_build diff --git a/libs/Common/CUDA.cpp b/libs/Common/CUDA.cpp index 736c3424b..2c31cabe7 100644 --- a/libs/Common/CUDA.cpp +++ b/libs/Common/CUDA.cpp @@ -42,6 +42,11 @@ int _convertSMVer2Cores(int major, int minor) {0x37, 192}, // Kepler Generation (SM 3.7) GK21x class {0x50, 128}, // Maxwell Generation (SM 5.0) GM10x class {0x52, 128}, // Maxwell Generation (SM 5.2) GM20x class + {0x53, 128}, // Maxwell Generation (SM 5.3) GM20x class + {0x60, 64 }, // Pascal Generation (SM 6.0) GP100 class + {0x61, 128}, // Pascal Generation (SM 6.1) GP10x class + {0x62, 128}, // Pascal Generation (SM 6.2) GP10x class + {0x70, 64 }, // Volta Generation (SM 7.0) GV100 class {-1, -1} }; diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index 15fddf7c8..1d8804e59 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -1095,6 +1095,7 @@ void MVS::EstimatePointNormals(const ImageArr& images, PointCloud& pointcloud, i // estimates normals direction; // Note: pca_estimate_normals() requires an iterator over points // as well as property maps to access each point's position and normal. + #if CGAL_VERSION_NR < 1041301000 #if CGAL_VERSION_NR < 1040800000 CGAL::pca_estimate_normals( #else @@ -1105,6 +1106,13 @@ void MVS::EstimatePointNormals(const ImageArr& images, PointCloud& pointcloud, i CGAL::Second_of_pair_property_map(), numNeighbors ); + #else + CGAL::pca_estimate_normals( + pointvectors, numNeighbors, + CGAL::parameters::point_map(CGAL::First_of_pair_property_map()) + .normal_map(CGAL::Second_of_pair_property_map()) + ); + #endif // store the point normals pointcloud.normals.Resize(pointcloud.points.GetSize()); FOREACH(i, pointcloud.normals) { From 0432d39abf28fb93e8e4e30a0503a92eb6105088 Mon Sep 17 00:00:00 2001 From: cDc Date: Mon, 4 Nov 2019 10:49:24 +0200 Subject: [PATCH 37/70] interface: log common COLMAP import problem --- apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp b/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp index 04406eeee..d450932df 100644 --- a/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp +++ b/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp @@ -449,6 +449,10 @@ bool ImportScene(const String& strFolder, Interface& scene) scene.platforms.push_back(platform); } } + if (mapCameras.empty()) { + VERBOSE("error: no valid cameras (make sure they are in PINHOLE model)"); + return false; + } // read images list typedef std::map ImagesMap; From 64e276ad478c4fa3102ec3f7c69dd164cd555b28 Mon Sep 17 00:00:00 2001 From: cDc Date: Fri, 6 Dec 2019 12:17:38 +0200 Subject: [PATCH 38/70] build: update Eigen repo --- .appveyor.yml | 4 ++-- BUILD.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 30592ae3e..37429cd5a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -67,7 +67,7 @@ install: # Ubuntu 18.04 LTS #------------------ - sh: sudo apt-get update -qq && sudo apt-get install -qq - - sh: sudo apt-get -y install build-essential git mercurial cmake libpng-dev libjpeg-dev libtiff-dev libglu1-mesa-dev libxmu-dev libxi-dev + - sh: sudo apt-get -y install build-essential git cmake libpng-dev libjpeg-dev libtiff-dev libglu1-mesa-dev libxmu-dev libxi-dev - sh: sudo apt-get -y install libboost-iostreams-dev libboost-program-options-dev libboost-system-dev libboost-serialization-dev - sh: sudo apt-get -y install libopencv-dev libcgal-dev libcgal-qt5-dev libatlas-base-dev #- sh: sudo apt-get -y install libsuitesparse-dev libceres-dev @@ -104,7 +104,7 @@ build_script: #------------------ # Ubuntu 18.04 LTS #------------------ - - sh: hg clone https://bitbucket.org/eigen/eigen#3.2 + - sh: git clone https://gitlab.com/libeigen/eigen.git --branch 3.2 - sh: mkdir eigen_build && cd eigen_build - sh: cmake . ../eigen - sh: make && sudo make install diff --git a/BUILD.md b/BUILD.md index 315f5fe7e..a0de2b439 100644 --- a/BUILD.md +++ b/BUILD.md @@ -3,7 +3,7 @@ Dependencies ------------ *OpenMVS* relies on a number of open source libraries, some of which are optional. For details on customizing the build process, see the compilation instructions. -* [Eigen](http://eigen.tuxfamily.org) version 3.2 or higher +* [Eigen](http://eigen.tuxfamily.org) version 3.2 (or higher on Windows only) * [OpenCV](http://opencv.org) version 2.4 or higher * [Ceres](http://ceres-solver.org) version 1.10 or higher * [CGAL](http://www.cgal.org) version 4.2 or higher @@ -60,11 +60,11 @@ Linux compilation ``` #Prepare and empty machine for building: sudo apt-get update -qq && sudo apt-get install -qq -sudo apt-get -y install git mercurial cmake libpng-dev libjpeg-dev libtiff-dev libglu1-mesa-dev +sudo apt-get -y install git cmake libpng-dev libjpeg-dev libtiff-dev libglu1-mesa-dev main_path=`pwd` #Eigen (Required) -hg clone https://bitbucket.org/eigen/eigen#3.2 +git clone https://gitlab.com/libeigen/eigen.git --branch 3.2 mkdir eigen_build && cd eigen_build cmake . ../eigen make && sudo make install From 6180ce116a330a4a7dc6ffcec4e87584e92ca624 Mon Sep 17 00:00:00 2001 From: cDc Date: Tue, 24 Dec 2019 11:28:04 +0200 Subject: [PATCH 39/70] mvs: fix color save --- libs/MVS/Scene.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/MVS/Scene.cpp b/libs/MVS/Scene.cpp index dd94c21d8..3b2fd8631 100644 --- a/libs/MVS/Scene.cpp +++ b/libs/MVS/Scene.cpp @@ -254,7 +254,7 @@ bool Scene::SaveInterface(const String & fileName) const vertexNormal.n = normal; } } - if (!pointcloud.normals.IsEmpty()) { + if (!pointcloud.colors.IsEmpty()) { obj.verticesColor.resize(pointcloud.colors.GetSize()); FOREACH(i, pointcloud.colors) { const PointCloud::Color& color = pointcloud.colors[i]; From 6947133183bbb313a486eaf5235c7a685fb11bd3 Mon Sep 17 00:00:00 2001 From: cDc Date: Sat, 28 Dec 2019 19:50:06 +0200 Subject: [PATCH 40/70] common: disable std iterator debugging --- libs/Common/Config.h | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/libs/Common/Config.h b/libs/Common/Config.h index 779b2bb3e..df28c109c 100644 --- a/libs/Common/Config.h +++ b/libs/Common/Config.h @@ -128,15 +128,11 @@ #if !defined(_DEBUG) && !defined(_PROFILE) -#define _RELEASE // exclude code useful only for debug +#define _RELEASE // exclude code useful only for debug #endif #if defined(_MSC_VER) && _MSC_VER >= 1400 -//disable the deprecated warnings for the CRT functions. -//#ifndef _CRT_SECURE_CPP_OVERLOAD_STANDARD_NAMES -//#define _CRT_SECURE_CPP_OVERLOAD_STANDARD_NAMES 1 -//#endif #ifndef _CRT_SECURE_NO_WARNINGS #define _CRT_SECURE_NO_WARNINGS 1 #endif @@ -149,6 +145,16 @@ #ifndef _CRT_NON_CONFORMING_SWPRINTFS #define _CRT_NON_CONFORMING_SWPRINTFS 1 #endif +#ifndef _CRT_SECURE_CPP_OVERLOAD_SECURE_NAMES +#define _CRT_SECURE_CPP_OVERLOAD_SECURE_NAMES 0 // disable automatically overloading CPP names to secure versions +#endif +#ifndef _ITERATOR_DEBUG_LEVEL +#ifndef _DEBUG +#define _ITERATOR_DEBUG_LEVEL 0 +#else +#define _ITERATOR_DEBUG_LEVEL 1 // disable std iterator debugging even in Debug, as it is very slow +#endif +#endif #endif From 05ab77fe7aed7ebffc9a1d81af148698507dddf4 Mon Sep 17 00:00:00 2001 From: cDc Date: Sat, 28 Dec 2019 19:56:46 +0200 Subject: [PATCH 41/70] common: remove deprecated define --- libs/Common/Config.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/libs/Common/Config.h b/libs/Common/Config.h index df28c109c..4a100f6cb 100644 --- a/libs/Common/Config.h +++ b/libs/Common/Config.h @@ -120,13 +120,6 @@ #endif // _MSC_VER -#define DECLARE_TEMPLATE_INSTANCE(mod, typ, lptyp, ...) \ - mod##_TPL template class mod##_API T##typ<__VA_ARGS__>; \ - typedef class T##typ<__VA_ARGS__> typ; \ - typedef typ *lptyp; \ - typedef cList lptyp##ARR; - - #if !defined(_DEBUG) && !defined(_PROFILE) #define _RELEASE // exclude code useful only for debug #endif From 4d7607a367afd49b56802fa344450dc8198ab010 Mon Sep 17 00:00:00 2001 From: cDc Date: Sun, 29 Dec 2019 00:26:48 +0200 Subject: [PATCH 42/70] common: include out to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3748365f0..1880d2e51 100644 --- a/.gitignore +++ b/.gitignore @@ -34,5 +34,6 @@ CMakeSettings.json .vs/ .idea/ .vscode/ +out/ bin/ binaries/ From a6dbdb3316c20762db245c9b95ff06a63969d85b Mon Sep 17 00:00:00 2001 From: cDc Date: Fri, 3 Jan 2020 16:52:06 +0200 Subject: [PATCH 43/70] common: place custom types inside namespace --- libs/Common/Types.h | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/libs/Common/Types.h b/libs/Common/Types.h index 3e6507fa9..396b2b0b7 100644 --- a/libs/Common/Types.h +++ b/libs/Common/Types.h @@ -267,19 +267,6 @@ int _vscprintf(LPCSTR format, va_list pargs); #define _T(s) s #endif //_MSC_VER -// signed and unsigned types of the size of the architecture -// (32 or 64 bit for x86 and respectively x64) -#ifdef _ENVIRONMENT64 -typedef int64_t int_t; -typedef uint64_t uint_t; -#else -typedef int32_t int_t; -typedef uint32_t uint_t; -#endif - -// type used for the size of the files -typedef int64_t size_f_t; - #define DECLARE_NO_INDEX(...) std::numeric_limits<__VA_ARGS__>::max() #define NO_ID DECLARE_NO_INDEX(uint32_t) @@ -318,14 +305,30 @@ typedef int64_t size_f_t; #endif #ifndef MINF -#define MINF std::min +#define MINF std::min #endif #ifndef MAXF -#define MAXF std::max +#define MAXF std::max #endif namespace SEACAVE { +// signed and unsigned types of the size of the architecture +// (32 or 64 bit for x86 and respectively x64) +#ifdef _ENVIRONMENT64 +typedef int64_t int_t; +typedef uint64_t uint_t; +#else +typedef int32_t int_t; +typedef uint32_t uint_t; +#endif + +// type used for the size of the files +typedef int64_t size_f_t; + +// type used as the default floating number precision +typedef double REAL; + template inline T MINF3(const T& x1, const T& x2, const T& x3) { return MINF(MINF(x1, x2), x3); @@ -1141,8 +1144,6 @@ namespace SEACAVE { // P R O T O T Y P E S ///////////////////////////////////////////// -typedef double REAL; - template class TMatrix; template class TAABB; template class TRay; From 0c0359da317c0acb8d085814c059e06049f9a44d Mon Sep 17 00:00:00 2001 From: cDc Date: Fri, 3 Jan 2020 18:34:41 +0200 Subject: [PATCH 44/70] common: improve RealType --- libs/Common/Types.h | 3 +++ libs/Common/Types.inl | 9 +-------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/libs/Common/Types.h b/libs/Common/Types.h index 396b2b0b7..57f59b918 100644 --- a/libs/Common/Types.h +++ b/libs/Common/Types.h @@ -329,6 +329,9 @@ typedef int64_t size_f_t; // type used as the default floating number precision typedef double REAL; +template +struct RealType { typedef typename std::conditional::value, TYPE, REALTYPE>::type type; }; + template inline T MINF3(const T& x1, const T& x2, const T& x3) { return MINF(MINF(x1, x2), x3); diff --git a/libs/Common/Types.inl b/libs/Common/Types.inl index 6a23b7c59..aa66a4714 100644 --- a/libs/Common/Types.inl +++ b/libs/Common/Types.inl @@ -768,13 +768,6 @@ inline TMatrix cross(const cv::Matx& l, const cv::Matx::Vec&)l).cross((const typename TMatrix::Vec&)r); } -template -struct RealType { typedef REAL type; }; -template <> -struct RealType { typedef float type; }; -template <> -struct RealType { typedef double type; }; - template inline typename RealType::type normSq(const cv::Point_& v) { typedef typename RealType::type real; @@ -2378,7 +2371,7 @@ inline void _ProcessScanLine(int y, const TPoint3& pa, const TPoint3& pb, } } // Raster the given triangle and output the position and depth of each pixel of the triangle; -// based on "Learning how to write a 3D software engine – Rasterization & Z-Buffering" by Nick (David Rousset) +// based on "Learning how to write a 3D software engine � Rasterization & Z-Buffering" by Nick (David Rousset) // http://blogs.msdn.com/b/davrous/archive/2013/06/21/tutorial-part-4-learning-how-to-write-a-3d-software-engine-in-c-ts-or-js-rasterization-amp-z-buffering.aspx template template From b200d3b7fd102495faf691f80a113789012dcbfa Mon Sep 17 00:00:00 2001 From: cDc Date: Sat, 4 Jan 2020 00:18:38 +0200 Subject: [PATCH 45/70] build: fix MacOS --- BUILD.md | 3 +++ CMakeLists.txt | 9 +++++++++ apps/DensifyPointCloud/CMakeLists.txt | 6 +----- apps/InterfaceCOLMAP/CMakeLists.txt | 8 ++------ apps/InterfaceOpenMVG/CMakeLists.txt | 6 +----- apps/InterfacePhotoScan/CMakeLists.txt | 6 +----- apps/InterfaceVisualSFM/CMakeLists.txt | 6 +----- apps/ReconstructMesh/CMakeLists.txt | 6 +----- apps/RefineMesh/CMakeLists.txt | 6 +----- apps/TextureMesh/CMakeLists.txt | 6 +----- apps/Viewer/CMakeLists.txt | 8 ++------ libs/MVS/SceneDensify.cpp | 2 +- 12 files changed, 24 insertions(+), 48 deletions(-) diff --git a/BUILD.md b/BUILD.md index a0de2b439..29ca640ba 100644 --- a/BUILD.md +++ b/BUILD.md @@ -118,6 +118,9 @@ brew tap homebrew/science brew install boost eigen opencv cgal ceres-solver main_path=`pwd` +#GLFW3 (Optional) +brew install glew glfw3 + #VCGLib (Required) git clone https://github.com/cdcseacave/VCG.git vcglib diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d9677e10..e40d7668c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,12 +55,20 @@ SET(OpenMVS_CONFIG_INCLUDE_DIR "${CMAKE_BINARY_DIR}/" CACHE PATH "Where to creat INCLUDE_DIRECTORIES("${OpenMVS_SOURCE_DIR}") # Find required packages +SET(OpenMVS_EXTRA_LIBS "") if(OpenMVS_USE_OPENMP) + SET(OpenMP_LIBS "") FIND_PACKAGE(OpenMP) if(OPENMP_FOUND) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") ADD_DEFINITIONS(-D_USE_OPENMP) SET(_USE_OPENMP TRUE) + #cmake only check for separate OpenMP library on AppleClang 7+ + #https://github.com/Kitware/CMake/blob/42212f7539040139ecec092547b7d58ef12a4d72/Modules/FindOpenMP.cmake#L252 + if (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "7.0") + SET(OpenMP_LIBS ${OpenMP_libomp_LIBRARY}) + LIST(APPEND OpenMVS_EXTRA_LIBS ${OpenMP_LIBS}) + endif() else() message("-- Can't find OpenMP. Continuing without it.") endif() @@ -97,6 +105,7 @@ if(OpenMVS_USE_BREAKPAD) INCLUDE_DIRECTORIES(${BREAKPAD_INCLUDE_DIRS}) ADD_DEFINITIONS(${BREAKPAD_DEFINITIONS} -D_USE_BREAKPAD) SET(_USE_BREAKPAD TRUE) + LIST(APPEND OpenMVS_EXTRA_LIBS ${BREAKPAD_LIBS}) else() MESSAGE("-- Can't find BreakPad. Continuing without it.") endif() diff --git a/apps/DensifyPointCloud/CMakeLists.txt b/apps/DensifyPointCloud/CMakeLists.txt index 90e7bc278..5dddebee4 100644 --- a/apps/DensifyPointCloud/CMakeLists.txt +++ b/apps/DensifyPointCloud/CMakeLists.txt @@ -5,11 +5,7 @@ else() endif() FILE(GLOB LIBRARY_FILES_H "*.h" "*.inl") -cxx_executable_with_flags_no_pch(DensifyPointCloud "Apps" "${cxx_default}" "MVS" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) - -if(OpenMVS_USE_BREAKPAD AND BREAKPAD_FOUND) - target_link_libraries(DensifyPointCloud ${BREAKPAD_LIBS}) -endif() +cxx_executable_with_flags_no_pch(DensifyPointCloud "Apps" "${cxx_default}" "MVS;${OpenMVS_EXTRA_LIBS}" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) # Install INSTALL(TARGETS DensifyPointCloud diff --git a/apps/InterfaceCOLMAP/CMakeLists.txt b/apps/InterfaceCOLMAP/CMakeLists.txt index 6c8ae3f3a..0a5b0fc0e 100644 --- a/apps/InterfaceCOLMAP/CMakeLists.txt +++ b/apps/InterfaceCOLMAP/CMakeLists.txt @@ -5,13 +5,9 @@ else() endif() FILE(GLOB LIBRARY_FILES_H "*.h" "*.inl") -cxx_executable_with_flags_no_pch(InterfaceCOLMAP "Apps" "${cxx_default}" "MVS" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) - -if(3Dnovator_USE_BREAKPAD AND BREAKPAD_FOUND) - target_link_libraries(InterfaceCOLMAP ${BREAKPAD_LIBS}) -endif() +cxx_executable_with_flags_no_pch(InterfaceCOLMAP "Apps" "${cxx_default}" "MVS;${OpenMVS_EXTRA_LIBS}" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) # Install INSTALL(TARGETS InterfaceCOLMAP - EXPORT 3DnovatorTargets + EXPORT OpenMVSTargets RUNTIME DESTINATION "${INSTALL_BIN_DIR}" COMPONENT bin) diff --git a/apps/InterfaceOpenMVG/CMakeLists.txt b/apps/InterfaceOpenMVG/CMakeLists.txt index cad5b125b..5dc56a0d9 100644 --- a/apps/InterfaceOpenMVG/CMakeLists.txt +++ b/apps/InterfaceOpenMVG/CMakeLists.txt @@ -15,11 +15,7 @@ else() endif() FILE(GLOB LIBRARY_FILES_H "*.h" "*.inl") -cxx_executable_with_flags_no_pch(InterfaceOpenMVG "Apps" "${cxx_default}" "${LIBS_DEPEND}" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) - -if(OpenMVS_USE_BREAKPAD AND BREAKPAD_FOUND) - target_link_libraries(InterfaceOpenMVG ${BREAKPAD_LIBS}) -endif() +cxx_executable_with_flags_no_pch(InterfaceOpenMVG "Apps" "${cxx_default}" "${LIBS_DEPEND};${OpenMVS_EXTRA_LIBS}" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) # Install INSTALL(TARGETS InterfaceOpenMVG diff --git a/apps/InterfacePhotoScan/CMakeLists.txt b/apps/InterfacePhotoScan/CMakeLists.txt index f9116b109..e030ce189 100644 --- a/apps/InterfacePhotoScan/CMakeLists.txt +++ b/apps/InterfacePhotoScan/CMakeLists.txt @@ -5,11 +5,7 @@ else() endif() FILE(GLOB LIBRARY_FILES_H "*.h" "*.inl") -cxx_executable_with_flags_no_pch(InterfacePhotoScan "Apps" "${cxx_default}" "${LIBS_DEPEND}" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) - -if(OpenMVS_USE_BREAKPAD AND BREAKPAD_FOUND) - target_link_libraries(InterfacePhotoScan ${BREAKPAD_LIBS}) -endif() +cxx_executable_with_flags_no_pch(InterfacePhotoScan "Apps" "${cxx_default}" "MVS;${OpenMVS_EXTRA_LIBS}" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) # Install INSTALL(TARGETS InterfacePhotoScan diff --git a/apps/InterfaceVisualSFM/CMakeLists.txt b/apps/InterfaceVisualSFM/CMakeLists.txt index 10e888d66..9bd8bdedf 100644 --- a/apps/InterfaceVisualSFM/CMakeLists.txt +++ b/apps/InterfaceVisualSFM/CMakeLists.txt @@ -5,11 +5,7 @@ else() endif() FILE(GLOB LIBRARY_FILES_H "*.h" "*.inl") -cxx_executable_with_flags_no_pch(InterfaceVisualSFM "Apps" "${cxx_default}" "MVS" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) - -if(OpenMVS_USE_BREAKPAD AND BREAKPAD_FOUND) - target_link_libraries(InterfaceVisualSFM ${BREAKPAD_LIBS}) -endif() +cxx_executable_with_flags_no_pch(InterfaceVisualSFM "Apps" "${cxx_default}" "MVS;${OpenMVS_EXTRA_LIBS}" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) # Install INSTALL(TARGETS InterfaceVisualSFM diff --git a/apps/ReconstructMesh/CMakeLists.txt b/apps/ReconstructMesh/CMakeLists.txt index 9def582eb..86cac3de4 100644 --- a/apps/ReconstructMesh/CMakeLists.txt +++ b/apps/ReconstructMesh/CMakeLists.txt @@ -5,11 +5,7 @@ else() endif() FILE(GLOB LIBRARY_FILES_H "*.h" "*.inl") -cxx_executable_with_flags_no_pch(ReconstructMesh "Apps" "${cxx_default}" "MVS" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) - -if(OpenMVS_USE_BREAKPAD AND BREAKPAD_FOUND) - target_link_libraries(ReconstructMesh ${BREAKPAD_LIBS}) -endif() +cxx_executable_with_flags_no_pch(ReconstructMesh "Apps" "${cxx_default}" "MVS;${OpenMVS_EXTRA_LIBS}" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) # Install INSTALL(TARGETS ReconstructMesh diff --git a/apps/RefineMesh/CMakeLists.txt b/apps/RefineMesh/CMakeLists.txt index 7902e19c2..eaffec244 100644 --- a/apps/RefineMesh/CMakeLists.txt +++ b/apps/RefineMesh/CMakeLists.txt @@ -5,11 +5,7 @@ else() endif() FILE(GLOB LIBRARY_FILES_H "*.h" "*.inl") -cxx_executable_with_flags_no_pch(RefineMesh "Apps" "${cxx_default}" "MVS" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) - -if(OpenMVS_USE_BREAKPAD AND BREAKPAD_FOUND) - target_link_libraries(RefineMesh ${BREAKPAD_LIBS}) -endif() +cxx_executable_with_flags_no_pch(RefineMesh "Apps" "${cxx_default}" "MVS;${OpenMVS_EXTRA_LIBS}" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) # Install INSTALL(TARGETS RefineMesh diff --git a/apps/TextureMesh/CMakeLists.txt b/apps/TextureMesh/CMakeLists.txt index eb8f3067d..b53e2f560 100644 --- a/apps/TextureMesh/CMakeLists.txt +++ b/apps/TextureMesh/CMakeLists.txt @@ -5,11 +5,7 @@ else() endif() FILE(GLOB LIBRARY_FILES_H "*.h" "*.inl") -cxx_executable_with_flags_no_pch(TextureMesh "Apps" "${cxx_default}" "MVS" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) - -if(OpenMVS_USE_BREAKPAD AND BREAKPAD_FOUND) - target_link_libraries(TextureMesh ${BREAKPAD_LIBS}) -endif() +cxx_executable_with_flags_no_pch(TextureMesh "Apps" "${cxx_default}" "MVS;${OpenMVS_EXTRA_LIBS}" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) # Install INSTALL(TARGETS TextureMesh diff --git a/apps/Viewer/CMakeLists.txt b/apps/Viewer/CMakeLists.txt index f1690a6e9..c519040d1 100644 --- a/apps/Viewer/CMakeLists.txt +++ b/apps/Viewer/CMakeLists.txt @@ -54,16 +54,12 @@ FILE(GLOB LIBRARY_FILES_H "*.h" "*.inl") LIST(REMOVE_ITEM LIBRARY_FILES_C ${PCH_C}) SET(LIBRARY_FILES_C "${PCH_C};${LIBRARY_FILES_C}") -cxx_executable_with_flags_no_pch(${VIEWER_NAME} "Apps" "${cxx_default}" "MVS;${OPENGL_LIBRARIES};${GLEW_LIBRARY};${GLFW_STATIC_LIBRARIES};GLEW::GLEW;${glfw3_LIBRARY};${GLFW3_LIBRARY};glfw" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) +cxx_executable_with_flags_no_pch(${VIEWER_NAME} "Apps" "${cxx_default}" "MVS;${OPENGL_LIBRARIES};${GLEW_LIBRARY};${GLFW_STATIC_LIBRARIES};GLEW::GLEW;${glfw3_LIBRARY};${GLFW3_LIBRARY};glfw;${OpenMVS_EXTRA_LIBS}" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) # Manually set Common.h as the precompiled header set_target_pch(${VIEWER_NAME} Common.h) -if(OpenMVS_USE_BREAKPAD AND BREAKPAD_FOUND) - target_link_libraries(${VIEWER_NAME} ${BREAKPAD_LIBS}) -endif() - # Install INSTALL(TARGETS ${VIEWER_NAME} - EXPORT 3DnovatorTargets + EXPORT OpenMVSTargets RUNTIME DESTINATION "${INSTALL_BIN_DIR}" COMPONENT bin) diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index fd5b518c0..82473bd05 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -2052,7 +2052,7 @@ void Scene::PointCloudFilter(int thRemove) for (int views: visibility) { if (views > 0) continue; - while (counts.size() <= -views) + while (counts.size() <= IDX(-views)) counts.push_back(0); ++counts[-views]; } From e56f2a3f9e833d24a9cf3b1f5a560708b2815c3e Mon Sep 17 00:00:00 2001 From: cDc Date: Sun, 5 Jan 2020 10:06:08 +0200 Subject: [PATCH 46/70] build: add MacOS to Appveyor --- .appveyor.yml | 159 +++++++++++++++++++++++++------------------------- BUILD.md | 3 +- 2 files changed, 82 insertions(+), 80 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 37429cd5a..84d82a06f 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,9 +1,10 @@ -# Specify version format +# specify version format version: "{build}" image: - Visual Studio 2017 - ubuntu1804 + - macos platform: - x64 @@ -17,105 +18,107 @@ configuration: - Debug - Release -# scripts that are called at very beginning, before repo cloning -init: - #------------------ - # Windows 10 - #------------------ - - cmd: ver - - cmd: cmake --version - - cmd: msbuild /version - - #------------------ - # Ubuntu 18.04 LTS - #------------------ - - sh: lsb_release -a - - sh: cmake --version - # branches to build branches: # blacklist except: - gh-pages -# scripts that run after cloning repository -install: +for: +- #------------------ # Windows 10 #------------------ - # update vcpkg - - cmd: cd C:\tools\vcpkg - - cmd: git pull - - cmd: .\bootstrap-vcpkg.bat - - - cmd: if "%platform%"=="Win32" set VCPKG_ARCH=x86-windows - - cmd: if "%platform%"=="x64" set VCPKG_ARCH=x64-windows - - # remove outdated versions - - cmd: vcpkg remove --outdated --recurse - - # install required dependencies - - cmd: vcpkg install --recurse --triplet %VCPKG_ARCH% zlib boost-iostreams boost-program-options boost-system boost-serialization eigen3 cgal[core] opencv - - # install optional dependencies - - cmd: vcpkg install --recurse --triplet %VCPKG_ARCH% glew glfw3 - - - cmd: vcpkg integrate install - - cmd: cd "%APPVEYOR_BUILD_FOLDER%" - - #------------------ - # Ubuntu 18.04 LTS - #------------------ - - sh: sudo apt-get update -qq && sudo apt-get install -qq - - sh: sudo apt-get -y install build-essential git cmake libpng-dev libjpeg-dev libtiff-dev libglu1-mesa-dev libxmu-dev libxi-dev - - sh: sudo apt-get -y install libboost-iostreams-dev libboost-program-options-dev libboost-system-dev libboost-serialization-dev - - sh: sudo apt-get -y install libopencv-dev libcgal-dev libcgal-qt5-dev libatlas-base-dev - #- sh: sudo apt-get -y install libsuitesparse-dev libceres-dev - - sh: sudo apt-get -y install freeglut3-dev libglew-dev libglfw3-dev - -# preserve contents of selected directories and files across project builds -for: -- matrix: only: - image: Visual Studio 2017 + # scripts that are called at very beginning, before repo cloning + init: + - ver + - cmake --version + - msbuild /version + # scripts that run after cloning repository + install: + # update vcpkg + - cd C:\tools\vcpkg + - git pull + - .\bootstrap-vcpkg.bat + - if "%platform%"=="Win32" set VCPKG_ARCH=x86-windows + - if "%platform%"=="x64" set VCPKG_ARCH=x64-windows + # remove outdated versions + - vcpkg remove --outdated --recurse + # install dependencies + - vcpkg install --recurse --triplet %VCPKG_ARCH% zlib boost-iostreams boost-program-options boost-system boost-serialization eigen3 cgal[core] opencv glew glfw3 + - vcpkg integrate install + - cd "%APPVEYOR_BUILD_FOLDER%" + # preserve contents of selected directories and files across project builds cache: - 'C:\tools\vcpkg\installed' + build_script: + - git clone https://github.com/cdcseacave/VCG.git + - if "%platform%"=="Win32" set CMAKE_GENERATOR=-G"Visual Studio 15 2017" + - if "%platform%"=="x64" set CMAKE_GENERATOR=-G"Visual Studio 15 2017 Win64" + - mkdir bin && cd bin + - cmake %CMAKE_GENERATOR% -DCMAKE_BUILD_TYPE=%Configuration% -DCMAKE_TOOLCHAIN_FILE="C:\tools\vcpkg\scripts\buildsystems\vcpkg.cmake" -DVCG_ROOT="%APPVEYOR_BUILD_FOLDER%\VCG" .. + - cmake --build . --target ALL_BUILD --config %Configuration% -- /maxcpucount:4 - + #------------------ + # Ubuntu 18.04 LTS + #------------------ matrix: only: - image: ubuntu1804 + # scripts that are called at very beginning, before repo cloning + init: + - lsb_release -a + - cmake --version + - gcc -v + # scripts that run after cloning repository + install: + - sudo apt-get update -qq && sudo apt-get install -qq + - sudo apt-get -y install build-essential git cmake libpng-dev libjpeg-dev libtiff-dev libglu1-mesa-dev libxmu-dev libxi-dev + - sudo apt-get -y install libboost-iostreams-dev libboost-program-options-dev libboost-system-dev libboost-serialization-dev + - sudo apt-get -y install libopencv-dev libcgal-dev libcgal-qt5-dev libatlas-base-dev + - sudo apt-get -y install freeglut3-dev libglew-dev libglfw3-dev + # preserve contents of selected directories and files across project builds cache: - '/usr/lib/x86_64-linux-gnu/' - -build_script: - # get VCG library - - git clone https://github.com/cdcseacave/VCG.git - - #------------------ - # Windows 10 - #------------------ - - cmd: if "%platform%"=="Win32" set CMAKE_GENERATOR=-G"Visual Studio 15 2017" - - cmd: if "%platform%"=="x64" set CMAKE_GENERATOR=-G"Visual Studio 15 2017 Win64" - - cmd: mkdir bin && cd bin - - cmd: cmake %CMAKE_GENERATOR% -DCMAKE_BUILD_TYPE=%Configuration% -DCMAKE_TOOLCHAIN_FILE="C:\tools\vcpkg\scripts\buildsystems\vcpkg.cmake" -DVCG_ROOT="%APPVEYOR_BUILD_FOLDER%\VCG" .. - - cmd: cmake --build . --target ALL_BUILD --config %Configuration% -- /maxcpucount:4 - + build_script: + - git clone https://github.com/cdcseacave/VCG.git + - git clone --single-branch --branch 3.2 https://gitlab.com/libeigen/eigen.git + - mkdir eigen_build && cd eigen_build + - cmake . ../eigen + - make && sudo make install + - cd .. + - mkdir bin && cd bin + - cmake -DCMAKE_BUILD_TYPE=$Configuration -DVCG_ROOT="$APPVEYOR_BUILD_FOLDER/VCG" .. + - make +- #------------------ - # Ubuntu 18.04 LTS + # MacOS #------------------ - - sh: git clone https://gitlab.com/libeigen/eigen.git --branch 3.2 - - sh: mkdir eigen_build && cd eigen_build - - sh: cmake . ../eigen - - sh: make && sudo make install - - sh: cd .. - - sh: mkdir bin && cd bin - - sh: cmake -DCMAKE_BUILD_TYPE=$Configuration -DVCG_ROOT="$APPVEYOR_BUILD_FOLDER/VCG" .. - - sh: make - -test_script: - #- cmd: ctest --build-config %Configuration% --parallel 4 --output-on-failure - #- sh: ctest -j4 + matrix: + only: + - image: macos + # scripts that are called at very beginning, before repo cloning + init: + - system_profiler SPSoftwareDataType + - cmake --version + - gcc -v + # scripts that run after cloning repository + install: + - brew update + - printf "#%s/bin/bash\nbrew install libomp boost eigen opencv cgal glew glfw3\nexit 0\n" "!" > install.sh + - chmod +x install.sh + - ./install.sh + # preserve contents of selected directories and files across project builds + cache: + - '/usr/local/Cellar/' + build_script: + - git clone https://github.com/cdcseacave/VCG.git + - mkdir bin && cd bin + - cmake -DCMAKE_BUILD_TYPE=$CONFIGURATION -DVCG_ROOT="$APPVEYOR_BUILD_FOLDER/VCG" .. + - make on_success: - cmd: 7z a OpenMVS_x64.7z "C:\projects\openmvs\bin\bin\vc15\x64\%Configuration%\*.exe" "C:\projects\openmvs\bin\bin\vc15\x64\%Configuration%\*.dll" diff --git a/BUILD.md b/BUILD.md index 29ca640ba..52e4dc0b8 100644 --- a/BUILD.md +++ b/BUILD.md @@ -114,8 +114,7 @@ Install dependencies, run CMake and make. ``` #Install dependencies brew update -brew tap homebrew/science -brew install boost eigen opencv cgal ceres-solver +brew install boost eigen opencv cgal main_path=`pwd` #GLFW3 (Optional) From 6d0f88173cd85e04ef6cad91dca0bc68e7ce4727 Mon Sep 17 00:00:00 2001 From: cDc Date: Sun, 5 Jan 2020 11:52:58 +0200 Subject: [PATCH 47/70] build: disable spam warnings on linux --- build/Utils.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/Utils.cmake b/build/Utils.cmake index d24b8453b..c828a28cf 100644 --- a/build/Utils.cmake +++ b/build/Utils.cmake @@ -497,10 +497,12 @@ macro(optimize_default_compiler_settings) add_extra_compiler_option(-Wno-narrowing) add_extra_compiler_option(-Wno-attributes) add_extra_compiler_option(-Wno-enum-compare) + add_extra_compiler_option(-Wno-misleading-indentation) add_extra_compiler_option(-Wno-missing-field-initializers) add_extra_compiler_option(-Wno-unused-parameter) add_extra_compiler_option(-Wno-delete-incomplete) add_extra_compiler_option(-Wno-unnamed-type-template-args) + add_extra_compiler_option(-Wno-int-in-bool-context) endif() add_extra_compiler_option(-fdiagnostics-show-option) add_extra_compiler_option(-ftemplate-backtrace-limit=0) From cda60fc7307686c82c7f8fd8b3dbabe97b13e720 Mon Sep 17 00:00:00 2001 From: cDc Date: Sun, 5 Jan 2020 15:23:40 +0200 Subject: [PATCH 48/70] common: increment version --- CMakeLists.txt | 6 +++--- build/Templates/ConfigLocal.h.in | 5 +++++ libs/MVS.h | 7 +++++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e40d7668c..356ed004b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,9 +12,9 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.1.0) # ${OpenMVS_BINARY_DIR}. PROJECT(OpenMVS) -set(OpenMVS_MAJOR_VERSION 0) -set(OpenMVS_MINOR_VERSION 7) -set(OpenMVS_PATCH_VERSION 0) +set(OpenMVS_MAJOR_VERSION 1) +set(OpenMVS_MINOR_VERSION 0) +set(OpenMVS_PATCH_VERSION 1) set(OpenMVS_VERSION ${OpenMVS_MAJOR_VERSION}.${OpenMVS_MINOR_VERSION}.${OpenMVS_PATCH_VERSION}) # Find dependencies: diff --git a/build/Templates/ConfigLocal.h.in b/build/Templates/ConfigLocal.h.in index 5fbc77b78..24ba104d3 100644 --- a/build/Templates/ConfigLocal.h.in +++ b/build/Templates/ConfigLocal.h.in @@ -1,3 +1,8 @@ +// OpenMVS version +#define OpenMVS_MAJOR_VERSION ${OpenMVS_MAJOR_VERSION} +#define OpenMVS_MINOR_VERSION ${OpenMVS_MINOR_VERSION} +#define OpenMVS_PATCH_VERSION ${OpenMVS_PATCH_VERSION} + // OpenMVS compiled as static or dynamic libs #cmakedefine BUILD_SHARED_LIBS diff --git a/libs/MVS.h b/libs/MVS.h index a8937c08f..598f6cba9 100644 --- a/libs/MVS.h +++ b/libs/MVS.h @@ -33,6 +33,13 @@ #define _MVS_MVS_H_ +// D E F I N E S /////////////////////////////////////////////////// + +#define OpenMVS_VERSION_AT_LEAST(x,y,z) \ + (OpenMVS_MAJOR_VERSION>x || (OpenMVS_MAJOR_VERSION==x && \ + (OpenMVS_MINOR_VERSION>y || (OpenMVS_MINOR_VERSION==y && OpenMVS_PATCH_VERSION>=z)))) + + // I N C L U D E S ///////////////////////////////////////////////// #include "ConfigLocal.h" From db9da0c2eb5264e4f4de97a50008881edcb2d92a Mon Sep 17 00:00:00 2001 From: cDc Date: Wed, 15 Jan 2020 13:42:06 +0200 Subject: [PATCH 49/70] common: save interface with version --- libs/MVS/Scene.cpp | 4 ++-- libs/MVS/Scene.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/MVS/Scene.cpp b/libs/MVS/Scene.cpp index 3b2fd8631..ff8cf696d 100644 --- a/libs/MVS/Scene.cpp +++ b/libs/MVS/Scene.cpp @@ -192,7 +192,7 @@ bool Scene::LoadInterface(const String & fileName) return true; } // LoadInterface -bool Scene::SaveInterface(const String & fileName) const +bool Scene::SaveInterface(const String & fileName, int version) const { TD_TIMER_STARTD(); Interface obj; @@ -264,7 +264,7 @@ bool Scene::SaveInterface(const String & fileName) const } // serialize out the current state - if (!ARCHIVE::SerializeSave(obj, fileName)) + if (!ARCHIVE::SerializeSave(obj, fileName, version>=0?uint32_t(version):MVSI_PROJECT_VER)) return false; DEBUG_EXTRA("Scene saved to interface format (%s):\n" diff --git a/libs/MVS/Scene.h b/libs/MVS/Scene.h index 7956525b5..48a6ca6e3 100644 --- a/libs/MVS/Scene.h +++ b/libs/MVS/Scene.h @@ -69,7 +69,7 @@ class MVS_API Scene bool IsEmpty() const; bool LoadInterface(const String& fileName); - bool SaveInterface(const String& fileName) const; + bool SaveInterface(const String& fileName, int version=-1) const; bool Import(const String& fileName); From 3ae6aa4999cb84cee4a6bd930df182278f859799 Mon Sep 17 00:00:00 2001 From: cDc Date: Fri, 17 Jan 2020 11:59:15 +0200 Subject: [PATCH 50/70] io: do not print normal index in OBJ if no normals (#516) --- libs/IO/OBJ.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/libs/IO/OBJ.cpp b/libs/IO/OBJ.cpp index 2d8bd4fda..83e9bbb91 100644 --- a/libs/IO/OBJ.cpp +++ b/libs/IO/OBJ.cpp @@ -160,12 +160,17 @@ bool ObjModel::Save(const String& fileName, unsigned precision, bool texLossless for (size_t i = 0; i < groups.size(); ++i) { out << "usemtl " << groups[i].material_name << "\n"; for (size_t j = 0; j < groups[i].faces.size(); ++j) { - Face const & face = groups[i].faces[j]; + const Face& face = groups[i].faces[j]; out << "f"; for (size_t k = 0; k < 3; ++k) { - out << " " << face.vertices[k] + OBJ_INDEX_OFFSET - << "/" << face.texcoords[k] + OBJ_INDEX_OFFSET - << "/" << face.normals[k] + OBJ_INDEX_OFFSET; + out << " " << face.vertices[k] + OBJ_INDEX_OFFSET; + if (!texcoords.empty()) { + out << "/" << face.texcoords[k] + OBJ_INDEX_OFFSET; + if (!normals.empty()) + out << "/" << face.normals[k] + OBJ_INDEX_OFFSET; + } else + if (!normals.empty()) + out << "//" << face.normals[k] + OBJ_INDEX_OFFSET; } out << "\n"; } From 27b1d32edb7e20522cb5eba773e7cef6308b1f5b Mon Sep 17 00:00:00 2001 From: cDc Date: Mon, 3 Feb 2020 16:12:32 +0200 Subject: [PATCH 51/70] io: fix read obj without material --- libs/IO/OBJ.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/IO/OBJ.cpp b/libs/IO/OBJ.cpp index 83e9bbb91..bf82ee43d 100644 --- a/libs/IO/OBJ.cpp +++ b/libs/IO/OBJ.cpp @@ -238,6 +238,8 @@ bool ObjModel::Load(const String& fileName) } if (in.fail()) return false; + if (groups.empty()) + AddGroup(""); groups.back().faces.push_back(f); } else if (keyword == "mtllib") { From 2378c9d4baeddb249df91fd55c0f0e774e46306e Mon Sep 17 00:00:00 2001 From: cDc Date: Mon, 3 Feb 2020 16:16:27 +0200 Subject: [PATCH 52/70] interface: add image mask support --- libs/MVS/Interface.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libs/MVS/Interface.h b/libs/MVS/Interface.h index 97d96240e..ea47cfcf9 100644 --- a/libs/MVS/Interface.h +++ b/libs/MVS/Interface.h @@ -12,7 +12,7 @@ // D E F I N E S /////////////////////////////////////////////////// #define MVSI_PROJECT_ID "MVSI" // identifies the project stream -#define MVSI_PROJECT_VER ((uint32_t)4) // identifies the version of a project stream +#define MVSI_PROJECT_VER ((uint32_t)5) // identifies the version of a project stream // set a default namespace name if none given #ifndef _INTERFACE_NAMESPACE @@ -453,6 +453,7 @@ struct Interface // structure describing an image struct Image { std::string name; // image file name + std::string maskName; // segmentation file name (optional) uint32_t platformID; // ID of the associated platform uint32_t cameraID; // ID of the associated camera on the associated platform uint32_t poseID; // ID of the pose of the associated platform @@ -465,6 +466,9 @@ struct Interface template void serialize(Archive& ar, const unsigned int version) { ar & name; + if (version > 4) { + ar & maskName; + } ar & platformID; ar & cameraID; ar & poseID; From fbedc56c06d20502b972f885eb4f1e9874b5281b Mon Sep 17 00:00:00 2001 From: cDc Date: Thu, 13 Feb 2020 07:17:20 -0800 Subject: [PATCH 53/70] build: set cmake policy --- CMakeLists.txt | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 356ed004b..41d145b9f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,13 @@ ######################################################################## # # Project-wide settings -CMAKE_MINIMUM_REQUIRED(VERSION 3.1.0) +CMAKE_MINIMUM_REQUIRED(VERSION 3.8.0) +if(POLICY CMP0011) + cmake_policy(SET CMP0011 NEW) +endif() +if(POLICY CMP0074) + cmake_policy(SET CMP0074 NEW) +endif() # Name of the project. # @@ -27,7 +33,6 @@ endif() SET(COTIRE_INTDIR "cotire") # Define helper functions and macros. -cmake_policy(SET CMP0011 OLD) INCLUDE(build/Utils.cmake) if(ENABLE_PRECOMPILED_HEADERS) INCLUDE(build/Cotire.cmake) @@ -75,6 +80,9 @@ if(OpenMVS_USE_OPENMP) endif() if(OpenMVS_USE_OPENGL) + if(POLICY CMP0072) + cmake_policy(SET CMP0072 NEW) + endif() FIND_PACKAGE(OpenGL) if(OPENGL_FOUND) INCLUDE_DIRECTORIES(${OPENGL_INCLUDE_DIR}) From 0638bf397d216e24d0bfc526c2d20e9bf0988d33 Mon Sep 17 00:00:00 2001 From: cDc Date: Thu, 13 Feb 2020 08:17:18 -0800 Subject: [PATCH 54/70] texture: decimate input mesh --- apps/TextureMesh/TextureMesh.cpp | 15 +++++++++++++++ libs/MVS/Mesh.cpp | 5 +++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/apps/TextureMesh/TextureMesh.cpp b/apps/TextureMesh/TextureMesh.cpp index 9e610a8ba..23f1421d1 100644 --- a/apps/TextureMesh/TextureMesh.cpp +++ b/apps/TextureMesh/TextureMesh.cpp @@ -47,6 +47,8 @@ namespace OPT { String strInputFileName; String strOutputFileName; String strMeshFileName; +float fDecimateMesh; +unsigned nCloseHoles; unsigned nResolutionLevel; unsigned nMinResolution; float fOutlierThreshold; @@ -98,6 +100,8 @@ bool Initialize(size_t argc, LPCTSTR* argv) config.add_options() ("input-file,i", boost::program_options::value(&OPT::strInputFileName), "input filename containing camera poses and image list") ("output-file,o", boost::program_options::value(&OPT::strOutputFileName), "output filename for storing the mesh") + ("decimate", boost::program_options::value(&OPT::fDecimateMesh)->default_value(1.f), "decimation factor in range [0..1] to be applied to the input surface before refinement (0 - auto, 1 - disabled)") + ("close-holes", boost::program_options::value(&OPT::nCloseHoles)->default_value(30), "try to close small holes in the input surface (0 - disabled)") ("resolution-level", boost::program_options::value(&OPT::nResolutionLevel)->default_value(0), "how many times to scale down the images before mesh refinement") ("min-resolution", boost::program_options::value(&OPT::nMinResolution)->default_value(640), "do not scale images lower than this resolution") ("outlier-threshold", boost::program_options::value(&OPT::fOutlierThreshold)->default_value(6e-2f), "threshold used to find and remove outlier face textures (0 - disabled)") @@ -226,6 +230,17 @@ int main(int argc, LPCTSTR* argv) } { + // decimate to the desired resolution + if (OPT::fDecimateMesh < 1.f) { + ASSERT(OPT::fDecimateMesh > 0.f); + scene.mesh.Clean(OPT::fDecimateMesh, 0.f, false, OPT::nCloseHoles, 0u, false); + scene.mesh.Clean(1.f, 0.f, false, 0, 0u, true); // extra cleaning to remove non-manifold problems created by closing holes + #if TD_VERBOSE != TD_VERBOSE_OFF + if (VERBOSITY_LEVEL > 3) + scene.mesh.Save(baseFileName +_T("_decim")+OPT::strExportType); + #endif + } + // compute mesh texture TD_TIMER_START(); if (!scene.TextureMesh(OPT::nResolutionLevel, OPT::nMinResolution, OPT::fOutlierThreshold, OPT::fRatioDataSmoothness, OPT::bGlobalSeamLeveling, OPT::bLocalSeamLeveling, OPT::nTextureSizeMultiple, OPT::nRectPackingHeuristic, Pixel8U(OPT::nColEmpty))) diff --git a/libs/MVS/Mesh.cpp b/libs/MVS/Mesh.cpp index 48ee5a1f9..f50997438 100644 --- a/libs/MVS/Mesh.cpp +++ b/libs/MVS/Mesh.cpp @@ -2852,9 +2852,10 @@ void Mesh::Subdivide(const AreaArr& maxAreas, uint32_t maxArea) FacetCountMap mapFaces; mapFaces.reserve(12*3); vertices.Reserve(vertices.GetSize()*2); faces.Reserve(faces.GetSize()*3); + const uint32_t maxAreaTh(2*maxArea); FOREACH(f, maxAreas) { const AreaArr::Type area(maxAreas[f]); - if (area <= maxArea) + if (area <= maxAreaTh) continue; // split face in four triangles // by adding a new vertex at the middle of each edge @@ -2892,7 +2893,7 @@ void Mesh::Subdivide(const AreaArr& maxAreas, uint32_t maxArea) ASSERT(fc.second.count <= 2 || (fc.second.count == 3 && fc.first == f)); if (fc.second.count != 2) continue; - if (fc.first < f && maxAreas[fc.first] > maxArea) { + if (fc.first < f && maxAreas[fc.first] > maxAreaTh) { // already fully split, nothing to do ASSERT(mapSplits[fc.first].idxVert[SplitFace::FindSharedEdge(faces[fc.first], face)] == newface[SplitFace::FindSharedEdge(face, faces[fc.first])]); continue; From 51b5bd057cf4a2fd16bb7a35743d40bbdf8b0418 Mon Sep 17 00:00:00 2001 From: cDc Date: Fri, 14 Feb 2020 19:59:36 +0200 Subject: [PATCH 55/70] script: add SfM/MVS pipeline (#526) --- MvgMvsPipeline.py | 349 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 349 insertions(+) create mode 100644 MvgMvsPipeline.py diff --git a/MvgMvsPipeline.py b/MvgMvsPipeline.py new file mode 100644 index 000000000..b9fb6896c --- /dev/null +++ b/MvgMvsPipeline.py @@ -0,0 +1,349 @@ +#!/usr/bin/python3 +# -*- encoding: utf-8 -*- +# +# Created by @FlachyJoe +""" +This script is for an easy use of OpenMVG and OpenMVS + +usage: MvgMvs_Pipeline.py [-h] [--steps STEPS [STEPS ...]] [--preset PRESET] + [--0 0 [0 ...]] [--1 1 [1 ...]] [--2 2 [2 ...]] + [--3 3 [3 ...]] [--4 4 [4 ...]] [--5 5 [5 ...]] + [--6 6 [6 ...]] [--7 7 [7 ...]] [--8 8 [8 ...]] + [--9 9 [9 ...]] [--10 10 [10 ...]] + [--11 11 [11 ...]] [--12 12 [12 ...]] + [--13 13 [13 ...]] + input_dir output_dir + +Photogrammetry reconstruction with these steps: + 0. Intrinsics analysis openMVG_main_SfMInit_ImageListing + 1. Compute features openMVG_main_ComputeFeatures + 2. Compute matches openMVG_main_ComputeMatches + 3. Incremental reconstruction openMVG_main_IncrementalSfM + 4. Global reconstruction openMVG_main_GlobalSfM + 5. Colorize Structure openMVG_main_ComputeSfM_DataColor + 6. Structure from Known Poses openMVG_main_ComputeStructureFromKnownPoses + 7. Colorized robust triangulation openMVG_main_ComputeSfM_DataColor + 8. Control Points Registration ui_openMVG_control_points_registration + 9. Export to openMVS openMVG_main_openMVG2openMVS + 10. Densify point cloud DensifyPointCloud + 11. Reconstruct the mesh ReconstructMesh + 12. Refine the mesh RefineMesh + 13. Texture the mesh TextureMesh + +positional arguments: + input_dir the directory wich contains the pictures set. + output_dir the directory wich will contain the resulting files. + +optional arguments: + -h, --help show this help message and exit + --steps STEPS [STEPS ...] + steps to process + --preset PRESET steps list preset in + SEQUENTIAL = [0, 1, 2, 3, 9, 10, 11, 12, 13] + GLOBAL = [0, 1, 2, 4, 9, 10, 11, 12, 13] + MVG_SEQ = [0, 1, 2, 3, 5, 6, 7] + MVG_GLOBAL = [0, 1, 2, 4, 5, 6, 7] + default : SEQUENTIAL + +Passthrough: + Option to be passed to command lines (remove - in front of option names) + e.g. --1 p ULTRA to use the ULTRA preset in openMVG_main_ComputeFeatures +""" + +import os +import subprocess +import sys +import argparse + +DEBUG = False + +# add current directory to PATH +if sys.platform.startswith('win'): + path_delim = ';' +else: + path_delim = ':' +os.environ['PATH'] += path_delim + os.getcwd() + + +def whereis(afile): + """ + return directory in which afile is, None if not found. Look in PATH + """ + if sys.platform.startswith('win'): + cmd = "where" + else: + cmd = "which" + try: + ret = subprocess.run([cmd, afile], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True) + return os.path.split(ret.stdout.decode())[0] + except subprocess.CalledProcessError: + return None + +def find(afile): + """ + As whereis look only for executable on linux, this find look for all file type + """ + for d in os.environ['PATH'].split(path_delim): + if os.path.isfile(os.path.join(d, afile)): + return d + return None + +# Try to find openMVG and openMVS binaries in PATH +OPENMVG_BIN = whereis("openMVG_main_SfMInit_ImageListing") +OPENMVS_BIN = whereis("ReconstructMesh") + +# Try to find openMVG camera sensor database +CAMERA_SENSOR_DB_FILE = "sensor_width_camera_database.txt" +CAMERA_SENSOR_DB_DIRECTORY = find(CAMERA_SENSOR_DB_FILE) + +# Ask user for openMVG and openMVS directories if not found +if not OPENMVG_BIN: + OPENMVG_BIN = input("openMVG binary folder?\n") +if not OPENMVS_BIN: + OPENMVS_BIN = input("openMVS binary folder?\n") +if not CAMERA_SENSOR_DB_DIRECTORY: + CAMERA_SENSOR_DB_DIRECTORY = input("openMVG camera database (%s) folder?\n" % CAMERA_SENSOR_DB_FILE) + + +PRESET = {'SEQUENTIAL': [0, 1, 2, 3, 9, 10, 11, 12, 13], + 'GLOBAL': [0, 1, 2, 4, 9, 10, 11, 12, 13], + 'MVG_SEQ': [0, 1, 2, 3, 5, 6, 7], + 'MVG_GLOBAL': [0, 1, 2, 4, 5, 6, 7]} + +PRESET_DEFAULT = 'SEQUENTIAL' + + +# HELPERS for terminal colors +BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) +NO_EFFECT, BOLD, UNDERLINE, BLINK, INVERSE, HIDDEN = (0, 1, 4, 5, 7, 8) + +# from Python cookbook, #475186 +def has_colours(stream): + ''' + Return stream colours capability + ''' + if not hasattr(stream, "isatty"): + return False + if not stream.isatty(): + return False # auto color only on TTYs + try: + import curses + curses.setupterm() + return curses.tigetnum("colors") > 2 + except Exception: + # guess false in case of error + return False + +HAS_COLOURS = has_colours(sys.stdout) + + +def printout(text, colour=WHITE, background=BLACK, effect=NO_EFFECT): + """ + print() with colour + """ + if HAS_COLOURS: + seq = "\x1b[%d;%d;%dm" % (effect, 30+colour, 40+background) + text + "\x1b[0m" + sys.stdout.write(seq+'\r\n') + else: + sys.stdout.write(text+'\r\n') + + +# OBJECTS to store config and data in +class ConfContainer: + """ + Container for all the config variables + """ + def __init__(self): + pass + + +class AStep: + def __init__(self, info, cmd, opt): + self.info = info + self.cmd = cmd + self.opt = opt + + +class StepsStore: + def __init__(self): + self.steps_data = [ + ["Intrinsics analysis", # 0 + os.path.join(OPENMVG_BIN, "openMVG_main_SfMInit_ImageListing"), + ["-i", "%input_dir%", "-o", "%matches_dir%", "-d", "%camera_file_params%"]], + ["Compute features", # 1 + os.path.join(OPENMVG_BIN, "openMVG_main_ComputeFeatures"), + ["-i", "%matches_dir%/sfm_data.json", "-o", "%matches_dir%", "-m", "SIFT", "-n", "4"]], + ["Compute matches", # 2 + os.path.join(OPENMVG_BIN, "openMVG_main_ComputeMatches"), + ["-i", "%matches_dir%/sfm_data.json", "-o", "%matches_dir%", "-n", "HNSWL2", "-r", ".8"]], + ["Incremental reconstruction", # 3 + os.path.join(OPENMVG_BIN, "openMVG_main_IncrementalSfM"), + ["-i", "%matches_dir%/sfm_data.json", "-m", "%matches_dir%", "-o", "%reconstruction_dir%"]], + ["Global reconstruction", # 4 + os.path.join(OPENMVG_BIN, "openMVG_main_GlobalSfM"), + ["-i", "%matches_dir%/sfm_data.json", "-m", "%matches_dir%", "-o", "%reconstruction_dir%"]], + ["Colorize Structure", # 5 + os.path.join(OPENMVG_BIN, "openMVG_main_ComputeSfM_DataColor"), + ["-i", "%reconstruction_dir%/sfm_data.bin", "-o", "%reconstruction_dir%/colorized.ply"]], + ["Structure from Known Poses", # 6 + os.path.join(OPENMVG_BIN, "openMVG_main_ComputeStructureFromKnownPoses"), + ["-i", "%reconstruction_dir%/sfm_data.bin", "-m", "%matches_dir%", "-f", "%matches_dir%/matches.f.bin", "-o", "%reconstruction_dir%/robust.bin"]], + ["Colorized robust triangulation", # 7 + os.path.join(OPENMVG_BIN, "openMVG_main_ComputeSfM_DataColor"), + ["-i", "%reconstruction_dir%/robust.bin", "-o", "%reconstruction_dir%/robust_colorized.ply"]], + ["Control Points Registration", # 8 + os.path.join(OPENMVG_BIN, "ui_openMVG_control_points_registration"), + ["-i", "%reconstruction_dir%/sfm_data.bin"]], + ["Export to openMVS", # 9 + os.path.join(OPENMVG_BIN, "openMVG_main_openMVG2openMVS"), + ["-i", "%reconstruction_dir%/sfm_data.bin", "-o", "%mvs_dir%/scene.mvs", "-d", "%mvs_dir%/images"]], + ["Densify point cloud", # 10 + os.path.join(OPENMVS_BIN, "DensifyPointCloud"), + ["--input-file", "%mvs_dir%/scene.mvs", "--resolution-level", "1", "-w", "%mvs_dir%"]], + ["Reconstruct the mesh", # 11 + os.path.join(OPENMVS_BIN, "ReconstructMesh"), + ["%mvs_dir%/scene_dense.mvs", "-w", "%mvs_dir%"]], + ["Refine the mesh", # 12 + os.path.join(OPENMVS_BIN, "RefineMesh"), + ["%mvs_dir%/scene_dense_mesh.mvs", "--scales", "2", "-w", "%mvs_dir%"]], + ["Texture the mesh", # 13 + os.path.join(OPENMVS_BIN, "TextureMesh"), + ["scene_dense_mesh_refine.mvs", "--decimate", "0.5", "-w", "%mvs_dir%"]] + ] + + def __getitem__(self, indice): + return AStep(*self.steps_data[indice]) + + def length(self): + return len(self.steps_data) + + def apply_conf(self, conf): + """ replace each %var% per conf.var value in steps data """ + for s in self.steps_data: + o2 = [] + for o in s[2]: + co = o.replace("%input_dir%", conf.input_dir) + co = co.replace("%output_dir%", conf.output_dir) + co = co.replace("%matches_dir%", conf.matches_dir) + co = co.replace("%reconstruction_dir%", conf.reconstruction_dir) + co = co.replace("%mvs_dir%", conf.mvs_dir) + co = co.replace("%camera_file_params%", conf.camera_file_params) + o2.append(co) + s[2] = o2 + + +CONF = ConfContainer() +STEPS = StepsStore() + +# ARGS +parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, + description="Photogrammetry reconstruction with these steps: \r\n" + + "\r\n".join(("\t%i. %s\t %s" % (t, STEPS[t].info, STEPS[t].cmd) for t in range(STEPS.length()))) + ) +parser.add_argument('input_dir', + help="the directory wich contains the pictures set.") +parser.add_argument('output_dir', + help="the directory wich will contain the resulting files.") +parser.add_argument('--steps', + type=int, + nargs="+", + help="steps to process") +parser.add_argument('--preset', + help="steps list preset in \r\n" + + " \r\n".join([k + " = " + str(PRESET[k]) for k in PRESET]) + + " \r\ndefault : " + PRESET_DEFAULT) + +group = parser.add_argument_group('Passthrough', description="Option to be passed to command lines (remove - in front of option names)\r\ne.g. --1 p ULTRA to use the ULTRA preset in openMVG_main_ComputeFeatures") +for n in range(STEPS.length()): + group.add_argument('--'+str(n), nargs='+') + +parser.parse_args(namespace=CONF) # store args in the ConfContainer + +# FOLDERS + +def mkdir_ine(dirname): + """Create the folder if not presents""" + if not os.path.exists(dirname): + os.mkdir(dirname) + +# Absolute path for input and ouput dirs +CONF.input_dir = os.path.abspath(CONF.input_dir) +CONF.output_dir = os.path.abspath(CONF.output_dir) + +if not os.path.exists(CONF.input_dir): + sys.exit("%s: path not found" % CONF.input_dir) + +CONF.reconstruction_dir = os.path.join(CONF.output_dir, "sfm") +CONF.matches_dir = os.path.join(CONF.reconstruction_dir, "matches") +CONF.mvs_dir = os.path.join(CONF.output_dir, "mvs") +CONF.camera_file_params = os.path.join(CAMERA_SENSOR_DB_DIRECTORY, CAMERA_SENSOR_DB_FILE) + +mkdir_ine(CONF.output_dir) +mkdir_ine(CONF.reconstruction_dir) +mkdir_ine(CONF.matches_dir) +mkdir_ine(CONF.mvs_dir) + +# Update directories in steps commandlines +STEPS.apply_conf(CONF) + +# PRESET +if CONF.steps and CONF.preset: + sys.exit("Steps and preset arguments can't be set together.") +elif CONF.preset: + try: + CONF.steps = PRESET[CONF.preset] + except KeyError: + sys.exit("Unkown preset %s, choose %s" % (CONF.preset, ' or '.join([s for s in PRESET]))) +elif not CONF.steps: + CONF.steps = PRESET[PRESET_DEFAULT] + +# WALK +print("# Using input dir: %s" % CONF.input_dir) +print("# output dir: %s" % CONF.output_dir) +print("# Steps: %s" % str(CONF.steps)) + +if 2 in CONF.steps: # ComputeMatches + if 4 in CONF.steps: # GlobalReconstruction + # Set the geometric_model of ComputeMatches to Essential + STEPS[2].opt.extend(["-g", "e"]) + +for cstep in CONF.steps: + printout("#%i. %s" % (cstep, STEPS[cstep].info), effect=INVERSE) + + # Retrieve "passthrough" commandline options + opt = getattr(CONF, str(cstep)) + if opt: + # add - sign to short options and -- to long ones + for o in range(0, len(opt), 2): + if len(opt[o]) > 1: + opt[o] = '-' + opt[o] + opt[o] = '-' + opt[o] + else: + opt = [] + + # Remove STEPS[cstep].opt options now defined in opt + for anOpt in STEPS[cstep].opt: + if anOpt in opt: + idx = STEPS[cstep].opt.index(anOpt) + if DEBUG: + print('#\tRemove ' + str(anOpt) + ' from defaults options at id ' + str(idx)) + del STEPS[cstep].opt[idx:idx+2] + + # create a commandline for the current step + cmdline = [STEPS[cstep].cmd] + STEPS[cstep].opt + opt + print('Cmd: ' + ' '.join(cmdline)) + + if not DEBUG: + # Launch the current step + try: + pStep = subprocess.Popen(cmdline) + pStep.wait() + if pStep.returncode != 0: + break + except KeyboardInterrupt: + sys.exit('\r\nProcess canceled by user, all files remains') + else: + print('\t'.join(cmdline)) + +printout("# Pipeline end #", effect=INVERSE) From efe02f4ad3d07357a3f08936b7d16644941dbce0 Mon Sep 17 00:00:00 2001 From: cDc Date: Sun, 16 Feb 2020 18:51:27 +0200 Subject: [PATCH 56/70] dense: implement Semi-Global Matching --- MvgMvsPipeline.py | 75 +- apps/DensifyPointCloud/DensifyPointCloud.cpp | 11 +- libs/Common/EventQueue.cpp | 13 + libs/Common/EventQueue.h | 14 + libs/Common/File.h | 92 +- libs/Common/List.h | 60 +- libs/Common/Thread.h | 84 +- libs/Common/Types.h | 142 +- libs/Common/Types.inl | 340 ++- libs/Common/Util.inl | 5 +- libs/MVS/Camera.cpp | 207 ++ libs/MVS/Camera.h | 9 + libs/MVS/DepthMap.cpp | 308 +++ libs/MVS/DepthMap.h | 9 + libs/MVS/Image.cpp | 229 ++ libs/MVS/Image.h | 10 + libs/MVS/Scene.h | 5 +- libs/MVS/SceneDensify.cpp | 238 +- libs/MVS/SceneDensify.h | 14 +- libs/MVS/SemiGlobalMatcher.cpp | 2361 ++++++++++++++++++ libs/MVS/SemiGlobalMatcher.h | 215 ++ 21 files changed, 4162 insertions(+), 279 deletions(-) create mode 100644 libs/MVS/SemiGlobalMatcher.cpp create mode 100644 libs/MVS/SemiGlobalMatcher.h diff --git a/MvgMvsPipeline.py b/MvgMvsPipeline.py index b9fb6896c..f4914bf86 100644 --- a/MvgMvsPipeline.py +++ b/MvgMvsPipeline.py @@ -9,41 +9,43 @@ [--0 0 [0 ...]] [--1 1 [1 ...]] [--2 2 [2 ...]] [--3 3 [3 ...]] [--4 4 [4 ...]] [--5 5 [5 ...]] [--6 6 [6 ...]] [--7 7 [7 ...]] [--8 8 [8 ...]] - [--9 9 [9 ...]] [--10 10 [10 ...]] - [--11 11 [11 ...]] [--12 12 [12 ...]] - [--13 13 [13 ...]] + [--9 9 [9 ...]] [--10 10 [10 ...]] [--11 11 [11 ...]] + [--12 12 [12 ...]] [--13 13 [13 ...]] + [--14 14 [14 ...]] [--15 15 [15 ...]] input_dir output_dir Photogrammetry reconstruction with these steps: - 0. Intrinsics analysis openMVG_main_SfMInit_ImageListing - 1. Compute features openMVG_main_ComputeFeatures - 2. Compute matches openMVG_main_ComputeMatches - 3. Incremental reconstruction openMVG_main_IncrementalSfM - 4. Global reconstruction openMVG_main_GlobalSfM - 5. Colorize Structure openMVG_main_ComputeSfM_DataColor - 6. Structure from Known Poses openMVG_main_ComputeStructureFromKnownPoses - 7. Colorized robust triangulation openMVG_main_ComputeSfM_DataColor - 8. Control Points Registration ui_openMVG_control_points_registration - 9. Export to openMVS openMVG_main_openMVG2openMVS - 10. Densify point cloud DensifyPointCloud - 11. Reconstruct the mesh ReconstructMesh - 12. Refine the mesh RefineMesh - 13. Texture the mesh TextureMesh + 0. Intrinsics analysis openMVG_main_SfMInit_ImageListing + 1. Compute features openMVG_main_ComputeFeatures + 2. Compute matches openMVG_main_ComputeMatches + 3. Incremental reconstruction openMVG_main_IncrementalSfM + 4. Global reconstruction openMVG_main_GlobalSfM + 5. Colorize Structure openMVG_main_ComputeSfM_DataColor + 6. Structure from Known Poses openMVG_main_ComputeStructureFromKnownPoses + 7. Colorized robust triangulation openMVG_main_ComputeSfM_DataColor + 8. Control Points Registration ui_openMVG_control_points_registration + 9. Export to openMVS openMVG_main_openMVG2openMVS + 10. Densify point-cloud DensifyPointCloud + 11. Reconstruct the mesh ReconstructMesh + 12. Refine the mesh RefineMesh + 13. Texture the mesh TextureMesh + 14. Estimate disparity-maps DensifyPointCloud + 15. Fuse disparity-maps DensifyPointCloud positional arguments: - input_dir the directory wich contains the pictures set. - output_dir the directory wich will contain the resulting files. + input_dir the directory wich contains the pictures set. + output_dir the directory wich will contain the resulting files. optional arguments: - -h, --help show this help message and exit - --steps STEPS [STEPS ...] - steps to process - --preset PRESET steps list preset in - SEQUENTIAL = [0, 1, 2, 3, 9, 10, 11, 12, 13] - GLOBAL = [0, 1, 2, 4, 9, 10, 11, 12, 13] - MVG_SEQ = [0, 1, 2, 3, 5, 6, 7] - MVG_GLOBAL = [0, 1, 2, 4, 5, 6, 7] - default : SEQUENTIAL + -h, --help show this help message and exit + --steps STEPS [STEPS ...] steps to process + --preset PRESET steps list preset in + SEQUENTIAL = [0, 1, 2, 3, 9, 10, 11, 12, 13] + GLOBAL = [0, 1, 2, 4, 9, 10, 11, 12, 13] + MVG_SEQ = [0, 1, 2, 3, 5, 6, 7] + MVG_GLOBAL = [0, 1, 2, 4, 5, 6, 7] + MVS_SGM = [14, 15] + default : SEQUENTIAL Passthrough: Option to be passed to command lines (remove - in front of option names) @@ -108,7 +110,8 @@ def find(afile): PRESET = {'SEQUENTIAL': [0, 1, 2, 3, 9, 10, 11, 12, 13], 'GLOBAL': [0, 1, 2, 4, 9, 10, 11, 12, 13], 'MVG_SEQ': [0, 1, 2, 3, 5, 6, 7], - 'MVG_GLOBAL': [0, 1, 2, 4, 5, 6, 7]} + 'MVG_GLOBAL': [0, 1, 2, 4, 5, 6, 7], + 'MVS_SGM': [14, 15]} PRESET_DEFAULT = 'SEQUENTIAL' @@ -199,16 +202,22 @@ def __init__(self): ["-i", "%reconstruction_dir%/sfm_data.bin", "-o", "%mvs_dir%/scene.mvs", "-d", "%mvs_dir%/images"]], ["Densify point cloud", # 10 os.path.join(OPENMVS_BIN, "DensifyPointCloud"), - ["--input-file", "%mvs_dir%/scene.mvs", "--resolution-level", "1", "-w", "%mvs_dir%"]], + ["scene.mvs", "--dense-config-file", "Densify.ini", "--resolution-level", "1", "-w", "%mvs_dir%"]], ["Reconstruct the mesh", # 11 os.path.join(OPENMVS_BIN, "ReconstructMesh"), - ["%mvs_dir%/scene_dense.mvs", "-w", "%mvs_dir%"]], + ["scene_dense.mvs", "-w", "%mvs_dir%"]], ["Refine the mesh", # 12 os.path.join(OPENMVS_BIN, "RefineMesh"), - ["%mvs_dir%/scene_dense_mesh.mvs", "--scales", "2", "-w", "%mvs_dir%"]], + ["scene_dense_mesh.mvs", "--scales", "2", "-w", "%mvs_dir%"]], ["Texture the mesh", # 13 os.path.join(OPENMVS_BIN, "TextureMesh"), - ["scene_dense_mesh_refine.mvs", "--decimate", "0.5", "-w", "%mvs_dir%"]] + ["scene_dense_mesh_refine.mvs", "--decimate", "0.5", "-w", "%mvs_dir%"]], + ["Estimate disparity-maps", # 14 + os.path.join(OPENMVS_BIN, "DensifyPointCloud"), + ["scene.mvs", "--dense-config-file", "Densify.ini", "--fusion-mode", "-1", "-w", "%mvs_dir%"]], + ["Fuse disparity-maps", # 15 + os.path.join(OPENMVS_BIN, "DensifyPointCloud"), + ["scene.mvs", "--dense-config-file", "Densify.ini", "--fusion-mode", "-2", "-w", "%mvs_dir%"]] ] def __getitem__(self, indice): diff --git a/apps/DensifyPointCloud/DensifyPointCloud.cpp b/apps/DensifyPointCloud/DensifyPointCloud.cpp index aeaa18cc7..9d225289c 100644 --- a/apps/DensifyPointCloud/DensifyPointCloud.cpp +++ b/apps/DensifyPointCloud/DensifyPointCloud.cpp @@ -50,6 +50,7 @@ String strMeshFileName; String strDenseConfigFileName; float fSampleMesh; int thFilterPointCloud; +int nFusionMode; int nArchiveType; int nProcessPriority; unsigned nMaxThreads; @@ -107,6 +108,7 @@ bool Initialize(size_t argc, LPCTSTR* argv) ("estimate-normals", boost::program_options::value(&nEstimateNormals)->default_value(0), "estimate the normals for the dense point-cloud") ("sample-mesh", boost::program_options::value(&OPT::fSampleMesh)->default_value(0.f), "uniformly samples points on a mesh (0 - disabled, <0 - number of points, >0 - sample density per square unit)") ("filter-point-cloud", boost::program_options::value(&OPT::thFilterPointCloud)->default_value(0), "filter dense point-cloud based on visibility (0 - disabled)") + ("fusion-mode", boost::program_options::value(&OPT::nFusionMode)->default_value(0), "depth map fusion mode (-2 - fuse disparity-maps, -1 - export disparity-maps only, 0 - depth-maps & fusion, 1 - export depth-maps only)") ; // hidden options, allowed both on command line and @@ -256,8 +258,13 @@ int main(int argc, LPCTSTR* argv) } if ((ARCHIVE_TYPE)OPT::nArchiveType != ARCHIVE_MVS) { TD_TIMER_START(); - if (!scene.DenseReconstruction()) - return EXIT_FAILURE; + if (!scene.DenseReconstruction(OPT::nFusionMode)) { + if (ABS(OPT::nFusionMode) != 1) + return EXIT_FAILURE; + VERBOSE("Depth-maps estimated (%s)", TD_TIMER_GET_FMT().c_str()); + Finalize(); + return EXIT_SUCCESS; + } VERBOSE("Densifying point-cloud completed: %u points (%s)", scene.pointcloud.GetSize(), TD_TIMER_GET_FMT().c_str()); } diff --git a/libs/Common/EventQueue.cpp b/libs/Common/EventQueue.cpp index dd090e63a..82f835ba0 100644 --- a/libs/Common/EventQueue.cpp +++ b/libs/Common/EventQueue.cpp @@ -86,3 +86,16 @@ uint_t EventQueue::GetSize() const return m_events.GetSize(); } /*----------------------------------------------------------------*/ + + + +/*-----------------------------------------------------------* +* EventThreadPool class implementation * +*-----------------------------------------------------------*/ + +void EventThreadPool::stop() +{ + ThreadPool::stop(); + EventQueue::Clear(); +} +/*----------------------------------------------------------------*/ diff --git a/libs/Common/EventQueue.h b/libs/Common/EventQueue.h index 31206cb72..b698785c3 100644 --- a/libs/Common/EventQueue.h +++ b/libs/Common/EventQueue.h @@ -71,6 +71,20 @@ class GENERAL_API EventQueue }; /*----------------------------------------------------------------*/ + +// basic event and thread pool +class GENERAL_API EventThreadPool : public ThreadPool, public EventQueue +{ +public: + inline EventThreadPool() {} + inline EventThreadPool(size_type nThreads) : ThreadPool(nThreads) {} + inline EventThreadPool(size_type nThreads, Thread::FncStart pfnStarter, void* pData=NULL) : ThreadPool(nThreads, pfnStarter, pData) {} + inline ~EventThreadPool() {} + + void stop(); //stop threads, reset locks state and empty event queue +}; +/*----------------------------------------------------------------*/ + } // namespace SEACAVE #endif // __SEACAVE_EVENTQUEUE_H__ diff --git a/libs/Common/File.h b/libs/Common/File.h index 674aed571..f369bb5df 100644 --- a/libs/Common/File.h +++ b/libs/Common/File.h @@ -13,10 +13,66 @@ #include "Streams.h" +// Under both Windows and Unix, the stat function is used for classification + +// Under Gnu/Linux, the following classifications are defined +// source: Gnu/Linux man page for stat(2) http://linux.die.net/man/2/stat +// S_IFMT 0170000 bitmask for the file type bitfields +// S_IFSOCK 0140000 socket (Note this overlaps with S_IFDIR) +// S_IFLNK 0120000 symbolic link +// S_IFREG 0100000 regular file +// S_IFBLK 0060000 block device +// S_IFDIR 0040000 directory +// S_IFCHR 0020000 character device +// S_IFIFO 0010000 FIFO +// There are also some Posix-standard macros: +// S_ISREG(m) is it a regular file? +// S_ISDIR(m) directory? +// S_ISCHR(m) character device? +// S_ISBLK(m) block device? +// S_ISFIFO(m) FIFO (named pipe)? +// S_ISLNK(m) symbolic link? (Not in POSIX.1-1996.) +// S_ISSOCK(m) socket? (Not in POSIX.1-1996.) +// Under Windows, the following are defined: +// source: Header file sys/stat.h distributed with Visual Studio 10 +// _S_IFMT (S_IFMT) 0xF000 file type mask +// _S_IFREG (S_IFREG) 0x8000 regular +// _S_IFDIR (S_IFDIR) 0x4000 directory +// _S_IFCHR (S_IFCHR) 0x2000 character special +// _S_IFIFO 0x1000 pipe + #ifdef _MSC_VER #include +// file type tests are not defined for some reason on Windows despite them providing the stat() function! +#define F_OK 0 +#define X_OK 1 +#define W_OK 2 +#define R_OK 4 +// Posix-style macros for Windows +#ifndef S_ISREG +#define S_ISREG(mode) ((mode & _S_IFMT) == _S_IFREG) +#endif +#ifndef S_ISDIR +#define S_ISDIR(mode) ((mode & _S_IFMT) == _S_IFDIR) +#endif +#ifndef S_ISCHR +#define S_ISCHR(mode) ((mode & _S_IFMT) == _S_IFCHR) +#endif +#ifndef S_ISBLK +#define S_ISBLK(mode) (false) +#endif +#ifndef S_ISFIFO +#define S_ISFIFO(mode) ((mode & _S_IFMT) == _S_IFIFO) +#endif +#ifndef S_ISLNK +#define S_ISLNK(mode) (false) +#endif +#ifndef S_ISSOCK +#define S_ISSOCK(mode) (false) +#endif #else #include +#include #define _taccess access #endif @@ -282,7 +338,7 @@ class GENERAL_API File : public IOStream { size_f_t totalSize = 0; String strPath(_strPath); Util::ensureFolderSlash(strPath); - //Find all the files in this folder. + // Find all the files in this folder. hFind = FindFirstFile((strPath + strMask).c_str(), &fd); if (hFind != INVALID_HANDLE_VALUE) { @@ -304,7 +360,7 @@ class GENERAL_API File : public IOStream { while (FindNextFile(hFind, &fd)); FindClose(hFind); } - //Process the subfolders also... + // Process the subfolders also... if (!bProcessSubdir) return totalSize; hFind = FindFirstFile((strPath + '*').c_str(), &fd); @@ -318,7 +374,7 @@ class GENERAL_API File : public IOStream { continue; if (!_tcscmp(fd.cFileName, _T(".."))) continue; - // Processe all subfolders recursively + // Process all subfolders recursively totalSize += findFiles(strPath + fd.cFileName + PATH_SEPARATOR, strMask, true, arrFiles); } while (FindNextFile(hFind, &fd)); @@ -490,7 +546,35 @@ class GENERAL_API File : public IOStream { #endif // _MSC_VER - static bool access(LPCTSTR aFileName, int mode=CA_EXIST) { return ::_taccess(aFileName, mode) == 0; } + // test for whether there's something (i.e. folder or file) with this name + // and what access mode is supported + static bool isPresent(LPCTSTR path) { + struct stat buf; + return stat(path, &buf) == 0; + } + static bool access(LPCTSTR path, int mode=CA_EXIST) { + return ::_taccess(path, mode) == 0; + } + // test for whether there's something present and its a folder + static bool isFolder(LPCTSTR path) { + struct stat buf; + if (!(stat(path, &buf) == 0)) + return false; + // If the object is present, see if it is a directory + // this is the Posix-approved way of testing + return S_ISDIR(buf.st_mode); + } + // test for whether there's something present and its a file + // a file can be a regular file, a symbolic link, a FIFO or a socket, but not a device + static bool isFile(LPCTSTR path) { + struct stat buf; + if (!(stat(path, &buf) == 0)) + return false; + // If the object is present, see if it is a file or file-like object + // Note that devices are neither folders nor files + // this is the Posix-approved way of testing + return S_ISREG(buf.st_mode) || S_ISLNK(buf.st_mode) || S_ISSOCK(buf.st_mode) || S_ISFIFO(buf.st_mode); + } template inline size_t write(const VECTOR& arr) { diff --git a/libs/Common/List.h b/libs/Common/List.h index 794b0a2a4..7b714308c 100644 --- a/libs/Common/List.h +++ b/libs/Common/List.h @@ -20,28 +20,24 @@ // cList index type #ifdef _SUPPORT_CPP11 -#ifdef _MSC_VER -#define ARR2IDX(arr) std::remove_reference::type::IDX -#else -#define ARR2IDX(arr) typename std::remove_reference::type::IDX -#endif +#define ARR2IDX(arr) typename std::remove_reference::type::size_type #else #define ARR2IDX(arr) IDX #endif // cList iterator by index #ifndef FOREACH -#define FOREACH(var, arr) for (ARR2IDX(arr) var=0, var##Size=(arr).GetSize(); var0; ) +#define RFOREACH(var, arr) for (ARR2IDX(arr) var=(arr).size(); var-->0; ) #endif // cList iterator by pointer #ifndef FOREACHPTR -#define FOREACHPTR(var, arr) for (auto var=(arr).Begin(), var##End=(arr).End(); var!=var##End; ++var) +#define FOREACHPTR(var, arr) for (auto var=(arr).begin(), var##End=(arr).end(); var!=var##End; ++var) #endif #ifndef RFOREACHPTR -#define RFOREACHPTR(var, arr) for (auto var=(arr).End(), var##Begin=(arr).Begin(); var--!=var##Begin; ) +#define RFOREACHPTR(var, arr) for (auto var=(arr).end(), var##Begin=(arr).begin(); var--!=var##Begin; ) #endif // raw data array iterator by index @@ -627,6 +623,41 @@ class cList return std::accumulate(Begin(), End(), TYPE(0)) / _size; } + inline ArgType GetMax() const { + return *std::max_element(Begin(), End()); + } + template + inline ArgType GetMax(const Functor& functor) const { + return *std::max_element(Begin(), End(), functor); + } + inline IDX GetMaxIdx() const { + return static_cast(std::max_element(Begin(), End()) - Begin()); + } + template + inline IDX GetMaxIdx(const Functor& functor) const { + return static_cast(std::max_element(Begin(), End(), functor) - Begin()); + } + #ifdef _SUPPORT_CPP11 + inline std::pair GetMinMax() const { + const auto minmax(std::minmax_element(Begin(), End())); + return std::pair(*minmax.first, *minmax.second); + } + template + inline std::pair GetMinMax(const Functor& functor) const { + const auto minmax(std::minmax_element(Begin(), End(), functor)); + return std::pair(*minmax.first, *minmax.second); + } + inline std::pair GetMinMaxIdx() const { + const auto minmax(std::minmax_element(Begin(), End())); + return std::make_pair(static_cast(minmax.first-Begin()), static_cast(minmax.second-Begin())); + } + template + inline std::pair GetMinMaxIdx(const Functor& functor) const { + const auto minmax(std::minmax_element(Begin(), End(), functor)); + return std::make_pair(static_cast(minmax.first-Begin()), static_cast(minmax.second-Begin())); + } + #endif + inline TYPE& PartialSort(IDX index) { TYPE* const nth(Begin()+index); @@ -1294,6 +1325,7 @@ class cList #if _USE_VECTORINTERFACE != 0 public: + typedef IDX size_type; typedef Type value_type; typedef value_type* iterator; typedef const value_type* const_iterator; @@ -1302,11 +1334,11 @@ class cList typedef std::vector VectorType; inline cList(const VectorType& rList) { CopyOf(&rList[0], rList.size()); } #ifdef _SUPPORT_CPP11 - inline cList(std::initializer_list l) : _size(0), _vectorSize((IDX)l.size()), _vector(NULL) { ASSERT(l.size() l) : _size(0), _vectorSize((size_type)l.size()), _vector(NULL) { ASSERT(l.size()_vector, elem); } #ifdef _SUPPORT_CPP11 @@ -1316,8 +1348,8 @@ class cList #endif inline void push_back(const_reference elem) { Insert(elem); } inline void pop_back() { RemoveLast(); } - inline void reserve(IDX newSize) { Reserve(newSize); } - inline void resize(IDX newSize) { Resize(newSize); } + inline void reserve(size_type newSize) { Reserve(newSize); } + inline void resize(size_type newSize) { Resize(newSize); } inline void erase(const_iterator it) { RemoveAtMove(it-this->_vector); } inline const_iterator cdata() const { return GetData(); } inline const_iterator cbegin() const { return Begin(); } diff --git a/libs/Common/Thread.h b/libs/Common/Thread.h index d05bdae2e..9ccacb37d 100644 --- a/libs/Common/Thread.h +++ b/libs/Common/Thread.h @@ -39,7 +39,7 @@ namespace SEACAVE { * basic thread control **************************************************************************************/ -class Thread +class GENERAL_API Thread { public: #ifdef _ENVIRONMENT64 @@ -252,6 +252,86 @@ class Thread DWORD threadId; #endif }; +/*----------------------------------------------------------------*/ + + + +/************************************************************************************** + * ThreadPool + * -------------- + * basic thread pool + **************************************************************************************/ + +class GENERAL_API ThreadPool +{ +public: + typedef uint32_t size_type; + typedef Thread value_type; + typedef value_type* iterator; + typedef const value_type* const_iterator; + typedef value_type& reference; + typedef const value_type& const_reference; + typedef CLISTDEFIDX(Thread,size_type) Threads; + +public: + inline ThreadPool() {} + inline ThreadPool(size_type nThreads) : _threads(nThreads>0?nThreads:Thread::hardwareConcurrency()) {} + inline ThreadPool(size_type nThreads, Thread::FncStart pfnStarter, void* pData=NULL) : _threads(nThreads>0?nThreads:Thread::hardwareConcurrency()) { start(pfnStarter, pData); } + inline ~ThreadPool() { join(); } + + #ifdef _SUPPORT_CPP11 + inline ThreadPool(ThreadPool&& rhs) : _threads(std::forward(rhs._threads)) { + } + + inline ThreadPool& operator=(ThreadPool&& rhs) { + _threads.Swap(rhs._threads); + return *this; + } + #endif + + // wait for all running threads to finish and resize threads array + void resize(size_type nThreads) { + join(); + _threads.resize(nThreads); + } + // start all threads with the given function + bool start(Thread::FncStart pfnStarter, void* pData=NULL) { + for (Thread& thread: _threads) + if (!thread.start(pfnStarter, pData)) + return false; + return true; + } + // wait for all running threads to finish + void join() { + for (Thread& thread: _threads) + thread.join(); + } + // stop all threads + void stop() { + for (Thread& thread: _threads) + thread.stop(); + } + // wait for all running threads to finish and release threads array + void Release() { + join(); + _threads.Release(); + } + + inline bool empty() const { return _threads.empty(); } + inline size_type size() const { return _threads.size(); } + inline const_iterator cbegin() const { return _threads.cbegin(); } + inline const_iterator cend() const { return _threads.cend(); } + inline const_iterator begin() const { return _threads.begin(); } + inline const_iterator end() const { return _threads.end(); } + inline iterator begin() { return _threads.begin(); } + inline iterator end() { return _threads.end(); } + inline const_reference operator[](size_type index) const { return _threads[index]; } + inline reference operator[](size_type index) { return _threads[index]; } + +protected: + Threads _threads; +}; +/*----------------------------------------------------------------*/ @@ -261,7 +341,7 @@ class Thread * basic process control **************************************************************************************/ -class Process +class GENERAL_API Process { public: enum Priority { diff --git a/libs/Common/Types.h b/libs/Common/Types.h index 57f59b918..37c28d615 100644 --- a/libs/Common/Types.h +++ b/libs/Common/Types.h @@ -380,9 +380,9 @@ typedef TAliasCast CastD2I; #include "Strings.h" #include "AutoPtr.h" +#include "List.h" #include "Thread.h" #include "SharedPtr.h" -#include "List.h" #include "Queue.h" #include "Hash.h" #include "Timer.h" @@ -605,6 +605,16 @@ namespace SEACAVE { // F U N C T I O N S /////////////////////////////////////////////// +template +struct MakeIdentity { using type = T; }; +template +using MakeSigned = typename std::conditional::value,std::make_signed,SEACAVE::MakeIdentity>::type; + +template +constexpr T1 Cast(const T2& v) { + return static_cast(v); +} + template constexpr T& NEGATE(T& a) { return (a = -a); @@ -808,6 +818,82 @@ FORCEINLINE double CBRT(const double& x) { /*----------------------------------------------------------------*/ +#if defined(__GNUC__) + +FORCEINLINE int PopCnt(uint32_t bb) { + return __builtin_popcount(bb); +} +FORCEINLINE int PopCnt(uint64_t bb) { + return __builtin_popcountll(bb); +} +FORCEINLINE int PopCnt15(uint64_t bb) { + return __builtin_popcountll(bb); +} +FORCEINLINE int PopCntSparse(uint64_t bb) { + return __builtin_popcountll(bb); +} + +#elif defined(_USE_SSE) && defined(_M_AMD64) // 64 bit windows + +FORCEINLINE int PopCnt(uint32_t bb) { + return (int)_mm_popcnt_u32(bb); +} +FORCEINLINE int PopCnt(uint64_t bb) { + return (int)_mm_popcnt_u64(bb); +} +FORCEINLINE int PopCnt15(uint64_t bb) { + return (int)_mm_popcnt_u64(bb); +} +FORCEINLINE int PopCntSparse(uint64_t bb) { + return (int)_mm_popcnt_u64(bb); +} + +#else + +// general purpose population count +template +constexpr int PopCnt(T bb) +{ + STATIC_ASSERT(std::is_integral::value && std::is_unsigned::value); + return std::bitset(bb).count(); +} +template<> +inline int PopCnt(uint64_t bb) { + const uint64_t k1 = (uint64_t)0x5555555555555555; + const uint64_t k2 = (uint64_t)0x3333333333333333; + const uint64_t k3 = (uint64_t)0x0F0F0F0F0F0F0F0F; + const uint64_t k4 = (uint64_t)0x0101010101010101; + bb -= (bb >> 1) & k1; + bb = (bb & k2) + ((bb >> 2) & k2); + bb = (bb + (bb >> 4)) & k3; + return (bb * k4) >> 56; +} +// faster version assuming not more than 15 bits set, used in mobility +// eval, posted on CCC forum by Marco Costalba of Stockfish team +inline int PopCnt15(uint64_t bb) { + unsigned w = unsigned(bb >> 32), v = unsigned(bb); + v -= (v >> 1) & 0x55555555; // 0-2 in 2 bits + w -= (w >> 1) & 0x55555555; + v = ((v >> 2) & 0x33333333) + (v & 0x33333333); // 0-4 in 4 bits + w = ((w >> 2) & 0x33333333) + (w & 0x33333333); + v += w; // 0-8 in 4 bits + v *= 0x11111111; + return int(v >> 28); +} +// version faster on sparsely populated bitboards +inline int PopCntSparse(uint64_t bb) { + int count = 0; + while (bb) { + count++; + bb &= bb - 1; + } + return count; +} + +#endif +/*----------------------------------------------------------------*/ + + #ifdef _FAST_FLOAT2INT // fast float to int conversion // (xs routines at stereopsis: http://www.stereopsis.com/sree/fpu2006.html by Sree Kotay) @@ -1371,6 +1457,37 @@ class TMatrix : public cv::Matx inline TMatrix(const EMat& rhs) { operator EMat& () = rhs; } #endif + TMatrix(TYPE v0); //!< 1x1 matrix + TMatrix(TYPE v0, TYPE v1); //!< 1x2 or 2x1 matrix + TMatrix(TYPE v0, TYPE v1, TYPE v2); //!< 1x3 or 3x1 matrix + TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3); //!< 1x4, 2x2 or 4x1 matrix + TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3, TYPE v4); //!< 1x5 or 5x1 matrix + TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3, TYPE v4, TYPE v5); //!< 1x6, 2x3, 3x2 or 6x1 matrix + TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3, TYPE v4, TYPE v5, TYPE v6); //!< 1x7 or 7x1 matrix + TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3, TYPE v4, TYPE v5, TYPE v6, TYPE v7); //!< 1x8, 2x4, 4x2 or 8x1 matrix + TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3, TYPE v4, TYPE v5, TYPE v6, TYPE v7, TYPE v8); //!< 1x9, 3x3 or 9x1 matrix + TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3, TYPE v4, TYPE v5, TYPE v6, TYPE v7, TYPE v8, TYPE v9); //!< 1x10, 2x5 or 5x2 or 10x1 matrix + TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3, + TYPE v4, TYPE v5, TYPE v6, TYPE v7, + TYPE v8, TYPE v9, TYPE v10, TYPE v11); //!< 1x12, 2x6, 3x4, 4x3, 6x2 or 12x1 matrix + TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3, + TYPE v4, TYPE v5, TYPE v6, TYPE v7, + TYPE v8, TYPE v9, TYPE v10, TYPE v11, + TYPE v12, TYPE v13); //!< 1x14, 2x7, 7x2 or 14x1 matrix + TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3, + TYPE v4, TYPE v5, TYPE v6, TYPE v7, + TYPE v8, TYPE v9, TYPE v10, TYPE v11, + TYPE v12, TYPE v13, TYPE v14, TYPE v15); //!< 1x16, 4x4 or 16x1 matrix + explicit TMatrix(const TYPE* vals); //!< initialize from a plain array + + TMatrix(const TMatrix& a, const TMatrix& b, cv::Matx_AddOp) : Base(a, b, cv::Matx_AddOp()) {} + TMatrix(const TMatrix& a, const TMatrix& b, cv::Matx_SubOp) : Base(a, b, cv::Matx_SubOp()) {} + template TMatrix(const TMatrix& a, TYPE2 alpha, cv::Matx_ScaleOp) : Base(a, alpha, cv::Matx_ScaleOp()) {} + TMatrix(const TMatrix& a, const TMatrix& b, cv::Matx_MulOp) : Base(a, b, cv::Matx_MulOp()) {} + TMatrix(const TMatrix& a, const TMatrix& b, cv::Matx_DivOp) : Base(a, b, cv::Matx_DivOp()) {} + template TMatrix(const TMatrix& a, const TMatrix& b, cv::Matx_MatMulOp) : Base(a, b, cv::Matx_MatMulOp()) {} + TMatrix(const TMatrix& a, cv::Matx_TOp) : Base(a, cv::Matx_TOp()) {} + template inline TMatrix& operator = (const cv::Matx& rhs) { Base::operator = (rhs); return *this; } inline TMatrix& operator = (const cv::Mat& rhs) { Base::operator = (rhs); return *this; } #ifdef _USE_EIGEN @@ -1822,6 +1939,10 @@ struct TPixel { template inline TPixel operator-(T v) const { return TPixel((TYPE)(r-v), (TYPE)(g-v), (TYPE)(b-v)); } template inline TPixel& operator-=(T v) { return (*this = operator-(v)); } inline uint32_t toDWORD() const { return RGBA((uint8_t)r, (uint8_t)g, (uint8_t)b, (uint8_t)0); } + // tools + template + static TPixel colorRamp(VT v, VT vmin, VT vmax); + static TPixel gray2color(ALT v); #ifdef _USE_BOOST // serialize template @@ -2008,14 +2129,14 @@ class TImage : public TDMatrix template TYPE sampleSafe(const TPoint2& pt) const; - template - bool sample(TYPE& v, const TPoint2& pt, bool (STCALL *fncCond)(const TYPE&)) const; - template - bool sampleSafe(TYPE& v, const TPoint2& pt, bool (STCALL *fncCond)(const TYPE&)) const; - template - TYPE sample(const TPoint2& pt, bool (STCALL *fncCond)(const TYPE&), const TYPE& dv) const; - template - TYPE sampleSafe(const TPoint2& pt, bool (STCALL *fncCond)(const TYPE&), const TYPE& dv) const; + template + bool sample(TV& v, const TPoint2& pt, const Functor& functor) const; + template + bool sampleSafe(TV& v, const TPoint2& pt, const Functor& functor) const; + template + TYPE sample(const TPoint2& pt, const Functor& functor, const TYPE& dv) const; + template + TYPE sampleSafe(const TPoint2& pt, const Functor& functor, const TYPE& dv) const; template INTERTYPE sample(const SAMPLER& sampler, const TPoint2& pt) const; @@ -2023,6 +2144,8 @@ class TImage : public TDMatrix template void toGray(TImage& out, int code, bool bNormalize=false, bool bSRGB=false) const; + static cv::Size computeResize(const cv::Size& size, REAL scale); + static cv::Size computeResize(const cv::Size& size, REAL scale, unsigned resizes); unsigned computeMaxResolution(unsigned& level, unsigned minImageSize=320, unsigned maxImageSize=INT_MAX) const; static unsigned computeMaxResolution(unsigned width, unsigned height, unsigned& level, unsigned minImageSize=320, unsigned maxImageSize=INT_MAX); @@ -2239,6 +2362,7 @@ struct TAccumulator { inline TAccumulator() : value(0), weight(0) {} inline TAccumulator(const Type& v, const WeightType& w) : value(v), weight(w) {} + inline bool IsEmpty() const { return weight <= 0; } // adds the given weighted value to the internal value inline void Add(const Type& v, const WeightType& w) { value += v*w; diff --git a/libs/Common/Types.inl b/libs/Common/Types.inl index aa66a4714..e9ef56086 100644 --- a/libs/Common/Types.inl +++ b/libs/Common/Types.inl @@ -1404,6 +1404,52 @@ inline TColor& operator*=(TColor& pt0, const TColor& pt1) { return pt0; } +// TMatrix operators +template +inline TMatrix operator + (const TMatrix& m1, const TMatrix& m2) { + return TMatrix(m1, m2, cv::Matx_AddOp()); +} +template +inline TMatrix operator + (const TMatrix& m1, const cv::Matx& m2) { + return cv::Matx(m1, m2, cv::Matx_AddOp()); +} +template +inline TMatrix operator + (const cv::Matx& m1, const TMatrix& m2) { + return TMatrix(m1, m2, cv::Matx_AddOp()); +} + +template +inline TMatrix operator - (const TMatrix& m1, const TMatrix& m2) { + return TMatrix(m1, m2, cv::Matx_SubOp()); +} +template +inline TMatrix operator - (const TMatrix& m1, const cv::Matx& m2) { + return TMatrix(m1, m2, cv::Matx_SubOp()); +} +template +inline TMatrix operator - (const cv::Matx& m1, const TMatrix& m2) { + return TMatrix(m1, m2, cv::Matx_SubOp()); +} +template +inline TMatrix operator - (const TMatrix& M) { + return TMatrix(M, TYPE(-1), cv::Matx_ScaleOp()); +} + +template +inline TMatrix operator * (const TMatrix& m1, const TMatrix& m2) { + return TMatrix(m1, m2, cv::Matx_MatMulOp()); +} + +template +inline TMatrix operator / (const TMatrix& mat, TYPE2 v) { + typedef typename std::conditional::value,TYPE2,REAL>::type real_t; + return TMatrix(mat, real_t(1)/v, cv::Matx_ScaleOp()); +} +template +inline TMatrix& operator /= (TMatrix& mat, TYPE2 v) { + return mat = mat/v; +} + // TImage operators template inline TImage cvtImage(const TImage& image) { @@ -1471,6 +1517,7 @@ DEFINE_CVDATADEPTH(SEACAVE::cuint32_t) // define specialized cv:DataType<> DEFINE_CVDATATYPE(SEACAVE::hfloat) DEFINE_CVDATATYPE(SEACAVE::cuint32_t) +DEFINE_GENERIC_CVDATATYPE(uint64_t,double) /*----------------------------------------------------------------*/ DEFINE_CVDATATYPE(SEACAVE::Point2i) DEFINE_CVDATATYPE(SEACAVE::Point2hf) @@ -1690,6 +1737,133 @@ TPoint3 TPoint3::RotateAngleAxis(const TPoint3& pt, const TPoint3& a // C L A S S ////////////////////////////////////////////////////// +template +inline TMatrix::TMatrix(TYPE v0) +{ + STATIC_ASSERT(channels >= 1); + val[0] = v0; + for (int i = 1; i < channels; i++) + val[i] = TYPE(0); +} +template +inline TMatrix::TMatrix(TYPE v0, TYPE v1) +{ + STATIC_ASSERT(channels >= 2); + val[0] = v0; val[1] = v1; + for (int i = 2; i < channels; i++) + val[i] = TYPE(0); +} +template +inline TMatrix::TMatrix(TYPE v0, TYPE v1, TYPE v2) +{ + STATIC_ASSERT(channels >= 3); + val[0] = v0; val[1] = v1; val[2] = v2; + for (int i = 3; i < channels; i++) + val[i] = TYPE(0); +} +template +inline TMatrix::TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3) +{ + STATIC_ASSERT(channels >= 4); + val[0] = v0; val[1] = v1; val[2] = v2; val[3] = v3; + for (int i = 4; i < channels; i++) + val[i] = TYPE(0); +} +template +inline TMatrix::TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3, TYPE v4) +{ + STATIC_ASSERT(channels >= 5); + val[0] = v0; val[1] = v1; val[2] = v2; val[3] = v3; val[4] = v4; + for (int i = 5; i < channels; i++) + val[i] = TYPE(0); +} +template +inline TMatrix::TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3, TYPE v4, TYPE v5) +{ + STATIC_ASSERT(channels >= 6); + val[0] = v0; val[1] = v1; val[2] = v2; val[3] = v3; + val[4] = v4; val[5] = v5; + for (int i = 6; i < channels; i++) + val[i] = TYPE(0); +} +template +inline TMatrix::TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3, TYPE v4, TYPE v5, TYPE v6) +{ + STATIC_ASSERT(channels >= 7); + val[0] = v0; val[1] = v1; val[2] = v2; val[3] = v3; + val[4] = v4; val[5] = v5; val[6] = v6; + for (int i = 7; i < channels; i++) + val[i] = TYPE(0); +} +template +inline TMatrix::TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3, TYPE v4, TYPE v5, TYPE v6, TYPE v7) +{ + STATIC_ASSERT(channels >= 8); + val[0] = v0; val[1] = v1; val[2] = v2; val[3] = v3; + val[4] = v4; val[5] = v5; val[6] = v6; val[7] = v7; + for (int i = 8; i < channels; i++) + val[i] = TYPE(0); +} +template +inline TMatrix::TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3, TYPE v4, TYPE v5, TYPE v6, TYPE v7, TYPE v8) +{ + STATIC_ASSERT(channels >= 9); + val[0] = v0; val[1] = v1; val[2] = v2; val[3] = v3; + val[4] = v4; val[5] = v5; val[6] = v6; val[7] = v7; + val[8] = v8; + for (int i = 9; i < channels; i++) + val[i] = TYPE(0); +} +template +inline TMatrix::TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3, TYPE v4, TYPE v5, TYPE v6, TYPE v7, TYPE v8, TYPE v9) +{ + STATIC_ASSERT(channels >= 10); + val[0] = v0; val[1] = v1; val[2] = v2; val[3] = v3; + val[4] = v4; val[5] = v5; val[6] = v6; val[7] = v7; + val[8] = v8; val[9] = v9; + for (int i = 10; i < channels; i++) + val[i] = TYPE(0); +} +template +inline TMatrix::TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3, TYPE v4, TYPE v5, TYPE v6, TYPE v7, TYPE v8, TYPE v9, TYPE v10, TYPE v11) +{ + STATIC_ASSERT(channels >= 12); + val[0] = v0; val[1] = v1; val[2] = v2; val[3] = v3; + val[4] = v4; val[5] = v5; val[6] = v6; val[7] = v7; + val[8] = v8; val[9] = v9; val[10] = v10; val[11] = v11; + for (int i = 12; i < channels; i++) + val[i] = TYPE(0); +} +template +inline TMatrix::TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3, TYPE v4, TYPE v5, TYPE v6, TYPE v7, TYPE v8, TYPE v9, TYPE v10, TYPE v11, TYPE v12, TYPE v13) +{ + STATIC_ASSERT(channels == 14); + val[0] = v0; val[1] = v1; val[2] = v2; val[3] = v3; + val[4] = v4; val[5] = v5; val[6] = v6; val[7] = v7; + val[8] = v8; val[9] = v9; val[10] = v10; val[11] = v11; + val[12] = v12; val[13] = v13; + for (int i = 14; i < channels; i++) + val[i] = TYPE(0); +} +template +inline TMatrix::TMatrix(TYPE v0, TYPE v1, TYPE v2, TYPE v3, TYPE v4, TYPE v5, TYPE v6, TYPE v7, TYPE v8, TYPE v9, TYPE v10, TYPE v11, TYPE v12, TYPE v13, TYPE v14, TYPE v15) +{ + STATIC_ASSERT(channels >= 16); + val[0] = v0; val[1] = v1; val[2] = v2; val[3] = v3; + val[4] = v4; val[5] = v5; val[6] = v6; val[7] = v7; + val[8] = v8; val[9] = v9; val[10] = v10; val[11] = v11; + val[12] = v12; val[13] = v13; val[14] = v14; val[15] = v15; + for (int i = 16; i < channels; i++) + val[i] = TYPE(0); +} +template +inline TMatrix::TMatrix(const TYPE* values) +{ + for (int i = 0; i < channels; i++) + val[i] = values[i]; +} + + template inline bool TMatrix::IsEqual(const Base& rhs) const { @@ -1990,6 +2164,67 @@ void TDVector::getKroneckerProduct(const TDVector& arg, TDVector +template +TPixel TPixel::colorRamp(VT v, VT vmin, VT vmax) +{ + if (v < vmin) + v = vmin; + if (v > vmax) + v = vmax; + const TYPE dv((TYPE)(vmax - vmin)); + TPixel c(1,1,1); // white + if (v < vmin + (VT)(TYPE(0.25) * dv)) { + c.r = TYPE(0); + c.g = TYPE(4) * (v - vmin) / dv; + } else if (v < vmin + (VT)(TYPE(0.5) * dv)) { + c.r = TYPE(0); + c.b = TYPE(1) + TYPE(4) * (vmin + TYPE(0.25) * dv - v) / dv; + } else if (v < vmin + (VT)(TYPE(0.75) * dv)) { + c.r = TYPE(4) * (v - vmin - TYPE(0.5) * dv) / dv; + c.b = TYPE(0); + } else { + c.g = TYPE(1) + TYPE(4) * (vmin + TYPE(0.75) * dv - v) / dv; + c.b = TYPE(0); + } + return c; +} + +// Gray values are expected in the range [0, 1] and converted to RGB values. +template +TPixel TPixel::gray2color(ALT gray) +{ + ASSERT(ALT(0) <= gray && gray <= ALT(1)); + // Jet colormap inspired by Matlab. + auto const Interpolate = [](ALT val, ALT y0, ALT x0, ALT y1, ALT x1) -> ALT { + return (val - x0) * (y1 - y0) / (x1 - x0) + y0; + }; + auto const Base = [&Interpolate](ALT val) -> ALT { + if (val <= ALT(0.125)) { + return ALT(0); + } else if (val <= ALT(0.375)) { + return Interpolate(ALT(2) * val - ALT(1), ALT(0), ALT(-0.75), ALT(1), ALT(-0.25)); + } else if (val <= ALT(0.625)) { + return ALT(1); + } else if (val <= ALT(0.87)) { + return Interpolate(ALT(2) * val - ALT(1), ALT(1), ALT(0.25), ALT(0), ALT(0.75)); + } else { + return ALT(0); + } + }; + return TPixel( + Base(gray + ALT(0.25)), + Base(gray), + Base(gray - ALT(0.25)) + ).cast(); +} +/*----------------------------------------------------------------*/ + + // C L A S S ////////////////////////////////////////////////////// // Find a pixel inside the image @@ -2037,71 +2272,71 @@ TYPE TImage::sampleSafe(const TPoint2& pt) const // sample by bilinear interpolation, using only pixels that meet the user condition template -template -bool TImage::sample(TYPE& v, const TPoint2& pt, bool (STCALL *fncCond)(const TYPE&)) const +template +bool TImage::sample(TV& v, const TPoint2& pt, const Functor& functor) const { const int lx((int)pt.x); const int ly((int)pt.y); const T x(pt.x-lx), x1(T(1)-x); const T y(pt.y-ly), y1(T(1)-y); - const TYPE& x0y0(BaseBase::operator()(ly , lx )); const bool b00(fncCond(x0y0)); - const TYPE& x1y0(BaseBase::operator()(ly , lx+1)); const bool b10(fncCond(x1y0)); - const TYPE& x0y1(BaseBase::operator()(ly+1, lx )); const bool b01(fncCond(x0y1)); - const TYPE& x1y1(BaseBase::operator()(ly+1, lx+1)); const bool b11(fncCond(x1y1)); + const TYPE& x0y0(BaseBase::operator()(ly , lx )); const bool b00(functor(x0y0)); + const TYPE& x1y0(BaseBase::operator()(ly , lx+1)); const bool b10(functor(x1y0)); + const TYPE& x0y1(BaseBase::operator()(ly+1, lx )); const bool b01(functor(x0y1)); + const TYPE& x1y1(BaseBase::operator()(ly+1, lx+1)); const bool b11(functor(x1y1)); if (!b00 && !b10 && !b01 && !b11) return false; - v = y1*(x1*(b00 ? x0y0 : (b10 ? x1y0 : (b01 ? x0y1 : x1y1))) + x*(b10 ? x1y0 : (b00 ? x0y0 : (b11 ? x1y1 : x0y1)))) + - y *(x1*(b01 ? x0y1 : (b11 ? x1y1 : (b00 ? x0y0 : x1y0))) + x*(b11 ? x1y1 : (b01 ? x0y1 : (b10 ? x1y0 : x0y0)))); + v = TV(y1*(x1*Cast(b00 ? x0y0 : (b10 ? x1y0 : (b01 ? x0y1 : x1y1))) + x*Cast(b10 ? x1y0 : (b00 ? x0y0 : (b11 ? x1y1 : x0y1)))) + + y *(x1*Cast(b01 ? x0y1 : (b11 ? x1y1 : (b00 ? x0y0 : x1y0))) + x*Cast(b11 ? x1y1 : (b01 ? x0y1 : (b10 ? x1y0 : x0y0))))); return true; } template -template -bool TImage::sampleSafe(TYPE& v, const TPoint2& pt, bool (STCALL *fncCond)(const TYPE&)) const +template +bool TImage::sampleSafe(TV& v, const TPoint2& pt, const Functor& functor) const { const int lx((int)pt.x); const int ly((int)pt.y); const T x(pt.x-lx), x1(T(1)-x); const T y(pt.y-ly), y1(T(1)-y); - const TYPE& x0y0(getPixel(ly , lx )); const bool b00(fncCond(x0y0)); - const TYPE& x1y0(getPixel(ly , lx+1)); const bool b10(fncCond(x1y0)); - const TYPE& x0y1(getPixel(ly+1, lx )); const bool b01(fncCond(x0y1)); - const TYPE& x1y1(getPixel(ly+1, lx+1)); const bool b11(fncCond(x1y1)); + const TYPE& x0y0(getPixel(ly , lx )); const bool b00(functor(x0y0)); + const TYPE& x1y0(getPixel(ly , lx+1)); const bool b10(functor(x1y0)); + const TYPE& x0y1(getPixel(ly+1, lx )); const bool b01(functor(x0y1)); + const TYPE& x1y1(getPixel(ly+1, lx+1)); const bool b11(functor(x1y1)); if (!b00 && !b10 && !b01 && !b11) return false; - v = y1*(x1*(b00 ? x0y0 : (b10 ? x1y0 : (b01 ? x0y1 : x1y1))) + x*(b10 ? x1y0 : (b00 ? x0y0 : (b11 ? x1y1 : x0y1)))) + - y *(x1*(b01 ? x0y1 : (b11 ? x1y1 : (b00 ? x0y0 : x1y0))) + x*(b11 ? x1y1 : (b01 ? x0y1 : (b10 ? x1y0 : x0y0)))); + v = TV(y1*(x1*Cast(b00 ? x0y0 : (b10 ? x1y0 : (b01 ? x0y1 : x1y1))) + x*Cast(b10 ? x1y0 : (b00 ? x0y0 : (b11 ? x1y1 : x0y1)))) + + y *(x1*Cast(b01 ? x0y1 : (b11 ? x1y1 : (b00 ? x0y0 : x1y0))) + x*Cast(b11 ? x1y1 : (b01 ? x0y1 : (b10 ? x1y0 : x0y0))))); return true; } // same as above, but using default value if the condition is not met template -template -TYPE TImage::sample(const TPoint2& pt, bool (STCALL *fncCond)(const TYPE&), const TYPE& dv) const +template +TYPE TImage::sample(const TPoint2& pt, const Functor& functor, const TYPE& dv) const { const int lx((int)pt.x); const int ly((int)pt.y); const T x(pt.x-lx), x1(T(1)-x); const T y(pt.y-ly), y1(T(1)-y); - const TYPE& x0y0(BaseBase::operator()(ly , lx )); const bool b00(fncCond(x0y0)); - const TYPE& x1y0(BaseBase::operator()(ly , lx+1)); const bool b10(fncCond(x1y0)); - const TYPE& x0y1(BaseBase::operator()(ly+1, lx )); const bool b01(fncCond(x0y1)); - const TYPE& x1y1(BaseBase::operator()(ly+1, lx+1)); const bool b11(fncCond(x1y1)); - return y1*(x1*(b00 ? x0y0 : dv) + x*(b10 ? x1y0 : dv)) + - y*(x1*(b01 ? x0y1 : dv) + x*(b11 ? x1y1 : dv)); + const TYPE& x0y0(BaseBase::operator()(ly , lx )); const bool b00(functor(x0y0)); + const TYPE& x1y0(BaseBase::operator()(ly , lx+1)); const bool b10(functor(x1y0)); + const TYPE& x0y1(BaseBase::operator()(ly+1, lx )); const bool b01(functor(x0y1)); + const TYPE& x1y1(BaseBase::operator()(ly+1, lx+1)); const bool b11(functor(x1y1)); + return TYPE(y1*(x1*Cast(b00 ? x0y0 : dv) + x*Cast(b10 ? x1y0 : dv)) + + y *(x1*Cast(b01 ? x0y1 : dv) + x*Cast(b11 ? x1y1 : dv))); } template -template -TYPE TImage::sampleSafe(const TPoint2& pt, bool (STCALL *fncCond)(const TYPE&), const TYPE& dv) const +template +TYPE TImage::sampleSafe(const TPoint2& pt, const Functor& functor, const TYPE& dv) const { const int lx((int)pt.x); const int ly((int)pt.y); const T x(pt.x-lx), x1(T(1)-x); const T y(pt.y-ly), y1(T(1)-y); - const TYPE& x0y0(getPixel(ly , lx )); const bool b00(fncCond(x0y0)); - const TYPE& x1y0(getPixel(ly , lx+1)); const bool b10(fncCond(x1y0)); - const TYPE& x0y1(getPixel(ly+1, lx )); const bool b01(fncCond(x0y1)); - const TYPE& x1y1(getPixel(ly+1, lx+1)); const bool b11(fncCond(x1y1)); - return y1*(x1*(b00 ? x0y0 : dv) + x*(b10 ? x1y0 : dv)) + - y*(x1*(b01 ? x0y1 : dv) + x*(b11 ? x1y1 : dv)); + const TYPE& x0y0(getPixel(ly , lx )); const bool b00(functor(x0y0)); + const TYPE& x1y0(getPixel(ly , lx+1)); const bool b10(functor(x1y0)); + const TYPE& x0y1(getPixel(ly+1, lx )); const bool b01(functor(x0y1)); + const TYPE& x1y1(getPixel(ly+1, lx+1)); const bool b11(functor(x1y1)); + return TYPE(y1*(x1*Cast(b00 ? x0y0 : dv) + x*Cast(b10 ? x1y0 : dv)) + + y *(x1*Cast(b01 ? x0y1 : dv) + x*Cast(b11 ? x1y1 : dv))); } // same as above, sample image at a specified position, but using the given sampler @@ -2119,10 +2354,11 @@ template void TImage::toGray(TImage& out, int code, bool bNormalize, bool bSRGB) const { #if 1 + typedef typename RealType::type Real; ASSERT(code==cv::COLOR_RGB2GRAY || code==cv::COLOR_RGBA2GRAY || code==cv::COLOR_BGR2GRAY || code==cv::COLOR_BGRA2GRAY); - static const T coeffsRGB[] = {T(0.299), T(0.587), T(0.114)}; - static const T coeffsBGR[] = {T(0.114), T(0.587), T(0.299)}; - const float* coeffs; + static const Real coeffsRGB[] = {Real(0.299), Real(0.587), Real(0.114)}; + static const Real coeffsBGR[] = {Real(0.114), Real(0.587), Real(0.299)}; + const Real* coeffs; switch (code) { case cv::COLOR_BGR2GRAY: case cv::COLOR_BGRA2GRAY: @@ -2135,7 +2371,7 @@ void TImage::toGray(TImage& out, int code, bool bNormalize, bool bSRGB) default: ASSERT("Unsupported image format" == NULL); } - const T &cb(coeffs[0]), &cg(coeffs[1]), &cr(coeffs[2]); + const Real &cb(coeffs[0]), &cg(coeffs[1]), &cr(coeffs[2]); if (out.rows!=rows || out.cols!=cols) out.create(rows, cols); ASSERT(cv::Mat::isContinuous()); @@ -2146,17 +2382,17 @@ void TImage::toGray(TImage& out, int code, bool bNormalize, bool bSRGB) typedef typename cv::DataType::channel_type ST; if (bSRGB) { if (bNormalize) { - typedef typename CONVERT::NormsRGB2RGB_t ColConv; + typedef typename CONVERT::NormsRGB2RGB_t ColConv; for (const ST* src=cv::Mat::template ptr(); dst!=dstEnd; src+=scn) *dst++ = T(cb*ColConv(src[0]) + cg*ColConv(src[1]) + cr*ColConv(src[2])); } else { - typedef typename CONVERT::NormsRGB2RGBUnNorm_t ColConv; + typedef typename CONVERT::NormsRGB2RGBUnNorm_t ColConv; for (const ST* src=cv::Mat::template ptr(); dst!=dstEnd; src+=scn) *dst++ = T(cb*ColConv(src[0]) + cg*ColConv(src[1]) + cr*ColConv(src[2])); } } else { if (bNormalize) { - typedef typename CONVERT::NormRGB_t ColConv; + typedef typename CONVERT::NormRGB_t ColConv; for (const ST* src=cv::Mat::template ptr(); dst!=dstEnd; src+=scn) *dst++ = T(cb*ColConv(src[0]) + cg*ColConv(src[1]) + cr*ColConv(src[2])); } else { @@ -2175,6 +2411,32 @@ void TImage::toGray(TImage& out, int code, bool bNormalize, bool bSRGB) /*----------------------------------------------------------------*/ +// compute scaled size such that the biggest dimension is scaled as desired +// and the smaller one maintains the aspect ratio as best as it can +template +cv::Size TImage::computeResize(const cv::Size& size, REAL scale) +{ + cv::Size scaledSize; + if (size.width > size.height) { + scaledSize.width = ROUND2INT(REAL(size.width)*scale); + scaledSize.height = ROUND2INT(REAL(size.height)*scaledSize.width/REAL(size.width)); + } else { + scaledSize.height = ROUND2INT(REAL(size.height)*scale); + scaledSize.width = ROUND2INT(REAL(size.width)*scaledSize.height/REAL(size.height)); + } + return scaledSize; +} +// compute the final scaled size by performing successive resizes +// with the given scale value +template +cv::Size TImage::computeResize(const cv::Size& size, REAL scale, unsigned resizes) +{ + cv::Size scaledSize(size); + while (resizes-- > 0) + scaledSize = computeResize(scaledSize, scale); + return scaledSize; +} + // compute image scale for a given max and min resolution template unsigned TImage::computeMaxResolution(unsigned width, unsigned height, unsigned& level, unsigned minImageSize, unsigned maxImageSize) diff --git a/libs/Common/Util.inl b/libs/Common/Util.inl index f64a1704e..d605b9173 100644 --- a/libs/Common/Util.inl +++ b/libs/Common/Util.inl @@ -886,8 +886,9 @@ inline std::pair ComputeX84Threshold(const TYPE* const values, size std::nth_element(data.Begin(), mid, data.End()); const TYPEW median(*mid); // threshold = 5.2 * MEDIAN(ABS(values-median)); - FOREACHPTR(pVal, data) - *pVal = ABS((*pVal)-median); + using TYPEI = typename MakeSigned::type; + for (TYPE& val: data) + val = TYPE(ABS(TYPEI(val)-TYPEI(median))); std::nth_element(data.Begin(), mid, data.End()); return std::make_pair(median, mul*TYPEW(*mid)); } // ComputeX84Threshold diff --git a/libs/MVS/Camera.cpp b/libs/MVS/Camera.cpp index e8d7d5e0c..cdcdbc154 100644 --- a/libs/MVS/Camera.cpp +++ b/libs/MVS/Camera.cpp @@ -187,3 +187,210 @@ void MVS::AssembleProjectionMatrix(const RMatrix& R, const CMatrix& C, PMatrix& eT = ((const Matrix3x3::EMat)R) * (-((const Point3::EVec)C)); //3x1 } // AssembleProjectionMatrix /*----------------------------------------------------------------*/ + + + +namespace MVS { + +namespace RECTIFY { + +// compute the ROIs for the two images based on the corresponding points +void GetImagePairROI(const Point3fArr& points1, const Point3fArr& points2, const Matrix3x3& K1, const Matrix3x3& K2, const Matrix3x3& R1, const Matrix3x3& R2, const Matrix3x3& invK1, const Matrix3x3& invK2, AABB2f& roi1h, AABB2f& roi2h) +{ + ASSERT(!points1.empty() && points1.size() && points2.size()); + + // compute rectification homography (from original to rectified image) + const Matrix3x3 H1(K1 * R1 * invK1); + const Matrix3x3 H2(K2 * R2 * invK2); + + // determine the ROIs in rectified images + roi1h.Reset(); roi2h.Reset(); + FOREACH(i, points1) { + Point2f xh; + const Point3f& x1 = points1[i]; + ProjectVertex_3x3_2_2(H1.val, x1.ptr(), xh.ptr()); + roi1h.InsertFull(xh); + const Point3f& x2 = points2[i]; + ProjectVertex_3x3_2_2(H2.val, x2.ptr(), xh.ptr()); + roi2h.InsertFull(xh); + } +} + +void SetCameraMatricesROI(const AABB2f& roi1h, const AABB2f& roi2h, cv::Size& size1, cv::Size& size2, Matrix3x3& K1, Matrix3x3& K2) +{ + // set the new image sizes such that they are equal and contain the entire ROI + const Point2f size1h(roi1h.GetSize()); + const Point2f size2h(roi2h.GetSize()); + const int maxSize(MAXF(size1.width+size2.width, size1.height+size2.height)/2); + size1.width = size2.width = MINF(ROUND2INT(MAXF(size1h.x,size2h.x)), maxSize); + size1.height = size2.height = MINF(ROUND2INT(MAXF(size1h.y,size2h.y)), maxSize); + + // set the new camera matrices such that the ROI is centered + const Point2f center1h(roi1h.GetCenter()); + const Point2f center2h(roi2h.GetCenter()); + K1(0,2) += size1.width /2-center1h[0]; + K1(1,2) += size1.height/2-center1h[1]; + K2(0,2) += size2.width /2-center2h[0]; + K2(1,2) += size2.height/2-center2h[1]; +} + +} // namespace RECTIFY + +} // namespace MVS + +// Compute stereo rectification homographies that transform two images, +// such that corresponding pixels in one image lie on the same scan-line in the other image; +// note: this function assumes that the two cameras are already undistorted +REAL Camera::StereoRectify(const cv::Size& size1, const Camera& camera1, const cv::Size& size2, const Camera& camera2, Matrix3x3& R1, Matrix3x3& R2, Matrix3x3& K1, Matrix3x3& K2) +{ + // compute relative pose + RMatrix poseR; + CMatrix poseC; + ComputeRelativePose(camera1.R, camera1.C, camera2.R, camera2.C, poseR, poseC); + + // compute the average rotation between the first and the second camera + const Point3 r(poseR.GetRotationAxisAngle()); + reinterpret_cast(R2).SetFromAxisAngle(r*REAL(-0.5)); + R1 = R2.t(); + + // compute the translation, such that it coincides with the X-axis + int idx(0); + Point3 t(R2 * (poseR*(-poseC))); + #if 0 + const Point3 unitX(t.x<0?-1:1, 0, 0); + const Point3 axis(t.cross(unitX)); + const REAL normAxis(norm(axis)); + if (normAxis > EPSILONTOLERANCE()) { + ASSERT(t.dot(unitX) >= 0); + const REAL angle(ACOS(ComputeAngle(t.ptr(), unitX.ptr()))); + const RMatrix Rx(axis*angle/normAxis); + #else + const REAL nt(norm(t)); + //idx = (ABS(t.x) > ABS(t.y) ? 0 : 1); + Point3 axis(Point3(0,0,1).cross(t)); + const REAL na(norm(axis)); + if (na < EPSILONTOLERANCE()) + return 0; + { + const Point3 w(normalized(t.cross(axis))); + RMatrix Rx; + for (int i = 0; i < 3; ++i) { + Rx(idx,i) = -t[i]/nt; + Rx(idx^1,i) = -axis[i]/na; + Rx(2,i) = w[i]*(1-2*idx); // if idx == 1 -> opposite direction + } + #endif + R1 = Rx * R1; + R2 = Rx * R2; + t = R2 * (poseR*(-poseC)); + } + ASSERT(ISEQUAL(-t.x, norm(camera2.C-camera1.C)) && ISZERO(t.y) && ISZERO(t.z)); + + // determine the intrinsic calibration matrix; + // vertical focal length must be the same for both images to keep the epipolar constraint + K1 = Matrix3x3::IDENTITY; + K1(0,0) = (camera1.K(0,0)+camera2.K(0,0))/2; + K1(1,1) = (camera1.K(1,1)+camera2.K(1,1))/2; + K2 = K1; + // set ROI to contain the whole transformed image + for (int k = 0; k < 2; k++) { + const Matrix3x3 H(k == 0 ? K1*R1*camera1.GetInvK() : K2*R2*camera2.GetInvK()); + Point2f ptMin(std::numeric_limits::max()); + for (int i = 0; i < 4; i++) { + const Point2f pt( + (float)((i%2)*size1.width), + (float)((i<2?0:1)*size1.height) + ); + Point2f ptH; + ProjectVertex_3x3_2_2(H.val, pt.ptr(), ptH.ptr()); + ptMin.x = MINF(ptMin.x,ptH.x); + ptMin.y = MINF(ptMin.y,ptH.y); + } + Matrix3x3& KNew(k == 0 ? K1 : K2); + KNew(0,2) = -ptMin.x; + KNew(1,2) = -ptMin.y; + } + K1(idx^1,2) = K2(idx^1,2) = MAXF(K1(idx^1,2), K2(idx^1,2)); + return t.x; +} // StereoRectify + +// see: "A compact algorithm for rectification of stereo pairs", A. Fusiello, E. Trucco, and A. Verri, 2000 +REAL Camera::StereoRectifyFusiello(const cv::Size& size1, const Camera& camera1, const cv::Size& size2, const Camera& camera2, Matrix3x3& R1, Matrix3x3& R2, Matrix3x3& K1, Matrix3x3& K2) +{ + // compute relative pose + RMatrix poseR; + CMatrix poseC; + ComputeRelativePose(camera1.R, camera1.C, camera2.R, camera2.C, poseR, poseC); + + // new x axis (baseline, from C1 to C2) + const Point3 v1(camera2.C-camera1.C); + // new y axes (orthogonal to old z and new x) + const Point3 v2(camera1.Direction().cross(v1)); + // new z axes (no choice, orthogonal to baseline and y) + const Point3 v3(v1.cross(v2)); + + // new extrinsic (translation unchanged) + RMatrix R; + R.SetFromRowVectors(normalized(v1), normalized(v2), normalized(v3)); + + // new intrinsic (arbitrary) + K1 = camera1.K; K1(0,1) = 0; + K2 = camera2.K; K2(0,1) = 0; + K1(1,1) = K2(1,1) = (camera1.K(1,1)+camera2.K(1,1))/2; + + // new rotations + R1 = R*camera1.R.t(); + R2 = R*camera2.R.t(); + + #if 0 + // new projection matrices + PMatrix P1, P2; + AssembleProjectionMatrix(K1, R, camera1.C, P1); + AssembleProjectionMatrix(K2, R, camera2.C, P2); + + // rectifying image transformation + #if 0 + const Matrix3x3 H1((PMatrix::EMat(P1).leftCols<3>()*(PMatrix::EMat(camera1.P).leftCols<3>().inverse())).eval()); + const Matrix3x3 H2((PMatrix::EMat(P2).leftCols<3>()*(PMatrix::EMat(camera2.P).leftCols<3>().inverse())).eval()); + #else + const Matrix3x3 H1(K1*R*camera1.R.t()*camera1.GetInvK()); + const Matrix3x3 H2(K2*R*camera2.R.t()*camera2.GetInvK()); + #endif + #endif + + const Point3 t(R2 * (poseR*(-poseC))); + ASSERT(ISEQUAL(-t.x, norm(v1)) && ISZERO(t.y) && ISZERO(t.z)); + return t.x; +} // StereoRectifyFusiello + +// adjust rectified camera matrices such that the entire area common to both source images is contained in the rectified images; +// as long as the focal-length on x and the skewness are equal in both cameras +// the point-scale is also equal in both images; +// note however that the surface-scale can still be different (most likely is) +// as it depends on its (unknown) normal too; +// - points1 and points2: contain the pairs of corresponding pairs of image projections and their depth +// - size1 and size2: input the size of the source images, output the size of the rectified images (rectified image sizes are equal) +void Camera::SetStereoRectificationROI(const Point3fArr& points1, cv::Size& size1, const Camera& camera1, const Point3fArr& points2, cv::Size& size2, const Camera& camera2, const Matrix3x3& R1, const Matrix3x3& R2, Matrix3x3& K1, Matrix3x3& K2) +{ + ASSERT(!points1.empty() && points1.size() && points2.size()); + + #if 1 + // ignore skewness + K1(0,1) = K2(0,1) = 0; + #else + // set same skewness + K1(0,1) = K2(0,1) = (K1(0,1)+K2(0,1))/2; + #endif + + // set same focal-length on x too + K1(0,0) = K2(0,0) = (K1(0,0)+K2(0,0))/2; + ASSERT(ISEQUAL(K1(1,1), K2(1,1))); + + // determine the ROIs in rectified images + AABB2f roi1h, roi2h; + RECTIFY::GetImagePairROI(points1, points2, K1, K2, R1, R2, camera1.GetInvK(), camera2.GetInvK(), roi1h, roi2h); + + // set the new camera matrices such that the ROI is centered + RECTIFY::SetCameraMatricesROI(roi1h, roi2h, size1, size2, K1, K2); +} // SetStereoRectificationROI +/*----------------------------------------------------------------*/ diff --git a/libs/MVS/Camera.h b/libs/MVS/Camera.h index aa9aebb6d..ee22d8a80 100644 --- a/libs/MVS/Camera.h +++ b/libs/MVS/Camera.h @@ -232,6 +232,10 @@ class MVS_API Camera : public CameraIntern REAL DistanceSq(const Point3& X) const; // compute the distance from the camera to the given 3D point inline REAL Distance(const Point3& X) const { return SQRT(DistanceSq(X)); } + static REAL StereoRectify(const cv::Size& size1, const Camera& camera1, const cv::Size& size2, const Camera& camera2, Matrix3x3& R1, Matrix3x3& R2, Matrix3x3& K1, Matrix3x3& K2); + static REAL StereoRectifyFusiello(const cv::Size& size1, const Camera& camera1, const cv::Size& size2, const Camera& camera2, Matrix3x3& R1, Matrix3x3& R2, Matrix3x3& K1, Matrix3x3& K2); + static void SetStereoRectificationROI(const Point3fArr& points1, cv::Size& size1, const Camera& camera1, const Point3fArr& points2, cv::Size& size2, const Camera& camera2, const Matrix3x3& R1, const Matrix3x3& R2, Matrix3x3& K1, Matrix3x3& K2); + // project 3D point by the camera template inline TPoint3 ProjectPointRT3(const TPoint3& X) const { @@ -337,6 +341,11 @@ class MVS_API Camera : public CameraIntern inline TPoint2 TransformPointW2I(const TPoint3& X) const { return TransformPointC2I(TransformPointW2C(X)); } + template + inline TPoint3 TransformPointW2I3(const TPoint3& X) const { + const TPoint3 camX(TransformPointW2C(X)); + return TPoint3(TransformPointC2I(camX), camX.z); + } // check if the given point (or its projection) is inside the camera view template diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index 1d8804e59..2e35cb3a3 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -32,6 +32,10 @@ #include "Common.h" #include "DepthMap.h" #include "../Common/AutoEstimator.h" +// CGAL: depth-map initialization +#include +#include +#include // CGAL: estimate normals #include #include @@ -908,6 +912,216 @@ DepthEstimator::PixelEstimate DepthEstimator::PerturbEstimate(const PixelEstimat // S T R U C T S /////////////////////////////////////////////////// +namespace CGAL { +typedef CGAL::Simple_cartesian kernel_t; +typedef CGAL::Projection_traits_xy_3 Geometry; +typedef CGAL::Delaunay_triangulation_2 Delaunay; +typedef CGAL::Delaunay::Face_circulator FaceCirculator; +typedef CGAL::Delaunay::Face_handle FaceHandle; +typedef CGAL::Delaunay::Vertex_circulator VertexCirculator; +typedef CGAL::Delaunay::Vertex_handle VertexHandle; +typedef kernel_t::Point_3 Point; +} + +// triangulate in-view points, generating a 2D mesh +// return also the estimated depth boundaries (min and max depth) +std::pair TriangulatePointsDelaunay(const DepthData::ViewData& image, const PointCloud& pointcloud, const IndexArr& points, CGAL::Delaunay& delaunay) +{ + ASSERT(sizeof(Point3) == sizeof(X3D)); + ASSERT(sizeof(Point3) == sizeof(CGAL::Point)); + std::pair depthBounds(FLT_MAX, 0.f); + for (uint32_t idx: points) { + const Point3f pt(image.camera.ProjectPointP3(pointcloud.points[idx])); + delaunay.insert(CGAL::Point(pt.x/pt.z, pt.y/pt.z, pt.z)); + if (depthBounds.first > pt.z) + depthBounds.first = pt.z; + if (depthBounds.second < pt.z) + depthBounds.second = pt.z; + } + // if full size depth-map requested + if (OPTDENSE::bAddCorners) { + typedef TIndexScore DepthDist; + typedef CLISTDEF0(DepthDist) DepthDistArr; + typedef Eigen::Map< Eigen::VectorXf, Eigen::Unaligned, Eigen::InnerStride<2> > FloatMap; + // add the four image corners at the average depth + ASSERT(image.pImageData->IsValid() && ISINSIDE(image.pImageData->avgDepth, depthBounds.first, depthBounds.second)); + const CGAL::VertexHandle vcorners[] = { + delaunay.insert(CGAL::Point(0, 0, image.pImageData->avgDepth)), + delaunay.insert(CGAL::Point(image.image.width(), 0, image.pImageData->avgDepth)), + delaunay.insert(CGAL::Point(0, image.image.height(), image.pImageData->avgDepth)), + delaunay.insert(CGAL::Point(image.image.width(), image.image.height(), image.pImageData->avgDepth)) + }; + // compute average depth from the closest 3 directly connected faces, + // weighted by the distance + const size_t numPoints = 3; + for (int i=0; i<4; ++i) { + const CGAL::VertexHandle vcorner = vcorners[i]; + CGAL::FaceCirculator cfc(delaunay.incident_faces(vcorner)); + if (cfc == 0) + continue; // normally this should never happen + const CGAL::FaceCirculator done(cfc); + Point3d& poszA = reinterpret_cast(vcorner->point()); + const Point2d& posA = reinterpret_cast(poszA); + const Ray3d rayA(Point3d::ZERO, normalized(image.camera.TransformPointI2C(poszA))); + DepthDistArr depths(0, numPoints); + do { + CGAL::FaceHandle fc(cfc->neighbor(cfc->index(vcorner))); + if (fc == delaunay.infinite_face()) + continue; + for (int j=0; j<4; ++j) + if (fc->has_vertex(vcorners[j])) + goto Continue; + // compute the depth as the intersection of the corner ray with + // the plane defined by the face's vertices + { + const Point3d& poszB0 = reinterpret_cast(fc->vertex(0)->point()); + const Point3d& poszB1 = reinterpret_cast(fc->vertex(1)->point()); + const Point3d& poszB2 = reinterpret_cast(fc->vertex(2)->point()); + const Planed planeB( + image.camera.TransformPointI2C(poszB0), + image.camera.TransformPointI2C(poszB1), + image.camera.TransformPointI2C(poszB2) + ); + const Point3d poszB(rayA.Intersects(planeB)); + if (poszB.z <= 0) + continue; + const Point2d posB(( + reinterpret_cast(poszB0)+ + reinterpret_cast(poszB1)+ + reinterpret_cast(poszB2))/3.f + ); + const double dist(norm(posB-posA)); + depths.StoreTop(DepthDist(CLAMP((float)poszB.z,depthBounds.first,depthBounds.second), INVERT((float)dist))); + } + Continue:; + } while (++cfc != done); + if (depths.size() != numPoints) + continue; // normally this should never happen + FloatMap vecDists(&depths[0].score, numPoints); + vecDists *= 1.f/vecDists.sum(); + FloatMap vecDepths(&depths[0].idx, numPoints); + poszA.z = vecDepths.dot(vecDists); + } + } + return depthBounds; +} + +// roughly estimate depth and normal maps by triangulating the sparse point cloud +// and interpolating normal and depth for all pixels +bool MVS::TriangulatePoints2DepthMap( + const DepthData::ViewData& image, const PointCloud& pointcloud, const IndexArr& points, + DepthMap& depthMap, NormalMap& normalMap, Depth& dMin, Depth& dMax) +{ + ASSERT(image.pImageData != NULL); + + // triangulate in-view points + CGAL::Delaunay delaunay; + const std::pair thDepth(TriangulatePointsDelaunay(image, pointcloud, points, delaunay)); + dMin = thDepth.first; + dMax = thDepth.second; + + // create rough depth-map by interpolating inside triangles + const Camera& camera = image.camera; + depthMap.create(image.image.size()); + normalMap.create(image.image.size()); + if (!OPTDENSE::bAddCorners) { + depthMap.memset(0); + normalMap.memset(0); + } + struct RasterDepthDataPlaneData { + const Camera& P; + DepthMap& depthMap; + NormalMap& normalMap; + Point3f normal; + Point3f normalPlane; + inline void operator()(const ImageRef& pt) { + if (!depthMap.isInside(pt)) + return; + const Depth z(INVERT(normalPlane.dot(P.TransformPointI2C(Point2f(pt))))); + if (z <= 0) // due to numerical instability + return; + depthMap(pt) = z; + normalMap(pt) = normal; + } + }; + RasterDepthDataPlaneData data = {camera, depthMap, normalMap}; + for (CGAL::Delaunay::Face_iterator it=delaunay.faces_begin(); it!=delaunay.faces_end(); ++it) { + const CGAL::Delaunay::Face& face = *it; + const Point3f i0(reinterpret_cast(face.vertex(0)->point())); + const Point3f i1(reinterpret_cast(face.vertex(1)->point())); + const Point3f i2(reinterpret_cast(face.vertex(2)->point())); + // compute the plane defined by the 3 points + const Point3f c0(camera.TransformPointI2C(i0)); + const Point3f c1(camera.TransformPointI2C(i1)); + const Point3f c2(camera.TransformPointI2C(i2)); + const Point3f edge1(c1-c0); + const Point3f edge2(c2-c0); + data.normal = normalized(edge2.cross(edge1)); + data.normalPlane = data.normal * INVERT(data.normal.dot(c0)); + // draw triangle and for each pixel compute depth as the ray intersection with the plane + Image8U::RasterizeTriangle( + reinterpret_cast(i2), + reinterpret_cast(i1), + reinterpret_cast(i0), data); + } + return true; +} // TriangulatePoints2DepthMap +// same as above, but does not estimate the normal-map +bool MVS::TriangulatePoints2DepthMap( + const DepthData::ViewData& image, const PointCloud& pointcloud, const IndexArr& points, + DepthMap& depthMap, Depth& dMin, Depth& dMax) +{ + ASSERT(image.pImageData != NULL); + + // triangulate in-view points + CGAL::Delaunay delaunay; + const std::pair thDepth(TriangulatePointsDelaunay(image, pointcloud, points, delaunay)); + dMin = thDepth.first; + dMax = thDepth.second; + + // create rough depth-map by interpolating inside triangles + const Camera& camera = image.camera; + depthMap.create(image.image.size()); + if (!OPTDENSE::bAddCorners) + depthMap.memset(0); + struct RasterDepthDataPlaneData { + const Camera& P; + DepthMap& depthMap; + Point3f normalPlane; + inline void operator()(const ImageRef& pt) { + if (!depthMap.isInside(pt)) + return; + const Depth z((Depth)INVERT(normalPlane.dot(P.TransformPointI2C(Point2f(pt))))); + if (z <= 0) // due to numerical instability + return; + depthMap(pt) = z; + } + }; + RasterDepthDataPlaneData data = {camera, depthMap}; + for (CGAL::Delaunay::Face_iterator it=delaunay.faces_begin(); it!=delaunay.faces_end(); ++it) { + const CGAL::Delaunay::Face& face = *it; + const Point3f i0(reinterpret_cast(face.vertex(0)->point())); + const Point3f i1(reinterpret_cast(face.vertex(1)->point())); + const Point3f i2(reinterpret_cast(face.vertex(2)->point())); + // compute the plane defined by the 3 points + const Point3f c0(camera.TransformPointI2C(i0)); + const Point3f c1(camera.TransformPointI2C(i1)); + const Point3f c2(camera.TransformPointI2C(i2)); + const Point3f edge1(c1-c0); + const Point3f edge2(c2-c0); + const Normal normal(normalized(edge2.cross(edge1))); + data.normalPlane = normal * INVERT(normal.dot(c0)); + // draw triangle and for each pixel compute depth as the ray intersection with the plane + Image8U::RasterizeTriangle( + reinterpret_cast(i2), + reinterpret_cast(i1), + reinterpret_cast(i0), data); + } + return true; +} // TriangulatePoints2DepthMap +/*----------------------------------------------------------------*/ + + namespace MVS { class PlaneSolverAdaptor @@ -1131,6 +1345,100 @@ void MVS::EstimatePointNormals(const ImageArr& images, PointCloud& pointcloud, i } // EstimatePointNormals /*----------------------------------------------------------------*/ +bool MVS::EstimateNormalMap(const Matrix3x3f& K, const DepthMap& depthMap, NormalMap& normalMap) +{ + normalMap.create(depthMap.size()); + struct Tool { + static bool IsDepthValid(Depth d, Depth nd) { + return nd > 0 && IsDepthSimilar(d, nd, Depth(0.03f)); + } + // computes depth gradient (first derivative) at current pixel + static bool DepthGradient(const DepthMap& depthMap, const ImageRef& ir, Point3f& ws) { + float& w = ws[0]; + float& wx = ws[1]; + float& wy = ws[2]; + w = depthMap(ir); + if (w <= 0) + return false; + // loop over neighborhood and finding least squares plane, + // the coefficients of which give gradient of depth + int whxx(0), whxy(0), whyy(0); + float wgx(0), wgy(0); + const int Radius(1); + int n(0); + for (int y = -Radius; y <= Radius; ++y) { + for (int x = -Radius; x <= Radius; ++x) { + if (x == 0 && y == 0) + continue; + const ImageRef pt(ir.x+x, ir.y+y); + if (!depthMap.isInside(pt)) + continue; + const float wi(depthMap(pt)); + if (!IsDepthValid(w, wi)) + continue; + whxx += x*x; whxy += x*y; whyy += y*y; + wgx += (wi - w)*x; wgy += (wi - w)*y; + ++n; + } + } + if (n < 3) + return false; + // solve 2x2 system, generated from depth gradient + const int det(whxx*whyy - whxy*whxy); + if (det == 0) + return false; + const float invDet(1.f/float(det)); + wx = (float( whyy)*wgx - float(whxy)*wgy)*invDet; + wy = (float(-whxy)*wgx + float(whxx)*wgy)*invDet; + return true; + } + // computes normal to the surface given the depth and its gradient + static Normal ComputeNormal(const Matrix3x3f& K, int x, int y, Depth d, Depth dx, Depth dy) { + ASSERT(ISZERO(K(0,1))); + return normalized(Normal( + K(0,0)*dx, + K(1,1)*dy, + (K(0,2)-float(x))*dx+(K(1,2)-float(y))*dy-d + )); + } + }; + for (int r=0; r1?cv::INTER_CUBIC:cv::INTER_AREA); + else + scaledImage.image.release(); + } + return scaledImage; +} // GetImage // compute the camera extrinsics from the platform pose and the relative camera pose to the platform Camera Image::GetCamera(const PlatformArr& platforms, const Image8U::Size& resolution) const { @@ -209,3 +225,216 @@ REAL Image::ComputeFOV(int dir) const return 0; } // ComputeFOV /*----------------------------------------------------------------*/ + + +// stereo-rectify the image pair +// - input leftPoints and rightPoints contains the pairs of corresponding image projections and their depth (optional) +// - H converts a pixel from original to rectified left-image +// - Q converts [x' y' disparity 1] in rectified coordinates to [x*z y*z z 1]*w in original image coordinates +// where disparity=x1-x2 +bool Image::StereoRectifyImages(const Image& image1, const Image& image2, const Point3fArr& points1, const Point3fArr& points2, Image8U3& rectifiedImage1, Image8U3& rectifiedImage2, Image8U& mask1, Image8U& mask2, Matrix3x3& H, Matrix4x4& Q) +{ + ASSERT(image1.IsValid() && image2.IsValid()); + ASSERT(image1.GetSize() == image1.image.size() && image2.GetSize() == image2.image.size()); + ASSERT(points1.size() && points2.size()); + + #if 0 + { // display projection pairs + std::vector matches1, matches2; + FOREACH(i, points1) { + matches1.emplace_back(reinterpret_cast(points1[i])); + matches2.emplace_back(reinterpret_cast(points2[i])); + } + RECTIFY::DrawMatches(const_cast(image1.image), const_cast(image2.image), matches1, matches2); + } + #endif + + // compute rectification + Matrix3x3 K1, K2, R1, R2; + #if 0 + const REAL t(Camera::StereoRectify(image1.GetSize(), image1.camera, image2.GetSize(), image2.camera, R1, R2, K1, K2)); + #elif 1 + const REAL t(Camera::StereoRectifyFusiello(image1.GetSize(), image1.camera, image2.GetSize(), image2.camera, R1, R2, K1, K2)); + #else + Pose pose; + ComputeRelativePose(image1.camera.R, image1.camera.C, image2.camera.R, image2.camera.C, pose.R, pose.C); + cv::Mat P1, P2; + cv::stereoRectify(image1.camera.K, cv::noArray(), image2.camera.K, cv::noArray(), image1.GetSize(), pose.R, Vec3(pose.GetTranslation()), R1, R2, P1, P2, Q, 0/*cv::CALIB_ZERO_DISPARITY*/, -1); + K1 = P1(cv::Rect(0,0,3,3)); + K2 = P2(cv::Rect(0,0,3,3)); + const Point3 _t(R2 * pose.GetTranslation()); + ASSERT((ISZERO(_t.x) || ISZERO(_t.y)) && ISZERO(_t.z)); + const REAL t(ISZERO(_t.x)?_t.y:_t.x); + #if 0 + cv::Mat map1, map2; + cv::initUndistortRectifyMap(image1.camera.K, cv::noArray(), R1, K1, image1.GetSize(), CV_16SC2, map1, map2); + cv::remap(image1.image, rectifiedImage1, map1, map2, cv::INTER_CUBIC); + cv::initUndistortRectifyMap(image2.camera.K, cv::noArray(), R2, K2, image1.GetSize(), CV_16SC2, map1, map2); + cv::remap(image2.image, rectifiedImage2, map1, map2, cv::INTER_CUBIC); + return; + #endif + #endif + if (ISZERO(t)) + return false; + + // adjust rectified camera matrices such that the entire area common to both source images is contained in the rectified images + cv::Size size1(image1.GetSize()), size2(image2.GetSize()); + if (!points1.empty()) + Camera::SetStereoRectificationROI(points1, size1, image1.camera, points2, size2, image2.camera, R1, R2, K1, K2); + ASSERT(size1 == size2); + + // compute rectification homography (from original to rectified image) + const Matrix3x3 H1(K1 * R1 * image1.camera.GetInvK()); H = H1; + const Matrix3x3 H2(K2 * R2 * image2.camera.GetInvK()); + + #if 0 + { // display epipolar lines before and after rectification + Pose pose; + ComputeRelativePose(image1.camera.R, image1.camera.C, image2.camera.R, image2.camera.C, pose.R, pose.C); + const Matrix3x3 F(CreateF(pose.R, pose.C, image1.camera.K, image2.camera.K)); + std::vector matches1, matches2; + #if 1 + FOREACH(i, points1) { + matches1.emplace_back(reinterpret_cast(points1[i])); + matches2.emplace_back(reinterpret_cast(points2[i])); + } + #endif + RECTIFY::DrawRectifiedImages(image1.image.clone(), image2.image.clone(), F, H1, H2, matches1, matches2); + } + #endif + + // rectify images (apply homographies) + rectifiedImage1.create(size1); + cv::warpPerspective(image1.image, rectifiedImage1, H1, rectifiedImage1.size()); + rectifiedImage2.create(size2); + cv::warpPerspective(image2.image, rectifiedImage2, H2, rectifiedImage2.size()); + + // mark valid regions covered by the rectified images + struct Compute { + static void Mask(Image8U& mask, const cv::Size& sizeh, const cv::Size& size, const Matrix3x3& H) { + mask.create(sizeh); + mask.memset(0); + std::vector corners(4); + corners[0] = Point2f(0,0); + corners[1] = Point2f((float)size.width,0); + corners[2] = Point2f((float)size.width,(float)size.height); + corners[3] = Point2f(0,(float)size.height); + cv::perspectiveTransform(corners, corners, H); + std::vector> contours(1); + for (int i=0; i<4; ++i) + contours.front().emplace_back(ROUND2INT(corners[i])); + cv::drawContours(mask, contours, 0, cv::Scalar(255), cv::FILLED); + } + }; + Compute::Mask(mask1, size1, image1.GetSize(), H1); + Compute::Mask(mask2, size2, image2.GetSize(), H2); + + // from the formula that relates disparity to depth as z=B*f/d where B=-t and d=x_l-x_r + // and the formula that converts the image projection from right to left x_r=K1*K2.inv()*x_l + // compute the inverse projection matrix that transforms image coordinates in image 1 and its + // corresponding disparity value to the 3D point in camera 1 coordinates as: + ASSERT(ISEQUAL(K1(1,1),K2(1,1))); + Q = Matrix4x4::ZERO; + // Q * [x, y, disparity, 1] = [X, Y, Z, 1] * w + ASSERT(ISEQUAL(K1(0,0),K2(0,0)) && ISZERO(K1(0,1)) && ISZERO(K2(0,1))); + Q(0,0) = Q(1,1) = REAL(1); + Q(0,3) = -K1(0,2); + Q(1,3) = -K1(1,2); + Q(2,3) = K1(0,0); + Q(3,2) = -REAL(1)/t; + Q(3,3) = (K1(0,2)-K2(0,2))/t; + + // compute Q that converts disparity from rectified to depth in original image + Matrix4x4 P(Matrix4x4::IDENTITY); + cv::Mat(image1.camera.K*R1.t()).copyTo(cv::Mat(4,4,cv::DataType::type,P.val)(cv::Rect(0,0,3,3))); + Q = P*Q; + return true; +} + +// adjust H and Q such that they correspond to rectified images scaled by the given factor +void Image::ScaleStereoRectification(Matrix3x3& H, Matrix4x4& Q, REAL scale) +{ + { + Matrix3x3 S(Matrix3x3::IDENTITY); + S(0,0) = S(1,1) = scale; + H = S*H*S.inv(); + } + { + Matrix4x4 Sinv(Matrix4x4::IDENTITY); + Sinv(0,0) = Sinv(1,1) = Sinv(2,2) = REAL(1)/scale; + Matrix4x4 S(Matrix4x4::IDENTITY); + S(0,0) = S(1,1) = scale*scale; + S(2,2) = S(3,3) = scale; + Q = S*Q*Sinv; + } +} + + +// note: disparity has to be inverted as the formula is z=B*f/d where d=x_l-x_r +// while we store d as the value that fulfills x_l+d=x_r +// +// converts the given disparity at the rectified image coordinates to +// the depth corresponding to the un-rectified image coordinates +template +float TDisparity2Depth(const Matrix4x4& Q, const TPoint2& u, float d) +{ + const REAL w(Q(3,0)*u.x + Q(3,1)*u.y - Q(3,2)*d + Q(3,3)); + if (ISZERO(w)) + return 0; + const REAL z(Q(2,0)*u.x + Q(2,1)*u.y - Q(2,2)*d + Q(2,3)); + const float depth((float)(z/w)); + return depth < ZEROTOLERANCE() ? float(0) : depth; +} +float Image::Disparity2Depth(const Matrix4x4& Q, const ImageRef& u, float d) +{ + return TDisparity2Depth(Q, u, d); +} +float Image::Disparity2Depth(const Matrix4x4& Q, const Point2f& u, float d) +{ + return TDisparity2Depth(Q, u, d); +} +// same as above, but converts also the coordinates from rectified to original image +template +float TDisparity2Depth(const Matrix4x4& Q, const TPoint2& u, float d, Point2f& pt) +{ + const REAL w(Q(3,0)*u.x + Q(3,1)*u.y - Q(3,2)*d + Q(3,3)); + if (ISZERO(w)) + return 0; + const REAL z((Q(2,0)*u.x + Q(2,1)*u.y - Q(2,2)*d + Q(2,3))/w); + if (z < ZEROTOLERANCE()) + return 0; + const REAL nrm(REAL(1)/(w*z)); + pt.x = (float)((Q(0,0)*u.x + Q(0,1)*u.y - Q(0,2)*d + Q(0,3))*nrm); + pt.y = (float)((Q(1,0)*u.x + Q(1,1)*u.y - Q(1,2)*d + Q(1,3))*nrm); + return (float)z; +} +float Image::Disparity2Depth(const Matrix4x4& Q, const ImageRef& u, float d, Point2f& pt) +{ + return TDisparity2Depth(Q, u, d, pt); +} +float Image::Disparity2Depth(const Matrix4x4& Q, const Point2f& u, float d, Point2f& pt) +{ + return TDisparity2Depth(Q, u, d, pt); +} +// converts the given disparity at the rectified image coordinates into +// the distance from the camera position to the corresponding 3D point; +// note: the distance is the same in both rectified and un-rectified cameras +// - K is the camera matrix of the un-rectified image +float Image::Disparity2Distance(const Matrix3x3& K, const Matrix4x4& Q, const Point2f& u, float d) +{ + Point2f pt; + const float depth(Disparity2Depth(Q, u, d, pt)); + return (float)(SQRT(SQUARE(pt.x-K(0,2))+SQUARE(pt.y-K(1,2))+SQUARE(K(0,0)))*depth/K(0,0)); +} +// converts the given depth at the un-rectified image coordinates to +// the disparity corresponding to the rectified image coordinates +bool Image::Depth2Disparity(const Matrix4x4& Q, const Point2f& u, float d, float& disparity) +{ + const REAL w((Q(3,0)*u.x + Q(3,1)*u.y + Q(3,2))*d + Q(3,3)); + if (ISZERO(w)) + return false; + const REAL z((Q(2,0)*u.x + Q(2,1)*u.y + Q(2,2))*d + Q(2,3)); + disparity = -(float)(z/w); + return true; +} +/*----------------------------------------------------------------*/ diff --git a/libs/MVS/Image.h b/libs/MVS/Image.h index 627ec5cee..85a4f5eaf 100644 --- a/libs/MVS/Image.h +++ b/libs/MVS/Image.h @@ -108,10 +108,20 @@ class MVS_API Image float ResizeImage(unsigned nMaxResolution=0); unsigned RecomputeMaxResolution(unsigned& level, unsigned minImageSize, unsigned maxImageSize=INT_MAX) const; + Image GetImage(const PlatformArr& platforms, double scale, bool bUseImage=true) const; Camera GetCamera(const PlatformArr& platforms, const Image8U::Size& resolution) const; void UpdateCamera(const PlatformArr& platforms); REAL ComputeFOV(int dir) const; + static bool StereoRectifyImages(const Image& image1, const Image& image2, const Point3fArr& points1, const Point3fArr& points2, Image8U3& rectifiedImage1, Image8U3& rectifiedImage2, Image8U& mask1, Image8U& mask2, Matrix3x3& H, Matrix4x4& Q); + static void ScaleStereoRectification(Matrix3x3& H, Matrix4x4& Q, REAL scale); + static float Disparity2Depth(const Matrix4x4& Q, const ImageRef& u, float d); + static float Disparity2Depth(const Matrix4x4& Q, const Point2f& u, float d); + static float Disparity2Depth(const Matrix4x4& Q, const ImageRef& u, float d, Point2f& pt); + static float Disparity2Depth(const Matrix4x4& Q, const Point2f& u, float d, Point2f& pt); + static float Disparity2Distance(const Matrix3x3& K, const Matrix4x4& Q, const Point2f& u, float d); + static bool Depth2Disparity(const Matrix4x4& Q, const Point2f& u, float d, float& disparity); + float GetNormalizationScale() const { ASSERT(width > 0 && height > 0); return camera.GetNormalizationScale(width, height); diff --git a/libs/MVS/Scene.h b/libs/MVS/Scene.h index 48a6ca6e3..b10286029 100644 --- a/libs/MVS/Scene.h +++ b/libs/MVS/Scene.h @@ -35,9 +35,8 @@ // I N C L U D E S ///////////////////////////////////////////////// -#include "DepthMap.h" -#include "Mesh.h" #include "SceneDensify.h" +#include "Mesh.h" // D E F I N E S /////////////////////////////////////////////////// @@ -82,7 +81,7 @@ class MVS_API Scene bool ExportCamerasMLP(const String& fileName, const String& fileNameScene) const; // Dense reconstruction - bool DenseReconstruction(); + bool DenseReconstruction(int nFusionMode=0); bool ComputeDepthMaps(DenseDepthMapData& data); void DenseReconstructionEstimate(void*); void DenseReconstructionFilter(void*); diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index 82473bd05..9843e3bdd 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -31,12 +31,9 @@ #include "Common.h" #include "Scene.h" +#include "SceneDensify.h" // MRF: view selection #include "../Math/TRWS/MRFEnergy.h" -// CGAL: depth-map initialization -#include -#include -#include using namespace MVS; @@ -364,99 +361,6 @@ bool DepthMapsData::InitViews(DepthData& depthData, IIndex idxNeighbor, IIndex n } // InitViews /*----------------------------------------------------------------*/ -namespace CGAL { -typedef CGAL::Simple_cartesian kernel_t; -typedef CGAL::Projection_traits_xy_3 Geometry; -typedef CGAL::Delaunay_triangulation_2 Delaunay; -typedef CGAL::Delaunay::Face_circulator FaceCirculator; -typedef CGAL::Delaunay::Face_handle FaceHandle; -typedef CGAL::Delaunay::Vertex_circulator VertexCirculator; -typedef CGAL::Delaunay::Vertex_handle VertexHandle; -typedef kernel_t::Point_3 Point; -} - -// triangulate in-view points, generating a 2D mesh -// return also the estimated depth boundaries (min and max depth) -std::pair TriangulatePointsDelaunay(CGAL::Delaunay& delaunay, const Scene& scene, const DepthData::ViewData& image, const IndexArr& points) -{ - ASSERT(sizeof(Point3) == sizeof(X3D)); - ASSERT(sizeof(Point3) == sizeof(CGAL::Point)); - std::pair depthBounds(FLT_MAX, 0.f); - for (uint32_t idx: points) { - const Point3f pt(image.camera.ProjectPointP3(scene.pointcloud.points[idx])); - delaunay.insert(CGAL::Point(pt.x/pt.z, pt.y/pt.z, pt.z)); - if (depthBounds.first > pt.z) - depthBounds.first = pt.z; - if (depthBounds.second < pt.z) - depthBounds.second = pt.z; - } - // if full size depth-map requested - if (OPTDENSE::bAddCorners) { - typedef TIndexScore DepthDist; - typedef CLISTDEF0(DepthDist) DepthDistArr; - typedef Eigen::Map< Eigen::VectorXf, Eigen::Unaligned, Eigen::InnerStride<2> > FloatMap; - // add the four image corners at the average depth - const CGAL::VertexHandle vcorners[] = { - delaunay.insert(CGAL::Point(0, 0, image.pImageData->avgDepth)), - delaunay.insert(CGAL::Point(image.image.width(), 0, image.pImageData->avgDepth)), - delaunay.insert(CGAL::Point(0, image.image.height(), image.pImageData->avgDepth)), - delaunay.insert(CGAL::Point(image.image.width(), image.image.height(), image.pImageData->avgDepth)) - }; - // compute average depth from the closest 3 directly connected faces, - // weighted by the distance - const size_t numPoints = 3; - for (int i=0; i<4; ++i) { - const CGAL::VertexHandle vcorner = vcorners[i]; - CGAL::FaceCirculator cfc(delaunay.incident_faces(vcorner)); - if (cfc == 0) - continue; // normally this should never happen - const CGAL::FaceCirculator done(cfc); - Point3d& poszA = reinterpret_cast(vcorner->point()); - const Point2d& posA = reinterpret_cast(poszA); - const Ray3d rayA(Point3d::ZERO, normalized(image.camera.TransformPointI2C(poszA))); - DepthDistArr depths(0, numPoints); - do { - CGAL::FaceHandle fc(cfc->neighbor(cfc->index(vcorner))); - if (fc == delaunay.infinite_face()) - continue; - for (int j=0; j<4; ++j) - if (fc->has_vertex(vcorners[j])) - goto Continue; - // compute the depth as the intersection of the corner ray with - // the plane defined by the face's vertices - { - const Point3d& poszB0 = reinterpret_cast(fc->vertex(0)->point()); - const Point3d& poszB1 = reinterpret_cast(fc->vertex(1)->point()); - const Point3d& poszB2 = reinterpret_cast(fc->vertex(2)->point()); - const Planed planeB( - image.camera.TransformPointI2C(poszB0), - image.camera.TransformPointI2C(poszB1), - image.camera.TransformPointI2C(poszB2) - ); - const Point3d poszB(rayA.Intersects(planeB)); - if (poszB.z <= 0) - continue; - const Point2d posB(( - reinterpret_cast(poszB0)+ - reinterpret_cast(poszB1)+ - reinterpret_cast(poszB2))/3.f - ); - const double dist(norm(posB-posA)); - depths.StoreTop(DepthDist(CLAMP((float)poszB.z,depthBounds.first,depthBounds.second), INVERT((float)dist))); - } - Continue:; - } while (++cfc != done); - if (depths.GetSize() != numPoints) - continue; // normally this should never happen - FloatMap vecDists(&depths[0].score, numPoints); - vecDists *= 1.f/vecDists.sum(); - FloatMap vecDepths(&depths[0].idx, numPoints); - poszA.z = vecDepths.dot(vecDists); - } - } - return depthBounds; -} - // roughly estimate depth and normal maps by triangulating the sparse point cloud // and interpolating normal and depth for all pixels bool DepthMapsData::InitDepthMap(DepthData& depthData) @@ -464,58 +368,21 @@ bool DepthMapsData::InitDepthMap(DepthData& depthData) TD_TIMER_STARTD(); ASSERT(depthData.images.GetSize() > 1 && !depthData.points.IsEmpty()); - const DepthData::ViewData& image(depthData.images.First()); - ASSERT(!image.image.empty()); - - // triangulate in-view points - CGAL::Delaunay delaunay; - const std::pair thDepth(TriangulatePointsDelaunay(delaunay, scene, image, depthData.points)); - depthData.dMin = thDepth.first*0.9f; - depthData.dMax = thDepth.second*1.1f; - - // create rough depth-map by interpolating inside triangles - const Camera& camera = image.camera; - depthData.depthMap.create(image.image.size()); - depthData.normalMap.create(image.image.size()); - if (!OPTDENSE::bAddCorners) { - depthData.depthMap.setTo(Depth(0)); - depthData.normalMap.setTo(0.f); - } - struct RasterDepthDataPlaneData { - const Camera& P; - DepthMap& depthMap; - NormalMap& normalMap; - Point3f normal; - Point3f normalPlane; - inline void operator()(const ImageRef& pt) { - if (!depthMap.isInside(pt)) - return; - const Depth z(INVERT(normalPlane.dot(P.TransformPointI2C(Point2f(pt))))); - if (z <= 0) // due to numerical instability - return; - depthMap(pt) = z; - normalMap(pt) = normal; - } - }; - RasterDepthDataPlaneData data = {camera, depthData.depthMap, depthData.normalMap}; - for (CGAL::Delaunay::Face_iterator it=delaunay.faces_begin(); it!=delaunay.faces_end(); ++it) { - const CGAL::Delaunay::Face& face = *it; - const Point3f i0(reinterpret_cast(face.vertex(0)->point())); - const Point3f i1(reinterpret_cast(face.vertex(1)->point())); - const Point3f i2(reinterpret_cast(face.vertex(2)->point())); - // compute the plane defined by the 3 points - const Point3f c0(camera.TransformPointI2C(i0)); - const Point3f c1(camera.TransformPointI2C(i1)); - const Point3f c2(camera.TransformPointI2C(i2)); - const Point3f edge1(c1-c0); - const Point3f edge2(c2-c0); - data.normal = normalized(edge2.cross(edge1)); - data.normalPlane = data.normal * INVERT(data.normal.dot(c0)); - // draw triangle and for each pixel compute depth as the ray intersection with the plane - Image8U::RasterizeTriangle(reinterpret_cast(i2), reinterpret_cast(i1), reinterpret_cast(i0), data); + const DepthData::ViewData& image(depthData.GetView()); + TriangulatePoints2DepthMap(image, scene.pointcloud, depthData.points, depthData.depthMap, depthData.normalMap, depthData.dMin, depthData.dMax); + depthData.dMin *= 0.9f; + depthData.dMax *= 1.1f; + + #if TD_VERBOSE != TD_VERBOSE_OFF + // save rough depth map as image + if (g_nVerbosityLevel > 4) { + ExportDepthMap(ComposeDepthFilePath(image.GetID(), "init.png"), depthData.depthMap); + ExportNormalMap(ComposeDepthFilePath(image.GetID(), "init.normal.png"), depthData.normalMap); + ExportPointCloud(ComposeDepthFilePath(image.GetID(), "init.ply"), *depthData.images.First().pImageData, depthData.depthMap, depthData.normalMap); } + #endif - DEBUG_ULTIMATE("Depth-map %3u roughly estimated from %u sparse points: %dx%d (%s)", &depthData-arrDepthData.Begin(), depthData.points.GetSize(), image.image.width(), image.image.height(), TD_TIMER_GET_FMT().c_str()); + DEBUG_ULTIMATE("Depth-map %3u roughly estimated from %u sparse points: %dx%d (%s)", image.GetID(), depthData.points.size(), image.image.width(), image.image.height(), TD_TIMER_GET_FMT().c_str()); return true; } // InitDepthMap /*----------------------------------------------------------------*/ @@ -668,14 +535,6 @@ bool DepthMapsData::EstimateDepthMap(IIndex idxImage) } else { // compute rough estimates using the sparse point-cloud InitDepthMap(depthData); - #if TD_VERBOSE != TD_VERBOSE_OFF - // save rough depth map as image - if (g_nVerbosityLevel > 4) { - ExportDepthMap(ComposeDepthFilePath(image.GetID(), "init.png"), depthData.depthMap); - ExportNormalMap(ComposeDepthFilePath(image.GetID(), "init.normal.png"), depthData.normalMap); - ExportPointCloud(ComposeDepthFilePath(image.GetID(), "init.ply"), *depthData.images.First().pImageData, depthData.depthMap, depthData.normalMap); - } - #endif } // init integral images and index to image-ref map for the reference data @@ -1316,6 +1175,7 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b // find best connected images IndexScoreArr connections(0, scene.images.GetSize()); size_t nPointsEstimate(0); + bool bNormalMap(true); FOREACH(i, scene.images) { DepthData& depthData = arrDepthData[i]; if (!depthData.IsValid()) @@ -1327,6 +1187,8 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b connection.idx = i; connection.score = (float)scene.images[i].neighbors.GetSize(); nPointsEstimate += ROUND2INT(depthData.depthMap.area()*(0.5f/*valid*/*0.3f/*new*/)); + if (depthData.normalMap.empty()) + bNormalMap = false; } connections.Sort(); @@ -1339,6 +1201,8 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b typedef cList DepthIndexArr; DepthIndexArr arrDepthIdx(scene.images.GetSize()); ProjsArr projs(0, nPointsEstimate); + if (bEstimateNormal && !bNormalMap) + bEstimateNormal = false; pointcloud.points.Reserve(nPointsEstimate); pointcloud.pointViews.Reserve(nPointsEstimate); pointcloud.pointWeights.Reserve(nPointsEstimate); @@ -1394,7 +1258,7 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b REAL confidence(weights.emplace_back(Conf2Weight(depthData.confMap(x),depth))); ProjArr& pointProjs = projs.AddEmpty(); pointProjs.Insert(Proj(x)); - const PointCloud::Normal normal(imageData.camera.R.t()*Cast(depthData.normalMap(x))); + const PointCloud::Normal normal(bNormalMap ? imageData.camera.R.t()*Cast(depthData.normalMap(x)) : Normal(0,0,-1)); ASSERT(ISEQUAL(norm(normal), 1.f)); // check the projection in the neighbor depth-maps Point3 X(point*confidence); @@ -1422,7 +1286,7 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b continue; if (IsDepthSimilar(pt.z, depthB, OPTDENSE::fDepthDiffThreshold)) { // check if normals agree - const PointCloud::Normal normalB(imageDataB.camera.R.t()*Cast(depthDataB.normalMap(xB))); + const PointCloud::Normal normalB(bNormalMap ? imageDataB.camera.R.t()*Cast(depthDataB.normalMap(xB)) : Normal(0,0,-1)); ASSERT(ISEQUAL(norm(normalB), 1.f)); if (normal.dot(normalB) > normalError) { // add view to the 3D point @@ -1520,19 +1384,48 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b +// S T R U C T S /////////////////////////////////////////////////// + +DenseDepthMapData::DenseDepthMapData(Scene& _scene, int _nFusionMode) + : scene(_scene), depthMaps(_scene), idxImage(0), sem(1), nFusionMode(_nFusionMode) +{ + if (nFusionMode < 0) { + STEREO::SemiGlobalMatcher::CreateThreads(scene.nMaxThreads); + if (nFusionMode == -1) + OPTDENSE::nOptimize &= ~OPTDENSE::OPTIMIZE; + } +} +DenseDepthMapData::~DenseDepthMapData() +{ + if (nFusionMode < 0) + STEREO::SemiGlobalMatcher::DestroyThreads(); +} + +void DenseDepthMapData::SignalCompleteDepthmapFilter() +{ + ASSERT(idxImage > 0); + if (Thread::safeDec(idxImage) == 0) + sem.Signal((unsigned)images.GetSize()*2); +} +/*----------------------------------------------------------------*/ + + + // S T R U C T S /////////////////////////////////////////////////// static void* DenseReconstructionEstimateTmp(void*); static void* DenseReconstructionFilterTmp(void*); -bool Scene::DenseReconstruction() +bool Scene::DenseReconstruction(int nFusionMode) { - DenseDepthMapData data(*this); - + DenseDepthMapData data(*this, nFusionMode); + // estimate depth-maps if (!ComputeDepthMaps(data)) return false; - + if (ABS(nFusionMode) == 1) + return true; + // fuse all depth-maps pointcloud.Release(); data.depthMaps.FuseDepthMaps(pointcloud, OPTDENSE::nEstimateColors == 2, OPTDENSE::nEstimateNormals == 2); @@ -1761,7 +1654,7 @@ void Scene::DenseReconstructionEstimate(void* pData) break; } // try to load already compute depth-map for this image - if (File::access(ComposeDepthFilePath(depthData.GetView().GetID(), "dmap"))) { + if (data.nFusionMode >= 0 && File::access(ComposeDepthFilePath(depthData.GetView().GetID(), "dmap"))) { if (OPTDENSE::nOptimize & OPTDENSE::OPTIMIZE) { if (!depthData.Load(ComposeDepthFilePath(depthData.GetView().GetID(), "dmap"))) { VERBOSE("error: invalid depth-map '%s'", ComposeDepthFilePath(depthData.GetView().GetID(), "dmap").c_str()); @@ -1784,7 +1677,23 @@ void Scene::DenseReconstructionEstimate(void* pData) data.events.AddEvent(new EVTProcessImage((uint32_t)Thread::safeInc(data.idxImage))); // extract depth map data.sem.Wait(); - data.depthMaps.EstimateDepthMap(data.images[evtImage.idxImage]); + if (data.nFusionMode >= 0) { + // extract depth-map using Patch-Match algorithm + data.depthMaps.EstimateDepthMap(data.images[evtImage.idxImage]); + } else { + // extract disparity-maps using SGM algorithm + if (data.nFusionMode == -1) { + data.sgm.Match(*this, data.images[evtImage.idxImage], OPTDENSE::nNumViews); + } else { + // fuse existing disparity-maps + const IIndex idx(data.images[evtImage.idxImage]); + DepthData& depthData(data.depthMaps.arrDepthData[idx]); + data.sgm.Fuse(*this, data.images[evtImage.idxImage], OPTDENSE::nNumViews, 2, depthData.depthMap, depthData.confMap); + if (OPTDENSE::nEstimateNormals == 2) + EstimateNormalMap(depthData.images.front().camera.K, depthData.depthMap, depthData.normalMap); + depthData.dMin = ZEROTOLERANCE(); depthData.dMax = FLT_MAX; + } + } data.sem.Signal(); if (OPTDENSE::nOptimize & OPTDENSE::OPTIMIZE) { // optimize depth-map @@ -1838,7 +1747,8 @@ void Scene::DenseReconstructionEstimate(void* pData) } #endif // save compute depth-map for this image - depthData.Save(ComposeDepthFilePath(depthData.GetView().GetID(), "dmap")); + if (!depthData.depthMap.empty()) + depthData.Save(ComposeDepthFilePath(depthData.GetView().GetID(), "dmap")); depthData.ReleaseImages(); depthData.Release(); data.progress->operator++(); diff --git a/libs/MVS/SceneDensify.h b/libs/MVS/SceneDensify.h index 6d7d0efe0..7a42215bd 100644 --- a/libs/MVS/SceneDensify.h +++ b/libs/MVS/SceneDensify.h @@ -35,6 +35,8 @@ // I N C L U D E S ///////////////////////////////////////////////// +#include "SemiGlobalMatcher.h" + // S T R U C T S /////////////////////////////////////////////////// @@ -89,15 +91,13 @@ struct MVS_API DenseDepthMapData { SEACAVE::EventQueue events; // internal events queue (processed by the working threads) Semaphore sem; CAutoPtr progress; + int nFusionMode; + STEREO::SemiGlobalMatcher sgm; - DenseDepthMapData(Scene& _scene) - : scene(_scene), depthMaps(_scene), idxImage(0), sem(1) {} + DenseDepthMapData(Scene& _scene, int _nFusionMode=0); + ~DenseDepthMapData(); - void SignalCompleteDepthmapFilter() { - ASSERT(idxImage > 0); - if (Thread::safeDec(idxImage) == 0) - sem.Signal((unsigned)images.GetSize()*2); - } + void SignalCompleteDepthmapFilter(); }; /*----------------------------------------------------------------*/ diff --git a/libs/MVS/SemiGlobalMatcher.cpp b/libs/MVS/SemiGlobalMatcher.cpp new file mode 100644 index 000000000..6549acbda --- /dev/null +++ b/libs/MVS/SemiGlobalMatcher.cpp @@ -0,0 +1,2361 @@ +/* +* SemiGlobalMatcher.cpp +* +* Copyright (c) 2014-2015 SEACAVE +* +* Author(s): +* +* cDc +* +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* +* Additional Terms: +* +* You are required to preserve legal notices and author attributions in +* that material or in the Appropriate Legal Notices displayed by works +* containing it. +*/ + +#include "Common.h" +#include "SemiGlobalMatcher.h" +#include "Scene.h" + +using namespace MVS; + +using namespace STEREO; + + +// D E F I N E S /////////////////////////////////////////////////// + +// uncomment to enable OpenCV filter demo +//#define _USE_FILTER_DEMO + + +// S T R U C T S /////////////////////////////////////////////////// + +#ifdef _USE_FILTER_DEMO +#include "opencv2/ximgproc/disparity_filter.hpp" + +const cv::String keys = +"{help h usage ? | | print this message }" +"{GT |None | optional ground-truth disparity (MPI-Sintel or Middlebury format) }" +"{dst_path |None | optional path to save the resulting filtered disparity map }" +"{dst_raw_path |None | optional path to save raw disparity map before filtering }" +"{algorithm |bm | stereo matching method (bm or sgbm) }" +"{filter |wls_conf| used post-filtering (wls_conf or wls_no_conf) }" +"{no-display | | don't display results }" +"{no-downscale | | force stereo matching on full-sized views to improve quality }" +"{dst_conf_path |None | optional path to save the confidence map used in filtering }" +"{vis_mult |1.0 | coefficient used to scale disparity map visualizations }" +"{max_disparity |160 | parameter of stereo matching }" +"{window_size |-1 | parameter of stereo matching }" +"{wls_lambda |8000.0 | parameter of post-filtering }" +"{wls_sigma |1.5 | parameter of post-filtering }" +; + +cv::Rect computeROI(cv::Size2i src_sz, cv::Ptr matcher_instance) +{ + int min_disparity = matcher_instance->getMinDisparity(); + int num_disparities = matcher_instance->getNumDisparities(); + int block_size = matcher_instance->getBlockSize(); + + int bs2 = block_size/2; + int minD = min_disparity, maxD = min_disparity + num_disparities - 1; + + int xmin = maxD + bs2; + int xmax = src_sz.width + minD - bs2; + int ymin = bs2; + int ymax = src_sz.height - bs2; + + return cv::Rect(xmin, ymin, xmax - xmin, ymax - ymin); +} + +int disparityFiltering(cv::Mat left, cv::Mat right, int argc, const LPCSTR* argv) +{ + cv::CommandLineParser parser(argc,argv,keys); + parser.about("Disparity Filtering Demo"); + if(parser.has("help")) + { + parser.printMessage(); + return 0; + } + + cv::String GT_path = parser.get("GT"); + cv::String dst_path = parser.get("dst_path"); + cv::String dst_raw_path = parser.get("dst_raw_path"); + cv::String dst_conf_path = parser.get("dst_conf_path"); + cv::String algo = parser.get("algorithm"); + cv::String filter = parser.get("filter"); + bool no_display = parser.has("no-display"); + bool no_downscale = parser.has("no-downscale"); + int max_disp = parser.get("max_disparity"); + double lambda = parser.get("wls_lambda"); + double sigma = parser.get("wls_sigma"); + double vis_mult = parser.get("vis_mult"); + + int wsize; + if(parser.get("window_size")>=0) //user provided window_size value + wsize = parser.get("window_size"); + else + { + if(algo=="sgbm") + wsize = 3; //default window size for SGBM + else if(!no_downscale && algo=="bm" && filter=="wls_conf") + wsize = 7; //default window size for BM on downscaled views (downscaling is performed only for wls_conf) + else + wsize = 15; //default window size for BM on full-sized views + } + if(!parser.check()) + { + parser.printErrors(); + return -1; + } + + cv::Mat GT_disp; + bool noGT(cv::ximgproc::readGT(GT_path,GT_disp)!=0); + + cv::Mat left_for_matcher, right_for_matcher; + cv::Mat left_disp,right_disp; + cv::Mat filtered_disp; + cv::Mat conf_map(left.rows,left.cols,CV_8U); + conf_map = cv::Scalar(255); + cv::Rect ROI; + cv::Ptr wls_filter; + double matching_time, filtering_time; + if(max_disp<=0 || max_disp%16!=0) + { + std::cout<<"Incorrect max_disparity value: it should be positive and divisible by 16"; + return -1; + } + if(wsize<=0 || wsize%2!=1) + { + std::cout<<"Incorrect window_size value: it should be positive and odd"; + return -1; + } + if(filter=="wls_conf") // filtering with confidence (significantly better quality than wls_no_conf) + { + if(!no_downscale) + { + // downscale the views to speed-up the matching stage, as we will need to compute both left + // and right disparity maps for confidence map computation + //! [downscale] + max_disp/=2; + if(max_disp%16!=0) + max_disp += 16-(max_disp%16); + resize(left ,left_for_matcher ,cv::Size(),0.5,0.5, cv::INTER_LINEAR_EXACT); + resize(right,right_for_matcher,cv::Size(),0.5,0.5, cv::INTER_LINEAR_EXACT); + //! [downscale] + } + else + { + left_for_matcher = left.clone(); + right_for_matcher = right.clone(); + } + + if(algo=="bm") + { + //! [matching] + cv::Ptr left_matcher = cv::StereoBM::create(max_disp,wsize); + wls_filter = cv::ximgproc::createDisparityWLSFilter(left_matcher); + cv::Ptr right_matcher = cv::ximgproc::createRightMatcher(left_matcher); + + cvtColor(left_for_matcher, left_for_matcher, cv::COLOR_BGR2GRAY); + cvtColor(right_for_matcher, right_for_matcher, cv::COLOR_BGR2GRAY); + + matching_time = (double)cv::getTickCount(); + left_matcher-> compute(left_for_matcher, right_for_matcher,left_disp); + right_matcher->compute(right_for_matcher,left_for_matcher, right_disp); + matching_time = ((double)cv::getTickCount() - matching_time)/cv::getTickFrequency(); + //! [matching] + } + else if(algo=="sgbm") + { + cv::Ptr left_matcher = cv::StereoSGBM::create(-max_disp/2+1,max_disp,wsize); + left_matcher->setP1(24*wsize*wsize); + left_matcher->setP2(96*wsize*wsize); + left_matcher->setPreFilterCap(63); + left_matcher->setMode(cv::StereoSGBM::MODE_SGBM_3WAY); + wls_filter = cv::ximgproc::createDisparityWLSFilter(left_matcher); + cv::Ptr right_matcher = cv::ximgproc::createRightMatcher(left_matcher); + + matching_time = (double)cv::getTickCount(); + left_matcher-> compute(left_for_matcher, right_for_matcher,left_disp); + right_matcher->compute(right_for_matcher,left_for_matcher, right_disp); + matching_time = ((double)cv::getTickCount() - matching_time)/cv::getTickFrequency(); + } + else + { + std::cout<<"Unsupported algorithm"; + return -1; + } + + //! [filtering] + wls_filter->setLambda(lambda); + wls_filter->setSigmaColor(sigma); + filtering_time = (double)cv::getTickCount(); + wls_filter->filter(left_disp,left,filtered_disp,right_disp); + filtering_time = ((double)cv::getTickCount() - filtering_time)/cv::getTickFrequency(); + //! [filtering] + conf_map = wls_filter->getConfidenceMap(); + + // Get the ROI that was used in the last filter call: + ROI = wls_filter->getROI(); + if(!no_downscale) + { + // upscale raw disparity and ROI back for a proper comparison: + resize(left_disp,left_disp,cv::Size(),2.0,2.0,cv::INTER_LINEAR_EXACT); + left_disp = left_disp*2.0; + ROI = cv::Rect(ROI.x*2,ROI.y*2,ROI.width*2,ROI.height*2); + } + } + else if(filter=="wls_no_conf") + { + /* There is no convenience function for the case of filtering with no confidence, so we + will need to set the ROI and matcher parameters manually */ + + left_for_matcher = left.clone(); + right_for_matcher = right.clone(); + + if(algo=="bm") + { + cv::Ptr matcher = cv::StereoBM::create(max_disp,wsize); + matcher->setTextureThreshold(0); + matcher->setUniquenessRatio(0); + cvtColor(left_for_matcher, left_for_matcher, cv::COLOR_BGR2GRAY); + cvtColor(right_for_matcher, right_for_matcher, cv::COLOR_BGR2GRAY); + ROI = computeROI(left_for_matcher.size(),matcher); + wls_filter = cv::ximgproc::createDisparityWLSFilterGeneric(false); + wls_filter->setDepthDiscontinuityRadius((int)ceil(0.33*wsize)); + + matching_time = (double)cv::getTickCount(); + matcher->compute(left_for_matcher,right_for_matcher,left_disp); + matching_time = ((double)cv::getTickCount() - matching_time)/cv::getTickFrequency(); + } + else if(algo=="sgbm") + { + cv::Ptr matcher = cv::StereoSGBM::create(0,max_disp,wsize); + matcher->setUniquenessRatio(0); + matcher->setDisp12MaxDiff(1000000); + matcher->setSpeckleWindowSize(0); + matcher->setP1(24*wsize*wsize); + matcher->setP2(96*wsize*wsize); + matcher->setMode(cv::StereoSGBM::MODE_SGBM_3WAY); + ROI = computeROI(left_for_matcher.size(),matcher); + wls_filter = cv::ximgproc::createDisparityWLSFilterGeneric(false); + wls_filter->setDepthDiscontinuityRadius((int)ceil(0.5*wsize)); + + matching_time = (double)cv::getTickCount(); + matcher->compute(left_for_matcher,right_for_matcher,left_disp); + matching_time = ((double)cv::getTickCount() - matching_time)/cv::getTickFrequency(); + } + else + { + std::cout<<"Unsupported algorithm"; + return -1; + } + + wls_filter->setLambda(lambda); + wls_filter->setSigmaColor(sigma); + filtering_time = (double)cv::getTickCount(); + wls_filter->filter(left_disp,left,filtered_disp,cv::Mat(),ROI); + filtering_time = ((double)cv::getTickCount() - filtering_time)/cv::getTickFrequency(); + } + else + { + std::cout<<"Unsupported filter"; + return -1; + } + + //collect and print all the stats: + std::cout.precision(2); + std::cout<<"Matching time: "< ftzero ? ftzero * 2 : x - OFS + ftzero); + uint8_t val0 = tab[0 + OFS]; + + #ifdef _USE_SSE + volatile bool useSIMD = cv::checkHardwareSupport(CV_CPU_SSE2); + #endif + + ASSERT(src.type() == CV_8U); + const cv::Size size(src.size()); + dst.create(size, src.type()); + int y; + for (y = 0; y < size.height - 1; y += 2) { + const uint8_t* srow1 = src.ptr(y); + const uint8_t* srow0 = y > 0 ? srow1 - src.step : size.height > 1 ? srow1 + src.step : srow1; + const uint8_t* srow2 = y < size.height - 1 ? srow1 + src.step : size.height > 1 ? srow1 - src.step : srow1; + const uint8_t* srow3 = y < size.height - 2 ? srow1 + src.step * 2 : srow1; + uint8_t* dptr0 = dst.ptr(y); + uint8_t* dptr1 = dptr0 + dst.step; + + dptr0[0] = dptr0[size.width - 1] = dptr1[0] = dptr1[size.width - 1] = val0; + int x = 1; + + #ifdef _USE_SSE + if (useSIMD) { + __m128i z = _mm_setzero_si128(), ftz = _mm_set1_epi16((short)ftzero); + __m128i ftz2 = _mm_set1_epi8(cv::saturate_cast(ftzero * 2)); + for (; x <= size.width - 9; x += 8) { + __m128i c0 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)(srow0 + x - 1)), z); + __m128i c1 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)(srow1 + x - 1)), z); + __m128i d0 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)(srow0 + x + 1)), z); + __m128i d1 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)(srow1 + x + 1)), z); + + d0 = _mm_sub_epi16(d0, c0); + d1 = _mm_sub_epi16(d1, c1); + + __m128i c2 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)(srow2 + x - 1)), z); + __m128i c3 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)(srow3 + x - 1)), z); + __m128i d2 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)(srow2 + x + 1)), z); + __m128i d3 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)(srow3 + x + 1)), z); + + d2 = _mm_sub_epi16(d2, c2); + d3 = _mm_sub_epi16(d3, c3); + + __m128i v0 = _mm_add_epi16(d0, _mm_add_epi16(d2, _mm_add_epi16(d1, d1))); + __m128i v1 = _mm_add_epi16(d1, _mm_add_epi16(d3, _mm_add_epi16(d2, d2))); + v0 = _mm_packus_epi16(_mm_add_epi16(v0, ftz), _mm_add_epi16(v1, ftz)); + v0 = _mm_min_epu8(v0, ftz2); + + _mm_storel_epi64((__m128i*)(dptr0 + x), v0); + _mm_storel_epi64((__m128i*)(dptr1 + x), _mm_unpackhi_epi64(v0, v0)); + } + } + #endif + + for (; x < size.width - 1; x++) { + int d0 = srow0[x + 1] - srow0[x - 1], d1 = srow1[x + 1] - srow1[x - 1], + d2 = srow2[x + 1] - srow2[x - 1], d3 = srow3[x + 1] - srow3[x - 1]; + int v0 = tab[d0 + d1 * 2 + d2 + OFS]; + int v1 = tab[d1 + d2 * 2 + d3 + OFS]; + dptr0[x] = (uint8_t)v0; + dptr1[x] = (uint8_t)v1; + } + } + + for (; y < size.height; y++) { + uint8_t* dptr = dst.ptr(y); + for (int x = 0; x < size.width; x++) + dptr[x] = val0; + } +} +/*----------------------------------------------------------------*/ + + + +// S T R U C T S /////////////////////////////////////////////////// + +enum EVENT_TYPE { + EVT_JOB = 0, + EVT_CLOSE, +}; + +class EVTClose : public Event +{ +public: + EVTClose() : Event(EVT_CLOSE) {} +}; +class EVTPixelProcess : public Event +{ +public: + typedef std::function FncPixel; + const cv::Size size; + volatile Thread::safe_t& idxPixel; + const FncPixel fncPixel; + bool Run(void*) override { + const int numPixels(size.area()); + int idx; + while ((idx=(int)Thread::safeInc(idxPixel)) < numPixels) + fncPixel(idx, idx/size.width, idx%size.width); + return true; + } + EVTPixelProcess(cv::Size s, volatile Thread::safe_t& idx, FncPixel f) : Event(EVT_JOB), size(s), idxPixel(idx), fncPixel(f) {} +}; +class EVTPixelAccumInc : public Event +{ +public: + typedef std::function FncPixel; + const int numPixels; + volatile Thread::safe_t& idxPixel; + const FncPixel fncPixel; + bool Run(void*) override { + int idx; + while ((idx=(int)Thread::safeInc(idxPixel)) < numPixels) + fncPixel(idx); + return true; + } + EVTPixelAccumInc(int s, volatile Thread::safe_t& idx, FncPixel f) : Event(EVT_JOB), numPixels(s), idxPixel(idx), fncPixel(f) {} +}; +class EVTPixelAccumDec : public Event +{ +public: + typedef std::function FncPixel; + volatile Thread::safe_t& idxPixel; + const FncPixel fncPixel; + bool Run(void*) override { + int idx; + while ((idx=(int)Thread::safeDec(idxPixel)) >= 0) + fncPixel(idx); + return true; + } + EVTPixelAccumDec(volatile Thread::safe_t& idx, FncPixel f) : Event(EVT_JOB), idxPixel(idx), fncPixel(f) {} +}; +/*----------------------------------------------------------------*/ + + + +// S T R U C T S /////////////////////////////////////////////////// + +// - P1 and P2s are algorithm constants very similar to those from the original SGM algorithm; +// they are set to defaults according to the patch size +// - alpha & beta form the final P2 as P2*(1+alpha*e^(-DI^2/(2*beta^2))) +// where DI is the difference in image intensity I(x)-I(x_prev) in [0,255] range +// - subpixelSteps represents how much sub-pixel accuracy is searched/stored; +// if 1 no sub-pixel precision, if for example 4 a 0.25 sub-pixel accuracy is stored; +// the stored value is quantized and represented as integer: val=(float)valStored/subpixelSteps +SemiGlobalMatcher::SemiGlobalMatcher(SgmSubpixelMode _subpixelMode, Disparity _subpixelSteps, AccumCost _P1, AccumCost P2, float P2alpha, float P2beta) + : + subpixelMode(_subpixelMode), + subpixelSteps(_subpixelSteps), + P1(_P1), P2s(GenerateP2s(P2, P2alpha, P2beta)) +{ +} + +SemiGlobalMatcher::~SemiGlobalMatcher() +{ +} + +CLISTDEF0IDX(SemiGlobalMatcher::AccumCost,int) SemiGlobalMatcher::GenerateP2s(AccumCost P2, float P2alpha, float P2beta) +{ + CLISTDEF0IDX(AccumCost,int) P2s(256); + FOREACH(i, P2s) + P2s[i] = (AccumCost)ROUND2INT(P2*(1.f+P2alpha*EXP(-SQUARE((float)i)/(2.f*SQUARE(P2beta))))); + return P2s; +} + + +// Compute SGM stereo for this image and each of the neighbor views: +// - minResolution is the resolution of the top of the pyramid for tSGM; +// can be 0 to force the standard SGM algorithm +void SemiGlobalMatcher::Match(const Scene& scene, IIndex idxImage, IIndex numNeighbors, unsigned minResolution) +{ + const Image& leftImage = scene.images[idxImage]; + const float fMinScore(MAXF(leftImage.neighbors.front().score*(OPTDENSE::fViewMinScoreRatio*0.1f), OPTDENSE::fViewMinScore)); + FOREACH(idxNeighbor, leftImage.neighbors) { + const ViewScore& neighbor = leftImage.neighbors[idxNeighbor]; + // exclude neighbors that over the limit or too small score + ASSERT(scene.images[neighbor.idx.ID].IsValid()); + if ((numNeighbors && idxNeighbor >= numNeighbors) || + (neighbor.score < fMinScore)) + break; + // check if the disparity-map was already estimated for the same image pairs + const Image& rightImage = scene.images[neighbor.idx.ID]; + const String pairName(MAKE_PATH(String::FormatString("%04u_%04u", leftImage.ID, rightImage.ID))); + if (File::isPresent((pairName+".dimap").c_str()) || File::isPresent(MAKE_PATH(String::FormatString("%04u_%04u.dimap", rightImage.ID, leftImage.ID)))) + continue; + TD_TIMER_STARTD(); + IndexArr points; + Matrix3x3 H; Matrix4x4 Q; + ViewData leftData, rightData; + MaskMap leftMaskMap, rightMaskMap; { + // fetch pairs of corresponding image points + //TODO: use precomputed points from SelectViews() + Point3fArr leftPoints, rightPoints; + FOREACH(idxPoint, scene.pointcloud.points) { + const PointCloud::ViewArr& views = scene.pointcloud.pointViews[idxPoint]; + if (views.FindFirst(idxImage) != PointCloud::ViewArr::NO_INDEX) { + points.push_back((uint32_t)idxPoint); + if (views.FindFirst(neighbor.idx.ID) != PointCloud::ViewArr::NO_INDEX) { + const Point3 X(scene.pointcloud.points[idxPoint]); + leftPoints.emplace_back(leftImage.camera.TransformPointW2I3(X)); + rightPoints.emplace_back(rightImage.camera.TransformPointW2I3(X)); + } + } + } + // stereo-rectify image pair + if (!Image::StereoRectifyImages(leftImage, rightImage, leftPoints, rightPoints, leftData.imageColor, rightData.imageColor, leftMaskMap, rightMaskMap, H, Q)) + continue; + ASSERT(leftData.imageColor.size() == rightData.imageColor.size()); + } + #ifdef _USE_FILTER_DEMO + // run openCV implementation + const LPCSTR argv[] = {"disparityFiltering", "-algorithm=sgbm", "-max_disparity=160", "-no-downscale"}; + disparityFiltering(leftData.imageColor, rightData.imageColor, (int)SizeOfArray(argv), argv); + #endif + // color to gray conversion + #if SGM_SIMILARITY == SGM_SIMILARITY_CENSUS + leftData.imageColor.toGray(leftData.imageGray, cv::COLOR_BGR2GRAY, false, true); + rightData.imageColor.toGray(rightData.imageGray, cv::COLOR_BGR2GRAY, false, true); + #else + leftData.imageColor.toGray(leftData.imageGray, cv::COLOR_BGR2GRAY, true, true); + rightData.imageColor.toGray(rightData.imageGray, cv::COLOR_BGR2GRAY, true, true); + #endif + // compute scale used for the disparity estimation + REAL scale(1); + if (minResolution) { + unsigned resolutionLevel(8); + Image8U::computeMaxResolution(leftData.imageGray.width(), leftData.imageGray.height(), resolutionLevel, minResolution); + scale = REAL(1)/MAXF(2,POWI(2,resolutionLevel)); + } + const bool tSGM(!ISEQUAL(scale, REAL(1))); + DisparityMap leftDisparityMap, rightDisparityMap; AccumCostMap costMap; + do { + #if 0 + // export the intermediate disparity-maps + if (!leftDisparityMap.empty()) { + const REAL _scale(scale*0.5); + Matrix3x3 _H(H); Matrix4x4 _Q(Q); + Image::ScaleStereoRectification(_H, _Q, _scale); + ExportDisparityDataRawFull(String::FormatString("%s_%d.dimap", pairName.c_str(), log2i(ROUND2INT(REAL(1)/_scale))), leftDisparityMap, costMap, Image8U::computeResize(leftImage.GetSize(), _scale), H, _Q, 1); + } + #endif + // initialize + const ViewData leftDataLevel(leftData.GetImage(scale)); + const ViewData rightDataLevel(rightData.GetImage(scale)); + const cv::Size size(leftDataLevel.imageGray.size()); + const cv::Size sizeValid(size.width-2*halfWindowSizeX, size.height-2*halfWindowSizeY); + const bool bFirstLevel(leftDisparityMap.empty()); + if (bFirstLevel) { + // initialize the disparity-map with a rough estimate based on the sparse point-cloud + //TODO: remove DepthData::ViewData dependency + Image leftImageLevel(leftImage.GetImage(scene.platforms, scale*0.5, false)); + DepthData::ViewData image; + image.pImageData = &leftImageLevel; // used only for avgDepth + image.image.create(leftImageLevel.GetSize()); + image.camera = leftImageLevel.camera; + DepthMap depthMap; + Depth dMin, dMax; + TriangulatePoints2DepthMap(image, scene.pointcloud, points, depthMap, dMin, dMax); + points.Release(); + Matrix3x3 H2(H); Matrix4x4 Q2(Q); + Image::ScaleStereoRectification(H2, Q2, scale*0.5); + const cv::Size sizeHalf(Image8U::computeResize(size, 0.5)); + const cv::Size sizeValidHalf(sizeHalf.width-2*halfWindowSizeX, sizeHalf.height-2*halfWindowSizeY); + leftDisparityMap.create(sizeValidHalf); + Depth2DisparityMap(depthMap, H2.inv(), Q2.inv(), 1, leftDisparityMap); + // resize masks + cv::resize(leftMaskMap, leftMaskMap, size, 0, 0, cv::INTER_NEAREST); + cv::resize(rightMaskMap, rightMaskMap, size, 0, 0, cv::INTER_NEAREST); + const cv::Rect ROI(halfWindowSizeX,halfWindowSizeY, sizeValid.width,sizeValid.height); + leftMaskMap(ROI).copyTo(leftMaskMap); + rightMaskMap(ROI).copyTo(rightMaskMap); + } else { + // upscale masks + UpscaleMask(leftMaskMap, sizeValid); + UpscaleMask(rightMaskMap, sizeValid); + } + // estimate right-left disparity-map + Index numCosts; + if (tSGM) { + // upscale the disparity-map from the previous level + FlipDirection(leftDisparityMap, rightDisparityMap); + numCosts = Disparity2RangeMap(rightDisparityMap, rightMaskMap, bFirstLevel?11:5, bFirstLevel?33:7); + } else { + // extract global min and max disparities + Range range{std::numeric_limits::max(), std::numeric_limits::min()}; + ASSERT(leftDisparityMap.isContinuous()); + const Disparity* pd = leftDisparityMap.ptr(); + const Disparity* const pde = pd+leftDisparityMap.area(); + do { + const Disparity d(*pd); + if (range.minDisp > d) + range.minDisp = d; + if (range.maxDisp < d) + range.maxDisp = d; + } while (++pd < pde); + // set disparity search range to the global min/max range + const Disparity numDisp(range.numDisp()+16); + const Disparity disp(range.minDisp+range.maxDisp); + range.minDisp = disp-numDisp; + range.maxDisp = disp+numDisp; + maxNumDisp = range.numDisp(); + numCosts = 0; + imagePixels.resize(sizeValid.area()); + for (PixelData& pixel: imagePixels) { + pixel.range = range; + pixel.idx = numCosts; + numCosts += maxNumDisp; + } + } + imageCosts.resize(numCosts); + imageAccumCosts.resize(numCosts); + Match(rightDataLevel, leftDataLevel, rightDisparityMap, costMap); + // estimate left-right disparity-map + if (tSGM) { + numCosts = Disparity2RangeMap(leftDisparityMap, leftMaskMap, bFirstLevel?11:5, bFirstLevel?33:7); + imageCosts.resize(numCosts); + imageAccumCosts.resize(numCosts); + } else { + for (PixelData& pixel: imagePixels) { + const Disparity maxDisp(-pixel.range.minDisp); + pixel.range.minDisp = -pixel.range.maxDisp; + pixel.range.maxDisp = maxDisp; + } + } + Match(leftDataLevel, rightDataLevel, leftDisparityMap, costMap); + // check disparity-map cross-consistency + #if 0 + if (ISEQUAL(scale, REAL(1))) { + cv::Ptr filter = cv::ximgproc::createDisparityWLSFilterGeneric(true); + const cv::Rect rcValid(halfWindowSizeX,halfWindowSizeY, sizeValid.width,sizeValid.height); + Image32F leftDisparityMap32F, rightDisparityMap32F, filtered32F; + leftDisparityMap.convertTo(leftDisparityMap32F, CV_32F, 16); + rightDisparityMap.convertTo(rightDisparityMap32F, CV_32F, 16); + filter->filter(leftDisparityMap32F, leftData.imageColor(rcValid), filtered32F, rightDisparityMap32F); + filtered32F.convertTo(leftDisparityMap, CV_16S, 1.0/16, 0.5); + } else + #endif + if (bFirstLevel) { + // perform a rigorous filtering of the estimated disparity maps in order to + // estimate the common region or interest and set the validity masks + ConsistencyCrossCheck(leftDisparityMap, rightDisparityMap); + ConsistencyCrossCheck(rightDisparityMap, leftDisparityMap); + cv::filterSpeckles(leftDisparityMap, NO_DISP, OPTDENSE::nSpeckleSize, 5); + cv::filterSpeckles(rightDisparityMap, NO_DISP, OPTDENSE::nSpeckleSize, 5); + ExtractMask(leftDisparityMap, leftMaskMap); + ExtractMask(rightDisparityMap, rightMaskMap); + } else { + // simply run a left-right consistency check + ConsistencyCrossCheck(leftDisparityMap, rightDisparityMap); + } + } while ((scale*=2) < REAL(1)+ZEROTOLERANCE()); + #if 0 + // remove speckles + if (OPTDENSE::nSpeckleSize > 0) + cv::filterSpeckles(leftDisparityMap, NO_DISP, OPTDENSE::nSpeckleSize, 5); + #endif + // sub-pixel disparity-map estimation + RefineDisparityMap(leftDisparityMap); + #if 1 + // export disparity-map for the left image + DEBUG_EXTRA("Disparity-map for images %3u and %3u: %dx%d (%s)", leftImage.ID, rightImage.ID, + leftImage.width, leftImage.height, TD_TIMER_GET_FMT().c_str()); + ExportPointCloud(pairName+".ply", leftImage, leftDisparityMap, Q, subpixelSteps); + ExportDisparityMap(pairName+".png", leftDisparityMap); + ExportDisparityDataRawFull(pairName+".dimap", leftDisparityMap, costMap, leftImage.GetSize(), H, Q, subpixelSteps); + #else + // convert disparity-map to final depth-map for the left image + DepthMap depthMap(leftImage.image.size()); ConfidenceMap confMap; + Disparity2DepthMap(leftDisparityMap, costMap, H, Q, subpixelSteps, depthMap, confMap); + DEBUG_EXTRA("Depth-map for images %3u and %3u: %dx%d (%s)", leftImage.ID, rightImage.ID, + depthMap.width(), depthMap.height(), TD_TIMER_GET_FMT().c_str()); + ExportDepthMap(pairName+".png", depthMap); + MVS::ExportPointCloud(pairName+".ply", leftImage, depthMap, NormalMap()); + ExportDepthDataRaw(pairName+".dmap", leftImage.name, IIndexArr{leftImage.ID,rightImage.ID}, leftImage.GetSize(), leftImage.camera.K, leftImage.camera.R, leftImage.camera.C, 0, FLT_MAX, depthMap, NormalMap(), confMap); + #endif + } +} + +void SemiGlobalMatcher::Fuse(const Scene& scene, IIndex idxImage, IIndex numNeighbors, unsigned minViews, DepthMap& depthMap, ConfidenceMap& confMap) +{ + TD_TIMER_STARTD(); + const Image& leftImage = scene.images[idxImage]; + const float fMinScore(MAXF(leftImage.neighbors.front().score*(OPTDENSE::fViewMinScoreRatio*0.1f), OPTDENSE::fViewMinScore)); + struct PairData { + DepthMap depthMap; + DepthRangeMap depthRangeMap; + ConfidenceMap confMap; + PairData(const cv::Size& size) : depthMap(size) {} + }; + // load and put in same space all disparity-maps containing this view + CLISTDEFIDX(PairData,IIndex) pairs; + pairs.reserve(numNeighbors); + FOREACH(idxNeighbor, leftImage.neighbors) { + const ViewScore& neighbor = leftImage.neighbors[idxNeighbor]; + // exclude neighbors that over the limit or too small score + ASSERT(scene.images[neighbor.idx.ID].IsValid()); + if ((numNeighbors && idxNeighbor >= numNeighbors) || + (neighbor.score < fMinScore)) + break; + // check if the disparity-map was estimated for this images pair + const Image& rightImage = scene.images[neighbor.idx.ID]; + Disparity subpixelSteps; + cv::Size imageSize; Matrix3x3 H; Matrix4x4 Q; + DisparityMap disparityMap; AccumCostMap costMap; + if (!ImportDisparityDataRawFull(MAKE_PATH(String::FormatString("%04u_%04u.dimap", leftImage.ID, rightImage.ID)), disparityMap, costMap, imageSize, H, Q, subpixelSteps)) { + if (!ImportDisparityDataRawFull(MAKE_PATH(String::FormatString("%04u_%04u.dimap", rightImage.ID, leftImage.ID)), disparityMap, costMap, imageSize, H, Q, subpixelSteps)) { + DEBUG("warning: no disparity-data file found for image pair (%u,%u)", leftImage.ID, rightImage.ID); + continue; + } + // adapt Q to project the 3D point from right into the left-image + RMatrix poseR; + CMatrix poseC; + ComputeRelativePose(rightImage.camera.R, rightImage.camera.C, leftImage.camera.R, leftImage.camera.C, poseR, poseC); + Matrix4x4 P(Matrix4x4::IDENTITY); + AssembleProjectionMatrix(leftImage.camera.K, poseR, poseC, reinterpret_cast(P)); + Matrix4x4 invK(Matrix4x4::IDENTITY); + cv::Mat(rightImage.camera.GetInvK()).copyTo(cv::Mat(4,4,cv::DataType::type,invK.val)(cv::Rect(0,0,3,3))); + Q = P*invK*Q; + } + // convert disparity-map to final depth-map for the left image + PairData& pair = pairs.emplace_back(leftImage.image.size()); + if (!ProjectDisparity2DepthMap(disparityMap, costMap, Q, subpixelSteps, pair.depthMap, pair.depthRangeMap, pair.confMap)) { + pairs.RemoveLast(); + continue; + } + #if TD_VERBOSE != TD_VERBOSE_OFF + // save pair depth-map + if (VERBOSITY_LEVEL > 3) { + const String pairName(MAKE_PATH(String::FormatString("depth%04u_%04u", leftImage.ID, rightImage.ID))); + ExportDepthMap(pairName+".png", pair.depthMap); + MVS::ExportPointCloud(pairName+".ply", leftImage, pair.depthMap, NormalMap()); + } + #endif + } + // fuse available depth-maps such that for each pixel set its depth as the average of the largest cluster of agreeing depths; + // pixel depths values agree if their trust regions overlap + depthMap.create(leftImage.image.size()); confMap.create(depthMap.size()); + for (int r=0; r range.y) + cluster.range.y = range.y; + ++numClusters; + } + if (numClusters == 0) + clusters.emplace_back(Cluster{IIndexArr{p}, range}); + } + if (clusters.empty()) { + depthMap(r,c) = Depth(0); + confMap(r,c) = float(0); + continue; + } + const Cluster& cluster = clusters.GetMax([](const Cluster& i, const Cluster& j) { return i.views.size() < j.views.size(); }); + if (cluster.views.size() < minViews) { + depthMap(r,c) = Depth(0); + confMap(r,c) = float(0); + continue; + } + Depth& depth = depthMap(r,c); depth = 0; + float& conf = confMap(r,c); conf = 0; + unsigned numDepths(0); + for (IIndex p: cluster.views) { + const PairData& pair = pairs[p]; + depth += pair.depthMap(r,c); + conf += pair.confMap(r,c); + ++numDepths; + } + depth /= numDepths; + conf /= numDepths; + } + } + DEBUG_EXTRA("Depth-map for image %3u fused: %dx%d (%s)", leftImage.ID, + depthMap.width(), depthMap.height(), TD_TIMER_GET_FMT().c_str()); + #if TD_VERBOSE != TD_VERBOSE_OFF + // save depth-map + if (VERBOSITY_LEVEL > 2) { + const String fileName(MAKE_PATH(String::FormatString("depth%04u", leftImage.ID))); + ExportDepthMap(fileName+".png", depthMap); + MVS::ExportPointCloud(fileName+".ply", leftImage, depthMap, NormalMap()); + } + #endif +} + + +// Compute SGM stereo on the images +void SemiGlobalMatcher::Match(const ViewData& leftImage, const ViewData& rightImage, DisparityMap& disparityMap, AccumCostMap& costMap) +{ + const cv::Size size(leftImage.imageGray.size()); + const cv::Size sizeValid(size.width-2*halfWindowSizeX, size.height-2*halfWindowSizeY); + ASSERT(leftImage.imageColor.size() == size); + + #if 0 + // display search info (average disparity and range) + DisplayState(sizeValid); + #endif + + // compute costs + { + ASSERT(!imageCosts.empty()); + const float eps(1e-3f); // used suppress the effect of noise in untextured regions + auto pixel = [&](int idx, int r, int c) { + // ignore pixel if not valid + const PixelData& pixel = imagePixels[idx]; + if (!pixel.range.isValid()) + return; + #if SGM_SIMILARITY == SGM_SIMILARITY_CENSUS + struct Compute { + static inline int HammingDistance(uint64_t c1, uint64_t c2) { + return PopCnt(c1 ^ c2); + } + }; + // compute pixel cost + Cost* costs = imageCosts.data()+pixel.idx; + const Census lc(leftImage.imageCensus(r,c)); + for (int d=pixel.range.minDisp; d(windowSizeX,windowSizeY)))); + return float(SQUARE(x) + SQUARE(y)) * sigmaSpatial; + } + }; + const ImageRef u(c+halfWindowSizeX,r+halfWindowSizeY); + // initialize pixel patch weights + WeightedPatch w; + w.normSq0 = 0; + w.sumWeights = 0; + int n = 0; + const Pixel8U& colCenter = leftImage.imageColor(u); + for (int i=-halfWindowSizeY; i<=halfWindowSizeY; ++i) { + for (int j=-halfWindowSizeX; j<=halfWindowSizeX; ++j) { + const ImageRef x(u.x+j,u.y+i); + WeightedPatch::Pixel& pw = w.weights[n++]; + w.normSq0 += + (pw.tempWeight = leftImage.imageGray(x)) * + (pw.weight = EXP(Compute::WeightColor(leftImage.imageColor, colCenter, x)+Compute::WeightSpatial(j,i))); + w.sumWeights += pw.weight; + } + } + ASSERT(n == numTexels); + const float tm(w.normSq0/w.sumWeights); + w.normSq0 = 0; + n = 0; + do { + WeightedPatch::Pixel& pw = w.weights[n]; + const float t(pw.tempWeight - tm); + w.normSq0 += (pw.tempWeight = pw.weight * t) * t; + } while (++n < numTexels); + // compute pixel cost + Cost* costs = imageCosts.data()+pixel.idx; + for (int d=pixel.range.minDisp; d v) m = v; } + }; + ASSERT(Ls.R.isValid()); + #if SGM_SIMILARITY == SGM_SIMILARITY_CENSUS + const AccumCost P2(P2s[DI]); + #else + const AccumCost P2(P2s[ABS(ROUND2INT(255.f*DI))]); + #endif + const Disparity minDisp(MAXF(Lp.R.minDisp, Ls.R.minDisp)); + const Disparity maxDisp(MINF(Lp.R.maxDisp, Ls.R.maxDisp)); + if (minDisp >= maxDisp) { + // the disparity ranges for the two pixels do not intersect; + // fill all accumulated costs with L(d)=C(d)+P2 + const Disparity numDisp(Ls.R.numDisp()); + for (int idxDisp=0; idxDisp1 + AccumCost minLp(std::numeric_limits::max()); + for (const AccumCost *L=Lp.L+(minDisp-Lp.R.minDisp), *endL=L+(maxDisp-minDisp); L::max(); + for (Disparity dp=minDisp; dp= 0); + }; + volatile Thread::safe_t idxPixel(-1); + FOREACH(i, threads) + threads.AddEvent(new EVTPixelAccumInc(sizeValid.width, idxPixel, pixels)); + WaitThreadWorkers(threads.size()); + } + { // height-left + auto pixels = [&](int y) { + ImageRef u(sizeValid.width-1,y); + ACCUM_PIXELS(--u.x >= 0); + }; + volatile Thread::safe_t idxPixel(-1); + FOREACH(i, threads) + threads.AddEvent(new EVTPixelAccumInc(sizeValid.height, idxPixel, pixels)); + WaitThreadWorkers(threads.size()); + } + if (numDirs == 4) { + { // width-right-down + auto pixels = [&](int x) { + ImageRef u(x,0); + ACCUM_PIXELS(++u.x < sizeValid.width && ++u.y < sizeValid.height); + }; + volatile Thread::safe_t idxPixel(-1); + FOREACH(i, threads) + threads.AddEvent(new EVTPixelAccumInc(sizeValid.width, idxPixel, pixels)); + } + { // height-right-down + auto pixels = [&](int y) { + ImageRef u(0,y); + ACCUM_PIXELS(++u.x < sizeValid.width && ++u.y < sizeValid.height); + }; + volatile Thread::safe_t idxPixel(0); + FOREACH(i, threads) + threads.AddEvent(new EVTPixelAccumInc(sizeValid.height, idxPixel, pixels)); + } + WaitThreadWorkers(threads.size()*2); + { // width-left-down + auto pixels = [&](int x) { + ImageRef u(x,0); + ACCUM_PIXELS(--u.x >= 0 && ++u.y < sizeValid.height); + }; + volatile Thread::safe_t idxPixel(-1); + FOREACH(i, threads) + threads.AddEvent(new EVTPixelAccumInc(sizeValid.width-1, idxPixel, pixels)); + } + { // height-left-down + auto pixels = [&](int y) { + ImageRef u(sizeValid.width-1,y); + ACCUM_PIXELS(--u.x >= 0 && ++u.y < sizeValid.height); + }; + volatile Thread::safe_t idxPixel(-1); + FOREACH(i, threads) + threads.AddEvent(new EVTPixelAccumInc(sizeValid.height, idxPixel, pixels)); + } + WaitThreadWorkers(threads.size()*2); + { // width-right-up + auto pixels = [&](int x) { + ImageRef u(x,sizeValid.height-1); + ACCUM_PIXELS(++u.x < sizeValid.width && --u.y >= 0); + }; + volatile Thread::safe_t idxPixel(0); + FOREACH(i, threads) + threads.AddEvent(new EVTPixelAccumInc(sizeValid.width, idxPixel, pixels)); + } + { // height-right-up + auto pixels = [&](int y) { + ImageRef u(0,y); + ACCUM_PIXELS(++u.x < sizeValid.width && --u.y >= 0); + }; + volatile Thread::safe_t idxPixel(sizeValid.height); + FOREACH(i, threads) + threads.AddEvent(new EVTPixelAccumDec(idxPixel, pixels)); + } + WaitThreadWorkers(threads.size()*2); + { // width-left-up + auto pixels = [&](int x) { + ImageRef u(x,sizeValid.height-1); + ACCUM_PIXELS(--u.x >= 0 && --u.y >= 0); + }; + volatile Thread::safe_t idxPixel(sizeValid.width); + FOREACH(i, threads) + threads.AddEvent(new EVTPixelAccumDec(idxPixel, pixels)); + } + { // height-left-up + auto pixels = [&](int y) { + ImageRef u(sizeValid.width-1,y); + ACCUM_PIXELS(--u.x >= 0 && --u.y >= 0); + }; + volatile Thread::safe_t idxPixel(sizeValid.height-1); + FOREACH(i, threads) + threads.AddEvent(new EVTPixelAccumDec(idxPixel, pixels)); + } + WaitThreadWorkers(threads.size()*2); + } + #undef ACCUM_PIXELS + } else { + const ImageRef dirs[] = {{-1,0}, {0,-1}, {-1,-1}, {1,-1}}; + struct AccumLines { + const Disparity maxNumDisp; + LineData* linesBuffer; + LineData* lines[numDirs][2]; + AccumLines(Disparity _maxNumDisp) : maxNumDisp(_maxNumDisp), linesBuffer(NULL) {} + ~AccumLines() { delete[] linesBuffer; } + void Init(int w) { + const int linewidth(w+2); + const int buffersize(2*linewidth*numDirs); + if (linesBuffer == NULL) { + linesBuffer = new LineData[buffersize]; + for (int i=0; i=0; ) { + for (int c=sizeValid.width; --c>=0; ) { + ACCUM_PIXELS(-dir.x, -dir.y, sizeValid.width-1-c); + } + lines.NextLine(); + } + #undef ACCUM_PIXELS + } + } + + // select best disparity and cost + { + disparityMap.create(sizeValid); + costMap.create(sizeValid); + auto pixel = [&](int idx) { + const PixelData& pixel = imagePixels[idx]; + if (pixel.range.isValid()) { + const AccumCost* accums = imageAccumCosts.cdata()+pixel.idx; + const AccumCost* bestAccum = accums; + for (const AccumCost *accum=accums+1, *accumEnd=accums+pixel.range.numDisp(); accum *accum) + bestAccum = accum; + } + disparityMap(idx) = pixel.range.minDisp+(Disparity)(bestAccum-accums); + costMap(idx) = *bestAccum; + } else { + disparityMap(idx) = pixel.range.minDisp; + costMap(idx) = NO_ACCUMCOST; + } + }; + ASSERT(threads.IsEmpty()); + if (!threads.empty()) { + volatile Thread::safe_t idxPixel(-1); + FOREACH(i, threads) + threads.AddEvent(new EVTPixelAccumInc(sizeValid.area(), idxPixel, pixel)); + WaitThreadWorkers(threads.size()); + } else + for (int r=0; r(r*2+halfWindowSizeY, halfWindowSizeX)); + for (int c=0, c2=0; c maxNumDisp) { + range.minDisp = disp-(maxNumDisp*(disp-minmax.first*2)+1)/numDisp; + range.maxDisp = disp+(maxNumDisp*(minmax.second*2+1-disp)+1)/numDisp; + numDisp = range.numDisp(); + } else { + range.minDisp = disp-numDisp/2; + range.maxDisp = disp+(numDisp+1)/2; + } + } + } + ASSERT(range.numDisp() == numDisp); + if (maxNumDisp < numDisp) + maxNumDisp = numDisp; + } + c2e += 2; + do { + ASSERT(c2 < size2x.width); + PixelData& pixel = imagePixels[offset+c2]; + pixel.range = range; + pixel.idx = numCosts; + numCosts += numDisp; + } while (++c2 < c2e); + } + ASSERT(c2e < size2x.width); + do { + const PixelData& pixel = imagePixels[offset+c2e-1]; + PixelData& _pixel = imagePixels[offset+c2e]; + _pixel.range = pixel.range; + _pixel.idx = numCosts; + numCosts += pixel.range.numDisp(); + } while (++c2e < size2x.width); + const int _offsete((r+1 == disparityMap.rows ? size2x.height : r*2+halfWindowSizeY+2)*size2x.width); + for (int _offset=offset+size2x.width; _offset<_offsete; _offset+=size2x.width) { + for (int c2=0; c2= 0); + ASSERT(!l2r.empty() && !r2l.empty()); + ASSERT(l2r.height() == r2l.height()); + + auto pixel = [&](int, int r, int c) { + Disparity& ld = l2r(r,c); + if (ld == NO_DISP) + return; + // compute the corresponding disparity pixel according to the disparity value + const ImageRef v(c+ld,r); + // check image bounds + if (v.x < 0 || v.x >= r2l.width()) { + ld = NO_DISP; + return; + } + // check right disparity is valid + const Disparity rd = r2l(v); + if (r2l(v) == NO_DISP) { + ld = NO_DISP; + return; + } + // check disparity consistency: + // since the left and right disparities are opposite in sign, + // we determine their similarity by *summing* them, rather + // than differencing them as you might expect + if (ABS(ld + rd) > thCross) + ld = NO_DISP; + }; + ASSERT(threads.IsEmpty()); + if (!threads.empty()) { + volatile Thread::safe_t idxPixel(-1); + FOREACH(i, threads) + threads.AddEvent(new EVTPixelProcess(l2r.size(), idxPixel, pixel)); + WaitThreadWorkers(threads.size()); + } else + for (int r=0; r 0); + ASSERT(!disparityMap.empty() && disparityMap.size() == costMap.size()); + + auto pixel = [&](int, int r, int c) { + Disparity& d = disparityMap(r,c); + if (d == NO_DISP) + return; + if (costMap(r,c) > th) + d = NO_DISP; + }; + ASSERT(threads.IsEmpty()); + if (!threads.empty()) { + volatile Thread::safe_t idxPixel(-1); + FOREACH(i, threads) + threads.AddEvent(new EVTPixelProcess(disparityMap.size(), idxPixel, pixel)); + WaitThreadWorkers(threads.size()); + } else + for (int r=0; r= thValid) \ + break + + // left-right direction + { + auto pixel = [&](int r) { + int numValid(0); + for (int c=0; c=0; ) { + MASK_PIXEL(); + } + }; + ASSERT(threads.IsEmpty()); + if (!threads.empty()) { + volatile Thread::safe_t idxPixel(-1); + FOREACH(i, threads) + threads.AddEvent(new EVTPixelAccumInc(disparityMap.height(), idxPixel, pixel)); + WaitThreadWorkers(threads.size()); + } else + for (int r=0; r=0; ) { + MASK_PIXEL(); + } + }; + ASSERT(threads.IsEmpty()); + if (!threads.empty()) { + volatile Thread::safe_t idxPixel(-1); + FOREACH(i, threads) + threads.AddEvent(new EVTPixelAccumInc(disparityMap.width(), idxPixel, pixel)); + WaitThreadWorkers(threads.size()); + } else + for (int c=0; c (int)std::numeric_limits::min()); + ASSERT((int)d*subpixelSteps < (int)std::numeric_limits::max()); + d *= subpixelSteps; + }; + ASSERT(threads.IsEmpty()); + if (!threads.empty()) { + volatile Thread::safe_t idxPixel(-1); + FOREACH(i, threads) + threads.AddEvent(new EVTPixelProcess(disparityMap.size(), idxPixel, pixel)); + WaitThreadWorkers(threads.size()); + } else + for (int r=0; r(primary) / static_cast(other)); + } + // compute the offset to be added to the integer disparity to get the final result + static real subpixelMode(AccumCost prev, AccumCost center, AccumCost next, SgmSubpixelMode subpixelMode) { + ASSERT(prev != NO_ACCUMCOST && center != NO_ACCUMCOST && next != NO_ACCUMCOST); + // use a lower quality two value interpolation if only two values are available + if (prev == center) + return center == next ? real(0) : semisubpixel(center, next); + if (center == next) + return prev == center ? real(0) : -semisubpixel(center, prev); + // pick which direction to interpolate in + const AccumCost ld(prev-center); + const AccumCost rd(next-center); + real x, mult; + if (ld < rd) { + x = static_cast(ld) / static_cast(rd); + mult = real(1); + } else { + x = static_cast(rd) / static_cast(ld); + mult = real(-1); + } + // use the selected subpixelMode function + real value(0); + switch (subpixelMode) { + case SUBPIXEL_LINEAR: value = linear(x); break; + case SUBPIXEL_POLY4: value = poly4(x); break; + case SUBPIXEL_PARABOLA: value = parabola(x); break; + case SUBPIXEL_SINE: value = sine(x); break; + case SUBPIXEL_COSINE: value = cosine(x); break; + case SUBPIXEL_LC_BLEND: value = lcBlend(x); break; + }; + // complete computation + return (value - real(0.5))*mult; + } + }; + // estimate sub-pixel disparity based on the cost values + auto pixel = [&](int idx) { + const PixelData& pixel = imagePixels[idx]; + if (pixel.range.numDisp() < 2) + return; + Disparity& d = disparityMap(idx); + if (d == NO_DISP) + return; + const AccumCost* accums = imageAccumCosts.cdata()+pixel.idx; + const int idxDisp(d-pixel.range.minDisp); + real disparity((real)d); + if (d == pixel.range.minDisp) + disparity += Fit::semisubpixel(accums[idxDisp], accums[idxDisp+1]); + else if (d+1 == pixel.range.maxDisp) + disparity -= Fit::semisubpixel(accums[idxDisp], accums[idxDisp-1]); + else + disparity += Fit::subpixelMode(accums[idxDisp-1], accums[idxDisp], accums[idxDisp+1], subpixelMode); + ASSERT(ROUND2INT(disparity*subpixelSteps) > (int)std::numeric_limits::min()); + ASSERT(ROUND2INT(disparity*subpixelSteps) < (int)std::numeric_limits::max()); + d = (Disparity)ROUND2INT(disparity*subpixelSteps); + }; + ASSERT(threads.IsEmpty()); + if (!threads.empty()) { + volatile Thread::safe_t idxPixel(-1); + FOREACH(i, threads) + threads.AddEvent(new EVTPixelAccumInc(disparityMap.size().area(), idxPixel, pixel)); + WaitThreadWorkers(threads.size()); + } else + for (int r=0; r 0; }) || !Image::Depth2Disparity(invQ, u, depth, disparity)) + disparityMap(r,c) = NO_DISP; + else + disparityMap(r,c) = (Disparity)ROUND2INT(disparity*subpixelSteps); + }; + ASSERT(threads.IsEmpty()); + if (!threads.empty()) { + volatile Thread::safe_t idxPixel(-1); + FOREACH(i, threads) + threads.AddEvent(new EVTPixelProcess(disparityMap.size(), idxPixel, pixel)); + WaitThreadWorkers(threads.size()); + } else + for (int r=0; r overlapBorder || absDist.y > overlapBorder) + continue; + const int idx((dist.x<0?1:0)+(dist.y<0?2:0)); + DepthData& depthData = depthDatas[idx]; + const float deistSq(normSq(dist)); + float& ndeistSq = depthData.distMap(nx); + Depth& ndepth = depthData.depthMap(nx); + if (ndepth > 0 && ndeistSq <= deistSq) + continue; + ndeistSq = deistSq; + ndepth = depth; + depthData.depthRangeMap(nx) = depthRange; + depthData.confMap(nx) = cost; + } + } + } + } + typedef TAccumulator ValueAccumulator; + depthRangeMap.create(depthMap.size()); + confMap.create(depthMap.size()); + const float thDist(SQUARE(0.75f)); + const Depth thDepth(0.02f); + unsigned numDepths(0); + for (int r=0; r::max()); + Depth depthCenter; + for (int i=0; i<4; ++i) { + const DepthData& depthData = depthDatas[i]; + const Depth depth(depthData.depthMap(r,c)); + if (depth <= 0) + continue; + const float dist(depthData.distMap(r,c)); + if (distCenter > dist) { + distCenter = dist; + depthCenter = depth; + } + } + if (distCenter > thDist) { + depthMap(r,c) = Depth(0); + continue; + } + ValueAccumulator acc(Vec4f::ZERO, 0.f); + for (int i=0; i<4; ++i) { + const DepthData& depthData = depthDatas[i]; + const Depth depth(depthData.depthMap(r,c)); + if (depth <= 0 || !IsDepthSimilar(depthCenter, depth, thDepth)) + continue; + const DepthRange& depthRange(depthData.depthRangeMap(r,c)); + acc.Add(Vec4f((float)depth,depthRange.x,depthRange.y,depthData.confMap(r,c)), SQRT(depthData.distMap(r,c))); + } + ASSERT(!acc.IsEmpty()); + const Vec4f value(acc.Normalized()); + depthMap(r,c) = value(0); + depthRangeMap(r,c) = DepthRange(value(1),value(2)); + confMap(r,c) = value(3); + ++numDepths; + } + } + if (costMap.empty()) + confMap.release(); + return numDepths > 0; +} + + +EventThreadPool SemiGlobalMatcher::threads; +Semaphore SemiGlobalMatcher::sem; + +// start worker threads +void SemiGlobalMatcher::CreateThreads(unsigned nMaxThreads) +{ + ASSERT(nMaxThreads > 0); + ASSERT(threads.IsEmpty() && threads.empty()); + if (nMaxThreads > 1) { + threads.resize(nMaxThreads); + threads.start(ThreadWorker); + } +} +// destroy worker threads +void SemiGlobalMatcher::DestroyThreads() +{ + ASSERT(threads.IsEmpty()); + if (!threads.empty()) { + FOREACH(i, threads) + threads.AddEvent(new EVTClose()); + threads.Release(); + } +} + +void* SemiGlobalMatcher::ThreadWorker(void*) { + while (true) { + CAutoPtr evt(threads.GetEvent()); + switch (evt->GetID()) { + case EVT_JOB: + evt->Run(); + break; + case EVT_CLOSE: + return NULL; + default: + ASSERT("Should not happen!" == NULL); + } + sem.Signal(); + } + return NULL; +} +void SemiGlobalMatcher::WaitThreadWorkers(unsigned nJobs) +{ + while (nJobs-- > 0) + sem.Wait(); + ASSERT(threads.IsEmpty()); +} +/*----------------------------------------------------------------*/ + + + +// S T R U C T S /////////////////////////////////////////////////// + +bool SemiGlobalMatcher::ExportDisparityDataRaw(const String& fileName, const DisparityMap& disparityMap, const AccumCostMap& costMap, const cv::Size& imageSize, const Matrix3x3& H, const Matrix4x4& Q, Disparity subpixelSteps) +{ + ASSERT(!disparityMap.empty()); + ASSERT(costMap.empty() || disparityMap.size() == costMap.size()); + + FILE *f = fopen(fileName, "wb"); + if (f == NULL) + return false; + + // write info + fwrite(&imageSize.width, sizeof(int), 2, f); + fwrite(H.val, sizeof(REAL), 9, f); + fwrite(Q.val, sizeof(REAL), 16, f); + fwrite(&subpixelSteps, sizeof(Disparity), 1, f); + + // write resolution + fwrite(&disparityMap.cols, sizeof(int), 1, f); + fwrite(&disparityMap.rows, sizeof(int), 1, f); + + // write disparity-map + fwrite(disparityMap.getData(), sizeof(Disparity), disparityMap.area(), f); + + // write cost-map + if (!costMap.empty()) + fwrite(costMap.getData(), sizeof(AccumCost), costMap.area(), f); + + fclose(f); + return true; +} // ExportDisparityDataRaw +// same as above, but exports also the empty border +bool SemiGlobalMatcher::ExportDisparityDataRawFull(const String& fileName, const DisparityMap& disparityMap, const AccumCostMap& costMap, const cv::Size& imageSize, const Matrix3x3& H, const Matrix4x4& Q, Disparity subpixelSteps) +{ + ASSERT(!disparityMap.empty()); + ASSERT(costMap.empty() || disparityMap.size() == costMap.size()); + + const cv::Size size(disparityMap.width()+2*halfWindowSizeX,disparityMap.height()+2*halfWindowSizeY); + const cv::Rect ROI(halfWindowSizeX,halfWindowSizeY, disparityMap.width(),disparityMap.height()); + DisparityMap disparityMapFull(size, NO_DISP); + disparityMap.copyTo(disparityMapFull(ROI)); + if (costMap.empty()) + return ExportDisparityDataRaw(fileName, disparityMapFull, costMap, imageSize, H, Q, subpixelSteps); + AccumCostMap costMapFull(size, NO_ACCUMCOST); + costMap.copyTo(costMapFull(ROI)); + return ExportDisparityDataRaw(fileName, disparityMapFull, costMapFull, imageSize, H, Q, subpixelSteps); +} // ExportDisparityDataRawFull + +bool SemiGlobalMatcher::ImportDisparityDataRaw(const String& fileName, DisparityMap& disparityMap, AccumCostMap& costMap, cv::Size& imageSize, Matrix3x3& H, Matrix4x4& Q, Disparity& subpixelSteps) +{ + FILE *f = fopen(fileName, "rb"); + if (f == NULL) + return false; + + // read info + fread(&imageSize.width, sizeof(int), 2, f); + fread(H.val, sizeof(REAL), 9, f); + fread(Q.val, sizeof(REAL), 16, f); + fread(&subpixelSteps, sizeof(Disparity), 1, f); + ASSERT(imageSize.width > 0 && imageSize.height > 0); + + // read resolution + int w, h; + fread(&w, sizeof(int), 1, f); + fread(&h, sizeof(int), 1, f); + ASSERT(w > 0 && h > 0); + + // read disparity-map + disparityMap.create(h,w); + fread(disparityMap.getData(), sizeof(Disparity), w*h, f); + + // read cost-map + if (fgetc(f) != EOF) { + fseek(f, -1, SEEK_CUR); + costMap.create(h,w); + fread(costMap.getData(), sizeof(AccumCost), w*h, f); + } + + fclose(f); + return true; +} // ImportDisparityDataRaw +// same as above, but imports also the empty border +bool SemiGlobalMatcher::ImportDisparityDataRawFull(const String& fileName, DisparityMap& disparityMap, AccumCostMap& costMap, cv::Size& imageSize, Matrix3x3& H, Matrix4x4& Q, Disparity& subpixelSteps) +{ + if (!ImportDisparityDataRaw(fileName, disparityMap, costMap, imageSize, H, Q, subpixelSteps)) + return false; + const cv::Size sizeValid(disparityMap.width()-2*halfWindowSizeX,disparityMap.height()-2*halfWindowSizeY); + const cv::Rect ROI(halfWindowSizeX,halfWindowSizeY, sizeValid.width,sizeValid.height); + disparityMap = disparityMap(ROI).clone(); + if (!costMap.empty()) + costMap = costMap(ROI).clone(); + return true; +} // ImportDisparityDataRawFull + + +// export disparity-map as an image (red - maximum disparity, blue - minimum disparity) +Image8U3 SemiGlobalMatcher::DisparityMap2Image(const DisparityMap& disparityMap, Disparity minDisparity, Disparity maxDisparity) +{ + ASSERT(!disparityMap.empty()); + // find min and max values + if (minDisparity == NO_DISP || maxDisparity == NO_DISP) { + CLISTDEF0(Disparity) disparities(0, disparityMap.area()); + for (int i=disparityMap.area(); i-- > 0; ) { + const Disparity disparity = disparityMap[i]; + if (disparity != NO_DISP) + disparities.emplace_back(disparity); + } + if (!disparities.empty()) { + const std::pair th(ComputeX84Threshold(disparities.data(), disparities.size(), 5.2f)); + minDisparity = (Disparity)ROUND2INT(th.first-th.second); + maxDisparity = (Disparity)ROUND2INT(th.first+th.second); + } + DEBUG_ULTIMATE("\tdisparity range: [%d, %d]", minDisparity, maxDisparity); + } + const float sclDepth(1.f/(float)(maxDisparity - minDisparity)); + // create color image + Image8U3 img(cv::Size(disparityMap.width()+2*halfWindowSizeX,disparityMap.height()+2*halfWindowSizeY), Pixel8U::BLACK); + for (int r=0; r(u)) + continue; + const Point3f X(imageData.camera.TransformPointI2W(Point3(u,depth))); + vertex.x = X.x; vertex.y = X.y; vertex.z = X.z; + const Pixel8U C(imageData.image.empty() ? Pixel8U::WHITE : imageData.image.sample(u)); + vertex.r = C.r; vertex.g = C.g; vertex.b = C.b; + ply.put_element(&vertex); + } + } + if (ply.get_current_element_count() == 0) + return false; + + // write to file + return ply.header_complete(); +} // ExportPointCloud + +// imports a DIMAP file and converts it to point-cloud +bool SemiGlobalMatcher::ImportPointCloud(const String& fileName, const ImageArr& images, PointCloud& pointcloud) +{ + // load disparity-map + Disparity subpixelSteps; + cv::Size imageSize; Matrix3x3 H; Matrix4x4 Q; + DisparityMap disparityMap; AccumCostMap costMap; + if (!ImportDisparityDataRawFull(fileName, disparityMap, costMap, imageSize, H, Q, subpixelSteps)) + return false; + // parse image index from the file name + const String name(Util::getFileName(fileName)); + IIndex idxImage(NO_ID), idxImagePair(NO_ID); + if (sscanf(name, "%u_%u", &idxImage, &idxImagePair) != 2 || idxImage == NO_ID || idxImagePair == NO_ID) + return false; + const Image& imageData = images[idxImage]; + ASSERT(imageData.image.size() == imageSize); + // import the array of 3D points + for (int r=0; r(u)) + continue; + pointcloud.points.emplace_back(Cast(imageData.camera.TransformPointI2W(Point3(u,depth)))); + pointcloud.colors.emplace_back(imageData.image.empty() ? Pixel8U::WHITE : imageData.image.sample(u)); + } + } + return true; +} // ImportPointCloud +/*----------------------------------------------------------------*/ + + +bool MVS::STEREO::ExportCamerasEngin(const Scene& scene, const String& fileName) +{ + ASSERT(!scene.IsEmpty()); + + File f(fileName, File::WRITE, File::CREATE | File::TRUNCATE); + if (!f.isOpen()) + return false; + + // write header + f.print("n_cameras %u\n", scene.images.size()); + f.print("n_points %u\n", 0); + + // write cameras + for (const Image& image: scene.images) { + if (!image.IsValid()) + continue; + const Point3 t(image.camera.GetT()); + f.print("%u %u %u %s " + "%g %g %g %g " + "%g %g %g %g %g %g %g %g %g " + "%g %g %g " + "%g %g " + "%u", + image.ID, image.width, image.height, image.name.c_str(), + image.camera.K(0,0), image.camera.K(1,1), image.camera.K(0,2), image.camera.K(1,2), + image.camera.R(0,0), image.camera.R(0,1), image.camera.R(0,2), + image.camera.R(1,0), image.camera.R(1,1), image.camera.R(1,2), + image.camera.R(2,0), image.camera.R(2,1), image.camera.R(2,2), + t.x, t.y, t.z, + 0, 0, + image.neighbors.size() + ); + for (const auto& neighbor: image.neighbors) + f.print(" %u", neighbor.idx.ID); + f.print("\n"); + } + + return true; +} // ExportCamerasEngin +/*----------------------------------------------------------------*/ diff --git a/libs/MVS/SemiGlobalMatcher.h b/libs/MVS/SemiGlobalMatcher.h new file mode 100644 index 000000000..740840db4 --- /dev/null +++ b/libs/MVS/SemiGlobalMatcher.h @@ -0,0 +1,215 @@ +/* +* SemiGlobalMatcher.h +* +* Copyright (c) 2014-2015 SEACAVE +* +* Author(s): +* +* cDc +* +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* +* Additional Terms: +* +* You are required to preserve legal notices and author attributions in +* that material or in the Appropriate Legal Notices displayed by works +* containing it. +*/ + +#ifndef _MVS_SEMIGLOBALMATCHER_H_ +#define _MVS_SEMIGLOBALMATCHER_H_ + + +// I N C L U D E S ///////////////////////////////////////////////// + +#include "DepthMap.h" + + +// D E F I N E S /////////////////////////////////////////////////// + +// similarity method +#define SGM_SIMILARITY_WZNCC 1 +#define SGM_SIMILARITY_CENSUS 2 +#define SGM_SIMILARITY SGM_SIMILARITY_WZNCC + + +// S T R U C T S /////////////////////////////////////////////////// + +namespace MVS { + +class Scene; + +namespace STEREO { + +// An implementation of the popular Semi-Global Matching (SGM) algorithm. +class MVS_API SemiGlobalMatcher +{ +public: + typedef uint8_t Mask; // used to mark the valid region to be estimated + typedef int16_t Disparity; // contains the allowable disparity range + typedef uint8_t Cost; // used to describe a single disparity cost + typedef uint16_t AccumCost; // used to accumulate CostType values + typedef uint64_t Index; // index pointing to pixel costs/accumulated-costs + + enum : Mask { INVALID = 0, VALID = DECLARE_NO_INDEX(Mask) }; // invalid/valid mask value + enum : Disparity { NO_DISP = DECLARE_NO_INDEX(Disparity) }; // invalid disparity value + enum : AccumCost { NO_ACCUMCOST = DECLARE_NO_INDEX(AccumCost) }; // invalid accumulated cost value + enum : Index { NO_INDEX = DECLARE_NO_INDEX(Index) }; // invalid index value + + struct Range { + Disparity minDisp; + Disparity maxDisp; + Disparity avgDisp() const { return (minDisp+maxDisp)>>1; } + Disparity numDisp() const { return maxDisp-minDisp; } + Disparity isValid() const { return minDisp MaskMap; // image of accumulated disparity cost + typedef TImage DisparityMap; // disparity image type + typedef TImage AccumCostMap; // image of accumulated disparity cost + typedef CLISTDEF0IDX(PixelData,Index) PixelMap; // map of pixel data + typedef CLISTDEF0IDX(Cost,Index) CostsMap; // map of pre-computed disparity costs + typedef CLISTDEF0IDX(AccumCost,Index) AccumCostsMap; // map of accumulated disparity costs + + enum : int { numDirs = 4 }; // 2 or 4 directions accumulated per pixel pass + #if SGM_SIMILARITY == SGM_SIMILARITY_CENSUS + enum : int { halfWindowSizeX = 3, halfWindowSizeY = 4 }; // patch kernel size + #else + enum : int { halfWindowSizeX = 3, halfWindowSizeY = 3 }; // patch kernel size + #endif + enum : int { windowSizeX = halfWindowSizeX*2+1, windowSizeY = halfWindowSizeY*2+1, numTexels = windowSizeX*windowSizeY }; // patch kernel info + + #if SGM_SIMILARITY == SGM_SIMILARITY_CENSUS + typedef uint64_t Census; // used to store Census transform + typedef TImage CensusMap; // image of Census transforms + typedef Image8U ImageGray; // used to store intensities image + STATIC_ASSERT(sizeof(Census)*8 >= numTexels); + #else + typedef WeightedPatchFix WeightedPatch; // pre-computed patch weights + typedef Image32F ImageGray; // used to store normalized float intensities image + #endif + + + // Available sub-pixel options + enum SgmSubpixelMode { + SUBPIXEL_NA = 0, // no sub-pixel processing + SUBPIXEL_LINEAR, + SUBPIXEL_POLY4, + SUBPIXEL_PARABOLA, + SUBPIXEL_SINE, + SUBPIXEL_COSINE, + SUBPIXEL_LC_BLEND // probably the best option + }; + + struct ViewData { + Image8U3 imageColor; // color image + ImageGray imageGray; // intensity image + #if SGM_SIMILARITY == SGM_SIMILARITY_CENSUS + CensusMap imageCensus; // image of Census transforms (valid per scale) + #endif + ViewData GetImage(REAL scale) const { + if (ISEQUAL(scale, REAL(1))) { + #if SGM_SIMILARITY == SGM_SIMILARITY_CENSUS + CensusTransform(imageGray, const_cast(imageCensus)); + #endif + return *this; + } + ASSERT(scale < 1); + ViewData scaledView; + cv::resize(imageColor, scaledView.imageColor, cv::Size(), scale, scale, cv::INTER_AREA); + cv::resize(imageGray, scaledView.imageGray, cv::Size(), scale, scale, cv::INTER_AREA); + #if SGM_SIMILARITY == SGM_SIMILARITY_CENSUS + CensusTransform(scaledView.imageGray, scaledView.imageCensus); + #endif + return scaledView; + } + }; + + typedef MVS_API Point2f DepthRange; + typedef MVS_API TImage DepthRangeMap; + +public: + SemiGlobalMatcher(SgmSubpixelMode subpixelMode=SUBPIXEL_LC_BLEND, Disparity subpixelSteps=4, AccumCost P1=3, AccumCost P2=4, float P2alpha=14, float P2beta=38); + ~SemiGlobalMatcher(); + + void Match(const Scene& scene, IIndex idxImage, IIndex numNeighbors, unsigned minResolution=320); + void Fuse(const Scene& scene, IIndex idxImage, IIndex numNeighbors, unsigned minViews, DepthMap& depthMap, ConfidenceMap& confMap); + + static void CreateThreads(unsigned nMaxThreads=1); + static void DestroyThreads(); + static void* ThreadWorker(void*); + static void WaitThreadWorkers(unsigned nJobs); + + static bool ExportDisparityDataRaw(const String& fileName, const DisparityMap&, const AccumCostMap&, const cv::Size& imageSize, const Matrix3x3& H, const Matrix4x4& Q, Disparity subpixelSteps); + static bool ExportDisparityDataRawFull(const String& fileName, const DisparityMap&, const AccumCostMap&, const cv::Size& imageSize, const Matrix3x3& H, const Matrix4x4& Q, Disparity subpixelSteps); + static bool ImportDisparityDataRaw(const String& fileName, DisparityMap&, AccumCostMap&, cv::Size& imageSize, Matrix3x3& H, Matrix4x4& Q, Disparity& subpixelSteps); + static bool ImportDisparityDataRawFull(const String& fileName, DisparityMap&, AccumCostMap&, cv::Size& imageSize, Matrix3x3& H, Matrix4x4& Q, Disparity& subpixelSteps); + static Image8U3 DisparityMap2Image(const DisparityMap&, Disparity minDisparity=NO_DISP, Disparity maxDisparity=NO_DISP); + static bool ExportDisparityMap(const String& fileName, const DisparityMap&, Disparity minDisparity=NO_DISP, Disparity maxDisparity=NO_DISP); + static bool ExportPointCloud(const String& fileName, const Image&, const DisparityMap&, const Matrix4x4& Q, Disparity subpixelSteps); + static bool ImportPointCloud(const String& fileName, const ImageArr& images, PointCloud&); + +protected: + void Match(const ViewData& leftImage, const ViewData& rightImage, DisparityMap& disparityMap, AccumCostMap& costMap); + Index Disparity2RangeMap(const DisparityMap& disparityMap, const MaskMap& maskMap, Disparity minNumDisp=3, Disparity minNumDispInvalid=16); + #if SGM_SIMILARITY == SGM_SIMILARITY_CENSUS + static void CensusTransform(const Image8U& imageGray, CensusMap& imageCensus); + #endif + static void ConsistencyCrossCheck(DisparityMap& l2r, const DisparityMap& r2l, Disparity thCross=1); + static void FilterByCost(DisparityMap&, const AccumCostMap&, AccumCost th); + static void ExtractMask(const DisparityMap&, MaskMap&, int thValid=3); + static void FlipDirection(const DisparityMap& l2r, DisparityMap& r2l); + static void UpscaleMask(MaskMap& maskMap, const cv::Size& size2x); + void RefineDisparityMap(DisparityMap& disparityMap) const; + void DisplayState(const cv::Size& size) const; + + static CLISTDEF0IDX(AccumCost,int) GenerateP2s(AccumCost P2, float P2alpha, float P2beta); + static void Depth2DisparityMap(const DepthMap&, const Matrix3x3& invH, const Matrix4x4& invQ, Disparity subpixelSteps, DisparityMap&); + static void Disparity2DepthMap(const DisparityMap&, const AccumCostMap&, const Matrix3x3& H, const Matrix4x4& Q, Disparity subpixelSteps, DepthMap&, ConfidenceMap&); + static bool ProjectDisparity2DepthMap(const DisparityMap&, const AccumCostMap&, const Matrix4x4& Q, Disparity subpixelSteps, DepthMap&, DepthRangeMap&, ConfidenceMap&); + +protected: + // parameters + SgmSubpixelMode subpixelMode; + Disparity subpixelSteps; + AccumCost P1; + CLISTDEF0IDX(AccumCost,int) P2s; + + // main memory buffers that must be allocated + PixelMap imagePixels; + CostsMap imageCosts; + AccumCostsMap imageAccumCosts; + Disparity maxNumDisp; // maximum number of disparities per-pixel + + // multi-threading + static EventThreadPool threads; // worker threads + static Semaphore sem; // signal job end +}; +/*----------------------------------------------------------------*/ + + +MVS_API bool ExportCamerasEngin(const Scene&, const String& fileName); +/*----------------------------------------------------------------*/ + +} // namespace STEREO + +} // namespace MVS + +#endif // _MVS_SEMIGLOBALMATCHER_H_ From d7c06163b003a3f50ff63ab02694ebf897f36411 Mon Sep 17 00:00:00 2001 From: cDc Date: Sun, 16 Feb 2020 19:42:47 +0200 Subject: [PATCH 57/70] dense: fix linux --- build/Utils.cmake | 5 +++++ libs/Common/Types.h | 1 + libs/Common/Types.inl | 2 +- libs/MVS/SceneDensify.cpp | 4 ++-- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/build/Utils.cmake b/build/Utils.cmake index c828a28cf..f41c9d89f 100644 --- a/build/Utils.cmake +++ b/build/Utils.cmake @@ -493,12 +493,17 @@ macro(optimize_default_compiler_settings) add_extra_compiler_option(-Wno-switch) add_extra_compiler_option(-Wno-switch-enum) add_extra_compiler_option(-Wno-switch-default) + add_extra_compiler_option(-Wno-implicit-fallthrough) add_extra_compiler_option(-Wno-comment) add_extra_compiler_option(-Wno-narrowing) add_extra_compiler_option(-Wno-attributes) + add_extra_compiler_option(-Wno-ignored-attributes) + add_extra_compiler_option(-Wno-maybe-uninitialized) add_extra_compiler_option(-Wno-enum-compare) add_extra_compiler_option(-Wno-misleading-indentation) add_extra_compiler_option(-Wno-missing-field-initializers) + add_extra_compiler_option(-Wno-unused-result) + add_extra_compiler_option(-Wno-unused-function) add_extra_compiler_option(-Wno-unused-parameter) add_extra_compiler_option(-Wno-delete-incomplete) add_extra_compiler_option(-Wno-unnamed-type-template-args) diff --git a/libs/Common/Types.h b/libs/Common/Types.h index 37c28d615..bcac65c1b 100644 --- a/libs/Common/Types.h +++ b/libs/Common/Types.h @@ -1440,6 +1440,7 @@ class TMatrix : public cv::Matx #endif using Base::val; + using Base::channels; enum { elems = m*n }; diff --git a/libs/Common/Types.inl b/libs/Common/Types.inl index e9ef56086..090f896dc 100644 --- a/libs/Common/Types.inl +++ b/libs/Common/Types.inl @@ -2220,7 +2220,7 @@ TPixel TPixel::gray2color(ALT gray) Base(gray + ALT(0.25)), Base(gray), Base(gray - ALT(0.25)) - ).cast(); + ).template cast(); } /*----------------------------------------------------------------*/ diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index 9843e3bdd..1e97e8806 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -1258,7 +1258,7 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b REAL confidence(weights.emplace_back(Conf2Weight(depthData.confMap(x),depth))); ProjArr& pointProjs = projs.AddEmpty(); pointProjs.Insert(Proj(x)); - const PointCloud::Normal normal(bNormalMap ? imageData.camera.R.t()*Cast(depthData.normalMap(x)) : Normal(0,0,-1)); + const PointCloud::Normal normal(bNormalMap ? Cast(imageData.camera.R.t()*Cast(depthData.normalMap(x))) : Normal(0,0,-1)); ASSERT(ISEQUAL(norm(normal), 1.f)); // check the projection in the neighbor depth-maps Point3 X(point*confidence); @@ -1286,7 +1286,7 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b continue; if (IsDepthSimilar(pt.z, depthB, OPTDENSE::fDepthDiffThreshold)) { // check if normals agree - const PointCloud::Normal normalB(bNormalMap ? imageDataB.camera.R.t()*Cast(depthDataB.normalMap(xB)) : Normal(0,0,-1)); + const PointCloud::Normal normalB(bNormalMap ? Cast(imageDataB.camera.R.t()*Cast(depthDataB.normalMap(xB))) : Normal(0,0,-1)); ASSERT(ISEQUAL(norm(normalB), 1.f)); if (normal.dot(normalB) > normalError) { // add view to the 3D point From 8603471b45c1a1da8835d0cb9fdedc5f07f2b4fb Mon Sep 17 00:00:00 2001 From: cDc Date: Sun, 16 Feb 2020 23:33:55 +0200 Subject: [PATCH 58/70] common: increase version --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 945103299..7966ed55b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,8 +19,8 @@ endif() PROJECT(OpenMVS) set(OpenMVS_MAJOR_VERSION 1) -set(OpenMVS_MINOR_VERSION 0) -set(OpenMVS_PATCH_VERSION 1) +set(OpenMVS_MINOR_VERSION 1) +set(OpenMVS_PATCH_VERSION 0) set(OpenMVS_VERSION ${OpenMVS_MAJOR_VERSION}.${OpenMVS_MINOR_VERSION}.${OpenMVS_PATCH_VERSION}) # Find dependencies: From 8875b38d2efa72774ea6c4623478c71e63e60bba Mon Sep 17 00:00:00 2001 From: Flachy Joe Date: Mon, 17 Feb 2020 21:20:16 +0100 Subject: [PATCH 59/70] script: MvgMvsPipeline.py improvement (#529) - add script directory to PATH in order to launch it from another location --- MvgMvsPipeline.py | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/MvgMvsPipeline.py b/MvgMvsPipeline.py index f4914bf86..106857f7e 100644 --- a/MvgMvsPipeline.py +++ b/MvgMvsPipeline.py @@ -59,12 +59,16 @@ DEBUG = False -# add current directory to PATH if sys.platform.startswith('win'): - path_delim = ';' + PATH_DELIM = ';' else: - path_delim = ':' -os.environ['PATH'] += path_delim + os.getcwd() + PATH_DELIM = ':' + +# add this script's directory to PATH +os.environ['PATH'] += PATH_DELIM + os.path.dirname(os.path.abspath(__file__)) + +# add current directory to PATH +os.environ['PATH'] += PATH_DELIM + os.getcwd() def whereis(afile): @@ -81,15 +85,17 @@ def whereis(afile): except subprocess.CalledProcessError: return None + def find(afile): """ As whereis look only for executable on linux, this find look for all file type """ - for d in os.environ['PATH'].split(path_delim): + for d in os.environ['PATH'].split(PATH_DELIM): if os.path.isfile(os.path.join(d, afile)): return d return None + # Try to find openMVG and openMVS binaries in PATH OPENMVG_BIN = whereis("openMVG_main_SfMInit_ImageListing") OPENMVS_BIN = whereis("ReconstructMesh") @@ -115,11 +121,11 @@ def find(afile): PRESET_DEFAULT = 'SEQUENTIAL' - # HELPERS for terminal colors BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) NO_EFFECT, BOLD, UNDERLINE, BLINK, INVERSE, HIDDEN = (0, 1, 4, 5, 7, 8) + # from Python cookbook, #475186 def has_colours(stream): ''' @@ -161,6 +167,7 @@ def __init__(self): class AStep: + """ Represents a process step to be run """ def __init__(self, info, cmd, opt): self.info = info self.cmd = cmd @@ -168,6 +175,7 @@ def __init__(self, info, cmd, opt): class StepsStore: + """ List of steps with facilities to configure them """ def __init__(self): self.steps_data = [ ["Intrinsics analysis", # 0 @@ -245,29 +253,30 @@ def apply_conf(self, conf): STEPS = StepsStore() # ARGS -parser = argparse.ArgumentParser( +PARSER = argparse.ArgumentParser( formatter_class=argparse.RawTextHelpFormatter, description="Photogrammetry reconstruction with these steps: \r\n" + "\r\n".join(("\t%i. %s\t %s" % (t, STEPS[t].info, STEPS[t].cmd) for t in range(STEPS.length()))) ) -parser.add_argument('input_dir', +PARSER.add_argument('input_dir', help="the directory wich contains the pictures set.") -parser.add_argument('output_dir', +PARSER.add_argument('output_dir', help="the directory wich will contain the resulting files.") -parser.add_argument('--steps', +PARSER.add_argument('--steps', type=int, nargs="+", help="steps to process") -parser.add_argument('--preset', +PARSER.add_argument('--preset', help="steps list preset in \r\n" + " \r\n".join([k + " = " + str(PRESET[k]) for k in PRESET]) + " \r\ndefault : " + PRESET_DEFAULT) -group = parser.add_argument_group('Passthrough', description="Option to be passed to command lines (remove - in front of option names)\r\ne.g. --1 p ULTRA to use the ULTRA preset in openMVG_main_ComputeFeatures") +GROUP = PARSER.add_argument_group('Passthrough', description="Option to be passed to command lines (remove - in front of option names)\r\ne.g. --1 p ULTRA to use the ULTRA preset in openMVG_main_ComputeFeatures") for n in range(STEPS.length()): - group.add_argument('--'+str(n), nargs='+') + GROUP.add_argument('--'+str(n), nargs='+') + +PARSER.parse_args(namespace=CONF) # store args in the ConfContainer -parser.parse_args(namespace=CONF) # store args in the ConfContainer # FOLDERS @@ -276,6 +285,7 @@ def mkdir_ine(dirname): if not os.path.exists(dirname): os.mkdir(dirname) + # Absolute path for input and ouput dirs CONF.input_dir = os.path.abspath(CONF.input_dir) CONF.output_dir = os.path.abspath(CONF.output_dir) From 9fd4c59dc756bd6f4a190a76ccb4d3d1e5849ce5 Mon Sep 17 00:00:00 2001 From: cDc Date: Tue, 3 Mar 2020 02:49:38 -0800 Subject: [PATCH 60/70] build: unify install folder usage --- libs/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/CMakeLists.txt b/libs/CMakeLists.txt index a8b41558a..3b6adb25c 100644 --- a/libs/CMakeLists.txt +++ b/libs/CMakeLists.txt @@ -5,4 +5,4 @@ ADD_SUBDIRECTORY(IO) ADD_SUBDIRECTORY(MVS) # Install -INSTALL(FILES "MVS.h" DESTINATION "include/${PROJECT_NAME}") +INSTALL(FILES "MVS.h" DESTINATION "${INSTALL_INCLUDE_DIR}") From 6a3546972ae0d1b8c5fd878f383a46cac699f78e Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 9 Mar 2020 11:12:08 +0100 Subject: [PATCH 61/70] interface: import binary COLMAP files (#538) - Easier to use (no need to create TXT files by hand) - BIN files store values with higher accuracy - Import of BIN files is slightly faster - backwards compatible: TXT files searched first, BIN files used if TXT files missing Authored-by: Sebastian --- apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp | 257 ++++++++++++++++++++--- apps/InterfaceCOLMAP/endian.h | 167 +++++++++++++++ 2 files changed, 394 insertions(+), 30 deletions(-) create mode 100644 apps/InterfaceCOLMAP/endian.h diff --git a/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp b/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp index d450932df..74e1f849c 100644 --- a/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp +++ b/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp @@ -34,6 +34,7 @@ #define _USE_OPENCV #include "../../libs/MVS/Interface.h" #include +#include "endian.h" using namespace MVS; @@ -44,9 +45,12 @@ using namespace MVS; #define MVS_EXT _T(".mvs") #define COLMAP_IMAGES_FOLDER _T("images/") #define COLMAP_SPARSE_FOLDER _T("sparse/") -#define COLMAP_CAMERAS COLMAP_SPARSE_FOLDER _T("cameras.txt") -#define COLMAP_IMAGES COLMAP_SPARSE_FOLDER _T("images.txt") -#define COLMAP_POINTS COLMAP_SPARSE_FOLDER _T("points3D.txt") +#define COLMAP_CAMERAS_TXT COLMAP_SPARSE_FOLDER _T("cameras.txt") +#define COLMAP_IMAGES_TXT COLMAP_SPARSE_FOLDER _T("images.txt") +#define COLMAP_POINTS_TXT COLMAP_SPARSE_FOLDER _T("points3D.txt") +#define COLMAP_CAMERAS_BIN COLMAP_SPARSE_FOLDER _T("cameras.bin") +#define COLMAP_IMAGES_BIN COLMAP_SPARSE_FOLDER _T("images.bin") +#define COLMAP_POINTS_BIN COLMAP_SPARSE_FOLDER _T("points3D.bin") #define COLMAP_DENSE_POINTS _T("fused.ply") #define COLMAP_DENSE_POINTS_VISIBILITY _T("fused.ply.vis") #define COLMAP_STEREO_FOLDER _T("stereo/") @@ -150,10 +154,9 @@ bool Initialize(size_t argc, LPCTSTR* argv) boost::program_options::options_description visible("Available options"); visible.add(generic).add(config); GET_LOG() << _T("\n" - "Import/export 3D reconstruction from/to COLMAP format as TXT (the only documented format).\n" + "Import/export 3D reconstruction from COLMAP (TXT/BIN format) and to COLMAP (TXT format). \n" "In order to import a scene, run COLMAP SfM and next undistort the images (only PINHOLE\n" - "camera model supported for the moment); and convert the BIN scene to TXT by importing in\n" - "COLMAP the sparse scene stored in 'dense' folder and exporting it as TXT.\n" + "camera model supported for the moment)." "\n") << visible; } @@ -204,6 +207,33 @@ void Finalize() } namespace COLMAP { + +using namespace colmap; + +// See colmap/src/util/types.h +typedef uint32_t camera_t; +typedef uint32_t image_t; +typedef uint64_t image_pair_t; +typedef uint32_t point2D_t; +typedef uint64_t point3D_t; + +typedef std::unordered_map CameraModelMap; +CameraModelMap mapCameraModel; + +void DefineCameraModels() { + COLMAP::mapCameraModel.emplace(0, "SIMPLE_PINHOLE"); + COLMAP::mapCameraModel.emplace(1, "PINHOLE"); + COLMAP::mapCameraModel.emplace(2, "SIMPLE_RADIAL"); + COLMAP::mapCameraModel.emplace(3, "RADIAL"); + COLMAP::mapCameraModel.emplace(4, "OPENCV"); + COLMAP::mapCameraModel.emplace(5, "OPENCV_FISHEYE"); + COLMAP::mapCameraModel.emplace(6, "FULL_OPENCV"); + COLMAP::mapCameraModel.emplace(7, "FOV"); + COLMAP::mapCameraModel.emplace(8, "SIMPLE_RADIAL_FISHEYE"); + COLMAP::mapCameraModel.emplace(9, "RADIAL_FISHEYE"); + COLMAP::mapCameraModel.emplace(10, "THIN_PRISM_FISHEYE"); +} + // tools bool NextLine(std::istream& stream, std::istringstream& in, bool bIgnoreEmpty=true) { String line; @@ -223,6 +253,7 @@ struct Camera { String model; // camera model name uint32_t width, height; // camera resolution std::vector params; // camera parameters + bool parsedNumCameras = false; Camera() {} Camera(uint32_t _ID) : ID(_ID) {} @@ -247,9 +278,17 @@ struct Camera { } }; + bool Read(std::istream& stream, bool binary) { + if (binary) { + return ReadBIN(stream); + } else { + return ReadTXT(stream); + } + } + // Camera list with one line of data per camera: // CAMERA_ID, MODEL, WIDTH, HEIGHT, PARAMS[] - bool Read(std::istream& stream) { + bool ReadTXT(std::istream& stream) { std::istringstream in; if (!NextLine(stream, in)) return false; @@ -262,6 +301,31 @@ struct Camera { in >> params[0] >> params[1] >> params[2] >> params[3]; return !in.fail(); } + + // See: colmap/src/base/reconstruction.cc + // void Reconstruction::ReadCamerasBinary(const std::string& path) + bool ReadBIN(std::istream& stream) { + + if (stream.peek() == EOF) + return false; + + if (!parsedNumCameras) { + // Read the first entry in the binary file + ReadBinaryLittleEndian(&stream); + parsedNumCameras = true; + } + + ID = ReadBinaryLittleEndian(&stream); + model = mapCameraModel.at(ReadBinaryLittleEndian(&stream)); + width = ReadBinaryLittleEndian(&stream); + height = ReadBinaryLittleEndian(&stream); + if (model != _T("PINHOLE")) + return false; + params.resize(4); + ReadBinaryLittleEndian(&stream, ¶ms); + return true; + } + bool Write(std::ostream& out) const { out << ID+1 << _T(" ") << model << _T(" ") << width << _T(" ") << height; if (out.fail()) @@ -288,15 +352,24 @@ struct Image { uint32_t idCamera; // ID of the associated camera String name; // image file name std::vector projs; // known image projections + bool parsedNumRegImages = false; Image() {} Image(uint32_t _ID) : ID(_ID) {} bool operator < (const Image& rhs) const { return ID < rhs.ID; } + bool Read(std::istream& stream, bool binary) { + if (binary) { + return ReadBIN(stream); + } else { + return ReadTXT(stream); + } + } + // Image list with two lines of data per image: // IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME // POINTS2D[] as (X, Y, POINT3D_ID) - bool Read(std::istream& stream) { + bool ReadTXT(std::istream& stream) { std::istringstream in; if (!NextLine(stream, in)) return false; @@ -319,6 +392,52 @@ struct Image { } return true; } + + // See: colmap/src/base/reconstruction.cc + // void Reconstruction::ReadImagesBinary(const std::string& path) + bool ReadBIN(std::istream& stream) { + + if (stream.peek() == EOF) + return false; + + if (!parsedNumRegImages) { + // Read the first entry in the binary file + ReadBinaryLittleEndian(&stream); + parsedNumRegImages = true; + } + + ID = ReadBinaryLittleEndian(&stream); + q.w() = ReadBinaryLittleEndian(&stream); + q.x() = ReadBinaryLittleEndian(&stream); + q.y() = ReadBinaryLittleEndian(&stream); + q.z() = ReadBinaryLittleEndian(&stream); + t(0) = ReadBinaryLittleEndian(&stream); + t(1) = ReadBinaryLittleEndian(&stream); + t(2) = ReadBinaryLittleEndian(&stream); + idCamera = ReadBinaryLittleEndian(&stream); + + name = ""; + char nameChar; + do { + stream.read(&nameChar, 1); + if (nameChar != '\0') { + name += nameChar; + } + } while (nameChar != '\0'); + Util::ensureValidPath(name); + + const size_t numPoints2D = ReadBinaryLittleEndian(&stream); + projs.clear(); + for (size_t j = 0; j < numPoints2D; ++j) { + Proj proj; + proj.p(0) = ReadBinaryLittleEndian(&stream); + proj.p(1) = ReadBinaryLittleEndian(&stream); + proj.idPoint = ReadBinaryLittleEndian(&stream); + projs.push_back(proj); + } + return true; + } + bool Write(std::ostream& out) const { out << ID+1 << _T(" ") << q.w() << _T(" ") << q.x() << _T(" ") << q.y() << _T(" ") << q.z() << _T(" ") @@ -346,14 +465,23 @@ struct Point { Interface::Col3 c; // BGR color float e; // error std::vector tracks; // point track + bool parsedNumPoints3D = false; Point() {} Point(uint32_t _ID) : ID(_ID) {} bool operator < (const Image& rhs) const { return ID < rhs.ID; } + bool Read(std::istream& stream, bool binary) { + if (binary) { + return ReadBIN(stream); + } else { + return ReadTXT(stream); + } + } + // 3D point list with one line of data per point: // POINT3D_ID, X, Y, Z, R, G, B, ERROR, TRACK[] as (IMAGE_ID, POINT2D_IDX) - bool Read(std::istream& stream) { + bool ReadTXT(std::istream& stream) { std::istringstream in; if (!NextLine(stream, in)) return false; @@ -377,6 +505,44 @@ struct Point { } return !tracks.empty(); } + + // See: colmap/src/base/reconstruction.cc + // void Reconstruction::ReadPoints3DBinary(const std::string& path) + bool ReadBIN(std::istream& stream) { + + if (stream.peek() == EOF) + return false; + + if (!parsedNumPoints3D) { + // Read the first entry in the binary file + ReadBinaryLittleEndian(&stream); + parsedNumPoints3D = true; + } + + int r,g,b; + ID = ReadBinaryLittleEndian(&stream); + p.x = ReadBinaryLittleEndian(&stream); + p.y = ReadBinaryLittleEndian(&stream); + p.z = ReadBinaryLittleEndian(&stream); + r = ReadBinaryLittleEndian(&stream); + g = ReadBinaryLittleEndian(&stream); + b = ReadBinaryLittleEndian(&stream); + e = ReadBinaryLittleEndian(&stream); + c.x = CLAMP(b,0,255); + c.y = CLAMP(g,0,255); + c.z = CLAMP(r,0,255); + + const size_t trackLength = ReadBinaryLittleEndian(&stream); + tracks.clear(); + for (size_t j = 0; j < trackLength; ++j) { + Track track; + track.idImage = ReadBinaryLittleEndian(&stream); + track.idProj = ReadBinaryLittleEndian(&stream); + tracks.push_back(track); + } + return !tracks.empty(); + } + bool Write(std::ostream& out) const { ASSERT(!tracks.empty()); const int r(c.z),g(c.y),b(c.x); @@ -399,23 +565,48 @@ typedef std::vector Points; typedef Eigen::Matrix EMat33d; typedef Eigen::Matrix EVec3d; + +bool DetermineInputSource(const String& filenameTXT, const String& filenameBIN, std::ifstream& file, String& filenameCamera, bool& binary) +{ + file.open(filenameTXT); + if (file.good()) { + filenameCamera = filenameTXT; + binary = false; + return true; + } + file.open(filenameBIN, std::ios::binary); + if (file.good()) { + filenameCamera = filenameBIN; + binary = true; + return true; + } + VERBOSE("error: unable to open file '%s'", filenameTXT.c_str()); + VERBOSE("error: unable to open file '%s'", filenameBIN.c_str()); + return false; +} + + bool ImportScene(const String& strFolder, Interface& scene) { + COLMAP::DefineCameraModels(); // read camera list typedef std::unordered_map CamerasMap; CamerasMap mapCameras; { - const String filenameCameras(strFolder+COLMAP_CAMERAS); - LOG_OUT() << "Reading cameras: " << filenameCameras << std::endl; - std::ifstream file(filenameCameras); - if (!file.good()) { - VERBOSE("error: unable to open file '%s'", filenameCameras.c_str()); + const String filenameCamerasTXT(strFolder+COLMAP_CAMERAS_TXT); + const String filenameCamerasBIN(strFolder+COLMAP_CAMERAS_BIN); + std::ifstream file; + bool binary; + String filenameCamera; + if (!DetermineInputSource(filenameCamerasTXT, filenameCamerasBIN, file, filenameCamera, binary)) { return false; } + LOG_OUT() << "Reading cameras: " << filenameCamera << std::endl; + typedef std::unordered_map CamerasSet; CamerasSet setCameras; COLMAP::Camera colmapCamera; - while (file.good() && colmapCamera.Read(file)) { + while (file.good() && colmapCamera.Read(file, binary)) { const auto setIt(setCameras.emplace(colmapCamera, (uint32_t)scene.platforms.size())); mapCameras.emplace(colmapCamera.ID, setIt.first->second); if (!setIt.second) { @@ -458,15 +649,18 @@ bool ImportScene(const String& strFolder, Interface& scene) typedef std::map ImagesMap; ImagesMap mapImages; { - const String filenameImages(strFolder+COLMAP_IMAGES); - LOG_OUT() << "Reading images: " << filenameImages << std::endl; - std::ifstream file(filenameImages); - if (!file.good()) { - VERBOSE("error: unable to open file '%s'", filenameImages.c_str()); + const String filenameImagesTXT(strFolder+COLMAP_IMAGES_TXT); + const String filenameImagesBIN(strFolder+COLMAP_IMAGES_BIN); + std::ifstream file; + bool binary; + String filenameImages; + if (!DetermineInputSource(filenameImagesTXT, filenameImagesBIN, file, filenameImages, binary)) { return false; } + LOG_OUT() << "Reading images: " << filenameImages << std::endl; + COLMAP::Image imageColmap; - while (file.good() && imageColmap.Read(file)) { + while (file.good() && imageColmap.Read(file, binary)) { mapImages.emplace(imageColmap, (uint32_t)scene.images.size()); Interface::Platform::Pose pose; Eigen::Map(pose.R.val) = imageColmap.q.toRotationMatrix(); @@ -489,15 +683,18 @@ bool ImportScene(const String& strFolder, Interface& scene) const String filenameDenseVisPoints(strFolder+COLMAP_DENSE_POINTS_VISIBILITY); if (!File::access(filenameDensePoints) || !File::access(filenameDenseVisPoints)) { // parse sparse point-cloud - const String filenamePoints(strFolder+COLMAP_POINTS); - LOG_OUT() << "Reading points: " << filenamePoints << std::endl; - std::ifstream file(filenamePoints); - if (!file.good()) { - VERBOSE("error: unable to open file '%s'", filenamePoints.c_str()); + const String filenamePointsTXT(strFolder+COLMAP_POINTS_TXT); + const String filenamePointsBIN(strFolder+COLMAP_POINTS_BIN); + std::ifstream file; + bool binary; + String filenamePoints; + if (!DetermineInputSource(filenamePointsTXT, filenamePointsBIN, file, filenamePoints, binary)) { return false; } + LOG_OUT() << "Reading images: " << filenamePoints << std::endl; + COLMAP::Point point; - while (file.good() && point.Read(file)) { + while (file.good() && point.Read(file, binary)) { Interface::Vertex vertex; vertex.X = point.p; for (const COLMAP::Point::Track& track: point.tracks) { @@ -560,7 +757,7 @@ bool ExportScene(const String& strFolder, const Interface& scene) CLISTDEF0IDX(KMatrix,uint32_t) Ks; CLISTDEF0IDX(COLMAP::Camera,uint32_t) cams; { - const String filenameCameras(strFolder+COLMAP_CAMERAS); + const String filenameCameras(strFolder+COLMAP_CAMERAS_TXT); LOG_OUT() << "Writing cameras: " << filenameCameras << std::endl; std::ofstream file(filenameCameras); if (!file.good()) { @@ -662,7 +859,7 @@ bool ExportScene(const String& strFolder, const Interface& scene) if (bSparsePointCloud) { // write points list { - const String filenamePoints(strFolder+COLMAP_POINTS); + const String filenamePoints(strFolder+COLMAP_POINTS_TXT); LOG_OUT() << "Writing points: " << filenamePoints << std::endl; std::ofstream file(filenamePoints); if (!file.good()) { @@ -774,7 +971,7 @@ bool ExportScene(const String& strFolder, const Interface& scene) // write images list { - const String filenameImages(strFolder+COLMAP_IMAGES); + const String filenameImages(strFolder+COLMAP_IMAGES_TXT); LOG_OUT() << "Writing images: " << filenameImages << std::endl; std::ofstream file(filenameImages); if (!file.good()) { diff --git a/apps/InterfaceCOLMAP/endian.h b/apps/InterfaceCOLMAP/endian.h new file mode 100644 index 000000000..12a22cab6 --- /dev/null +++ b/apps/InterfaceCOLMAP/endian.h @@ -0,0 +1,167 @@ +// Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of +// its contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) + +#ifndef COLMAP_SRC_UTIL_ENDIAN_H_ +#define COLMAP_SRC_UTIL_ENDIAN_H_ + +#include +#include +#include + +namespace colmap { + +// Reverse the order of each byte. +template +T ReverseBytes(const T& data); + +// Check the order in which bytes are stored in computer memory. +bool IsLittleEndian(); +bool IsBigEndian(); + +// Convert data between endianness and the native format. Note that, for float +// and double types, these functions are only valid if the format is IEEE-754. +// This is the case for pretty much most processors. +template +T LittleEndianToNative(const T x); +template +T BigEndianToNative(const T x); +template +T NativeToLittleEndian(const T x); +template +T NativeToBigEndian(const T x); + +// Read data in little endian format for cross-platform support. +template +T ReadBinaryLittleEndian(std::istream* stream); +template +void ReadBinaryLittleEndian(std::istream* stream, std::vector* data); + +// Write data in little endian format for cross-platform support. +template +void WriteBinaryLittleEndian(std::ostream* stream, const T& data); +template +void WriteBinaryLittleEndian(std::ostream* stream, const std::vector& data); + +//////////////////////////////////////////////////////////////////////////////// +// Implementation +//////////////////////////////////////////////////////////////////////////////// + +template +T ReverseBytes(const T& data) { + T data_reversed = data; + std::reverse(reinterpret_cast(&data_reversed), + reinterpret_cast(&data_reversed) + sizeof(T)); + return data_reversed; +} + +inline bool IsLittleEndian() { +#ifdef BOOST_BIG_ENDIAN + return false; +#else + return true; +#endif +} + +inline bool IsBigEndian() { +#ifdef BOOST_BIG_ENDIAN + return true; +#else + return false; +#endif +} + +template +T LittleEndianToNative(const T x) { + if (IsLittleEndian()) { + return x; + } else { + return ReverseBytes(x); + } +} + +template +T BigEndianToNative(const T x) { + if (IsBigEndian()) { + return x; + } else { + return ReverseBytes(x); + } +} + +template +T NativeToLittleEndian(const T x) { + if (IsLittleEndian()) { + return x; + } else { + return ReverseBytes(x); + } +} + +template +T NativeToBigEndian(const T x) { + if (IsBigEndian()) { + return x; + } else { + return ReverseBytes(x); + } +} + +template +T ReadBinaryLittleEndian(std::istream* stream) { + T data_little_endian; + stream->read(reinterpret_cast(&data_little_endian), sizeof(T)); + return LittleEndianToNative(data_little_endian); +} + +template +void ReadBinaryLittleEndian(std::istream* stream, std::vector* data) { + for (size_t i = 0; i < data->size(); ++i) { + (*data)[i] = ReadBinaryLittleEndian(stream); + } +} + +template +void WriteBinaryLittleEndian(std::ostream* stream, const T& data) { + const T data_little_endian = NativeToLittleEndian(data); + stream->write(reinterpret_cast(&data_little_endian), sizeof(T)); +} + +template +void WriteBinaryLittleEndian(std::ostream* stream, const std::vector& data) { + for (const auto& elem : data) { + WriteBinaryLittleEndian(stream, elem); + } +} + +} // namespace colmap + +#endif // COLMAP_SRC_UTIL_ENDIAN_H_ + From 60ba435e0c8a7df474188d8fbf741b4d847af18a Mon Sep 17 00:00:00 2001 From: cDc Date: Sat, 4 Apr 2020 14:11:52 +0300 Subject: [PATCH 62/70] interface: fix warnings --- apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp b/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp index 74e1f849c..dbe796ca6 100644 --- a/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp +++ b/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp @@ -317,8 +317,8 @@ struct Camera { ID = ReadBinaryLittleEndian(&stream); model = mapCameraModel.at(ReadBinaryLittleEndian(&stream)); - width = ReadBinaryLittleEndian(&stream); - height = ReadBinaryLittleEndian(&stream); + width = (uint32_t)ReadBinaryLittleEndian(&stream); + height = (uint32_t)ReadBinaryLittleEndian(&stream); if (model != _T("PINHOLE")) return false; params.resize(4); @@ -430,9 +430,9 @@ struct Image { projs.clear(); for (size_t j = 0; j < numPoints2D; ++j) { Proj proj; - proj.p(0) = ReadBinaryLittleEndian(&stream); - proj.p(1) = ReadBinaryLittleEndian(&stream); - proj.idPoint = ReadBinaryLittleEndian(&stream); + proj.p(0) = (float)ReadBinaryLittleEndian(&stream); + proj.p(1) = (float)ReadBinaryLittleEndian(&stream); + proj.idPoint = (uint32_t)ReadBinaryLittleEndian(&stream); projs.push_back(proj); } return true; @@ -520,14 +520,14 @@ struct Point { } int r,g,b; - ID = ReadBinaryLittleEndian(&stream); - p.x = ReadBinaryLittleEndian(&stream); - p.y = ReadBinaryLittleEndian(&stream); - p.z = ReadBinaryLittleEndian(&stream); + ID = (uint32_t)ReadBinaryLittleEndian(&stream); + p.x = (float)ReadBinaryLittleEndian(&stream); + p.y = (float)ReadBinaryLittleEndian(&stream); + p.z = (float)ReadBinaryLittleEndian(&stream); r = ReadBinaryLittleEndian(&stream); g = ReadBinaryLittleEndian(&stream); b = ReadBinaryLittleEndian(&stream); - e = ReadBinaryLittleEndian(&stream); + e = (float)ReadBinaryLittleEndian(&stream); c.x = CLAMP(b,0,255); c.y = CLAMP(g,0,255); c.z = CLAMP(r,0,255); From e1ea9c9602038c2c79eff37baeb089b8dbbab636 Mon Sep 17 00:00:00 2001 From: cDc Date: Sat, 4 Apr 2020 14:15:27 +0300 Subject: [PATCH 63/70] viewer: add support to visualize depth-maps --- apps/Viewer/Scene.cpp | 4 +- apps/Viewer/Window.cpp | 6 +-- apps/Viewer/Window.h | 2 +- libs/Common/Types.h | 26 ++++++------- libs/Common/Types.inl | 4 +- libs/MVS/Camera.h | 19 ++++++++++ libs/MVS/DepthMap.cpp | 42 ++++++++++++--------- libs/MVS/DepthMap.h | 1 + libs/MVS/Scene.cpp | 84 ++++++++++++++++++++++++++++++++++++++++++ libs/MVS/Scene.h | 1 + 10 files changed, 151 insertions(+), 38 deletions(-) diff --git a/apps/Viewer/Scene.cpp b/apps/Viewer/Scene.cpp index 8e63dbf9a..b40cb1bf2 100644 --- a/apps/Viewer/Scene.cpp +++ b/apps/Viewer/Scene.cpp @@ -334,7 +334,7 @@ bool Scene::Open(LPCTSTR fileName, LPCTSTR meshFileName) // init scene AABB3d bounds(true); if (!scene.pointcloud.IsEmpty()) { - bounds = scene.pointcloud.GetAABB(3); + bounds = scene.pointcloud.GetAABB(MINF(3u,scene.nCalibratedImages)); if (bounds.IsEmpty()) bounds = scene.pointcloud.GetAABB(); } @@ -378,7 +378,7 @@ bool Scene::Open(LPCTSTR fileName, LPCTSTR meshFileName) window.SetCamera(CameraPtr(new Camera(bounds))); window.camera->maxCamID = images.size(); window.SetName(String::FormatString((name + _T(": %s")).c_str(), Util::getFileName(fileName).c_str())); - window.Reset(); + window.Reset(MINF(2u, images.size())); return true; } diff --git a/apps/Viewer/Window.cpp b/apps/Viewer/Window.cpp index 6a9ada4f1..4cb3eeef4 100644 --- a/apps/Viewer/Window.cpp +++ b/apps/Viewer/Window.cpp @@ -102,12 +102,12 @@ void Window::SetVisible(bool v) else glfwHideWindow(window); } -void Window::Reset() +void Window::Reset(uint32_t _minViews) { if (camera) camera->Reset(); sparseType = SPR_ALL; - minViews = 2; + minViews = _minViews; pointSize = 2.f; cameraBlend = 0.5f; bRenderCameras = true; @@ -266,7 +266,7 @@ void Window::Key(int k, int /*scancode*/, int action, int mod) else if (mod & GLFW_MOD_SHIFT) camera->scaleF *= 1.11f; else - cameraBlend += MAXF(cameraBlend-0.1f, 0.f); + cameraBlend = MINF(cameraBlend+0.1f, 1.f); } break; } diff --git a/apps/Viewer/Window.h b/apps/Viewer/Window.h index 87f62eb76..0d9fe5034 100644 --- a/apps/Viewer/Window.h +++ b/apps/Viewer/Window.h @@ -100,7 +100,7 @@ class Window void SetCamera(CameraPtr); void SetName(LPCTSTR); void SetVisible(bool); - void Reset(); + void Reset(uint32_t minViews=2); inline GLFWwindow* GetWindow() { return window; } diff --git a/libs/Common/Types.h b/libs/Common/Types.h index bcac65c1b..631f024ce 100644 --- a/libs/Common/Types.h +++ b/libs/Common/Types.h @@ -1896,13 +1896,13 @@ struct TPixel { : r(TYPE((col>>16)&0xFF)), g(TYPE((col>>8)&0xFF)), b(TYPE(col&0xFF)) {} #endif // set/get from default type - inline void set(TYPE _r, TYPE _g, TYPE _b) { r = _r; g = _g; b = _b; } - inline void set(const TYPE* clr) { c[0] = clr[0]; c[1] = clr[1]; c[2] = clr[2]; } + inline TPixel& set(TYPE _r, TYPE _g, TYPE _b) { r = _r; g = _g; b = _b; return *this; } + inline TPixel& set(const TYPE* clr) { c[0] = clr[0]; c[1] = clr[1]; c[2] = clr[2]; return *this; } inline void get(TYPE& _r, TYPE& _g, TYPE& _b) const { _r = r; _g = g; _b = b; } inline void get(TYPE* clr) const { clr[0] = c[0]; clr[1] = c[1]; clr[2] = c[2]; } // set/get from alternative type - inline void set(ALT _r, ALT _g, ALT _b) { r = TYPE(_r); g = TYPE(_g); b = TYPE(_b); } - inline void set(const ALT* clr) { c[0] = TYPE(clr[0]); c[1] = TYPE(clr[1]); c[2] = TYPE(clr[2]); } + inline TPixel& set(ALT _r, ALT _g, ALT _b) { r = TYPE(_r); g = TYPE(_g); b = TYPE(_b); return *this; } + inline TPixel& set(const ALT* clr) { c[0] = TYPE(clr[0]); c[1] = TYPE(clr[1]); c[2] = TYPE(clr[2]); return *this; } inline void get(ALT& _r, ALT& _g, ALT& _b) const { _r = ALT(r); _g = ALT(g); _b = ALT(b); } inline void get(ALT* clr) const { clr[0] = ALT(c[0]); clr[1] = ALT(c[1]); clr[2] = ALT(c[2]); } template inline TPixel::value || !std::is_same::value,T>::type> cast() const { return TPixel(T(r), T(g), T(b)); } @@ -1952,9 +1952,9 @@ struct TPixel { } #endif }; -template <> inline void TPixel::set(uint8_t _r, uint8_t _g, uint8_t _b) { r = float(_r)/255; g = float(_g)/255; b = float(_b)/255; } -template <> inline void TPixel::get(uint8_t& _r, uint8_t& _g, uint8_t& _b) const { _r = uint8_t(r*255); _g = uint8_t(g*255); _b = uint8_t(b*255); } -template <> inline void TPixel::set(float _r, float _g, float _b) { r = uint8_t(_r*255); g = uint8_t(_g*255); b = uint8_t(_b*255); } +template <> inline TPixel& TPixel::set(uint8_t _r, uint8_t _g, uint8_t _b) { r = float(_r)/255; g = float(_g)/255; b = float(_b)/255; return *this; } +template <> inline void TPixel::get(uint8_t& _r, uint8_t& _g, uint8_t& _b) const { _r = (uint8_t)CLAMP(ROUND2INT(r*255), 0, 255); _g = (uint8_t)CLAMP(ROUND2INT(g*255), 0, 255); _b = (uint8_t)CLAMP(ROUND2INT(b*255), 0, 255); } +template <> inline TPixel& TPixel::set(float _r, float _g, float _b) { r = (uint8_t)CLAMP(ROUND2INT(_r*255), 0, 255); g = (uint8_t)CLAMP(ROUND2INT(_g*255), 0, 255); b = (uint8_t)CLAMP(ROUND2INT(_b*255), 0, 255); return *this; } template <> inline void TPixel::get(float& _r, float& _g, float& _b) const { _r = float(r)/255; _g = float(g)/255; _b = float(b)/255; } /*----------------------------------------------------------------*/ typedef TPixel Pixel8U; @@ -2019,13 +2019,13 @@ struct TColor { explicit inline TColor(uint32_t col) : r(TYPE((col>>16)&0xFF)), g(TYPE((col>>8)&0xFF)), b(TYPE(col&0xFF)), a(TYPE((col>>24)&0xFF)) {} // set/get from default type - inline void set(TYPE _r, TYPE _g, TYPE _b, TYPE _a=ColorType::ONE) { r = _r; g = _g; b = _b; a = _a; } - inline void set(const TYPE* clr) { c[0] = clr[0]; c[1] = clr[1]; c[2] = clr[2]; c[3] = clr[3]; } + inline TColor& set(TYPE _r, TYPE _g, TYPE _b, TYPE _a=ColorType::ONE) { r = _r; g = _g; b = _b; a = _a; return *this; } + inline TColor& set(const TYPE* clr) { c[0] = clr[0]; c[1] = clr[1]; c[2] = clr[2]; c[3] = clr[3]; return *this; } inline void get(TYPE& _r, TYPE& _g, TYPE& _b, TYPE& _a) const { _r = r; _g = g; _b = b; _a = a; } inline void get(TYPE* clr) const { clr[0] = c[0]; clr[1] = c[1]; clr[2] = c[2]; clr[3] = c[3]; } // set/get from alternative type - inline void set(ALT _r, ALT _g, ALT _b, ALT _a=ColorType::ALTONE) { r = TYPE(_r); g = TYPE(_g); b = TYPE(_b); a = TYPE(_a); } - inline void set(const ALT* clr) { c[0] = TYPE(clr[0]); c[1] = TYPE(clr[1]); c[2] = TYPE(clr[2]); c[3] = TYPE(clr[3]); } + inline TColor& set(ALT _r, ALT _g, ALT _b, ALT _a=ColorType::ALTONE) { r = TYPE(_r); g = TYPE(_g); b = TYPE(_b); a = TYPE(_a); return *this; } + inline TColor& set(const ALT* clr) { c[0] = TYPE(clr[0]); c[1] = TYPE(clr[1]); c[2] = TYPE(clr[2]); c[3] = TYPE(clr[3]); return *this; } inline void get(ALT& _r, ALT& _g, ALT& _b, ALT& _a) const { _r = ALT(r); _g = ALT(g); _b = ALT(b); _a = ALT(a); } inline void get(ALT* clr) const { clr[0] = ALT(c[0]); clr[1] = ALT(c[1]); clr[2] = ALT(c[2]); clr[3] = ALT(c[3]); } template inline TColor::value || !std::is_same::value,T>::type> cast() const { return TColor(T(r), T(g), T(b), T(a)); } @@ -2073,9 +2073,9 @@ struct TColor { } #endif }; -template <> inline void TColor::set(uint8_t _r, uint8_t _g, uint8_t _b, uint8_t _a) { r = float(_r)/255; g = float(_g)/255; b = float(_b)/255; a = float(_a)/255; } +template <> inline TColor& TColor::set(uint8_t _r, uint8_t _g, uint8_t _b, uint8_t _a) { r = float(_r)/255; g = float(_g)/255; b = float(_b)/255; a = float(_a)/255; return *this; } template <> inline void TColor::get(uint8_t& _r, uint8_t& _g, uint8_t& _b, uint8_t& _a) const { _r = uint8_t(r*255); _g = uint8_t(g*255); _b = uint8_t(b*255); _a = uint8_t(a*255); } -template <> inline void TColor::set(float _r, float _g, float _b, float _a) { r = uint8_t(_r*255); g = uint8_t(_g*255); b = uint8_t(_b*255); a = uint8_t(_a*255); } +template <> inline TColor& TColor::set(float _r, float _g, float _b, float _a) { r = uint8_t(_r*255); g = uint8_t(_g*255); b = uint8_t(_b*255); a = uint8_t(_a*255); return *this; } template <> inline void TColor::get(float& _r, float& _g, float& _b, float& _a) const { _r = float(r)/255; _g = float(g)/255; _b = float(b)/255; _a = float(a)/255; } template <> inline bool TColor::operator==(const TColor& col) const { return (*((const uint32_t*)c) == *((const uint32_t*)col.c)); } template <> inline bool TColor::operator!=(const TColor& col) const { return (*((const uint32_t*)c) != *((const uint32_t*)col.c)); } diff --git a/libs/Common/Types.inl b/libs/Common/Types.inl index 090f896dc..cbdf89fe1 100644 --- a/libs/Common/Types.inl +++ b/libs/Common/Types.inl @@ -2216,11 +2216,11 @@ TPixel TPixel::gray2color(ALT gray) return ALT(0); } }; - return TPixel( + return TPixel().set( Base(gray + ALT(0.25)), Base(gray), Base(gray - ALT(0.25)) - ).template cast(); + ); } /*----------------------------------------------------------------*/ diff --git a/libs/MVS/Camera.h b/libs/MVS/Camera.h index ee22d8a80..fabae38d4 100644 --- a/libs/MVS/Camera.h +++ b/libs/MVS/Camera.h @@ -107,6 +107,25 @@ class MVS_API CameraIntern return float(MAXF(width, height)); } + // return scaled K (assuming standard K format) + template + static inline TMatrix ScaleK(const TMatrix& K, TYPE s) { + #if 0 + TMatrix S(TMatrix::IDENTITY); + S(0,0) = S(1,1) = s; + return S*K; + #else + return TMatrix( + K(0,0)*s, K(0,1)*s, K(0,2)*s, + TYPE(0), K(1,1)*s, K(1,2)*s, + TYPE(0), TYPE(0), TYPE(1) + ); + #endif + } + inline KMatrix GetScaledK(REAL s) const { + return ScaleK(K, s); + } + // return K.inv() (assuming standard K format) inline KMatrix GetInvK() const { #if 0 diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index 0b3e6008d..286a03e22 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -1485,36 +1485,40 @@ bool MVS::LoadConfidenceMap(const String& fileName, ConfidenceMap& confMap) // export depth map as an image (dark - far depth, light - close depth) -bool MVS::ExportDepthMap(const String& fileName, const DepthMap& depthMap, Depth minDepth, Depth maxDepth) +Image8U3 MVS::DepthMap2Image(const DepthMap& depthMap, Depth minDepth, Depth maxDepth) { + ASSERT(!depthMap.empty()); // find min and max values if (minDepth == FLT_MAX && maxDepth == 0) { - cList depths(0, depthMap.area()); + cList depths(0, depthMap.area()); for (int i=depthMap.area(); --i >= 0; ) { const Depth depth = depthMap[i]; ASSERT(depth == 0 || depth > 0); if (depth > 0) depths.Insert(depth); } - if (!depths.IsEmpty()) { - const std::pair th(ComputeX84Threshold(depths.Begin(), depths.GetSize())); - maxDepth = th.first+th.second; - minDepth = th.first-th.second; + if (!depths.empty()) { + const std::pair th(ComputeX84Threshold(depths.data(), depths.size())); + const std::pair mm(depths.GetMinMax()); + maxDepth = MINF(th.first+th.second, mm.second); + minDepth = MAXF(th.first-th.second, mm.first); } - if (minDepth < 0.1f) - minDepth = 0.1f; - if (maxDepth < 0.1f) - maxDepth = 30.f; DEBUG_ULTIMATE("\tdepth range: [%g, %g]", minDepth, maxDepth); } - const Depth deltaDepth = maxDepth - minDepth; - // save image - Image8U img(depthMap.size()); + const Depth sclDepth(Depth(1)/(maxDepth - minDepth)); + // create color image + Image8U3 img(depthMap.size()); for (int i=depthMap.area(); --i >= 0; ) { const Depth depth = depthMap[i]; - img[i] = (depth > 0 ? (uint8_t)CLAMP((maxDepth-depth)*255.f/deltaDepth, 0.f, 255.f) : 0); + img[i] = (depth > 0 ? Pixel8U::gray2color(CLAMP((maxDepth-depth)*sclDepth, Depth(0), Depth(1))) : Pixel8U::BLACK); } - return img.Save(fileName); + return img; +} // DepthMap2Image +bool MVS::ExportDepthMap(const String& fileName, const DepthMap& depthMap, Depth minDepth, Depth maxDepth) +{ + if (depthMap.empty()) + return false; + return DepthMap2Image(depthMap, minDepth, maxDepth).Save(fileName); } // ExportDepthMap /*----------------------------------------------------------------*/ @@ -1829,8 +1833,12 @@ bool MVS::ImportDepthDataRaw(const String& fileName, String& imageFileName, dMax = header.dMax; imageSize.width = header.imageWidth; imageSize.height = header.imageHeight; - depthMap.create(header.depthHeight, header.depthWidth); - fread(depthMap.getData(), sizeof(float), depthMap.area(), f); + if ((flags & HeaderDepthDataRaw::HAS_DEPTH) != 0) { + depthMap.create(header.depthHeight, header.depthWidth); + fread(depthMap.getData(), sizeof(float), depthMap.area(), f); + } else { + fseek(f, sizeof(float)*header.depthWidth*header.depthHeight, SEEK_CUR); + } // read normal-map if ((header.type & HeaderDepthDataRaw::HAS_NORMAL) != 0) { diff --git a/libs/MVS/DepthMap.h b/libs/MVS/DepthMap.h index 2e71cc48e..6cdb72c30 100644 --- a/libs/MVS/DepthMap.h +++ b/libs/MVS/DepthMap.h @@ -464,6 +464,7 @@ MVS_API bool LoadNormalMap(const String& fileName, NormalMap& normalMap); MVS_API bool SaveConfidenceMap(const String& fileName, const ConfidenceMap& confMap); MVS_API bool LoadConfidenceMap(const String& fileName, ConfidenceMap& confMap); +MVS_API Image8U3 DepthMap2Image(const DepthMap& depthMap, Depth minDepth=FLT_MAX, Depth maxDepth=0); MVS_API bool ExportDepthMap(const String& fileName, const DepthMap& depthMap, Depth minDepth=FLT_MAX, Depth maxDepth=0); MVS_API bool ExportNormalMap(const String& fileName, const NormalMap& normalMap); MVS_API bool ExportConfidenceMap(const String& fileName, const ConfidenceMap& confMap); diff --git a/libs/MVS/Scene.cpp b/libs/MVS/Scene.cpp index ff8cf696d..4039377f2 100644 --- a/libs/MVS/Scene.cpp +++ b/libs/MVS/Scene.cpp @@ -277,10 +277,94 @@ bool Scene::SaveInterface(const String & fileName, int version) const } // SaveInterface /*----------------------------------------------------------------*/ + +// load depth-map and generate a Multi-View Stereo scene +bool Scene::LoadDMAP(const String& fileName) +{ + TD_TIMER_STARTD(); + + // load depth-map data + String imageFileName; + IIndexArr IDs; + cv::Size imageSize; + Camera camera; + Depth dMin, dMax; + DepthMap depthMap; + NormalMap normalMap; + ConfidenceMap confMap; + if (!ImportDepthDataRaw(fileName, imageFileName, IDs, imageSize, camera.K, camera.R, camera.C, dMin, dMax, depthMap, normalMap, confMap)) + return false; + + // create image + Platform& platform = platforms.AddEmpty(); + platform.name = _T("platform0"); + platform.cameras.emplace_back(CameraIntern::ScaleK(camera.K, REAL(1)/CameraIntern::GetNormalizationScale((uint32_t)imageSize.width,(uint32_t)imageSize.height)), RMatrix::IDENTITY, CMatrix::ZERO); + platform.poses.emplace_back(Platform::Pose{camera.R, camera.C}); + Image& image = images.AddEmpty(); + image.name = MAKE_PATH_FULL(WORKING_FOLDER_FULL, imageFileName); + image.platformID = 0; + image.cameraID = 0; + image.poseID = 0; + image.ID = IDs.front(); + image.scale = 1; + image.avgDepth = (dMin+dMax)/2; + image.width = (uint32_t)imageSize.width; + image.height = (uint32_t)imageSize.height; + image.UpdateCamera(platforms); + nCalibratedImages = 1; + + // load image pixels + const Image8U3 imageDepth(DepthMap2Image(depthMap)); + if (image.ReloadImage(MAXF(image.width,image.height))) + cv::resize(image.image, image.image, depthMap.size()); + else + image.image = imageDepth; + + // create point-cloud + pointcloud.points.reserve(depthMap.area()); + pointcloud.pointViews.reserve(depthMap.area()); + pointcloud.colors.reserve(depthMap.area()); + if (!normalMap.empty()) + pointcloud.normals.reserve(depthMap.area()); + if (!confMap.empty()) + pointcloud.pointWeights.reserve(depthMap.area()); + for (int r=0; r(camera.R.t()*Cast(normalMap(r,c)))); + if (!confMap.empty()) + pointcloud.pointWeights.emplace_back(PointCloud::WeightArr{confMap(r,c)}); + } + } + + // replace color-image with depth-image + image.image = imageDepth; + + DEBUG_EXTRA("Scene loaded from depth-map format - %dx%d size, %.2f%% coverage (%s):\n" + "\t1 images (1 calibrated) with a total of %.2f MPixels (%.2f MPixels/image)\n" + "\t%u points, 0 lines", + depthMap.width(), depthMap.height(), 100.0*pointcloud.GetSize()/depthMap.area(), TD_TIMER_GET_FMT().c_str(), + (double)image.image.area()/(1024.0*1024.0), (double)image.image.area()/(1024.0*1024.0*nCalibratedImages), + pointcloud.GetSize()); + return true; +} // LoadDMAP +/*----------------------------------------------------------------*/ + // try to load known point-cloud or mesh files bool Scene::Import(const String& fileName) { const String ext(Util::getFileExt(fileName).ToLower()); + if (ext == _T(".dmap")) { + // import point-cloud from dmap file + Release(); + return LoadDMAP(fileName); + } if (ext == _T(".obj")) { // import mesh from obj file Release(); diff --git a/libs/MVS/Scene.h b/libs/MVS/Scene.h index 055969ca7..71b94acab 100644 --- a/libs/MVS/Scene.h +++ b/libs/MVS/Scene.h @@ -70,6 +70,7 @@ class MVS_API Scene bool LoadInterface(const String& fileName); bool SaveInterface(const String& fileName, int version=-1) const; + bool LoadDMAP(const String& fileName); bool Import(const String& fileName); bool Load(const String& fileName, bool bImport=false); From 5f086ffa5257e4a6bfec7e8533bf7f854469402c Mon Sep 17 00:00:00 2001 From: cDc Date: Mon, 11 May 2020 18:07:05 +0300 Subject: [PATCH 64/70] dense: improve score aggregation --- apps/DensifyPointCloud/DensifyPointCloud.cpp | 3 --- libs/MVS/DepthMap.cpp | 22 ++++++++++---------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/apps/DensifyPointCloud/DensifyPointCloud.cpp b/apps/DensifyPointCloud/DensifyPointCloud.cpp index 9d225289c..62350b976 100644 --- a/apps/DensifyPointCloud/DensifyPointCloud.cpp +++ b/apps/DensifyPointCloud/DensifyPointCloud.cpp @@ -91,7 +91,6 @@ bool Initialize(size_t argc, LPCTSTR* argv) unsigned nMinResolution; unsigned nNumViews; unsigned nMinViewsFuse; - unsigned nOptimize; unsigned nEstimateColors; unsigned nEstimateNormals; boost::program_options::options_description config("Densify options"); @@ -103,7 +102,6 @@ bool Initialize(size_t argc, LPCTSTR* argv) ("min-resolution", boost::program_options::value(&nMinResolution)->default_value(640), "do not scale images lower than this resolution") ("number-views", boost::program_options::value(&nNumViews)->default_value(5), "number of views used for depth-map estimation (0 - all neighbor views available)") ("number-views-fuse", boost::program_options::value(&nMinViewsFuse)->default_value(3), "minimum number of images that agrees with an estimate during fusion in order to consider it inlier") - ("optimize", boost::program_options::value(&nOptimize)->default_value(7), "filter used after depth-map estimation (0 - disabled, 1 - remove speckles, 2 - fill gaps, 4 - cross-adjust)") ("estimate-colors", boost::program_options::value(&nEstimateColors)->default_value(2), "estimate the colors for the dense point-cloud") ("estimate-normals", boost::program_options::value(&nEstimateNormals)->default_value(0), "estimate the normals for the dense point-cloud") ("sample-mesh", boost::program_options::value(&OPT::fSampleMesh)->default_value(0.f), "uniformly samples points on a mesh (0 - disabled, <0 - number of points, >0 - sample density per square unit)") @@ -179,7 +177,6 @@ bool Initialize(size_t argc, LPCTSTR* argv) OPTDENSE::nMinResolution = nMinResolution; OPTDENSE::nNumViews = nNumViews; OPTDENSE::nMinViewsFuse = nMinViewsFuse; - OPTDENSE::nOptimize = nOptimize; OPTDENSE::nEstimateColors = nEstimateColors; OPTDENSE::nEstimateNormals = nEstimateNormals; if (!bValidConfig && !OPT::strDenseConfigFileName.IsEmpty()) diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index 286a03e22..991b26ece 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -449,7 +449,7 @@ float DepthEstimator::ScorePixelImage(const ViewData& image1, Depth depth, const const float num(texels0.dot(texels1)); #endif const float ncc(CLAMP(num/SQRT(nrmSq), -1.f, 1.f)); - float score = 1.f - ncc; + float score(1.f-ncc); #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA // encourage smoothness for (const NeighborEstimate& neighbor: neighborsClose) { @@ -500,17 +500,17 @@ float DepthEstimator::ScorePixel(Depth depth, const Normal& normal) #if 0 return std::accumulate(scores.cbegin(), &scores.GetNth(idxScore), 0.f) / idxScore; #elif 1 - const float* pescore(&scores.PartialSort(idxScore)); + const float* pescore(&scores.GetNth(idxScore)); const float* pscore(scores.cbegin()); - int n(0); float score(0); + int n(1); float score(*pscore); do { - const float s(*pscore); - if (s < thRobust) { - score += s; - ++n; - } - } while (++pscore <= pescore); - return n ? score/n : thRobust; + const float s(*(++pscore)); + if (s >= thRobust) + break; + score += s; + ++n; + } while (pscore < pescore); + return score/n; #else const float thScore(MAXF(*std::min_element(scores.cbegin(), scores.cend()), 0.05f)*2); const float* pscore(scores.cbegin()); @@ -658,7 +658,7 @@ void DepthEstimator::ProcessPixel(IDX idx) float& conf = confMap0(x0); Depth& depth = depthMap0(x0); Normal& normal = normalMap0(x0); - const Normal viewDir(Cast(static_cast(X0))); + const Normal viewDir(Cast(reinterpret_cast(X0))); ASSERT(depth > 0 && normal.dot(viewDir) <= 0); #if DENSE_REFINE == DENSE_REFINE_ITER // check if any of the neighbor estimates are better then the current estimate From 86356d8a6857757087457853704d939f27c24d22 Mon Sep 17 00:00:00 2001 From: Eberty Alves Date: Wed, 13 May 2020 11:06:17 -0300 Subject: [PATCH 65/70] interface: import undistorted Bundle OUT scene (#563 + #562) --- apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp | 5 +++-- apps/InterfaceVisualSFM/Util.h | 12 ++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp b/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp index 6230be3ce..c686abf8b 100644 --- a/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp +++ b/apps/InterfaceVisualSFM/InterfaceVisualSFM.cpp @@ -42,6 +42,7 @@ #define APPNAME _T("InterfaceVisualSFM") #define MVS_EXT _T(".mvs") #define VSFM_EXT _T(".nvm") +#define BUNDLE_EXT _T(".out") #define CMPMVS_EXT _T(".lst") @@ -88,7 +89,7 @@ bool Initialize(size_t argc, LPCTSTR* argv) // group of options allowed both on command line and in config file boost::program_options::options_description config("Main options"); config.add_options() - ("input-file,i", boost::program_options::value(&OPT::strInputFileName), "input filename containing camera poses and image list") + ("input-file,i", boost::program_options::value(&OPT::strInputFileName), "input filename containing camera poses and image list (NVM, undistorted OUT + image_list.TXT, LST)") ("output-file,o", boost::program_options::value(&OPT::strOutputFileName), "output filename for storing the mesh") ("output-image-folder", boost::program_options::value(&OPT::strOutputImageFolder)->default_value("undistorted_images"), "output folder to store undistorted images") ; @@ -508,7 +509,7 @@ int main(int argc, LPCTSTR* argv) return EXIT_FAILURE; const String strInputFileNameExt(Util::getFileExt(OPT::strInputFileName).ToLower()); - if (strInputFileNameExt == VSFM_EXT) { + if (strInputFileNameExt == VSFM_EXT || strInputFileNameExt == BUNDLE_EXT) { if (!ImportSceneVSFM()) return EXIT_FAILURE; } else diff --git a/apps/InterfaceVisualSFM/Util.h b/apps/InterfaceVisualSFM/Util.h index e5e8c43ff..3afd61f1c 100644 --- a/apps/InterfaceVisualSFM/Util.h +++ b/apps/InterfaceVisualSFM/Util.h @@ -198,7 +198,13 @@ bool LoadBundlerOut(const char* name, std::ifstream& in, std::vector& c std::ifstream listin(listpath); if(!listin.is_open()) { - listin.close(); listin.clear(); + listin.close(); listin.clear(); + strcpy(ext, ".txt\0"); + listin.open(listpath); + } + if(!listin.is_open()) + { + listin.close(); listin.clear(); char * slash = strrchr(listpath, '/'); if(slash == NULL) slash = strrchr(listpath, '\\'); slash = slash ? slash + 1 : listpath; @@ -231,9 +237,7 @@ bool LoadBundlerOut(const char* name, std::ifstream& in, std::vector& c if(listin >> filepath && f != 0) { - char* slash = strrchr(filepath , '/'); - if(slash == NULL) slash = strchr(filepath, '\\'); - names[i] = (slash? (slash + 1) : filepath); + names[i] = filepath; std::getline(listin, token); if(!det_checked) From 7a719f61dce821de95394782d169e74d33cca5d8 Mon Sep 17 00:00:00 2001 From: cDc Date: Mon, 25 May 2020 19:45:35 +0300 Subject: [PATCH 66/70] github: update issue templates --- .github/ISSUE_TEMPLATE.md | 3 --- .github/ISSUE_TEMPLATE/bug_report.md | 32 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 1 + .github/ISSUE_TEMPLATE/feature_request.md | 20 ++++++++++++++ 4 files changed, 53 insertions(+), 3 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index bfc5bfcee..000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,3 +0,0 @@ -### Specifications like the version of the project, operating system, and hardware - -### Steps to reproduce the problem diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..7f9c8d273 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,32 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. Use `Viewer` to verify/display the input/output. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. iOS] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..3ba13e0ce --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..bbcbbe7d6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. From 122397e1af32ac5660a31a61e3a0c9c0091355a3 Mon Sep 17 00:00:00 2001 From: cDc Date: Fri, 29 May 2020 10:34:18 +0300 Subject: [PATCH 67/70] interface: add helpful functions --- libs/MVS/Interface.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/libs/MVS/Interface.h b/libs/MVS/Interface.h index ea47cfcf9..d0e80a11f 100644 --- a/libs/MVS/Interface.h +++ b/libs/MVS/Interface.h @@ -383,6 +383,8 @@ struct Interface Camera() : width(0), height(0) {} bool HasResolution() const { return width > 0 && height > 0; } bool IsNormalized() const { return !HasResolution(); } + static uint32_t GetNormalizationScale(uint32_t width, uint32_t height) { return std::max(width, height); } + uint32_t GetNormalizationScale() const { return GetNormalizationScale(width, height); } template void serialize(Archive& ar, const unsigned int version) { @@ -429,6 +431,19 @@ struct Interface const Mat33d& GetK(uint32_t cameraID) const { return cameras[cameraID].K; } + static Mat33d ScaleK(const Mat33d& _K, double scale) { + Mat33d K(_K); + K(0,0) *= scale; + K(0,1) *= scale; + K(0,2) *= scale; + K(1,1) *= scale; + K(1,2) *= scale; + return K; + } + Mat33d GetFullK(uint32_t cameraID, uint32_t width, uint32_t height) const { + return ScaleK(GetK(cameraID), (double)Camera::GetNormalizationScale(width, height)/ + (cameras[cameraID].IsNormalized()?1.0:(double)cameras[cameraID].GetNormalizationScale())); + } Pose GetPose(uint32_t cameraID, uint32_t poseID) const { const Camera& camera = cameras[cameraID]; From 44c5eb00314b6483c1040174561e60a991271ef0 Mon Sep 17 00:00:00 2001 From: cDc Date: Tue, 30 Jun 2020 16:26:41 +0300 Subject: [PATCH 68/70] mvs: expose depth-map file format --- libs/MVS/DepthMap.cpp | 23 ++----------------- libs/MVS/Interface.h | 53 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 45 insertions(+), 31 deletions(-) diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index 991b26ece..f4068a292 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -31,6 +31,8 @@ #include "Common.h" #include "DepthMap.h" +#define _USE_OPENCV +#include "Interface.h" #include "../Common/AutoEstimator.h" // CGAL: depth-map initialization #include @@ -1696,27 +1698,6 @@ bool MVS::ExportPointCloud(const String& fileName, const Image& imageData, const /*----------------------------------------------------------------*/ -struct HeaderDepthDataRaw { - enum { - HAS_DEPTH = (1<<0), - HAS_NORMAL = (1<<1), - HAS_CONF = (1<<2), - }; - uint16_t name; // file type - uint8_t type; // content type - uint8_t padding; // reserve - uint32_t imageWidth, imageHeight; // image resolution - uint32_t depthWidth, depthHeight; // depth-map resolution - float dMin, dMax; // depth range for this view - // image file name length followed by the characters: uint16_t nFileNameSize; char* FileName - // number of view IDs followed by view ID and neighbor view IDs: uint32_t nIDs; uint32_t* IDs - // camera, rotation and translation matrices (row-major): double K[3][3], R[3][3], C[3] - // depth, normal, confidence maps - inline HeaderDepthDataRaw() : name(0), type(0), padding(0) {} - static uint16_t HeaderDepthDataRawName() { return *reinterpret_cast("DR"); } - int GetStep() const { return ROUND2INT((float)imageWidth/depthWidth); } -}; - bool MVS::ExportDepthDataRaw(const String& fileName, const String& imageFileName, const IIndexArr& IDs, const cv::Size& imageSize, const KMatrix& K, const RMatrix& R, const CMatrix& C, diff --git a/libs/MVS/Interface.h b/libs/MVS/Interface.h index d0e80a11f..6592a32dd 100644 --- a/libs/MVS/Interface.h +++ b/libs/MVS/Interface.h @@ -272,12 +272,12 @@ bool SerializeLoad(_Tp& obj, const std::string& fileName, uint32_t* pVersion=NUL #define ARCHIVE_DEFINE_TYPE(TYPE) \ template<> \ -bool Save(ArchiveSave& a, const TYPE& v) { \ +inline bool Save(ArchiveSave& a, const TYPE& v) { \ a.stream.write((const char*)&v, sizeof(TYPE)); \ return true; \ } \ template<> \ -bool Load(ArchiveLoad& a, TYPE& v) { \ +inline bool Load(ArchiveLoad& a, TYPE& v) { \ a.stream.read((char*)&v, sizeof(TYPE)); \ return true; \ } @@ -290,31 +290,31 @@ ARCHIVE_DEFINE_TYPE(double) // Serialization support for cv::Matx template -bool Save(ArchiveSave& a, const cv::Matx<_Tp,m,n>& _m) { +inline bool Save(ArchiveSave& a, const cv::Matx<_Tp,m,n>& _m) { a.stream.write((const char*)_m.val, sizeof(_Tp)*m*n); return true; } template -bool Load(ArchiveLoad& a, cv::Matx<_Tp,m,n>& _m) { +inline bool Load(ArchiveLoad& a, cv::Matx<_Tp,m,n>& _m) { a.stream.read((char*)_m.val, sizeof(_Tp)*m*n); return true; } // Serialization support for cv::Point3_ template -bool Save(ArchiveSave& a, const cv::Point3_<_Tp>& pt) { +inline bool Save(ArchiveSave& a, const cv::Point3_<_Tp>& pt) { a.stream.write((const char*)&pt.x, sizeof(_Tp)*3); return true; } template -bool Load(ArchiveLoad& a, cv::Point3_<_Tp>& pt) { +inline bool Load(ArchiveLoad& a, cv::Point3_<_Tp>& pt) { a.stream.read((char*)&pt.x, sizeof(_Tp)*3); return true; } // Serialization support for std::string template<> -bool Save(ArchiveSave& a, const std::string& s) { +inline bool Save(ArchiveSave& a, const std::string& s) { const uint64_t size(s.size()); Save(a, size); if (size > 0) @@ -322,7 +322,7 @@ bool Save(ArchiveSave& a, const std::string& s) { return true; } template<> -bool Load(ArchiveLoad& a, std::string& s) { +inline bool Load(ArchiveLoad& a, std::string& s) { uint64_t size; Load(a, size); if (size > 0) { @@ -334,7 +334,7 @@ bool Load(ArchiveLoad& a, std::string& s) { // Serialization support for std::vector template -bool Save(ArchiveSave& a, const std::vector<_Tp>& v) { +inline bool Save(ArchiveSave& a, const std::vector<_Tp>& v) { const uint64_t size(v.size()); Save(a, size); for (uint64_t i=0; i& v) { return true; } template -bool Load(ArchiveLoad& a, std::vector<_Tp>& v) { +inline bool Load(ArchiveLoad& a, std::vector<_Tp>& v) { uint64_t size; Load(a, size); if (size > 0) { @@ -616,6 +616,39 @@ struct Interface }; /*----------------------------------------------------------------*/ + +// interface used to export/import MVS depth-map data; +// see MVS::ExportDepthDataRaw() and MVS::ImportDepthDataRaw() for usage example: +// - image-resolution at which the depth-map was estimated +// - depth-map-resolution, for now only the same resolution as the image is supported +// - min/max-depth of the values in the depth-map +// - image-file-name is the path to the reference color image +// - image-IDs are the reference view ID and neighbor view IDs used to estimate the depth-map +// - camera/rotation/position matrices (row-major) is the absolute pose corresponding to the reference view +// - depth-map represents the pixel depth +// - normal-map (optional) represents the 3D point normal in camera space; same resolution as the depth-map +// - confidence-map (optional) represents the 3D point confidence (usually a value in [0,1]); same resolution as the depth-map +struct HeaderDepthDataRaw { + enum { + HAS_DEPTH = (1<<0), + HAS_NORMAL = (1<<1), + HAS_CONF = (1<<2), + }; + uint16_t name; // file type + uint8_t type; // content type + uint8_t padding; // reserve + uint32_t imageWidth, imageHeight; // image resolution + uint32_t depthWidth, depthHeight; // depth-map resolution + float dMin, dMax; // depth range for this view + // image file name length followed by the characters: uint16_t nFileNameSize; char* FileName + // number of view IDs followed by view ID and neighbor view IDs: uint32_t nIDs; uint32_t* IDs + // camera, rotation and position matrices (row-major): double K[3][3], R[3][3], C[3] + // depth, normal, confidence maps: float depthMap[height][width], normalMap[height][width][3], confMap[height][width] + inline HeaderDepthDataRaw() : name(0), type(0), padding(0) {} + static uint16_t HeaderDepthDataRawName() { return *reinterpret_cast("DR"); } +}; +/*----------------------------------------------------------------*/ + } // namespace _INTERFACE_NAMESPACE #endif // _INTERFACE_MVS_H_ From 8fcc3d05504ba24b5df2b5ad29cd11d9baba4881 Mon Sep 17 00:00:00 2001 From: cDc Date: Thu, 2 Jul 2020 10:26:49 +0300 Subject: [PATCH 69/70] dense: export normals by default --- apps/DensifyPointCloud/DensifyPointCloud.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/DensifyPointCloud/DensifyPointCloud.cpp b/apps/DensifyPointCloud/DensifyPointCloud.cpp index 62350b976..d82c07294 100644 --- a/apps/DensifyPointCloud/DensifyPointCloud.cpp +++ b/apps/DensifyPointCloud/DensifyPointCloud.cpp @@ -102,8 +102,8 @@ bool Initialize(size_t argc, LPCTSTR* argv) ("min-resolution", boost::program_options::value(&nMinResolution)->default_value(640), "do not scale images lower than this resolution") ("number-views", boost::program_options::value(&nNumViews)->default_value(5), "number of views used for depth-map estimation (0 - all neighbor views available)") ("number-views-fuse", boost::program_options::value(&nMinViewsFuse)->default_value(3), "minimum number of images that agrees with an estimate during fusion in order to consider it inlier") - ("estimate-colors", boost::program_options::value(&nEstimateColors)->default_value(2), "estimate the colors for the dense point-cloud") - ("estimate-normals", boost::program_options::value(&nEstimateNormals)->default_value(0), "estimate the normals for the dense point-cloud") + ("estimate-colors", boost::program_options::value(&nEstimateColors)->default_value(2), "estimate the colors for the dense point-cloud (0 - disabled, 1 - final, 2 - estimate)") + ("estimate-normals", boost::program_options::value(&nEstimateNormals)->default_value(2), "estimate the normals for the dense point-cloud (0 - disabled, 1 - final, 2 - estimate)") ("sample-mesh", boost::program_options::value(&OPT::fSampleMesh)->default_value(0.f), "uniformly samples points on a mesh (0 - disabled, <0 - number of points, >0 - sample density per square unit)") ("filter-point-cloud", boost::program_options::value(&OPT::thFilterPointCloud)->default_value(0), "filter dense point-cloud based on visibility (0 - disabled)") ("fusion-mode", boost::program_options::value(&OPT::nFusionMode)->default_value(0), "depth map fusion mode (-2 - fuse disparity-maps, -1 - export disparity-maps only, 0 - depth-maps & fusion, 1 - export depth-maps only)") From b6ecabefd71ed3f5315b978c1bb6446693fc5bc9 Mon Sep 17 00:00:00 2001 From: cDc Date: Thu, 2 Jul 2020 10:27:44 +0300 Subject: [PATCH 70/70] mvs: update version to v1.1.1 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7966ed55b..fff705bda 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ PROJECT(OpenMVS) set(OpenMVS_MAJOR_VERSION 1) set(OpenMVS_MINOR_VERSION 1) -set(OpenMVS_PATCH_VERSION 0) +set(OpenMVS_PATCH_VERSION 1) set(OpenMVS_VERSION ${OpenMVS_MAJOR_VERSION}.${OpenMVS_MINOR_VERSION}.${OpenMVS_PATCH_VERSION}) # Find dependencies: