From e868d7d199aa9db942f3e41441118217618e44a9 Mon Sep 17 00:00:00 2001 From: cpb8010 Date: Sat, 15 Jun 2013 18:11:42 -0400 Subject: [PATCH] import everything --- AUTHORS | 41 + CHANGES | 144 ++ CMakeLists.txt | 518 +++++ COPYRIGHT | 60 + HOWTOUSE | 76 + INSTALL | 122 + Package.cmake.in | 92 + README | 21 + TODO | 5 + VERSION | 1 + VOSM_changelist.txt | 65 + cmake_uninstall.cmake.in | 27 + cmd.txt | 1 + cvconfig.h.cmake | 45 + modules/CMakeLists.txt | 9 + modules/comalgs/CMakeLists.txt | 5 + modules/comalgs/comalgs.vcxproj | 107 + modules/comalgs/comalgs.vcxproj.filters | 72 + .../comalgs/include/VO_ClassificationAlgs.h | 152 ++ modules/comalgs/include/VO_DetectionAlgs.h | 197 ++ modules/comalgs/include/VO_EvaluationAlgs.h | 113 + .../comalgs/include/VO_FaceDetectionAlgs.h | 329 +++ .../comalgs/include/VO_HandDetectionAlgs.h | 98 + .../comalgs/include/VO_HumanDetectionAlgs.h | 139 ++ modules/comalgs/include/VO_LocalizationAlgs.h | 160 ++ modules/comalgs/include/VO_RecognitionAlgs.h | 213 ++ modules/comalgs/include/VO_TrackingAlgs.h | 204 ++ modules/comalgs/src/VO_ClassificationAlgs.cpp | 300 +++ modules/comalgs/src/VO_DetectionAlgs.cpp | 269 +++ modules/comalgs/src/VO_EvaluationAlgs.cpp | 266 +++ modules/comalgs/src/VO_FaceDetectionAlgs.cpp | 975 ++++++++ modules/comalgs/src/VO_HumanDetectionAlgs.cpp | 146 ++ modules/comalgs/src/VO_LocalizationAlgs.cpp | 183 ++ modules/comalgs/src/VO_RecognitionAlgs.cpp | 934 ++++++++ modules/comalgs/src/VO_TrackingAlgs.cpp | 353 +++ modules/common/CMakeLists.txt | 2 + modules/common/common.vcxproj | 83 + modules/common/common.vcxproj.filters | 30 + modules/common/include/VO_Common.h | 750 ++++++ modules/common/include/VO_ScanFilesInDir.h | 102 + modules/common/src/VO_ScanFilesInDir.cpp | 288 +++ modules/cvcommon/CMakeLists.txt | 3 + modules/cvcommon/cvcommon.vcxproj | 85 + modules/cvcommon/cvcommon.vcxproj.filters | 48 + modules/cvcommon/include/VO_AnnotationDBIO.h | 150 ++ modules/cvcommon/include/VO_CVCommon.h | 1154 +++++++++ modules/cvcommon/include/VO_DetectionDBIO.h | 184 ++ modules/cvcommon/include/VO_FaceCompPos.h | 146 ++ modules/cvcommon/include/VO_FaceKeyPoint.h | 121 + modules/cvcommon/src/VO_AnnotationDBIO.cpp | 464 ++++ modules/cvcommon/src/VO_DetectionDBIO.cpp | 471 ++++ modules/cvcommon/src/VO_FaceCompPos.cpp | 192 ++ modules/cvcommon/src/VO_FaceKeyPoint.cpp | 322 +++ modules/ensembletraining/CMakeLists.txt | 3 + .../ensembletraining/ensembletraining.vcxproj | 86 + .../ensembletraining.vcxproj.filters | 51 + .../include/VO_AdditiveStrongerClassifier.h | 106 + .../VO_BaggingRandomForestClassifier.h | 90 + .../include/VO_BoostingCascadeClassifier.h | 131 ++ .../VO_BoostingSingleEnsembleClassifier.h | 117 + .../include/VO_WeakClassifier.h | 98 + .../src/VO_AdditiveStrongerClassifier.cpp | 78 + .../src/VO_BaggingRandomForestClassifier.cpp | 62 + .../src/VO_BoostingCascadeClassifier.cpp | 103 + .../VO_BoostingSingleEnsembleClassifier.cpp | 68 + .../src/VO_WeakClassifier.cpp | 287 +++ modules/featureextraction/CMakeLists.txt | 3 + .../featureextraction.vcxproj | 91 + .../featureextraction.vcxproj.filters | 54 + .../include/VO_DaubechiesFeatures.h | 152 ++ .../include/VO_DirectFeatures.h | 110 + .../featureextraction/include/VO_Features.h | 260 +++ .../include/VO_GaborFeatures.h | 161 ++ .../include/VO_HaarFeatures.h | 149 ++ .../include/VO_LBPFeatures.h | 133 ++ .../src/VO_DaubechiesFeatures.cpp | 157 ++ .../src/VO_DirectFeatures.cpp | 129 ++ .../src/VO_GaborFeatures.cpp | 219 ++ .../featureextraction/src/VO_HaarFeatures.cpp | 322 +++ .../featureextraction/src/VO_LBPFeatures.cpp | 164 ++ modules/integraltransform/CMakeLists.txt | 2 + .../include/VO_CohenDaubechiesFeauveau.h | 100 + .../integraltransform/include/VO_Coiflets.h | 106 + .../integraltransform/include/VO_Daubechies.h | 107 + .../include/VO_DiscreteFIRMeyer.h | 71 + .../include/VO_DiscreteWavelet.h | 141 ++ modules/integraltransform/include/VO_Gabor.h | 118 + modules/integraltransform/include/VO_Gauss.h | 107 + modules/integraltransform/include/VO_Haar.h | 105 + .../include/VO_IntegralTransform.h | 108 + .../include/VO_ReverseBiorthogonal.h | 69 + modules/integraltransform/include/VO_STFT.h | 121 + .../integraltransform/include/VO_Symlets.h | 99 + .../integraltransform/include/VO_WindowFunc.h | 138 ++ .../integraltransform.vcxproj | 102 + .../integraltransform.vcxproj.filters | 93 + .../src/VO_CohenDaubechiesFeauveau.cpp | 110 + modules/integraltransform/src/VO_Coiflets.cpp | 156 ++ .../integraltransform/src/VO_Daubechies.cpp | 179 ++ .../src/VO_DiscreteFIRMeyer.cpp | 61 + modules/integraltransform/src/VO_Gabor.cpp | 143 ++ modules/integraltransform/src/VO_Gauss.cpp | 136 ++ modules/integraltransform/src/VO_Haar.cpp | 103 + .../src/VO_ReverseBiorthogonal.cpp | 61 + modules/integraltransform/src/VO_STFT.cpp | 159 ++ modules/integraltransform/src/VO_Symlets.cpp | 109 + .../integraltransform/src/VO_WindowFunc.cpp | 220 ++ modules/smbuilding/CMakeLists.txt | 5 + modules/smbuilding/include/VO_AAMBasic.h | 322 +++ modules/smbuilding/include/VO_AAMForwardIA.h | 132 ++ modules/smbuilding/include/VO_AAMInverseIA.h | 168 ++ modules/smbuilding/include/VO_AFM.h | 177 ++ modules/smbuilding/include/VO_ASMLTCs.h | 180 ++ modules/smbuilding/include/VO_ASMNDProfiles.h | 166 ++ modules/smbuilding/include/VO_AXM.h | 138 ++ modules/smbuilding/include/VO_Edge.h | 125 + modules/smbuilding/include/VO_Ellipse.h | 228 ++ modules/smbuilding/include/VO_FacePart.h | 168 ++ modules/smbuilding/include/VO_FaceParts.h | 244 ++ modules/smbuilding/include/VO_FeatureModel.h | 160 ++ .../include/VO_Point2DDistributionModel.h | 131 ++ modules/smbuilding/include/VO_Profile.h | 173 ++ modules/smbuilding/include/VO_Profiles.h | 100 + modules/smbuilding/include/VO_Shape.h | 378 +++ modules/smbuilding/include/VO_Shape2DInfo.h | 172 ++ modules/smbuilding/include/VO_ShapeFace.h | 106 + modules/smbuilding/include/VO_ShapeModel.h | 296 +++ modules/smbuilding/include/VO_Texture.h | 208 ++ modules/smbuilding/include/VO_TextureModel.h | 315 +++ modules/smbuilding/include/VO_Triangle2D.h | 163 ++ .../include/VO_Triangle2DStructure.h | 178 ++ modules/smbuilding/include/VO_WarpingPoint.h | 248 ++ modules/smbuilding/smbuilding.vcxproj | 124 + modules/smbuilding/smbuilding.vcxproj.filters | 159 ++ modules/smbuilding/src/VO_AAMBasic.cpp | 1525 ++++++++++++ modules/smbuilding/src/VO_AAMForwardIA.cpp | 171 ++ modules/smbuilding/src/VO_AAMInverseIA.cpp | 700 ++++++ modules/smbuilding/src/VO_AFM.cpp | 324 +++ modules/smbuilding/src/VO_ASMLTCs.cpp | 543 +++++ modules/smbuilding/src/VO_ASMNDProfiles.cpp | 523 +++++ modules/smbuilding/src/VO_AXM.cpp | 201 ++ modules/smbuilding/src/VO_Edge.cpp | 83 + modules/smbuilding/src/VO_Ellipse.cpp | 215 ++ modules/smbuilding/src/VO_FacePart.cpp | 272 +++ modules/smbuilding/src/VO_FaceParts.cpp | 295 +++ modules/smbuilding/src/VO_FeatureModel.cpp | 205 ++ .../src/VO_Point2DDistributionModel.cpp | 251 ++ modules/smbuilding/src/VO_Profile.cpp | 1027 ++++++++ modules/smbuilding/src/VO_Shape.cpp | 1829 +++++++++++++++ modules/smbuilding/src/VO_Shape2DInfo.cpp | 153 ++ modules/smbuilding/src/VO_ShapeFace.cpp | 102 + modules/smbuilding/src/VO_ShapeModel.cpp | 1149 +++++++++ modules/smbuilding/src/VO_Texture.cpp | 576 +++++ modules/smbuilding/src/VO_TextureModel.cpp | 2062 +++++++++++++++++ .../smbuilding/src/VO_Triangle2DStructure.cpp | 276 +++ modules/smbuilding/src/VO_WarpingPoint.cpp | 271 +++ modules/smfitting/CMakeLists.txt | 6 + modules/smfitting/include/VO_Fitting2DSM.h | 289 +++ .../smfitting/include/VO_FittingAAMBasic.h | 141 ++ .../include/VO_FittingAAMForwardIA.h | 108 + .../include/VO_FittingAAMInverseIA.h | 152 ++ modules/smfitting/include/VO_FittingAFM.h | 109 + modules/smfitting/include/VO_FittingASMLTCs.h | 152 ++ .../include/VO_FittingASMNDProfiles.h | 196 ++ modules/smfitting/smfitting.vcxproj | 90 + modules/smfitting/smfitting.vcxproj.filters | 63 + modules/smfitting/src/VO_Fitting2DSM.cpp | 453 ++++ modules/smfitting/src/VO_FittingAAMBasic.cpp | 621 +++++ .../smfitting/src/VO_FittingAAMForwardIA.cpp | 140 ++ .../smfitting/src/VO_FittingAAMInverseIA.cpp | 751 ++++++ modules/smfitting/src/VO_FittingAFM.cpp | 152 ++ modules/smfitting/src/VO_FittingASMLTCs.cpp | 530 +++++ .../smfitting/src/VO_FittingASMNDProfiles.cpp | 849 +++++++ tests/CMakeLists.txt | 5 + tests/test_ensembletraining/CMakeLists.txt | 54 + .../ensembletraining.cpp | 301 +++ .../test_ensembletraining.vcxproj | 88 + .../vo_ensembletraining.vcxproj.filters | 22 + tests/test_featureextraction/CMakeLists.txt | 52 + .../featureextraction.cpp | 95 + .../test_featureextraction.vcxproj | 89 + .../vo_featureextraction.vcxproj.filters | 22 + tests/test_integraltransform/CMakeLists.txt | 50 + tests/test_integraltransform/dft.jpg | 0 tests/test_integraltransform/gabored.jpg | Bin 0 -> 333 bytes tests/test_integraltransform/gaborkernel.jpg | Bin 0 -> 333 bytes tests/test_integraltransform/idft.jpg | 0 tests/test_integraltransform/igabored.jpg | Bin 0 -> 333 bytes .../integraltransform.cpp | 115 + tests/test_integraltransform/test.txt | 0 .../test_integraltransform.vcxproj | 84 + .../test_integraltransform.vcxproj.filters | 22 + tests/test_smbuilding/CMakeLists.txt | 60 + tests/test_smbuilding/smb.h | 84 + tests/test_smbuilding/smbuilding.cpp | 371 +++ tests/test_smbuilding/test_smbuilding.vcxproj | 85 + .../test_smbuilding.vcxproj.filters | 27 + tests/test_smfitting/smf.h | 75 + tests/test_smfitting/smfitting.cpp | 1078 +++++++++ tests/test_smfitting/subset.yaml | 21 + tests/test_smfitting/test_smfitting.vcxproj | 109 + .../test_smfitting.vcxproj.filters | 27 + vosm.pc.cmake.in | 13 + vosm.sln | 132 ++ vosmConfig.cmake.in | 83 + vosmModule.cmake | 61 + 206 files changed, 43189 insertions(+) create mode 100644 AUTHORS create mode 100644 CHANGES create mode 100644 CMakeLists.txt create mode 100644 COPYRIGHT create mode 100644 HOWTOUSE create mode 100644 INSTALL create mode 100644 Package.cmake.in create mode 100644 README create mode 100644 TODO create mode 100644 VERSION create mode 100644 VOSM_changelist.txt create mode 100644 cmake_uninstall.cmake.in create mode 100644 cmd.txt create mode 100644 cvconfig.h.cmake create mode 100644 modules/CMakeLists.txt create mode 100644 modules/comalgs/CMakeLists.txt create mode 100644 modules/comalgs/comalgs.vcxproj create mode 100644 modules/comalgs/comalgs.vcxproj.filters create mode 100644 modules/comalgs/include/VO_ClassificationAlgs.h create mode 100644 modules/comalgs/include/VO_DetectionAlgs.h create mode 100644 modules/comalgs/include/VO_EvaluationAlgs.h create mode 100644 modules/comalgs/include/VO_FaceDetectionAlgs.h create mode 100644 modules/comalgs/include/VO_HandDetectionAlgs.h create mode 100644 modules/comalgs/include/VO_HumanDetectionAlgs.h create mode 100644 modules/comalgs/include/VO_LocalizationAlgs.h create mode 100644 modules/comalgs/include/VO_RecognitionAlgs.h create mode 100644 modules/comalgs/include/VO_TrackingAlgs.h create mode 100644 modules/comalgs/src/VO_ClassificationAlgs.cpp create mode 100644 modules/comalgs/src/VO_DetectionAlgs.cpp create mode 100644 modules/comalgs/src/VO_EvaluationAlgs.cpp create mode 100644 modules/comalgs/src/VO_FaceDetectionAlgs.cpp create mode 100644 modules/comalgs/src/VO_HumanDetectionAlgs.cpp create mode 100644 modules/comalgs/src/VO_LocalizationAlgs.cpp create mode 100644 modules/comalgs/src/VO_RecognitionAlgs.cpp create mode 100644 modules/comalgs/src/VO_TrackingAlgs.cpp create mode 100644 modules/common/CMakeLists.txt create mode 100644 modules/common/common.vcxproj create mode 100644 modules/common/common.vcxproj.filters create mode 100644 modules/common/include/VO_Common.h create mode 100644 modules/common/include/VO_ScanFilesInDir.h create mode 100644 modules/common/src/VO_ScanFilesInDir.cpp create mode 100644 modules/cvcommon/CMakeLists.txt create mode 100644 modules/cvcommon/cvcommon.vcxproj create mode 100644 modules/cvcommon/cvcommon.vcxproj.filters create mode 100644 modules/cvcommon/include/VO_AnnotationDBIO.h create mode 100644 modules/cvcommon/include/VO_CVCommon.h create mode 100644 modules/cvcommon/include/VO_DetectionDBIO.h create mode 100644 modules/cvcommon/include/VO_FaceCompPos.h create mode 100644 modules/cvcommon/include/VO_FaceKeyPoint.h create mode 100644 modules/cvcommon/src/VO_AnnotationDBIO.cpp create mode 100644 modules/cvcommon/src/VO_DetectionDBIO.cpp create mode 100644 modules/cvcommon/src/VO_FaceCompPos.cpp create mode 100644 modules/cvcommon/src/VO_FaceKeyPoint.cpp create mode 100644 modules/ensembletraining/CMakeLists.txt create mode 100644 modules/ensembletraining/ensembletraining.vcxproj create mode 100644 modules/ensembletraining/ensembletraining.vcxproj.filters create mode 100644 modules/ensembletraining/include/VO_AdditiveStrongerClassifier.h create mode 100644 modules/ensembletraining/include/VO_BaggingRandomForestClassifier.h create mode 100644 modules/ensembletraining/include/VO_BoostingCascadeClassifier.h create mode 100644 modules/ensembletraining/include/VO_BoostingSingleEnsembleClassifier.h create mode 100644 modules/ensembletraining/include/VO_WeakClassifier.h create mode 100644 modules/ensembletraining/src/VO_AdditiveStrongerClassifier.cpp create mode 100644 modules/ensembletraining/src/VO_BaggingRandomForestClassifier.cpp create mode 100644 modules/ensembletraining/src/VO_BoostingCascadeClassifier.cpp create mode 100644 modules/ensembletraining/src/VO_BoostingSingleEnsembleClassifier.cpp create mode 100644 modules/ensembletraining/src/VO_WeakClassifier.cpp create mode 100644 modules/featureextraction/CMakeLists.txt create mode 100644 modules/featureextraction/featureextraction.vcxproj create mode 100644 modules/featureextraction/featureextraction.vcxproj.filters create mode 100644 modules/featureextraction/include/VO_DaubechiesFeatures.h create mode 100644 modules/featureextraction/include/VO_DirectFeatures.h create mode 100644 modules/featureextraction/include/VO_Features.h create mode 100644 modules/featureextraction/include/VO_GaborFeatures.h create mode 100644 modules/featureextraction/include/VO_HaarFeatures.h create mode 100644 modules/featureextraction/include/VO_LBPFeatures.h create mode 100644 modules/featureextraction/src/VO_DaubechiesFeatures.cpp create mode 100644 modules/featureextraction/src/VO_DirectFeatures.cpp create mode 100644 modules/featureextraction/src/VO_GaborFeatures.cpp create mode 100644 modules/featureextraction/src/VO_HaarFeatures.cpp create mode 100644 modules/featureextraction/src/VO_LBPFeatures.cpp create mode 100644 modules/integraltransform/CMakeLists.txt create mode 100644 modules/integraltransform/include/VO_CohenDaubechiesFeauveau.h create mode 100644 modules/integraltransform/include/VO_Coiflets.h create mode 100644 modules/integraltransform/include/VO_Daubechies.h create mode 100644 modules/integraltransform/include/VO_DiscreteFIRMeyer.h create mode 100644 modules/integraltransform/include/VO_DiscreteWavelet.h create mode 100644 modules/integraltransform/include/VO_Gabor.h create mode 100644 modules/integraltransform/include/VO_Gauss.h create mode 100644 modules/integraltransform/include/VO_Haar.h create mode 100644 modules/integraltransform/include/VO_IntegralTransform.h create mode 100644 modules/integraltransform/include/VO_ReverseBiorthogonal.h create mode 100644 modules/integraltransform/include/VO_STFT.h create mode 100644 modules/integraltransform/include/VO_Symlets.h create mode 100644 modules/integraltransform/include/VO_WindowFunc.h create mode 100644 modules/integraltransform/integraltransform.vcxproj create mode 100644 modules/integraltransform/integraltransform.vcxproj.filters create mode 100644 modules/integraltransform/src/VO_CohenDaubechiesFeauveau.cpp create mode 100644 modules/integraltransform/src/VO_Coiflets.cpp create mode 100644 modules/integraltransform/src/VO_Daubechies.cpp create mode 100644 modules/integraltransform/src/VO_DiscreteFIRMeyer.cpp create mode 100644 modules/integraltransform/src/VO_Gabor.cpp create mode 100644 modules/integraltransform/src/VO_Gauss.cpp create mode 100644 modules/integraltransform/src/VO_Haar.cpp create mode 100644 modules/integraltransform/src/VO_ReverseBiorthogonal.cpp create mode 100644 modules/integraltransform/src/VO_STFT.cpp create mode 100644 modules/integraltransform/src/VO_Symlets.cpp create mode 100644 modules/integraltransform/src/VO_WindowFunc.cpp create mode 100644 modules/smbuilding/CMakeLists.txt create mode 100644 modules/smbuilding/include/VO_AAMBasic.h create mode 100644 modules/smbuilding/include/VO_AAMForwardIA.h create mode 100644 modules/smbuilding/include/VO_AAMInverseIA.h create mode 100644 modules/smbuilding/include/VO_AFM.h create mode 100644 modules/smbuilding/include/VO_ASMLTCs.h create mode 100644 modules/smbuilding/include/VO_ASMNDProfiles.h create mode 100644 modules/smbuilding/include/VO_AXM.h create mode 100644 modules/smbuilding/include/VO_Edge.h create mode 100644 modules/smbuilding/include/VO_Ellipse.h create mode 100644 modules/smbuilding/include/VO_FacePart.h create mode 100644 modules/smbuilding/include/VO_FaceParts.h create mode 100644 modules/smbuilding/include/VO_FeatureModel.h create mode 100644 modules/smbuilding/include/VO_Point2DDistributionModel.h create mode 100644 modules/smbuilding/include/VO_Profile.h create mode 100644 modules/smbuilding/include/VO_Profiles.h create mode 100644 modules/smbuilding/include/VO_Shape.h create mode 100644 modules/smbuilding/include/VO_Shape2DInfo.h create mode 100644 modules/smbuilding/include/VO_ShapeFace.h create mode 100644 modules/smbuilding/include/VO_ShapeModel.h create mode 100644 modules/smbuilding/include/VO_Texture.h create mode 100644 modules/smbuilding/include/VO_TextureModel.h create mode 100644 modules/smbuilding/include/VO_Triangle2D.h create mode 100644 modules/smbuilding/include/VO_Triangle2DStructure.h create mode 100644 modules/smbuilding/include/VO_WarpingPoint.h create mode 100644 modules/smbuilding/smbuilding.vcxproj create mode 100644 modules/smbuilding/smbuilding.vcxproj.filters create mode 100644 modules/smbuilding/src/VO_AAMBasic.cpp create mode 100644 modules/smbuilding/src/VO_AAMForwardIA.cpp create mode 100644 modules/smbuilding/src/VO_AAMInverseIA.cpp create mode 100644 modules/smbuilding/src/VO_AFM.cpp create mode 100644 modules/smbuilding/src/VO_ASMLTCs.cpp create mode 100644 modules/smbuilding/src/VO_ASMNDProfiles.cpp create mode 100644 modules/smbuilding/src/VO_AXM.cpp create mode 100644 modules/smbuilding/src/VO_Edge.cpp create mode 100644 modules/smbuilding/src/VO_Ellipse.cpp create mode 100644 modules/smbuilding/src/VO_FacePart.cpp create mode 100644 modules/smbuilding/src/VO_FaceParts.cpp create mode 100644 modules/smbuilding/src/VO_FeatureModel.cpp create mode 100644 modules/smbuilding/src/VO_Point2DDistributionModel.cpp create mode 100644 modules/smbuilding/src/VO_Profile.cpp create mode 100644 modules/smbuilding/src/VO_Shape.cpp create mode 100644 modules/smbuilding/src/VO_Shape2DInfo.cpp create mode 100644 modules/smbuilding/src/VO_ShapeFace.cpp create mode 100644 modules/smbuilding/src/VO_ShapeModel.cpp create mode 100644 modules/smbuilding/src/VO_Texture.cpp create mode 100644 modules/smbuilding/src/VO_TextureModel.cpp create mode 100644 modules/smbuilding/src/VO_Triangle2DStructure.cpp create mode 100644 modules/smbuilding/src/VO_WarpingPoint.cpp create mode 100644 modules/smfitting/CMakeLists.txt create mode 100644 modules/smfitting/include/VO_Fitting2DSM.h create mode 100644 modules/smfitting/include/VO_FittingAAMBasic.h create mode 100644 modules/smfitting/include/VO_FittingAAMForwardIA.h create mode 100644 modules/smfitting/include/VO_FittingAAMInverseIA.h create mode 100644 modules/smfitting/include/VO_FittingAFM.h create mode 100644 modules/smfitting/include/VO_FittingASMLTCs.h create mode 100644 modules/smfitting/include/VO_FittingASMNDProfiles.h create mode 100644 modules/smfitting/smfitting.vcxproj create mode 100644 modules/smfitting/smfitting.vcxproj.filters create mode 100644 modules/smfitting/src/VO_Fitting2DSM.cpp create mode 100644 modules/smfitting/src/VO_FittingAAMBasic.cpp create mode 100644 modules/smfitting/src/VO_FittingAAMForwardIA.cpp create mode 100644 modules/smfitting/src/VO_FittingAAMInverseIA.cpp create mode 100644 modules/smfitting/src/VO_FittingAFM.cpp create mode 100644 modules/smfitting/src/VO_FittingASMLTCs.cpp create mode 100644 modules/smfitting/src/VO_FittingASMNDProfiles.cpp create mode 100644 tests/CMakeLists.txt create mode 100644 tests/test_ensembletraining/CMakeLists.txt create mode 100644 tests/test_ensembletraining/ensembletraining.cpp create mode 100644 tests/test_ensembletraining/test_ensembletraining.vcxproj create mode 100644 tests/test_ensembletraining/vo_ensembletraining.vcxproj.filters create mode 100644 tests/test_featureextraction/CMakeLists.txt create mode 100644 tests/test_featureextraction/featureextraction.cpp create mode 100644 tests/test_featureextraction/test_featureextraction.vcxproj create mode 100644 tests/test_featureextraction/vo_featureextraction.vcxproj.filters create mode 100644 tests/test_integraltransform/CMakeLists.txt create mode 100644 tests/test_integraltransform/dft.jpg create mode 100644 tests/test_integraltransform/gabored.jpg create mode 100644 tests/test_integraltransform/gaborkernel.jpg create mode 100644 tests/test_integraltransform/idft.jpg create mode 100644 tests/test_integraltransform/igabored.jpg create mode 100644 tests/test_integraltransform/integraltransform.cpp create mode 100644 tests/test_integraltransform/test.txt create mode 100644 tests/test_integraltransform/test_integraltransform.vcxproj create mode 100644 tests/test_integraltransform/test_integraltransform.vcxproj.filters create mode 100644 tests/test_smbuilding/CMakeLists.txt create mode 100644 tests/test_smbuilding/smb.h create mode 100644 tests/test_smbuilding/smbuilding.cpp create mode 100644 tests/test_smbuilding/test_smbuilding.vcxproj create mode 100644 tests/test_smbuilding/test_smbuilding.vcxproj.filters create mode 100644 tests/test_smfitting/smf.h create mode 100644 tests/test_smfitting/smfitting.cpp create mode 100644 tests/test_smfitting/subset.yaml create mode 100644 tests/test_smfitting/test_smfitting.vcxproj create mode 100644 tests/test_smfitting/test_smfitting.vcxproj.filters create mode 100644 vosm.pc.cmake.in create mode 100644 vosm.sln create mode 100644 vosmConfig.cmake.in create mode 100644 vosmModule.cmake diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..c25232b --- /dev/null +++ b/AUTHORS @@ -0,0 +1,41 @@ +AUTHORS +------------------------------------------------------------------------------- + +JIA Pei +contact: jp4work at gmail dot com +Website: http://www.visionopen.com +http://visionopen.com/members/jiapei/jiapei.html + + + +ACKNOWLEDGEMENT +------------------------------------------------------------------------------- + + +YAO Wei +Great much thanks are given to Mr. Yao Wei for his help on the following several points: +1) I implemented 1D ASM Profile after I read Mr's Yao Wei's ASMLibrary in 2008. + So, that's why his name appears as the first author in some functions + in file VO_FittingASMNDProfiles.cpp . +2) He helped to dramatically speed up my image warping function, namely, + function VO_TextureModel::VO_LoadOneTextureFromShape in file VO_TextureModel.cpp . + That's also why his name is listed as the 2nd author in file VO_TextureModel.cpp. +3) His two fabulous libraries, say ASMLibrary and AAMLibrary are partly built upon + my old version of statistical models, say, "aambuilding". +4) He did some further research in statistical models that I'm not familiar with, + which, I believe, enables him to be one of the best researchers in statistical models + for computer vision. + +His contact: njustyw at gmail dot com + + +HU XuBing +Great much thanks are given to Mr. HU XuBing, CEO of ShenYang YiMeng Ltd, China, +for his help on annotating two Chinese face databases: +a) EMOUNT b) JIAPEI +Some fitting results of the database EMOUNT can be viewed at +http://www.emount.com.cn/ShowNewsDetail.aspx?NewsId=9 + +His contact: +Website: http://www.emount.com.cn + diff --git a/CHANGES b/CHANGES new file mode 100644 index 0000000..6927534 --- /dev/null +++ b/CHANGES @@ -0,0 +1,144 @@ +Release notes of VOSM. + +--------------------------- Release 0.3.3 --May, 2012-------------------- +Minor improvements are made in this version VOSM 0.3.3, +in order to be compatible with OpenCV 2.4.0 above. + +1 General + - VOSM 0.3.3 is based upon: + + a) Boost (>= 1.48) for file and directory operations + b) OpenCV (2.4.0 or above) + + - VOSM 0.3.3 has been successfully tested on Boost 1.48 and OpenCV 2.4.0, + both under Ubuntu 12.04 and Windows 7 Professional. + + - Rename module "utils" in VOSM 0.3.2 to "common". + + - Add the following line in front of file VO_ShapeModel.h + #include "opencv/cvaux.h" + + - Don't forget to link opencv_legacy.lib when building modules or tests + based upon smbuilding. + + - In module "comalgs", all file names have been changed to begin with "VO_". + + - All files have been re-formated by the following 2 text rules + "tab -- 8 spaces" + "insert tab with spaces" + + - All testing files used to begin with "vo_", for instance "vo_smbuilding" + are now renamed to begin with "test_". + + - The reason why in the license, version "0.4" is used instead of "0.3.4" + is because version "0.4" is expected to be a major release. + + +--------------------------- Release 0.3.2 --August, 2011-------------------- +Minor improvements are made in this version VOSM 0.3.2, +in order to be compatible with OpenCV 2.3.0 above. + +1 General + - VOSM 0.3.2 is based upon: + + a) Boost (>= 1.42) for file and directory operations + b) OpenCV (subervsion R6267 or above) + + - VOSM 0.3.2 has been successfully tested on Boost 1.42 and OpenCV subversion 6267, + on Ubuntu 11.04 and Windows 7 Professional. + + - Two files, humandetectionalgs.h and humandetectionalgs.cpp have been + added to comalgs. + + - All #include "cv.h"s now have been changed to #include "opencv/cv.h"; + all #include "highgui.h"s now have been changed to #include "opencv/highgui.h"; + all #include "cxcore.h"s now have been changed to #include "opencv/cxcore.h". + + - INSTALL file has been changed to clarify how to install VOSM 0.3.2 . + + - HOWTOUSE file is updated to clarify how to use VOSM's 2 key command lines . + + +--------------------------- Release 0.3.1 --November, 2010-------------------- +Right after VOSM 0.3.0 was released on November 8, 2010, +due to prompt feedbacks from some VOSM users, +VOSM 0.3.1 is now released with the following improvements. + +1 General + - VOSM 0.3.1 is neither based on libgsl nor yaml-cpp any longer. + The only 3rd party libraries that VOSM 0.3.1 is based upon are + + a) Boost (>= 1.34.1) for file and directory operations + b) OpenCV (subervsion R3594 or above) as the basic but primary libraries + that VOSM 0.3.1 is built upon + + - A formal documentation is produced for reference + + - Add brief descriptions for every class + + +----------------------------- Release 0.3 --November, 2010-------------------- + +It has been a long time for Vision Open to release its recent +research fruits related to ASM/AAMs. +VOSM 0.3 offers a thoroughly fresh coding implementations, +and lots of improvements and new functionalities : + +1 General + + - Current VOSM can only be legally used under LGPL, namely, + Lesser General Public License. + + - Current VOSM libraries requires CMake for configuration. + + - Changes in the set of supported platforms: + + - Fully supported by GNU g++ 4.5 and tested. + + - Well supported by Microsoft Windows Visual Studio + C++ compiler 2008 (VC9). You may have to re-configure accordingly. + + - Changes in the set of supported OpenCV: + + - Must be OpenCV SVN R3594 or above + (OpenCV 2.1 is too old to be compile with current VOSM 0.3) + +2 Algorithms + + - Current VOSM affords 2D Profile ASM, which is dramatically faster + than traditional 1D Profile ASM. + + - Current VOSM proposes a conceptual novel idea named as + Active Feature Model. + + - Current VOSM implements all popular ASMs/AAMs, + both for static images and dynamic videos, including: + + - 1D profile ASM + + - 2D profile ASM + + - Direct Local texture contrained (LTC) ASM + + - Basic AAM + + - ICIA(Inverse compositional image alignment) AAM + + - IAIA(Inverse additive image alignment) AAM + +----------------------------- Release 0.2 -- April, 2008---------------------- + +Second release of "AAMBuilding": + + - No source, but a xwWidget based GUI application + with some binary static libraries for model building. + +----------------------------- Release 0.1 -- February, 2007------------------- + +Initial release of "AAMBuilding", + + - Contains only the model building code. + +------------------------------------------------------------------------------ + + diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..6aefe2d --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,518 @@ +# ---------------------------------------------------------------------------- +# Root CMake file for vosm +# +# From the off-tree build directory, invoke: +# $ cmake +# +# +# - AUG-2011: Revised version +# +# ---------------------------------------------------------------------------- + +set(CMAKE_ALLOW_LOOSE_LOOP_CONSTRUCTS true) +# Add these standard paths to the search paths for FIND_LIBRARY +# to find libraries from these locations first +if(UNIX) + set(CMAKE_LIBRARY_PATH ${CMAKE_LIBRARY_PATH} /lib /usr/lib) +endif() +# it _must_ go before PROJECT(vosm) in order to work +if (NOT CMAKE_INSTALL_PREFIX) + if (WIN32) + set(CMAKE_INSTALL_PREFIX ${CMAKE_BINARY_DIR} CACHE INTERNAL "" FORCE) + elseif() + set(CMAKE_INSTALL_PREFIX "/usr" CACHE INTERNAL "" FORCE) + endif() +endif() + + +set(CMAKE_CONFIGURATION_TYPES "Debug;Release" CACHE STRING "Configs" FORCE) +set(CMAKE_C_FLAGS_MINSIZEREL "" CACHE INTERNAL "" FORCE) +set(CMAKE_C_FLAGS_RELWITHDEBINFO "" CACHE INTERNAL "" FORCE) +set(CMAKE_CXX_FLAGS_MINSIZEREL "" CACHE INTERNAL "" FORCE) +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "" CACHE INTERNAL "" FORCE) +set(CMAKE_EXE_LINKER_FLAGS_MINSIZEREL "" CACHE INTERNAL "" FORCE) +set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "" CACHE INTERNAL "" FORCE) +set(CMAKE_MODULE_LINKER_FLAGS_MINSIZEREL "" CACHE INTERNAL "" FORCE) +set(CMAKE_MODULE_LINKER_FLAGS_RELWITHDEBINFO "" CACHE INTERNAL "" FORCE) +set(CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL "" CACHE INTERNAL "" FORCE) +set(CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO "" CACHE INTERNAL "" FORCE) +set(CMAKE_VERBOSE OFF CACHE BOOL "Verbose mode") + +if (CMAKE_VERBOSE) + set (CMAKE_VERBOSE_MAKEFILE 1) +endif() + +project(vosm) + +cmake_minimum_required(VERSION 2.4) + +if(MSVC) + set(CMAKE_USE_RELATIVE_PATHS ON CACHE INTERNAL "" FORCE) +endif() + +#set(CMAKE_C_COMPILER "/opt/BullseyeCoverage/bin/gcc") +#set(CMAKE_CXX_COMPILER "/opt/BullseyeCoverage/bin/g++") +#set(CMAKE_CXX_COMPILER_INIT "/opt/BullseyeCoverage/bin/gcc") + +# -------------------------------------------------------------- +# Indicate CMake 2.7 and above that we don't want to mix relative +# and absolute paths in linker lib lists. +# Run "cmake --help-policy CMP0003" for more information. +# -------------------------------------------------------------- +if(COMMAND cmake_policy) + cmake_policy(SET CMP0003 NEW) +endif() + +# ---------------------------------------------------------------------------- +# Current version number: +# ---------------------------------------------------------------------------- +set(VOSM_VERSION "0.3.2") + +string(REGEX MATCHALL "[0-9]" VOSM_VERSION_PARTS "${VOSM_VERSION}") + +list(GET VOSM_VERSION_PARTS 0 VOSM_VERSION_MAJOR) +list(GET VOSM_VERSION_PARTS 1 VOSM_VERSION_MINOR) +list(GET VOSM_VERSION_PARTS 2 VOSM_VERSION_PATCH) + +set(VOSM_SOVERSION "${VOSM_VERSION_MAJOR}.${VOSM_VERSION_MINOR}") + +if(WIN32) + # Postfix of DLLs: + set(VOSM_DLLVERSION "${VOSM_VERSION_MAJOR}${VOSM_VERSION_MINOR}${VOSM_VERSION_PATCH}") + set(VOSM_DEBUG_POSTFIX d) +else() + # Postfix of so's: + #set(VOSM_DLLVERSION "${VOSM_VERSION_MAJOR}${VOSM_VERSION_MINOR}${VOSM_VERSION_PATCH}") + set(VOSM_DLLVERSION "") + set(VOSM_DEBUG_POSTFIX) +endif() + + +# ---------------------------------------------------------------------------- +# Build static or dynamic libs? +# ---------------------------------------------------------------------------- +# Default: dynamic libraries: +SET(BUILD_SHARED_LIBS ON CACHE BOOL "Build shared libraries (.dll/.so) instead of static ones (.lib/.a)") +IF(BUILD_SHARED_LIBS) + SET(VOSM_BUILD_SHARED_LIB 1) # For cvconfig.h, etc. +ELSE(BUILD_SHARED_LIBS) + SET(VOSM_BUILD_SHARED_LIB 0) +ENDIF(BUILD_SHARED_LIBS) + +# ---------------------------------------------------------------------------- +# Variables for cvconfig.h.cmake +# ---------------------------------------------------------------------------- +set(PACKAGE "vosm") +set(PACKAGE_BUGREPORT "jp4work@gmail.com") +set(PACKAGE_NAME "vosm") +set(PACKAGE_STRING "${PACKAGE} ${VOSM_VERSION}") +set(PACKAGE_TARNAME "${PACKAGE}") +set(PACKAGE_VERSION "${VOSM_VERSION}") + +# ---------------------------------------------------------------------------- +# Autodetect if we are in a SVN repository +# ---------------------------------------------------------------------------- +find_program(SVNVERSION_PATH svnversion) +mark_as_advanced(force SVNVERSION_PATH) +if(SVNVERSION_PATH) + message(STATUS "Extracting svn version, please wait...") + execute_process(COMMAND ${SVNVERSION_PATH} -n ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE SVNVERSION_RESULT) + + if(SVNVERSION_RESULT MATCHES "exported") + # This is NOT a svn repository: + set(VOSM_SVNVERSION "") + else() + set(VOSM_SVNVERSION " svn:${SVNVERSION_RESULT}") + endif() + + message(STATUS "SVNVERSION: ${VOSM_SVNVERSION}") +else() + # We don't have svnversion: + set(VOSM_SVNVERSION "") +endif() + +# ---------------------------------------------------------------------------- +# Detect GNU version: +# ---------------------------------------------------------------------------- +if(CMAKE_COMPILER_IS_GNUCXX) + execute_process(COMMAND ${CMAKE_CXX_COMPILER} --version + OUTPUT_VARIABLE CMAKE_VOSM_GCC_VERSION_FULL + OUTPUT_STRIP_TRAILING_WHITESPACE) + + # Typical output in CMAKE_VOSM_GCC_VERSION_FULL: "c+//0 (whatever) 4.4.5 (...)" + # Look for the version number + string(REGEX MATCH "[0-9].[0-9].[0-9]" CMAKE_GCC_REGEX_VERSION "${CMAKE_VOSM_GCC_VERSION_FULL}") + + # Split the three parts: + string(REGEX MATCHALL "[0-9]" CMAKE_VOSM_GCC_VERSIONS "${CMAKE_GCC_REGEX_VERSION}") + + list(GET CMAKE_VOSM_GCC_VERSIONS 0 CMAKE_VOSM_GCC_VERSION_MAJOR) + list(GET CMAKE_VOSM_GCC_VERSIONS 1 CMAKE_VOSM_GCC_VERSION_MINOR) + + set(CMAKE_VOSM_GCC_VERSION ${CMAKE_VOSM_GCC_VERSION_MAJOR}${CMAKE_VOSM_GCC_VERSION_MINOR}) + math(EXPR CMAKE_VOSM_GCC_VERSION_NUM "${CMAKE_VOSM_GCC_VERSION_MAJOR}*100 + ${CMAKE_VOSM_GCC_VERSION_MINOR}") + message(STATUS "Detected version of GNU GCC: ${CMAKE_VOSM_GCC_VERSION} (${CMAKE_VOSM_GCC_VERSION_NUM})") + + if(WIN32) + execute_process(COMMAND ${CMAKE_CXX_COMPILER} -dumpmachine + OUTPUT_VARIABLE CMAKE_VOSM_GCC_TARGET_MACHINE + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(CMAKE_VOSM_GCC_TARGET_MACHINE MATCHES "64") + set(MINGW64 1) + endif() + endif() + +endif() + +# ---------------------------------------------------------------------------- +# CHECK FOR SYSTEM LIBRARIES, OPTIONS, ETC.. +# ---------------------------------------------------------------------------- + +# Build/install (or not) some apps: +# =================================================== +set(BUILD_EXAMPLES ON CACHE BOOL "Build all examples") +set(INSTALL_C_EXAMPLES ON CACHE BOOL "Install C examples") +#set(INSTALL_PYTHON_EXAMPLES OFF CACHE BOOL "Install Python examples") + +# Build tests: +# =================================================== +set(BUILD_TESTS ON CACHE BOOL "Build tests") + +include(vosmModule.cmake REQUIRED) + +if(UNIX) + include(VOSMFindPkgConfig.cmake OPTIONAL) + include(CheckFunctionExists) + include(CheckIncludeFile) +endif() + +# allow fine grained control over which libraries not to link, even if +# they are available on the system +# ==================================================================== +set(WITH_BOOST_REGEX ON CACHE BOOL "Include BOOST REGEX support") +set(WITH_BOOST_FILESYSTEM ON CACHE BOOL "Include BOOST FILESYSTEM support") +set(WITH_BOOST_SYSTEM ON CACHE BOOL "Include BOOST SYSTEM support") +set(WITH_OPENCV ON CACHE BOOL "Include OPENCV 2.x support") + +# =================================================== +# Macros that checks if module have been installed. +# After it adds module to build and define +# constants passed as second arg +# =================================================== + +macro(CHECK_MODULE module_name define) + set(${define} 0) + if(PKG_CONFIG_FOUND) + set(ALIAS ALIASOF_${module_name}) + set(ALIAS_FOUND ${ALIAS}_FOUND) + set(ALIAS_INCLUDE_DIRS ${ALIAS}_INCLUDE_DIRS) + set(ALIAS_LIBRARY_DIRS ${ALIAS}_LIBRARY_DIRS) + set(ALIAS_LIBRARIES ${ALIAS}_LIBRARIES) + + PKG_CHECK_MODULES(${ALIAS} ${module_name}) + if (${ALIAS_FOUND}) + set(${define} 1) + foreach(P "${ALIAS_INCLUDE_DIRS}") + if (${P}) + list(APPEND HIGHGUI_INCLUDE_DIRS ${${P}}) + endif() + endforeach() + + foreach(P "${ALIAS_LIBRARY_DIRS}") + if (${P}) + list(APPEND HIGHGUI_LIBRARY_DIRS ${${P}}) + endif() + endforeach() + + list(APPEND HIGHGUI_LIBRARIES ${${ALIAS_LIBRARIES}}) + endif() + endif() +endmacro() + +set(VOSM_LINKER_LIBS ${VOSM_LINKER_LIBS} ${OPENCV_LINKER_LIBS} ) + + +if(WITH_BOOST_REGEX) + CHECK_MODULE(libboost-regex HAVE_BOOST_REGEX) +else() + set(HAVE_BOOST_REGEX FALSE) +endif() + +if(WITH_BOOST_FILESYSTEM) + CHECK_MODULE(libboost-filesystem HAVE_BOOST_FILESYSTEM) +else() + set(HAVE_BOOST_FILESYSTEM FALSE) +endif() + +if(WITH_OPENCV) + find_path(OPENCV_INCLUDE_PATH "cv.h" + PATHS "/usr/include/opencv" "/usr/local/include/opencv" + DOC "The path to OPENCV headers") + if(OPENCV_INCLUDE_PATH) + include_directories(${OPENCV_INCLUDE_PATH}) + set(HAVE_OPENCV TRUE) + endif() + CHECK_MODULE(libopencv_calib3d HAVE_OPENCV_CALIB3D) + CHECK_MODULE(libopencv_contrib HAVE_OPENCV_CONTRIB) + CHECK_MODULE(libopencv_core HAVE_OPENCV_CORE) + CHECK_MODULE(libopencv_features2d HAVE_OPENCV_FEATURE2D) + CHECK_MODULE(libopencv_gpu HAVE_OPENCV_GPU) + CHECK_MODULE(libopencv_highgui HAVE_OPENCV_HIGHGUI) + CHECK_MODULE(libopencv_imgproc HAVE_OPENCV_IMGPROC) + CHECK_MODULE(libopencv_legacy HAVE_OPENCV_LEGACY) + CHECK_MODULE(libopencv_ml HAVE_OPENCV_ML) + CHECK_MODULE(libopencv_objdetect HAVE_OPENCV_OBJDETECT) + CHECK_MODULE(libopencv_video HAVE_OPENCV_VIDEO) + else() + set(HAVE_OPENCV FALSE) +endif() + + +# ---------------------------------------------------------------------------- +# UPDATE CONFIG FILES & SCRIPTS: +# +# CONFIGURE_FILE(InputFile OutputFile [COPYONLY] [ESCAPE_QUOTES] [@ONLY]) +# If @ONLY is specified, only variables of the form @VAR@ will be +# replaces and ${VAR} will be ignored. +# +# A directory will be created for each platform so the "cvconfig.h" file is +# not overwritten if cmake generates code in the same path. +# ---------------------------------------------------------------------------- +add_definitions(-DHAVE_CONFIG_H) + +set(VOSM_CONFIG_FILE_INCLUDE_DIR "${CMAKE_BINARY_DIR}/" CACHE PATH "Where to create the platform-dependant cvconfig.h") + +message(STATUS "Parsing 'cvconfig.h.cmake'") +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/cvconfig.h.cmake" "${VOSM_CONFIG_FILE_INCLUDE_DIR}/cvconfig.h") + +# --------------------------------------------------------------------------- +# The C+//0 include & link directories: +# --------------------------------------------------------------------------- +include_directories("." + "${OPENCV_INCLUDE_PATH}" + "${VOSM_CONFIG_FILE_INCLUDE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}/include" + "${CMAKE_CURRENT_SOURCE_DIR}/include/opencv" + ) + +# ---------------------------------------------------------------------------- +# Set the maximum level of warnings: +# ---------------------------------------------------------------------------- +# Should be set to true for development +set(VOSM_WARNINGS_ARE_ERRORS OFF CACHE BOOL "Treat warnings as errors") + +set(EXTRA_C_FLAGS "") +set(EXTRA_C_FLAGS_RELEASE "") +set(EXTRA_C_FLAGS_DEBUG "") +set(EXTRA_EXE_LINKER_FLAGS "") +set(EXTRA_EXE_LINKER_FLAGS_RELEASE "") +set(EXTRA_EXE_LINKER_FLAGS_DEBUG "") + +if(MSVC) + set(EXTRA_C_FLAGS "${EXTRA_C_FLAGS} /D _CRT_SECURE_NO_DEPRECATE /D _CRT_NONSTDC_NO_DEPRECATE /D _SCL_SECURE_NO_WARNINGS") + # 64-bit portability warnings, in MSVC8 + if(MSVC80) + set(EXTRA_C_FLAGS "${EXTRA_C_FLAGS} /Wp64") + endif() + #if(MSVC90) + # set(EXTRA_C_FLAGS "${EXTRA_C_FLAGS} /D _BIND_TO_CURRENT_CRT_VERSION=1 /D _BIND_TO_CURRENT_VCLIBS_VERSION=1") + #endif() + + set(EXTRA_EXE_LINKER_FLAGS_RELEASE "${EXTRA_EXE_LINKER_FLAGS_RELEASE} /debug") + + #if(ENABLE_OPENMP) + # set(EXTRA_C_FLAGS_RELEASE "${EXTRA_C_FLAGS_RELEASE} /openmp") + #endif() + + # Remove unreferenced functions: function level linking + set(EXTRA_C_FLAGS "${EXTRA_C_FLAGS} /Gy") + set(EXTRA_C_FLAGS_RELEASE "${EXTRA_C_FLAGS_RELEASE} /Zi") +endif() + +# ---------------------------------------------------------------------------- +# PROCESS SUBDIRECTORIES: +# ---------------------------------------------------------------------------- +# Save libs and executables in the same place +set(LIBRARY_OUTPUT_PATH ${CMAKE_BINARY_DIR}/lib CACHE PATH "Output directory for libraries" ) +set(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/bin CACHE PATH "Output directory for applications" ) + +if(WIN32) +set(VOSM_DOC_INSTALL_PATH doc) +else() +set(VOSM_DOC_INSTALL_PATH share/vosm/doc) +endif() + +# -------------------------------------------------------------------------------------------- +# Installation for CMake Module: vosmConfig.cmake +# Part 1/3: ${BIN_DIR}/vosmConfig.cmake -> For use *without* "make install" +# Part 2/3: ${BIN_DIR}/unix-install/vosmConfig.cmake -> For use with "make install" +# Part 3/3: ${BIN_DIR}/win-install/vosmConfig.cmake -> For use within binary installers/packages +# ------------------------------------------------------------------------------------------- + +# ------------------------------------------------------------------------------------------- +# Part 1/3: ${BIN_DIR}/vosmConfig.cmake -> For use *without* "make install" +# ------------------------------------------------------------------------------------------- + +# Name of libraries is: libcv.so.1.1.0, etc... +# VOSM_DLLVERSION must be set to "110", etc.. +# Already done above. + +# Set CMAKE_INCLUDE_DIRS_CONFIGCMAKE to the list of include directories: +set(CMAKE_INCLUDE_DIRS_CONFIGCMAKE "\"${VOSM_CONFIG_FILE_INCLUDE_DIR}\" \"${CMAKE_CURRENT_SOURCE_DIR}/include\" \"${CMAKE_CURRENT_SOURCE_DIR}/include/vosm\"") +set(CMAKE_BASE_INCLUDE_DIRS_CONFIGCMAKE "\"${CMAKE_CURRENT_SOURCE_DIR}\"") + +set(CMAKE_LIB_DIRS_CONFIGCMAKE "${LIBRARY_OUTPUT_PATH}") + +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/vosmConfig.cmake.in" "${CMAKE_BINARY_DIR}/vosmConfig.cmake" IMMEDIATE @ONLY) + +# -------------------------------------------------------------------------------------------- +# Part 2/3: ${BIN_DIR}/unix-install/vosmConfig.cmake -> For use *with* "make install" +# ------------------------------------------------------------------------------------------- +# Set CMAKE_INCLUDE_DIRS_CONFIGCMAKE to the list of include directories: + +set(CMAKE_INCLUDE_DIRS_CONFIGCMAKE "${CMAKE_INSTALL_PREFIX}/include/opencv" "${CMAKE_INSTALL_PREFIX}/include") +set(CMAKE_BASE_INCLUDE_DIRS_CONFIGCMAKE "\"\"") + +set(CMAKE_LIB_DIRS_CONFIGCMAKE "${CMAKE_INSTALL_PREFIX}/lib") + +#exec_program(${CMAKE_COMMAND} ARGS "-E make_directory \"${CMAKE_BINARY_DIR}/unix-install/\"") +exec_program(mkdir ARGS "-p \"${CMAKE_BINARY_DIR}/unix-install/\"") +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/vosmConfig.cmake.in" "${CMAKE_BINARY_DIR}/unix-install/vosmConfig.cmake" IMMEDIATE @ONLY) + +if(UNIX) + # For a command "FIND_PACKAGE(FOO)", CMake will look at the directory /usr/share|lib/FOO/FOOConfig.cmake, so: + install(FILES ${CMAKE_BINARY_DIR}/unix-install/vosmConfig.cmake DESTINATION ${CMAKE_INSTALL_PREFIX}/share/vosm/) +endif() + +# -------------------------------------------------------------------------------------------- +# Part 3/3: ${BIN_DIR}/win-install/vosmConfig.cmake -> For use within binary installers/packages +# ------------------------------------------------------------------------------------------- +if(WIN32) + # Set CMAKE_INCLUDE_DIRS_CONFIGCMAKE to the list of include directories: + + # This will expand to, for example, /vosm 0.3.2/include + set(CMAKE_INCLUDE_DIRS_CONFIGCMAKE "\"\${THIS_VOSM_CONFIG_PATH}/include\" \"\${THIS_VOSM_CONFIG_PATH}/include/vosm\"") + set(CMAKE_BASE_INCLUDE_DIRS_CONFIGCMAKE "\"${THIS_VOSM_CONFIG_PATH}\"") + + # This will expand to, for example, /vosm 0.3.2/lib + set(CMAKE_LIB_DIRS_CONFIGCMAKE "\"\${THIS_VOSM_CONFIG_PATH}/lib\"") + + #exec_program(${CMAKE_COMMAND} ARGS "-E make_directory \"${CMAKE_BINARY_DIR}/win-install/\"") + exec_program(mkdir ARGS "-p \"${CMAKE_BINARY_DIR}/win-install/\"") + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/vosmConfig.cmake.in" "${CMAKE_BINARY_DIR}/win-install/vosmConfig.cmake" IMMEDIATE @ONLY) +endif() + + +# -------------------------------------------------------------------------------------------- +# Part 2/2: ${BIN_DIR}/unix-install/vosm.pc -> For use *with* "make install" +# prefix=/usr +# exec_prefix=${prefix} +# libdir=${exec_prefix}/lib +# includedir=${prefix}/include/vosm +# ------------------------------------------------------------------------------------------- +set(opencvdir ${OPENCV_INCLUDE_PATH}) +set(prefix ${CMAKE_INSTALL_PREFIX}) +set(exec_prefix "\${prefix}") +set(libdir "\${exec_prefix}/lib") +set(includedir "\${prefix}/include") +set(VERSION ${VOSM_VERSION}) + + +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/vosm.pc.cmake.in" "${CMAKE_BINARY_DIR}/unix-install/vosm.pc" @ONLY IMMEDIATE) + +if(UNIX) + install(FILES ${CMAKE_BINARY_DIR}/unix-install/vosm.pc DESTINATION lib/pkgconfig) +endif() + + +# ---------------------------------------------------------------------------- +# Uninstall target, for "make uninstall" +# ---------------------------------------------------------------------------- +CONFIGURE_FILE( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" + IMMEDIATE @ONLY) + +ADD_CUSTOM_TARGET(uninstall + "${CMAKE_COMMAND}" -P "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake") + + +# ---------------------------------------------------------------------------- +# CPack target +# ---------------------------------------------------------------------------- + +set(BUILD_PACKAGE ON CACHE BOOL "Build a installer with the SDK") +message(STATUS "BUILD_PACKAGE FALSE") +if(BUILD_PACKAGE) +message(STATUS "BUILD_PACKAGE TRUE") + configure_file( + Package.cmake.in + ${CMAKE_BINARY_DIR}/.cpack/Package.cmake + @ONLY + ) + + include(${CMAKE_BINARY_DIR}/.cpack/Package.cmake) + + set(root_files README) + + file(GLOB root_src_files *.in *.cmake CMakeLists.txt) + + if (NOT WIN32) + install(FILES ${root_files} + DESTINATION ${VOSM_DOC_INSTALL_PATH} + COMPONENT main) + else() + install(FILES ${root_files} + DESTINATION "." + COMPONENT main) + install(FILES ${root_src_files} + DESTINATION "." + COMPONENT src) + # Add the "win-install/OpenCVConfig.cmake" file to the "main" install component + install(FILES + "${CMAKE_BINARY_DIR}/win-install/OpenCVConfig.cmake" + DESTINATION "." + COMPONENT main + ) + + if(CMAKE_INSTALL_DEBUG_LIBRARIES) + foreach(m cxcore cv cvaux ml highgui opencv_ffmpeg) + install(FILES ${CMAKE_BINARY_DIR}/bin/Debug/${m}${VOSM_DLLVERSION}d.dll + DESTINATION bin COMPONENT main) + install(FILES ${CMAKE_BINARY_DIR}/lib/Debug/${m}${VOSM_DLLVERSION}d.lib + DESTINATION lib COMPONENT main) + endforeach() + endif() + + install(FILES AUTHORS CHANGES COPYRIGHT HOWTOUSE INSTALL README TODO VERSION DESTINATION share/doc/vosm) + install(DIRECTORY doc/ DESTINATION doc COMPONENT src FILES_MATCHING PATTERN "*.tex") + + endif() +endif() + + +#----------------------------------- +# Subdirectories: +#----------------------------------- +add_subdirectory(modules) + + +if(BUILD_LATEX_DOCS) + include(OpenCVFindLATEX.cmake REQUIRED) + + if(PDFLATEX_COMPILER) + message(STATUS "PDF LaTeX found!") + endif() +endif() + + +#if(BUILD_EXAMPLES OR INSTALL_PYTHON_EXAMPLES) +if(BUILD_EXAMPLES) + add_subdirectory(tests) +endif() + + +#---------------------------------------------------------------------------- +# Generate the vosmModule.cmake file for unix +# installation in CMAKE_INSTALL_PREFIX +#---------------------------------------------------------------------------- diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 0000000..3d8eda4 --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,60 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-16 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + diff --git a/HOWTOUSE b/HOWTOUSE new file mode 100644 index 0000000..a2cbbe8 --- /dev/null +++ b/HOWTOUSE @@ -0,0 +1,76 @@ + +How to use vosm to +a) train a face model +b) fit an untrained face using the trained face model + +First of all, in order to use vosm, you need to find and download +respective ShapeInfo.txt files summarized in the table at +http://www.visionopen.com/resources/computer-vision/statistical-models/ + +Secondly, you have to find and download or buy the standard +annotated face databases by yourself. + +Help on two commands "test_smbuilding" and "test_smfitting". +./test_smbuilding --help +Usage: smbuilding [options] save_directory annotation_directory image_directory shapeinfo_path database channels type levels percentage +options: + -o output directory (default './') + -a annotation directory (required) + -i image directory (required) + -s path of the file shapeinfo (required) + -d training database (required, IMM, AGING, BIOID, XM2VTS, FRANCK, EMOUNT, JIAPEI ) + -c channels (1 or 3, default 3) + -t statistical model type (SM, TM, AM, IA, FM, SMLTC, SMNDPROFILE. default SMNDPROFILE ) + -l level of parymid (default 4) + -p percentage of shape, texture and appearance PCA (default 0.95) +Note: If you are building SMLTC or SMNDPROFILE, you must specify + -c 1, namely, SMLTC and SMNDPROFILE can only deal with gray-level images. + + + +./test_smfitting --help +Usage: smfitting [options] trained_data type testing_images testing_annotations database staticORdynamic recording +options: + -o trained data directory (required) + -t fitting method to be used (ASM_PROFILEND, ASM_LTC, AAM_BASIC, AAM_CMUICIA, AAM_IAIA. default ASM_PROFILEND ) + -i testing image directory containing at least one image (required) + -a testing annotation directory (can be ignored) + -d testing database -- if annotation directory is specified, database should also be specified for further evaluation on fitting performance (can be ignored) + -s static image sequence or dynamic image sequence (default value true) + -r recording the fitting results or not (default value false) + + + + +Examples are given as: + +1) For training: +test_smbuilding -a "F:\Research\Databases\face\2D\EMOUNT\annotations\training" +-i "F:\Research\Databases\face\2D\EMOUNT\images\training" +-s "F:\Research\Databases\face\2D\EMOUNT\annotations\training\ShapeInfo.txt" +-d "EMOUNT" +-c 1 +-t "SMNDPROFILE" + + +2) For testing: + +a) For static image database EMOUNT: +test_smfitting -o "F:\Research\Databases\face\2D\EMOUNT\traineddata\60Frontal\BGR" +-t "ASM_PROFILEND" +-i "F:\Research\Databases\face\2D\EMOUNT\images\testing" +-s "true" +-r "true" + +b) For dynamic image database JIAPEI: +test_smfitting -o "F:\Research\Databases\face\2D\JIAPEI\traineddata\33Frontal\BGR" +-t "ASM_PROFILEND" +-i "F:\Research\Databases\face\2D\JIAPEI\images\testing" +-a "F:\Research\Databases\face\2D\JIAPEI\annotations\testing" +-d "JIAPEI" +-s "true" +-r "true" + + + + diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000..43ffc4b --- /dev/null +++ b/INSTALL @@ -0,0 +1,122 @@ +INSTALL +------------------------------------------------------------------------------- + + +INTRODUCTION +============ + +This file describes how to install VOSM. The instructions in this file +are for the most common use cases, and cover the command line tools. + +The documentation of VOSM is available in HTML formats. +It is not bundled with the software but can be viewed separately at +. + +For more information about VOSM, refer to + + + + +PREREQUISITES +============= + +To install VOSM, you need 'cmake' and several third-party libraries are pre-requisite. + + * CMake (>= 2.8), the build system used by VOSM + Required for building VOSM + + * OpenCV (>= 2.4.0) + + * Boost (>= 1.48) + Required for building VOSM and for applications using VOSM + Required compiled Boost library: Boost.Regex + Required compiled Boost library: Boost.Filesystem + Required compiled Boost library: Boost.System + http://www.boost.org/ + + * Visual Studio 2010 + Recommended under Windows. + + +Windows + vosm-0.3.3 comes with a 7z compressed file "vosm-0.3.3.7z". + To test it out, just unpack it and open the solution with Visual Studio 2010. + + + +Linux + vosm-0.3.3 also comes with a tar ball file "vosm-0.3.3.tar.gz". + You may have to follow the following steps to make it run under Linux. + +CONFIGURATION +============= + +To configure VOSM, type + + cmake . + +in the directory that contains this INSTALL file. You can add several options +to this command. The most important ones are + + -DCMAKE_INSTALL_PREFIX= installation directory [/usr/local] + -DCMAKE_BUILD_TYPE= build type [Release] + -DBUILD_SHARED_LIBS= shared or static libraries [TRUE] + -DCMAKE_C_COMPILER= C compiler [gcc] + -DCMAKE_CXX_COMPILER= C++ compiler [g++] + +In case you want to add additional compiler and linker flags, you can use + + -DCGAL_CXX_FLAGS additional compiler flags + -DCGAL_MODULE_LINKER_FLAGS add. linker flags (static libraries) + -DCGAL_SHARED_LINKER_FLAGS add. linker flags (shared libraries) + -DCGAL_EXE_LINKER_FLAGS add. linker flags (executables) + +Variants with the additional suffix "_DEBUG" and "_RELEASE" allow to set +separate values for debug and release builds. In case you do not want to add +additional flags, but to override the default flags, replace "VOSM" by +"CMAKE" in the variable names above. + +Note that CMake maintains a cache name "CMakeCache.txt". If you change options +(or your environment changes), it is best to remove that file to avoid +problems. + + +BUILDING +======== + +To build the VOSM libraries, type + + make + +If you want, you can install the VOSM header and libraries. To do so, type + + make install + +A list of all available make targets can be obtained by + + make help + + +OUT-OF-SOURCE BUILDS +==================== + +The above instructions build the VOSM library in the same directory tree as +the VOSM sources. Sometimes it is advisable to place all the generated files +somewhere else. For example, if you want to build the library in several +configurations (debug and release, different compilers, and so on). Using +different build directories keeps all the generated files separated for each +configuration. + +In the following, $VOSM_SRC denotes the directory with the VOSM sources; +$VOSM_BUILD is an arbitrary directory where the generated files will be +placed. You can perform an out-of-source build as follows: + + mkdir $VOSM_BUILD + cd $VOSM_BUILD + cmake [options] $VOSM_SRC + make + make install (if desired) + +Basically, the only difference is the last parameter of the "cmake" command: +$VOSM_SRC instead of "." . + diff --git a/Package.cmake.in b/Package.cmake.in new file mode 100644 index 0000000..f0d31ea --- /dev/null +++ b/Package.cmake.in @@ -0,0 +1,92 @@ +if(MSVC) + set(CMAKE_INSTALL_DEBUG_LIBRARIES 1) +endif() +if(ENABLE_OPENMP) + set(CMAKE_INSTALL_OPENMP_LIBRARIES 1) +endif() +#include(VOSMInstallRequiredSystemLibraries.cmake) +#include(InstallRequiredSystemLibraries) + +set(CPACK_PACKAGE_NAME "VOSM") +set(CPACK_PACKAGE_VENDOR "VOSM project Vision Open Statistical Models") + +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "VOSM SDK for ${CMAKE_GENERATOR} is an vosm based open source for developing statistical models") + +#set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_SOURCE_DIR}/README") +set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/doc/license.txt") + +set(CPACK_PACKAGE_VERSION_MAJOR "${VOSM_VERSION_MAJOR}") +set(CPACK_PACKAGE_VERSION_MINOR "${VOSM_VERSION_MINOR}") +set(CPACK_PACKAGE_VERSION_PATCH "${VOSM_VERSION_PATCH}") + +set(CPACK_PACKAGE_INSTALL_DIRECTORY "vosm${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}") + +set(CPACK_PACKAGE_EXECUTABLES "") + +set(CPACK_COMPONENTS_ALL main src Unspecified) + +set(CPACK_COMPONENT_main_DISPLAY_NAME "Binaries and the Documentation") +set(CPACK_COMPONENT_src_DISPLAY_NAME "The source code") +#set(CPACK_COMPONENT_py_DISPLAY_NAME "Python Bindings") + +set(CPACK_ALL_INSTALL_TYPES Full) + +set(CPACK_COMPONENT_MAIN_INSTALL_TYPES Full) +set(CPACK_COMPONENT_SRC_INSTALL_TYPES Full) +#set(CPACK_COMPONENT_py_INSTALL_TYPES Full) + +set(CPACK_SOURCE_IGNORE_FILES + "/\\\\.svn/" + "/autom4te.cache/" + "/build/" + "/lib/" + "~$" + "\\\\.aux$" + "\\\\.bbl$" + "\\\\.blg$" + "\\\\.idx$" + "\\\\.ilg$" + "\\\\.ind$" + "\\\\.log$" + "\\\\.toc$" + "\\\\.out$" + "\\\\.vcproj$" + "/1$" + "${CPACK_SOURCE_IGNORE_FILES}") + +set(CPACK_SOURCE_PACKAGE_FILE_NAME + "${CMAKE_PROJECT_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}") + +if(WIN32) + set(CPACK_GENERATOR "NSIS") + set(CPACK_SOURCE_GENERATOR "ZIP") + set(CPACK_NSIS_PACKAGE_NAME "vosm ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}") + set(CPACK_NSIS_MUI_ICON "${CMAKE_SOURCE_DIR}\\\\common\\\\vo.ico") + set(CPACK_NSIS_MUI_UNIICON "${CMAKE_SOURCE_DIR}\\\\common\\\\vo.ico") + #set(CPACK_PACKAGE_ICON "common/vo.ico") + + set(CPACK_NSIS_INSTALLED_ICON_NAME "${CMAKE_SOURCE_DIR}\\\\common\\\\vo.ico") + set(CPACK_NSIS_HELP_LINK "http:\\\\\\\\visionopen.com") + set(CPACK_NSIS_URL_INFO_ABOUT "http:\\\\\\\\visionopen.com") + set(CPACK_NSIS_CONTACT "jp4work@gmail.com") + set(CPACK_NSIS_DISPLAY_NAME "vosm SDK") + + set(CPACK_NSIS_MENU_LINKS + "doc\\\\index.htm" "Start Page" + "doc\\\\vosm.pdf" "The Reference Manual" + "CMakeLists.txt" "The Build Script (open with CMake)" + "samples\\\\c" "C Samples" + "samples\\\\python" "Python Samples") + + # Add "/bin" to the system PATH + set(CPACK_NSIS_MODIFY_PATH ON) +else() + set(CPACK_GENERATOR "TBZ2") + set(CPACK_SOURCE_GENERATOR "TBZ2") + + if(APPLE) + set(CPACK_GENERATOR "PackageMaker;TBZ2") + endif() +endif() + +include(CPack) diff --git a/README b/README new file mode 100644 index 0000000..33da7f4 --- /dev/null +++ b/README @@ -0,0 +1,21 @@ +README +VOSM is the abbreviation of Vision Open Statistical Models +------------------------------------------------------------------------------- + +This distribution of VOSM includes: + + AUTHORS - current and former authors of VOSM + CHANGES - history of changes for the library + CMakeLists.txt - main script of the build system + COPYRIGHT - describes the copyright of VOSM + doc/ - documentations + HOWTOUSE - how to use vosm + INSTALL - information about the installation process + README - this file + TODO - what's to be done in future + VERSION - version number of this release + modules/ - the source code for building the VOSM libraries + tests/ - the source code for building command line executives, + such as test_smbuilding and test_smfitting, etc. + +------------------------------------------------------------------------------- diff --git a/TODO b/TODO new file mode 100644 index 0000000..65d9549 --- /dev/null +++ b/TODO @@ -0,0 +1,5 @@ +1) use namespace "vo->sm->", rather than current "VO_" for every class +2) use "XML/YML" files to store trained data, rather than current '.txt" files. +3) 3D statistical models are not covered yet (tough to build up a 3D face database but this is to be done!) +4) documentations are still to be improved + diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..1c09c74 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.3.3 diff --git a/VOSM_changelist.txt b/VOSM_changelist.txt new file mode 100644 index 0000000..bca362d --- /dev/null +++ b/VOSM_changelist.txt @@ -0,0 +1,65 @@ +General + Fixed numerous numeric type precision warnings through helper methods and static casts in VO_Common.h + static_cast + static_cast + safeDoubleToFloat + + Added VO_Fitting2DSM::DrawMeshInfo class to move the drawing of points on the image outside of the timed fitting process; which should improve the reported fitting time with large images while recording fitting results + + Moved boost::filesystem calls to a helper method in VO_Common.h, this provides the same functionality with less duplicated code and allows OS specific filesystem methods to improve performance if necessary. + +test_smfitting + Changes + YAML Config file can be used to specify call of the command line arguments + Refactored fitting calls to reduce code duplication + Loaded pyramid levels and profile dimensions/directions from saved model files + + +VO_FittingASMNDProfiles + + Changes + Includes fittingTechnique specifier, an enum declared in VO_FittingASMNDProfiles.h + + VO_ASMNDProfileFitting - Added call to StagedPyramidFit to allow for different fitting techniques during fitting + + UpdateShape - Refactored to call FindBestOffset that selects the appropriate fitting technique. + + New + VO_FindBestMatchingProfile2D - Checks 2 profiles per direction and chooses the profile with the smallest Mahalnobis distance. + + StagedPtramidFit - Chooses different techniques for different points in the pyramid fitting process (if applicable) + +VO_RecognitionAlgs + Changes + CalcShapeFittingEffect - Per point distance calculations as + New + SaveShapeResults - Improves SaveShapeRecogResults by calculating error per point and saving the given point placements, used for saving point updates after each iteration + + SaveFittingResults - More comprehensive SaveShapeResults that saves candidate locations and fitting time + +VO_TextureModel.cpp + New + VO_CalcSubPixelTexture - Two channel texture calculation + +VO_Profile + New + Set2DimProfile - Sets two channel mat with index + + Get2DProfileInMat4OneLandmark - Only supports 2 channel + + Changed + Get1DProfileInMat4OneLandmark - Full hybrid, dual channel support + + GetNDProfiles4OneLandmark - Special case for 2 channels + +VO_ASMNDProfies + Changed + VO_BuildASMNDProfiles - Two channels will produce 4 profiles + + VO_LoadProfileTrainingData - Takes an argument to control OpenCV imread functionality to load multi-channel images + +VO_ASMLTCs + Changed + VO_HardSaveWaveletSingleChannelImage - prevented run from obvious bug in code, no functional solution. + + \ No newline at end of file diff --git a/cmake_uninstall.cmake.in b/cmake_uninstall.cmake.in new file mode 100644 index 0000000..14e6010 --- /dev/null +++ b/cmake_uninstall.cmake.in @@ -0,0 +1,27 @@ +# ----------------------------------------------- +# File that provides "make uninstall" target +# We use the file 'install_manifest.txt' +# ----------------------------------------------- +IF(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") + MESSAGE(FATAL_ERROR "Cannot find install manifest: \"@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt\"") +ENDIF(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") + +FILE(READ "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt" files) +STRING(REGEX REPLACE "\n" ";" files "${files}") +FOREACH(file ${files}) + MESSAGE(STATUS "Uninstalling \"$ENV{DESTDIR}${file}\"") + IF(EXISTS "$ENV{DESTDIR}${file}") + EXEC_PROGRAM( + "@CMAKE_COMMAND@" ARGS "-E remove \"$ENV{DESTDIR}${file}\"" + OUTPUT_VARIABLE rm_out + RETURN_VALUE rm_retval + ) + IF(NOT "${rm_retval}" STREQUAL 0) + MESSAGE(FATAL_ERROR "Problem when removing \"$ENV{DESTDIR}${file}\"") + ENDIF(NOT "${rm_retval}" STREQUAL 0) + ELSE(EXISTS "$ENV{DESTDIR}${file}") + MESSAGE(STATUS "File \"$ENV{DESTDIR}${file}\" does not exist.") + ENDIF(EXISTS "$ENV{DESTDIR}${file}") +ENDFOREACH(file) + + diff --git a/cmd.txt b/cmd.txt new file mode 100644 index 0000000..475178d --- /dev/null +++ b/cmd.txt @@ -0,0 +1 @@ +C:/data/vosm-0.3.3/Debug/test_smfitting.exe -o "C:/data/CrossVerify/Set_0/Models/CR" -c depth -i "C:/data/CrossVerify/Set_0/Faces/Test/E_SADNESS" -a "C:/data/CrossVerify/Set_0/Faces/Test/E_SADNESS" -t ASM_PROFILEND -s true -r true -l min-max diff --git a/cvconfig.h.cmake b/cvconfig.h.cmake new file mode 100644 index 0000000..46322b9 --- /dev/null +++ b/cvconfig.h.cmake @@ -0,0 +1,45 @@ +/* Boost Regex */ +#cmakedefine HAVE_BOOST_REGEX + +/* Boost FileSystem */ +#cmakedefine HAVE_BOOST_FILESYSTEM + +/* OpenCV library */ +#cmakedefine HAVE_OPENCV + +/* VOSM compiled as static or dynamic libs */ +#cmakedefine VOSM_BUILD_SHARED_LIB + +/* Name of package */ +#define PACKAGE "${PACKAGE}" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "${PACKAGE_BUGREPORT}" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "${PACKAGE_NAME}" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "${PACKAGE_STRING}" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "${PACKAGE_TARNAME}" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "${PACKAGE_VERSION}" + +/* If using the C implementation of alloca, define if you know the + direction of stack growth for your system; otherwise it will be + automatically deduced at runtime. + STACK_DIRECTION > 0 => grows toward higher addresses + STACK_DIRECTION < 0 => grows toward lower addresses + STACK_DIRECTION = 0 => direction of growth unknown */ +#cmakedefine STACK_DIRECTION + +/* Version number of package */ +#define VERSION "${PACKAGE_VERSION}" + +/* Define to 1 if your processor stores words with the most significant byte + first (like Motorola and SPARC, unlike Intel and VAX). */ +#cmakedefine WORDS_BIGENDIAN + diff --git a/modules/CMakeLists.txt b/modules/CMakeLists.txt new file mode 100644 index 0000000..d325d40 --- /dev/null +++ b/modules/CMakeLists.txt @@ -0,0 +1,9 @@ +ADD_SUBDIRECTORY(common) +ADD_SUBDIRECTORY(integraltransform) +ADD_SUBDIRECTORY(featureextraction) +ADD_SUBDIRECTORY(ensembletraining) +ADD_SUBDIRECTORY(cvcommon) +ADD_SUBDIRECTORY(comalgs) +ADD_SUBDIRECTORY(smbuilding) +ADD_SUBDIRECTORY(smfitting) + diff --git a/modules/comalgs/CMakeLists.txt b/modules/comalgs/CMakeLists.txt new file mode 100644 index 0000000..337bab7 --- /dev/null +++ b/modules/comalgs/CMakeLists.txt @@ -0,0 +1,5 @@ +include_directories("${CMAKE_CURRENT_SOURCE_DIR}/../common/include" "${CMAKE_CURRENT_SOURCE_DIR}/../cvcommon/include" +"${CMAKE_CURRENT_SOURCE_DIR}/../ensembletraining/include" "${CMAKE_CURRENT_SOURCE_DIR}/../smbuilding/include") +set(deps opencv_core opencv_imgproc opencv_highgui) +define_vosm_module(comalgs vosm_common vosm_featureextraction ${deps}) + diff --git a/modules/comalgs/comalgs.vcxproj b/modules/comalgs/comalgs.vcxproj new file mode 100644 index 0000000..32516bc --- /dev/null +++ b/modules/comalgs/comalgs.vcxproj @@ -0,0 +1,107 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {5F3443C8-6018-489E-9972-86C08813580B} + comalgs + + + + StaticLibrary + true + MultiByte + v110 + + + Application + false + true + MultiByte + v110 + + + + + + + + + + + + + + + Level3 + Disabled + ./include;../include;../common/include;../cvcommon/include;../ensembletraining/include;../smbuilding/include;C:\OpenCV2.4.5\opencv\build\include;C:\boost\boost_1_53_0;%(AdditionalIncludeDirectories) + NOMINMAX;_MBCS;%(PreprocessorDefinitions) + + + true + + + true + + + + + Level3 + MaxSpeed + true + true + + + true + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + {34e3a0e0-a9aa-4338-a68c-5b086e505117} + + + {6dc6813e-859a-4759-94cd-a1f1a6466b7b} + + + {4e713018-b49c-44ee-bf6a-4aaecbd0d148} + + + {838e7724-4e9b-4489-b26c-b419547473d2} + + + + + + \ No newline at end of file diff --git a/modules/comalgs/comalgs.vcxproj.filters b/modules/comalgs/comalgs.vcxproj.filters new file mode 100644 index 0000000..d5dc0c6 --- /dev/null +++ b/modules/comalgs/comalgs.vcxproj.filters @@ -0,0 +1,72 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + \ No newline at end of file diff --git a/modules/comalgs/include/VO_ClassificationAlgs.h b/modules/comalgs/include/VO_ClassificationAlgs.h new file mode 100644 index 0000000..6f8d5dc --- /dev/null +++ b/modules/comalgs/include/VO_ClassificationAlgs.h @@ -0,0 +1,152 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_CLASSIFICATIONALGS_H__ +#define __VO_CLASSIFICATIONALGS_H__ + +#include +#include +#include "opencv/cv.h" +#include "opencv/cvaux.h" +#include "opencv/highgui.h" +#include "VO_CVCommon.h" +#include "VO_AdditiveStrongerClassifier.h" + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Classification algorithms. + */ +class CClassificationAlgs +{ +protected: + /** classification method */ + unsigned int m_iClassificationMethod; + + /** How many classes in total */ + unsigned int m_iNbOfCategories; + + /** Decision tree */ + CvDTree m_CVDtree; + + /** boosting */ + CvBoost m_CVBoost; + + /** random tree */ + CvRTrees m_CVRTrees; + + /** extreme random tree */ + CvERTrees m_CVERTrees; + + /** SVM */ + CvSVM m_CVSVM; + + /** Initialization */ + void init(unsigned int mtd) + { + this->m_iClassificationMethod = mtd; + } + +public: + enum{ + NONE = 0, + SVM = 1, + DecisionTree = 2, + Boost = 3, + RandomForest = 4, + ExtremeRandomForest = 5 + }; + + /** Constructor */ + CClassificationAlgs(unsigned int mtd=CClassificationAlgs::NONE) + { + this->init(mtd); + } + + /** Destructor */ + ~CClassificationAlgs() {} + + void SetConfiguration( const string& trainedclassifier, + unsigned int mtd) + { + this->init(mtd); + this->Load(trainedclassifier); + } + + /** Gets and sets */ + unsigned int GetClassificationMethod() const + { + return this->m_iClassificationMethod; + } + + void Training(const Mat_& data, const Mat_& categories); + + int Classification( const Mat_& sample); + + void Save(const string& fn) const; + + void Load(const string& fn); +}; + +#endif // __VO_CLASSIFICATIONALGS_H__ diff --git a/modules/comalgs/include/VO_DetectionAlgs.h b/modules/comalgs/include/VO_DetectionAlgs.h new file mode 100644 index 0000000..ffff438 --- /dev/null +++ b/modules/comalgs/include/VO_DetectionAlgs.h @@ -0,0 +1,197 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_DETECTIONALGS_H__ +#define __VO_DETECTIONALGS_H__ + +#include +#include "opencv/cv.h" +#include "opencv/cvaux.h" +#include "opencv/highgui.h" +#include "VO_CVCommon.h" +#include "VO_AdditiveStrongerClassifier.h" + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Object detection algorithms. + */ +class CDetectionAlgs +{ +friend class CLocalizationAlgs; +protected: + /** Detected object rectangles */ + vector m_vDetectedObjectRects; + + /** Detection Method */ + unsigned int m_iDetectionMethod; + + /** Either load a cascade file for boosting, or a boostrap file for rtree */ + string m_sFile2BLoad; + + /** bagging random forest classifier */ + RTreeClassifier m_rtreeClassifier; + + /** boosting cascade classifier */ + CascadeClassifier m_cascadeClassifier; + + /** Whether .... is detected */ + bool m_bObjectDetected; + + /** Initialization */ + void init(const string& str, unsigned int mtd) + { + this->SetConfiguration(str, mtd); + this->m_vDetectedObjectRects.clear(); + this->m_bObjectDetected = false; + } + +public: + /** Constructor */ + CDetectionAlgs( const string& str="", + unsigned int mtd=VO_AdditiveStrongerClassifier::BOOSTING) + { + this->init(str, mtd); + } + + /** Destructor */ + ~CDetectionAlgs() {} + + void SetConfiguration(const string& str, unsigned int mtd) + { + this->m_iDetectionMethod = mtd; + switch(this->m_iDetectionMethod) + { + case VO_AdditiveStrongerClassifier::BAGGING: + { + if(str!="") this->SetBaggingRTree(str); + } + break; + case VO_AdditiveStrongerClassifier::BOOSTING: + { + if(str!="") this->SetBoostingCascade(str); + } + break; + default: + break; + } + } + + void SetBaggingRTree(const string& str) + { + this->m_sFile2BLoad = str; + this->m_rtreeClassifier.read( this->m_sFile2BLoad.c_str() ); + } + + void SetBoostingCascade(const string& str) + { + this->m_sFile2BLoad = str; + this->m_cascadeClassifier.load( this->m_sFile2BLoad ); + } + + double Detection( const Mat& img, + const Rect* confinedArea = NULL, + const double scale = 1.0, + Size sSize = Size( FACESMALLESTSIZE, + FACESMALLESTSIZE), + Size bSize = Size( FACEBIGGESTSIZE, + FACEBIGGESTSIZE) ); + + static double BaggingDetection( vector& objs, + const RTreeClassifier& rtree, + const Mat& img, + const Rect* confinedArea = NULL, + const double scale = 1.0, + Size sSize = Size(FACESMALLESTSIZE, + FACESMALLESTSIZE), + Size bSize = Size(FACEBIGGESTSIZE, + FACEBIGGESTSIZE)); + + static double BoostingDetection( vector& objs, + const CascadeClassifier& cascade, + const Mat& img, + const Rect* confinedArea = NULL, + const double scale = 1.0, + Size sSize = Size(FACESMALLESTSIZE, + FACESMALLESTSIZE), + Size bSize = Size(FACEBIGGESTSIZE, + FACEBIGGESTSIZE)); + + /** Draw all detected objects on the image */ + void VO_DrawDetection(Mat& ioImg, Scalar color = colors[6]); + + /** Is object detected? */ + bool IsObjectDetected() const + { + return this->m_bObjectDetected; + } + + /** Return detected face parts rectangles */ + vector GetDetectedObjectRects() const + { + return this->m_vDetectedObjectRects; + } +}; + +#endif // __VO_DETECTIONALGS_H__ diff --git a/modules/comalgs/include/VO_EvaluationAlgs.h b/modules/comalgs/include/VO_EvaluationAlgs.h new file mode 100644 index 0000000..7e053dd --- /dev/null +++ b/modules/comalgs/include/VO_EvaluationAlgs.h @@ -0,0 +1,113 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_EVALUATIONALGS_H__ +#define __VO_EVALUATIONALGS_H__ + +#include +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" + + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Some concerned evaluation algorithms to evaluate + * the detection performance or the fitting performance. + */ +class CEvaluationAlgs +{ +public: + static bool IsPointInRect( const Point2d& pt, + const Rect& rect); + + static void StandardDetectionEvaluation( + const Rect& dObjRect, + const Rect& lObjRect, + double& oar, + double& co); + + static void PridictEyeCenters( const Rect& face, + Point2d& lEyeCent, + Point2d& rEyeCent); + + static vector CristinacceDEyes( + const vector& detectedFaces, + const vector& lEyeCents, + const vector& rEyeCents ); + + static vector DEyesEval( const vector& dEyess, + unsigned int& dEyeWrong, + double mindEyes = 0.0, + double maxdEyes = 0.5, + unsigned int nb = 100 ); + + static vector MSEFaceComp(const vector& detectedFaceComp, + const vector& faceCompCenters ); + +}; + +#endif // __VO_EVALUATIONALGS_H__ + diff --git a/modules/comalgs/include/VO_FaceDetectionAlgs.h b/modules/comalgs/include/VO_FaceDetectionAlgs.h new file mode 100644 index 0000000..49b2986 --- /dev/null +++ b/modules/comalgs/include/VO_FaceDetectionAlgs.h @@ -0,0 +1,329 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_FACEDETECTIONALGS_H__ +#define __VO_FACEDETECTIONALGS_H__ + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_CVCommon.h" +#include "VO_FaceCompPos.h" +#include "VO_DetectionAlgs.h" + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief Face detection algorithms. +*/ +class CFaceDetectionAlgs : public CDetectionAlgs +{ +private: + /** Original detected face rectangle */ + vector m_vDetectedFaceRects; + + /** Detected face type: + 0 - no face detected; 1 - frontal; 2 - left; 3 - right */ + unsigned int m_iFaceType; + + /** Whether .... is detected */ + bool m_bFaceDetected; + bool m_bLeftEyeDetected; + bool m_bRightEyeDetected; + bool m_bNoseDetected; + bool m_bMouthDetected; + + /** Possible windows for further face parts detection */ + Rect m_CVLeftEyePossibleWindow; + Rect m_CVRightEyePossibleWindow; + Rect m_CVNosePossibleWindow; + Rect m_CVMouthPossibleWindow; + Rect m_CVNoseCentralArea; + + /** Detected windows for respective face parts */ + // haven't translated back to the whole image yet + VO_FaceCompPos m_VOFaceComponents0; + VO_FaceCompPos m_VOFaceComponents; + Rect m_CVDetectedFaceWindow2SM; + Mat m_CVDetectedFaceImagePatch2SM; + + /** Detected Key Points */ + Point2f m_CVDetectedCenterOfGravity; + Point2f m_CVDetectedLeftEyeLeftCorner; + Point2f m_CVDetectedLeftEyeRightCorner; + Point2f m_CVDetectedLeftEyeCenter; + Point2f m_CVDetectedRightEyeLeftCorner; + Point2f m_CVDetectedRightEyeRightCorner; + Point2f m_CVDetectedRightEyeCenter; + Point2f m_CVDetectedNoseTip; + Point2f m_CVDetectedNostrilLeft; + Point2f m_CVDetectedNostrilRight; + Point2f m_CVDetectedNoseCenter; + Point2f m_CVDetectedMouthLeftCorner; + Point2f m_CVDetectedMouthRightCorner; + Point2f m_CVDetectedMouthCenter; + + string m_sFile2BLoadFrontalFace; + string m_sFile2BLoadProfileFace; + string m_sFile2BLoadLeftEye; + string m_sFile2BLoadRightEye; + string m_sFile2BLoadNose; + string m_sFile2BLoadMouth; + string m_sFile2BLoadMouthBeard; + + CascadeClassifier m_cascadeClassifierFrontalFace; + CascadeClassifier m_cascadeClassifierProfileFace; + CascadeClassifier m_cascadeClassifierEyes; + CascadeClassifier m_cascadeClassifierLeftEye; + CascadeClassifier m_cascadeClassifierRightEye; + CascadeClassifier m_cascadeClassifierNose; + CascadeClassifier m_cascadeClassifierMouth; + CascadeClassifier m_cascadeClassifierMouthBeard; + RTreeClassifier m_rtreeClassifierFrontalFace; + RTreeClassifier m_rtreeClassifierProfileFace; + RTreeClassifier m_rtreeClassifierEyes; + RTreeClassifier m_rtreeClassifierLeftEye; + RTreeClassifier m_rtreeClassifierRightEye; + RTreeClassifier m_rtreeClassifierNose; + RTreeClassifier m_rtreeClassifierMouth; + RTreeClassifier m_rtreeClassifierMouthBeard; + + /** Initialization */ + void init(const string& str, unsigned int mtd) + { + CDetectionAlgs::init(str, mtd); + + this->m_vDetectedFaceRects.clear(); + this->m_iFaceType = UNDETECTED; + this->m_bFaceDetected = false; + this->m_bLeftEyeDetected = false; + this->m_bRightEyeDetected = false; + this->m_bNoseDetected = false; + this->m_bMouthDetected = false; + } + +public: + /** frontal face, left profile face, right profile face, etc. */ + enum { + FRONTAL = 0, + LEFTPROFILE = 1, + RIGHTPROFILE = 2, + UPPROFILE = 3, + DOWNPROFILE = 4, + ALL = 9 + }; + + /** face directions */ + enum { + UNDETECTED = -1, + DIR_DOWNFRONTAL = 0, + DIR_DOWNLEFT = 1, + DIR_DOWNRIGHT = 2, + DIR_UPFRONTAL = 3, + DIR_UPLEFT = 4, + DIR_UPRIGHT = 5, + DIR_FRONTAL = 6, + DIR_LEFT = 7, + DIR_RIGHT = 8 + }; + + /** Constructor */ + CFaceDetectionAlgs( const string& str="", + unsigned int mtd= + VO_AdditiveStrongerClassifier::BOOSTING) + { + this->init(str, mtd); + } + + /** Destructor */ + ~CFaceDetectionAlgs() {} + + /** set detection configurations */ + void SetConfiguration( const string& strfrontalface="", + const string& strprofileface="", + const string& strlefteye="", + const string& strrighteye="", + const string& strnose="", + const string& strmouth="", + unsigned int mtd = + VO_AdditiveStrongerClassifier::BOOSTING, + unsigned int facetype = FRONTAL); + + /** only two main methods are adopted here, random forest or + boosting cascade. Every detected face will be recorded */ + double FaceDetection( const Mat& img, + const Rect* confinedArea = NULL, + const double scale = 1.0, + Size sSize + = Size(FACESMALLESTSIZE, FACESMALLESTSIZE), + Size bSize + = Size(FACEBIGGESTSIZE, FACEBIGGESTSIZE)); + + /** only a single face will be recorded */ + double FullFaceDetection ( const Mat& iImg, + const Rect* confinedArea = NULL, + bool lefteye = true, + bool righteye = true, + bool nose = true, + bool mouth = true, + const double scale = 1.0, + Size sSize + = Size(FACESMALLESTSIZE, FACESMALLESTSIZE), + Size bSize + = Size(FACEBIGGESTSIZE, FACEBIGGESTSIZE) ); + + /** detect face components */ + double VO_FaceComponentsDetection( const Mat& iImg, + int faceOrient = FRONTAL, + bool lefteye = true, + bool righteye = true, + bool nose = true, + bool mouth = true); + + /** Detect one specific face part */ + bool VO_FacePartDetection( const Mat& iImg, + const Rect& iWindow, + Rect& oWindow, + unsigned int facepart); + + /** Estimate face directions just based */ + int VO_DetectFaceDirection(int faceOrient = FRONTAL); + static int VO_DetectFaceDirection( + const VO_FaceCompPos& facecomponents, + const Rect& possiblenose, + const Rect& nosecentralarea); + + /** Draw the detected face details */ + void VO_DrawDetection( Mat& ioImg, + bool face = true, + bool lefteye = false, + bool righteye = false, + bool nose = false, + bool mouth = false, + Scalar color = colors[0]); + + /** Calculate face key points by detected face components */ + void CalcFaceKeyPoints(); + + /** Return detected face key point */ + Point2f GetDetectedFaceKeyPoint(unsigned int ptType) const; + + /** Is face parts detected */ + bool IsFaceDetected() const {return this->m_bFaceDetected;} + bool IsLeftEyeDetected() const {return this->m_bLeftEyeDetected;} + bool IsRightEyeDetected() const {return this->m_bRightEyeDetected;} + bool IsNoseDetected() const {return this->m_bNoseDetected;} + bool IsMouthDetected() const {return this->m_bMouthDetected;} + + + /** 0 - no face detected; 1 - frontal; 2 - left; 3 - right */ + unsigned int GetFaceType() const {return this->m_iFaceType;} + + /** Return detected face parts rectangles */ + vector GetDetectedFaceRects() const + { + return this->m_vDetectedFaceRects; + } + VO_FaceCompPos GetFaceComponents() const + { + return this->m_VOFaceComponents; + } + Rect GetDetectedFaceWindow() const + { + return this->m_VOFaceComponents.m_rectObject; + } + Rect GetDetectedFaceWindow2SM() const + { + return this->m_CVDetectedFaceWindow2SM; + } + Rect GetDetectedLeftEyeWindow() const + { + return this->m_VOFaceComponents.m_rectLeftEye; + } + Rect GetDetectedRightEyeWindow() const + { + return this->m_VOFaceComponents.m_rectRightEye; + } + Rect GetDetectedNoseWindow() const + { + return this->m_VOFaceComponents.m_rectNose; + } + Rect GetDetectedMouthWindow() const + { + return this->m_VOFaceComponents.m_rectMouth; + } + + /** Reasonably adjust detected face rectangle to AAM rectangle */ + static Rect VO_FaceRectFromDetection2SM(const Size& imgSize, + const Rect& iFaceRect); + + /** Reasonably set respective window for various face parts */ + static Rect VO_SetDetectedFacePartsPossibleWindow( + int imgWidth, + int imgHeight, + unsigned int facepart, + unsigned int dir = FRONTAL); +}; + +#endif // __VO_FACEDETECTIONALGS_H__ diff --git a/modules/comalgs/include/VO_HandDetectionAlgs.h b/modules/comalgs/include/VO_HandDetectionAlgs.h new file mode 100644 index 0000000..3f5afaa --- /dev/null +++ b/modules/comalgs/include/VO_HandDetectionAlgs.h @@ -0,0 +1,98 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_HANDDETECTIONALGS_H__ +#define __VO_HANDDETECTIONALGS_H__ + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_CVCommon.h" +#include "VO_FaceCompPos.h" +#include "VO_DetectionAlgs.h" + +/** +* @author JIA Pei +* @brief Hand detection algorithms. +*/ +class CHandDetectionAlgs : public CDetectionAlgs +{ +private: + /** Original detected human rectangle */ + vector m_vDetectedHandRects; + + /** Initialization */ + void init(const string& str, unsigned int mtd); + +public: + /** Constructor */ + CHandDetectionAlgs( const string& str="", + unsigned int mtd= + VO_AdditiveStrongerClassifier::BOOSTING) + { + this->init(str, mtd); + } + + /** Destructor */ + ~CHandDetectionAlgs() {} +}; + +#endif // __VO_HANDDETECTIONALGS_H__ diff --git a/modules/comalgs/include/VO_HumanDetectionAlgs.h b/modules/comalgs/include/VO_HumanDetectionAlgs.h new file mode 100644 index 0000000..f2d94b6 --- /dev/null +++ b/modules/comalgs/include/VO_HumanDetectionAlgs.h @@ -0,0 +1,139 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_HUMANDETECTIONALGS_H__ +#define __VO_HUMANDETECTIONALGS_H__ + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_CVCommon.h" +#include "VO_FaceCompPos.h" +#include "VO_DetectionAlgs.h" + +/** + * @author JIA Pei + * @brief Human detection algorithms. + */ +class CHumanDetectionAlgs : public CDetectionAlgs +{ +private: + /** Original detected human rectangle */ + vector m_vDetectedHumanRects; + + /** Whether .... is detected */ + bool m_bHumanDetected; + + /** HOG detector */ + HOGDescriptor m_hog; + + /** Initialization */ + void init(const string& str, unsigned int mtd) + { + CDetectionAlgs::init(str, mtd); + this->m_bHumanDetected = false; + this->m_vDetectedHumanRects.clear(); + this->m_hog.setSVMDetector(HOGDescriptor::getDefaultPeopleDetector()); + } + +public: + /** Constructor */ + CHumanDetectionAlgs(const string& str="", + unsigned int mtd= + VO_AdditiveStrongerClassifier::BOOSTING) + { + this->init(str, mtd); + } + + /** Destructor */ + ~CHumanDetectionAlgs() {this->m_vDetectedHumanRects.clear();} + + /** only two main methods are adopted here, random forest + or boosting cascade. Every detected face will be recorded */ + double HumanDetection( const Mat& img, + const Rect* confinedArea = NULL, + const double scale = 1.0, + Size sSize = Size(8, 8), + Size bSize = Size(32, 32)); + + /** Draw the detected human */ + void VO_DrawDetection( Mat& ioImg, + Scalar color = colors[0]); + + /** Return detected face parts rectangles */ + vector GetDetectedHumanRects() const + { + return this->m_vDetectedHumanRects; + } + Rect GetDetectedHumanWindow() const + { + return this->m_vDetectedHumanRects[0]; + } + + /** Is human body detected */ + bool IsHumanDetected() const + { + return this->m_bHumanDetected; + } + +}; + +#endif // __VO_HUMANDETECTIONALGS_H__ diff --git a/modules/comalgs/include/VO_LocalizationAlgs.h b/modules/comalgs/include/VO_LocalizationAlgs.h new file mode 100644 index 0000000..b434a5e --- /dev/null +++ b/modules/comalgs/include/VO_LocalizationAlgs.h @@ -0,0 +1,160 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_LOCALIZATIONALGS_H__ +#define __VO_LOCALIZATIONALGS_H__ + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_CVCommon.h" + +#include "VO_DetectionAlgs.h" +#include "VO_TrackingAlgs.h" + + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief Object localization algorithms, currently, +* can deal with only a single concerned object. +*/ +class CLocalizationAlgs +{ +protected: + /** Original localized face rectangle */ + Rect m_CVLocalizedObjectRect; + + /** Whether the objects is localized */ + bool m_bObjectLocalized; + + /** The detection algorithm */ + CDetectionAlgs m_detectionAlgs; + + /** The tracking algorithm */ + CTrackingAlgs m_trackingAlgs; + + /** Initialization */ + void init( const string& str, + unsigned int detectionMtd, + unsigned int trackingMtd) + { + this->m_detectionAlgs.SetConfiguration(str, detectionMtd); + this->m_trackingAlgs.SetConfiguration(trackingMtd); + this->m_bObjectLocalized = false; + } + +public: + enum {DETECTIONONLY = 0, DETECTIONTRACKING = 1}; + + CLocalizationAlgs( const string& str="", + unsigned int detectionMtd + =VO_AdditiveStrongerClassifier::BOOSTING, + unsigned int trackingMtd + =CTrackingAlgs::CAMSHIFT) + { + this->init(str, detectionMtd, trackingMtd); + } + ~CLocalizationAlgs() {} + + void SetConfiguration( const string& str, + unsigned int detectionMtd, + unsigned int trackingMtd) + { + this->m_detectionAlgs.SetConfiguration(str, detectionMtd); + this->m_trackingAlgs.SetConfiguration(trackingMtd); + } + + double Localization( const Mat& img, + Size sSize = + Size(FACESMALLESTSIZE, FACESMALLESTSIZE), + Size bSize = + Size(FACEBIGGESTSIZE, FACEBIGGESTSIZE) ); + + static double Localization( const Mat& img, + CDetectionAlgs& detectAlg, + CTrackingAlgs& trackAlg, + bool& isTracked, + Rect& objPos, + const Size& size1, + const Size& size2); + + /** Draw all detected objects on the image */ + void VO_DrawLocalization(Mat& ioImg, + Scalar color = colors[6]); + + /** Is object detected? */ + bool IsObjectLocalized() const + { + return this->m_bObjectLocalized; + } + + /** Return localized object */ + Rect GetLocalizedObjectRect () const + { + return this->m_CVLocalizedObjectRect; + } +}; + +#endif // __VO_LOCALIZATIONALGS_H__ diff --git a/modules/comalgs/include/VO_RecognitionAlgs.h b/modules/comalgs/include/VO_RecognitionAlgs.h new file mode 100644 index 0000000..6f0b467 --- /dev/null +++ b/modules/comalgs/include/VO_RecognitionAlgs.h @@ -0,0 +1,213 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_RECOGNITIONALGS_H__ +#define __VO_RECOGNITIONALGS_H__ + +#include +#include "opencv/cv.h" +#include "opencv/cvaux.h" +#include "opencv/highgui.h" +#include "VO_CVCommon.h" +#include "VO_AdditiveStrongerClassifier.h" +#include "VO_FaceDetectionAlgs.h" +#include "VO_TrackingAlgs.h" +#include "VO_FaceParts.h" +#include "VO_Shape2DInfo.h" +#include "VO_Shape.h" + +#define FOCAL_LENGTH 10000000000 + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief Recognition algorithms. +*/ +class CRecognitionAlgs +{ +protected: + void init() {} +public: + /** Constructor */ + CRecognitionAlgs() {this->init();} + + /** Destructor */ + ~CRecognitionAlgs(); + + /** Global Texture Constraint using Probability image, + basically an evaluation method for current fitting effect */ + static bool EvaluateFaceTrackedByProbabilityImage( + CTrackingAlgs* trackalg, + const Mat& iImg, + const VO_Shape& iShape, + Size smallSize + = Size(FACESMALLESTSIZE, FACESMALLESTSIZE), + Size bigSize + = Size(FACEBIGGESTSIZE, FACEBIGGESTSIZE) ); + + /** An evaluation method for current fitting effect, + based on face components detection */ + static bool EvaluateFaceTrackedByCascadeDetection( + const CFaceDetectionAlgs* fd, + const Mat& iImg, + const VO_Shape& iShape, + const vector& iShapeInfo, + const VO_FaceParts& iFaceParts); + + /** Calculate shape distance */ + static float ShapeDistance( const VO_Shape& shape1, + const VO_Shape& shape2); + + /** Calculate fitting effect for static images */ + static bool CalcFittingEffect4StaticImage( + const Mat_& avgSParam, + const Mat_& icovSParam, + const Mat_& avgTParam, + const Mat_& icovTParam, + const Mat_& iSParams, + const Mat_& iTParams, + const Scalar& ShapeDistMean, + const Scalar& ShapeDistStddev, + const Scalar& TextureDistMean, + const Scalar& TextureDistStddev, + float& sDist, + float& tDist, + bool WeakFitting = true ); + + /** Calculate fitting effect for dynamic images sequence */ + static bool CalcFittingEffect4ImageSequence( + const Mat_& avgSParam, + const Mat_& icovSParam, + const Mat_& avgTParam, + const Mat_& icovTParam, + const Mat_& iSParams, + const Mat_& iTParams, + const Scalar& ShapeDistMean, + const Scalar& ShapeDistStddev, + const Scalar& TextureDistMean, + const Scalar& TextureDistStddev, + bool WeakFitting = true ); + + /** Calculate face fitting effect */ + static void CalcShapeFittingEffect( const VO_Shape& refShape, + const VO_Shape& fittedShape, + float& deviation, + vector& ptErrorFreq, + int nb = 20, + vector* ptDists = 0); + + /** Save shape recognition results */ + static void SaveShapeRecogResults( const string& fd, + const string& fnIdx, + float deviation, + vector& ptErrorFreq); + /** Save shape data results */ + static void SaveShapeResults( const string& fd, + const string& fnIdx, + float deviation, + vector& ptErrorFreq, + vector& ptErrorPerPoint, + const VO_Shape& fittedShape); + /** Save a myriad of results from the fitting process*/ + static void CRecognitionAlgs::SaveFittingResults( const string& fd, + const string& fnIdx, + float deviation, + vector& ptDists, + vector& ptErrorFreq, + const VO_Shape& fittedShape, + cv::Point2f* gt_canidatePoints, + cv::Point2f* t_canidatePoints, + float fitTime); + + /** Calculate face absolute orientations */ + static vector CalcAbsoluteOrientations( + const VO_Shape& iShape2D, + const VO_Shape& iShape3D, + VO_Shape& oShape2D); + + /** Recognize face roll angle */ + static float CalcFaceRoll(const vector& iLine); + + /** Recognize face yaw angle */ + static float CalcFaceYaw(const vector& iLine, + const VO_Shape& iShape, + const VO_FaceParts& iFaceParts); + + /** Recognize face pitch angle */ + static float CalcFacePitch( const VO_Shape& iShape, + const VO_FaceParts& iFaceParts); + + /** Recognize face angles using 2D model */ + static void CalcFittedFaceAngle2D( vector& angles, + const VO_Shape& iShape, + const VO_FaceParts& iFaceParts); + + /** Recognize face angles using 3D model */ + static void CalcFittedFaceAngle3D( vector& angles, + const VO_Shape& iShape, + const VO_FaceParts& iFaceParts); + +}; + +#endif // __VO_RECOGNITIONALGS_H__ diff --git a/modules/comalgs/include/VO_TrackingAlgs.h b/modules/comalgs/include/VO_TrackingAlgs.h new file mode 100644 index 0000000..059cc4c --- /dev/null +++ b/modules/comalgs/include/VO_TrackingAlgs.h @@ -0,0 +1,204 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_TRACKINGALGS_H__ +#define __VO_TRACKINGALGS_H__ + + +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_CVCommon.h" + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief Object tracking algorithms, unlike CDetectionAlgs, +* CTrackingAlgs can deal with only one single concerned object. +*/ +class CTrackingAlgs +{ + friend class CLocalizationAlgs; +protected: + /** Tracked object rectangle */ + Rect m_CVTrackedObjectRect; + + /** Tracking Method */ + unsigned int m_iTrackingMethod; + + /** Tracker Method */ + unsigned int m_iTrackerMethod; + + /** Whether the tracker has been initialized */ + bool m_bTrackerInitialized; + + /** Whether the objects is tracked */ + bool m_bObjectTracked; + + /** Initialization */ + void init(unsigned int trackingmtd, unsigned int trackermtd) + { + this->SetConfiguration(trackingmtd, trackermtd); + this->m_bTrackerInitialized = false; + this->m_bObjectTracked = false; + } + +public: + enum { PROBABILITYIMAGE = 1}; + + enum { + NONE = 0, + CAMSHIFT = 1, + KALMANFILTER = 2, + PARTICLEFILTER = 3, + ASMAAM = 4 + }; + + CTrackingAlgs( unsigned int trackingmtd + = CTrackingAlgs::CAMSHIFT, + unsigned int trackermtd + = CTrackingAlgs::PROBABILITYIMAGE) + { + this->init(trackingmtd, trackermtd); + } + + ~CTrackingAlgs() {} + + void SetConfiguration( unsigned int trackingmtd + = CTrackingAlgs::CAMSHIFT, + unsigned int trackermtd + = CTrackingAlgs::PROBABILITYIMAGE) + { + this->m_iTrackingMethod = trackingmtd; + this->m_iTrackerMethod = trackermtd; + } + + void UpdateTracker(const Mat& img, const Rect& obj); + + double Tracking( Rect& obj, + const Mat& img, + Size smallSize + = Size(FACESMALLESTSIZE, FACESMALLESTSIZE), + Size bigSize + = Size(FACEBIGGESTSIZE, FACEBIGGESTSIZE)); + + static bool CamshiftUpdateTracker( const Mat& img, + const Rect& obj, + MatND& hist); + + static double CamshiftTracking( Rect& obj, + const Mat& img, + MatND& hist, + bool& isTracked, + Size smallSize, + Size bigSize); + + static double KalmanTracking( Rect& obj, + const Mat& img, + bool& isTracked, + Size smallSize, + Size bigSize); + + static double ParticleFilterTracking( Rect& obj, + const Mat& img, + bool& isTracked, + Size smallSize, + Size bigSize); + + static double ASMAAMTracking( Rect& obj, + const Mat& img, + bool& isTracked, + Size smallSize, + Size bigSize); + + /** Draw all detected objects on the image */ + void VO_DrawTracking(Mat& ioImg, + Scalar color = colors[6]); + + /** Are objects tracked? */ + bool IsObjectTracked() const + { + return this->m_bObjectTracked; + } + + /** Return tracked objects rectangles */ + Rect GetTrackedObjectRect() const + { + return this->m_CVTrackedObjectRect; + } + + static const int vmin = 10; + static const int vmax = 256; + static const int smin = 30; + static const int hbins = 32; // quantize the hue to 32 levels + static const int sbins = 32; // quantize the saturation to 32 levels + static int histSize[] ; + static float hranges[]; + static float sranges[]; + static const float* ranges[]; + static int channels[]; + + MatND m_hist; +}; + +#endif // __VO_TRACKINGALGS_H__ diff --git a/modules/comalgs/src/VO_ClassificationAlgs.cpp b/modules/comalgs/src/VO_ClassificationAlgs.cpp new file mode 100644 index 0000000..094c86b --- /dev/null +++ b/modules/comalgs/src/VO_ClassificationAlgs.cpp @@ -0,0 +1,300 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_ClassificationAlgs.h" + + +/************************************************************************/ +/*@author JIA Pei */ +/*@version 2010-11-04 */ +/*@brief Training */ +/*@param data Input - input data */ +/*@param categories Input - a column vector */ +/*@return void */ +/************************************************************************/ +void CClassificationAlgs::Training( const Mat_& data, + const Mat_& categories) +{ + unsigned int NbOfSamples = data.rows; + set ClassSet; + for(int i = 0; i < categories.rows; i++) + { + ClassSet.insert(categories(i, 0)); + } + this->m_iNbOfCategories = ClassSet.size(); + + switch(this->m_iClassificationMethod) + { + case CClassificationAlgs::DecisionTree: + this->m_CVDtree.train( data, + CV_ROW_SAMPLE, + categories, + Mat(), + Mat(), + Mat(), + Mat(), + CvDTreeParams( INT_MAX, + 2, + 0.0f, + false, + this->m_iNbOfCategories, + 0, + false, + false, + 0 ) ); + break; + case CClassificationAlgs::Boost: + this->m_CVBoost.train( data, + CV_ROW_SAMPLE, + categories, + Mat(), + Mat(), + Mat(), + Mat(), + CvBoostParams( CvBoost::DISCRETE, + 50, + 0.95, + INT_MAX, + false, + 0), + false ); + break; + case CClassificationAlgs::RandomForest: + this->m_CVRTrees.train( data, + CV_ROW_SAMPLE, + categories, + Mat(), + Mat(), + Mat(), + Mat(), + CvRTParams( INT_MAX, + 2, + 0.0f, + false, + this->m_iNbOfCategories, + 0, + true, + 0, + 100, + 0.0f, + CV_TERMCRIT_ITER ) ); + break; + case CClassificationAlgs::ExtremeRandomForest: + this->m_CVERTrees.train(data, + CV_ROW_SAMPLE, + categories, + Mat(), + Mat(), + Mat(), + Mat(), + CvRTParams( INT_MAX, + 2, + 0.0f, + false, + this->m_iNbOfCategories, + 0, + true, + 0, + 100, + 0.0f, + CV_TERMCRIT_ITER ) ); + break; + case CClassificationAlgs::SVM: + this->m_CVSVM.train(data, + categories, + Mat(), + Mat(), + CvSVMParams(CvSVM::C_SVC, + CvSVM::RBF, + 0, + 1, + 0, + 1, + 0, + 0, + NULL, + cvTermCriteria( CV_TERMCRIT_ITER, + 1000, + 1E-6) ) ); + break; + } +} + + +/************************************************************************/ +/*@author JIA Pei */ +/*@version 2010-11-04 */ +/*@brief Classification */ +/*@param sample Input - a sample to be classified */ +/*@return the classified category */ +/************************************************************************/ +int CClassificationAlgs::Classification(const Mat_& sample ) +{ + int res = -1; + switch(this->m_iClassificationMethod) + { + case CClassificationAlgs::DecisionTree: + { + CvDTreeNode* node = this->m_CVDtree.predict( sample ); + res = node->class_idx; + } + break; + case CClassificationAlgs::Boost: + { + res = (int) this->m_CVBoost.predict( sample ); + } + break; + case CClassificationAlgs::RandomForest: + { + res = (int) this->m_CVRTrees.predict( sample ); + } + break; + case CClassificationAlgs::ExtremeRandomForest: + { + res = (int) this->m_CVERTrees.predict( sample ); + } + break; + case CClassificationAlgs::SVM: + default: + { + res = (int) this->m_CVSVM.predict( sample ); + } + break; + } + + return res; +} + + +/** Save the classifier */ +void CClassificationAlgs::Save(const string& fn ) const +{ + switch(this->m_iClassificationMethod) + { + case CClassificationAlgs::DecisionTree: + { + this->m_CVDtree.save(fn.c_str()); + } + break; + case CClassificationAlgs::Boost: + { + this->m_CVBoost.save(fn.c_str()); + } + break; + case CClassificationAlgs::RandomForest: + { + this->m_CVRTrees.save(fn.c_str()); + } + break; + case CClassificationAlgs::ExtremeRandomForest: + { + this->m_CVERTrees.save(fn.c_str()); + } + break; + case CClassificationAlgs::SVM: + default: + { + this->m_CVSVM.save(fn.c_str()); + } + break; + } +} + + +/** Load the classifier */ +void CClassificationAlgs::Load(const string& fn) +{ + switch(this->m_iClassificationMethod) + { + case CClassificationAlgs::DecisionTree: + { + this->m_CVDtree.load(fn.c_str()); + } + break; + case CClassificationAlgs::Boost: + { + this->m_CVBoost.load(fn.c_str()); + } + break; + case CClassificationAlgs::RandomForest: + { + this->m_CVRTrees.load(fn.c_str()); + } + break; + case CClassificationAlgs::ExtremeRandomForest: + { + this->m_CVERTrees.load(fn.c_str()); + } + break; + case CClassificationAlgs::SVM: + { + this->m_CVSVM.load(fn.c_str()); + } + case CClassificationAlgs::NONE: + default: + break; + } +} + diff --git a/modules/comalgs/src/VO_DetectionAlgs.cpp b/modules/comalgs/src/VO_DetectionAlgs.cpp new file mode 100644 index 0000000..c704ddd --- /dev/null +++ b/modules/comalgs/src/VO_DetectionAlgs.cpp @@ -0,0 +1,269 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_DetectionAlgs.h" + + + +/************************************************************************/ +/*@author JIA Pei */ +/*@version 2009-10-04 */ +/*@brief Object Detection */ +/*@param iImg Input - image to be searched in */ +/*@param confinedArea Input - only detect the object in this */ +/* confined area */ +/*@param scale Input - scalar for img scaling */ +/*@param sSize Input - detected obj must be bigger than sSize */ +/*@param bSize Input - detected object must be smaller than bSize */ +/*@return detection time cost */ +/************************************************************************/ +double CDetectionAlgs::Detection( const Mat& iImg, + const Rect* confinedArea, + const double scale, + Size sSize, + Size bSize) +{ + double res = (double)cvGetTickCount(); + + switch(this->m_iDetectionMethod) + { + case VO_AdditiveStrongerClassifier::BAGGING: + CDetectionAlgs::BaggingDetection( this->m_vDetectedObjectRects, + this->m_rtreeClassifier, + iImg, + confinedArea, + scale, + sSize, + bSize); + break; + case VO_AdditiveStrongerClassifier::BOOSTING: + default: + CDetectionAlgs::BoostingDetection( this->m_vDetectedObjectRects, + this->m_cascadeClassifier, + iImg, + confinedArea, + scale, + sSize, + bSize); + break; + } + + if(this->m_vDetectedObjectRects.size() >= 1) + this->m_bObjectDetected = true; + else + this->m_bObjectDetected = false; + + res = ((double)cvGetTickCount() - res) + / ((double)cvGetTickFrequency()*1000.); + return res; +} + + +/************************************************************************/ +/*@author JIA Pei */ +/*@version 2009-10-04 */ +/*@brief Boosting based Object Detection */ +/*@param objs Output - detected objects */ +/*@param cascade Input - cascade classifier to be used */ +/*@param iImg Input - image to be searched in */ +/*@param confinedArea Input - only detect the object in this */ +/* confined area */ +/*@param scale Input - scalar for img scaling */ +/*@param sSize Input - detected obj must be bigger than sSize */ +/*@param bSize Input - detected object must be smaller than bSize */ +/*@return detection time cost */ +/************************************************************************/ +double CDetectionAlgs::BoostingDetection( vector& objs, + const CascadeClassifier& cascade, + const Mat& img, + const Rect* confinedArea, + const double scale, + Size sSize, + Size bSize) +{ + double res = (double)cvGetTickCount(); + objs.clear(); + + Mat confinedImg; + if(confinedArea) + confinedImg = img(*confinedArea); + else + confinedImg = img; + + Mat gray, smallImg( cvRound (confinedImg.rows/scale), + cvRound(confinedImg.cols/scale), + CV_8UC1 ); + + if(confinedImg.channels() == 3) + cvtColor( confinedImg, gray, CV_BGR2GRAY ); + else + gray = confinedImg; + resize( gray, smallImg, smallImg.size(), 0, 0, INTER_LINEAR ); + equalizeHist( smallImg, smallImg ); + + /////////////////detection///////////////////////////////////////// + //t = (double)cvGetTickCount(); + const_cast(cascade).detectMultiScale( + smallImg, + objs, + 1.1, + 2, + 0, + //|CascadeClassifier::DO_CANNY_PRUNING + //|CascadeClassifier::FIND_BIGGEST_OBJECT + //|CascadeClassifier::DO_ROUGH_SEARCH + //|CascadeClassifier::SCALE_IMAGE, + sSize, + bSize); + + ///////////////////////sort/////////////////////////////////////// + if (objs.size() > 0) + { + qsort( (void *)&(objs[0]), objs.size(), sizeof(Size), cvSizeCompare ); + // re-position + if (confinedArea) + { + for (unsigned int i = 0; i < objs.size(); ++i) + { + objs[i].x = static_cast(objs[i].x*scale+confinedArea->x); + objs[i].y = static_cast(objs[i].y*scale+confinedArea->y); + } + } + else + { + for (unsigned int i = 0; i < objs.size(); ++i) + { + objs[i].x = static_cast(objs[i].x*scale); + objs[i].y = static_cast(objs[i].y*scale); + } + } + + //scale back + for ( unsigned int i = 0; i < objs.size(); ++i) + { + objs[i].width = static_cast( objs[i].width * scale); + objs[i].height = static_cast( objs[i].height * scale); + } + } + + res = ((double)cvGetTickCount() - res) + / ((double)cvGetTickFrequency()*1000.); + return res; +} + + +/************************************************************************/ +/*@author JIA Pei */ +/*@version 2009-10-04 */ +/*@brief Bagging based Object Detection */ +/*@param objs Output - detected objects */ +/*@param rtree Input - random tree classifier to be used */ +/*@param iImg Input - image to be searched in */ +/*@param confinedArea Input - only detect the object in this */ +/* confined area */ +/*@param scale Input - scalar for img scaling */ +/*@param sSize Input - detected obj must be bigger than sSize */ +/*@param bSize Input - detected object must be smaller than bSize */ +/*@return detection time cost */ +/************************************************************************/ +double CDetectionAlgs::BaggingDetection(vector& objs, + const RTreeClassifier& rtree, + const Mat& img, + const Rect* confinedArea, + const double scale, + Size sSize, + Size bSize) +{ + double res = (double)cvGetTickCount(); + + // To be finished.... + + res = ((double)cvGetTickCount() - res) + / ((double)cvGetTickFrequency()*1000.); + return res; +} + + +void CDetectionAlgs::VO_DrawDetection(Mat& ioImg, Scalar color) +{ + unsigned int NbOfDetectedObjs = this->m_vDetectedObjectRects.size(); + + Rect curRect; + Point lefttop, rightbottom; + + if ( NbOfDetectedObjs > 0 ) + { + for(unsigned int i = 0; i < NbOfDetectedObjs; ++i) + { + curRect = this->m_vDetectedObjectRects[i]; + lefttop.x = cvRound(curRect.x); + lefttop.y = cvRound(curRect.y); + rightbottom.x = cvRound((curRect.x+curRect.width)); + rightbottom.y = cvRound((curRect.y+curRect.height)); + cv::rectangle(ioImg, lefttop, rightbottom, color, 2, 8, 0); + } + } +} + diff --git a/modules/comalgs/src/VO_EvaluationAlgs.cpp b/modules/comalgs/src/VO_EvaluationAlgs.cpp new file mode 100644 index 0000000..2523633 --- /dev/null +++ b/modules/comalgs/src/VO_EvaluationAlgs.cpp @@ -0,0 +1,266 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_EvaluationAlgs.h" +#include "VO_CVCommon.h" + +/** +* @brief to judge whether point "pt" is within rectangle "rect" +* @param pt - input the concerned point +* @param rect - input the concerned rectangle +*/ +bool CEvaluationAlgs::IsPointInRect(const Point2d& pt, const Rect& rect) +{ + if(pt.x >= rect.x && pt.x < rect.x+rect.width && + pt.y >= rect.y && pt.y < rect.y+rect.height) + return true; + else + return false; +} + +/** +* @brief calculate Overlapped Area Ratio and Center Offset +* @param dObjRect - input, detected object rectangle + rectangle that encompasses the detected object +* @param lObjRect - input, located object rectangle + rectangle loaded from annotated object (realistic object) +* @param oar - output overlapped area ratio, + overlapped area / entire area covered by two rectangles +* @param co - output center offset + the distance between two centers of dObjRect and lObjRect +*/ +void CEvaluationAlgs::StandardDetectionEvaluation( const Rect& dObjRect, + const Rect& lObjRect, + double& oar, + double& co) +{ + unsigned int LeftMost, TopMost, RightMost, BottomMost; + LeftMost = (dObjRect.x < lObjRect.x) ? dObjRect.x : lObjRect.x; + TopMost = (dObjRect.y < lObjRect.y) ? dObjRect.y : lObjRect.y; + RightMost = (dObjRect.x + dObjRect.width > lObjRect.x + lObjRect.width) + ? dObjRect.x + dObjRect.width : lObjRect.x + lObjRect.width; + BottomMost = (dObjRect.y + dObjRect.height > lObjRect.y + lObjRect.height) + ? dObjRect.y + dObjRect.height : lObjRect.y + lObjRect.height; + + unsigned int overlappedarea = 0; + unsigned int entirearea = 0; + Point2d pt; + bool isInFirst, isInSecond; + for(unsigned int i = LeftMost; i < RightMost; ++i) + { + for(unsigned int j = TopMost; j < BottomMost; ++j) + { + pt.x = i; + pt.y = j; + if(CEvaluationAlgs::IsPointInRect(pt, dObjRect) ) + isInFirst = true; + else + isInFirst = false; + if(CEvaluationAlgs::IsPointInRect(pt, lObjRect) ) + isInSecond = true; + else + isInSecond = false; + if(isInFirst || isInSecond) + entirearea++; + if(isInFirst && isInSecond) + overlappedarea++; + } + } + oar = (double)overlappedarea / (double)entirearea; + + double dxCenter = (double)dObjRect.x + (double)dObjRect.width/2.0; + double dyCenter = (double)dObjRect.y + (double)dObjRect.height/2.0; + double lxCenter = (double)lObjRect.x + (double)lObjRect.width/2.0; + double lyCenter = (double)lObjRect.y + (double)lObjRect.height/2.0; + co = sqrt ( pow( (dxCenter-lxCenter), 2.0) + + pow( (dyCenter-lyCenter), 2.0) ); +} + + +/** +* @param face - input detected face rectangle +* @param lEyeCent - output center of the left eye +* @param rEyeCent - output center of the right eye +*/ +void CEvaluationAlgs::PridictEyeCenters(const Rect& face, + Point2d& lEyeCent, + Point2d& rEyeCent) +{ + lEyeCent.x = (double)face.width/20.0 + 9.0*(double)face.width/20.0*0.5+face.x; + lEyeCent.y = (double)face.height/10.0 + (double)face.height/3.0*0.5+face.y; + rEyeCent.x = (double)face.width/2.0 + 9.0*(double)face.width/20.0*0.5+face.x; + rEyeCent.y = (double)face.height/10.0 + (double)face.height/3.0*0.5+face.y; +} + + +/** +* @brief Evaluation for all image files from an image sequence. +* @param detectedFaces - input a vector of detected faces +* @param lEyeCents - input a vector of detected left eye centers +* @param rEyeCents - input a vector of detected right eye centers +* @return a vector of dEyes -- refer to Cristinacce paper +*/ +vector CEvaluationAlgs::CristinacceDEyes( + const vector& detectedFaces, + const vector& lEyeCents, + const vector& rEyeCents ) +{ + assert(detectedFaces.size() == lEyeCents.size() ); + assert(detectedFaces.size() == rEyeCents.size() ); + + Point2d predictedLEyeCent, predictedREyeCent; + unsigned int NbOfSamples = detectedFaces.size(); + vector dEyes(NbOfSamples, 0.0); + double dLeft, dRight, dLeftRight; + for (unsigned int i = 0; i < NbOfSamples; i++) + { + // if face hasn't been detected + if(detectedFaces[i].x < 0 && detectedFaces[i].y < 0 && + detectedFaces[i].width < 0 && detectedFaces[i].height < 0) + { + dEyes[i] = DBL_MAX; + } + else + { + CEvaluationAlgs::PridictEyeCenters( detectedFaces[i], + predictedLEyeCent, + predictedREyeCent); + dLeft = sqrt ( pow( (predictedLEyeCent.x - lEyeCents[i].x), 2.0) + + pow( (predictedLEyeCent.y - lEyeCents[i].y), 2.0) ); + dRight = sqrt ( pow( (predictedREyeCent.x - rEyeCents[i].x), 2.0) + + pow( (predictedREyeCent.y - rEyeCents[i].y), 2.0) ); + dLeftRight = sqrt ( pow( (lEyeCents[i].x - rEyeCents[i].x), 2.0) + + pow( (lEyeCents[i].y - rEyeCents[i].y), 2.0) ); + dEyes[i] = (dLeft+dRight)/(2*dLeftRight); + } + } + + return dEyes; +} + + +/** +* @param dEyess - input a vector of dEyes, refer to Cristinacce paper +* @param wrongDet - output how many wrong detections +* @param mindEyes - input min dEyes +* @param maxdEyes - input max dEyes +* @param nb - input how many evaluation levels to be used +*/ +vector CEvaluationAlgs::DEyesEval( const vector& dEyess, + unsigned int& wrongDet, + double mindEyes, + double maxdEyes, + unsigned int nb ) +{ + vector res(nb, 0); + double interval = (maxdEyes-mindEyes)/(double)nb; + double curdEye = 0.0; + + for(unsigned int i = 0; i < nb; i++) + { + curdEye = mindEyes + i*interval; + for (unsigned int j = 0; j < dEyess.size(); j++) + { + if (dEyess[j] < curdEye) + { + res[i]++; + } + } + } + + wrongDet = 0; + for (unsigned int j = 0; j < dEyess.size(); j++) + { + if (dEyess[j] > maxdEyes) + { + wrongDet++; + } + } + + return res; +} + + +/** +* @param detectedFaceComp - input detected face components +* @param faceCompCenters - input loaded face components' centers (realistic face information) +* @return a vector of MSEs +*/ +vector CEvaluationAlgs::MSEFaceComp( + const vector& detectedFaceComp, + const vector& faceCompCenters ) +{ + assert(detectedFaceComp.size() == faceCompCenters.size() ); + unsigned int size = detectedFaceComp.size(); + double xCenter, yCenter; + vector mse(size, 0.0); + + for(unsigned int i = 0; i < size; i++) + { + xCenter = detectedFaceComp[i].x + (double)detectedFaceComp[i].width/2.0; + yCenter = detectedFaceComp[i].y + (double)detectedFaceComp[i].height/2.0; + mse[i] = sqrt ( pow( (xCenter-faceCompCenters[i].x), 2.0) + + pow( (yCenter-faceCompCenters[i].y), 2.0) ); + } + + return mse; +} + diff --git a/modules/comalgs/src/VO_FaceDetectionAlgs.cpp b/modules/comalgs/src/VO_FaceDetectionAlgs.cpp new file mode 100644 index 0000000..cebf575 --- /dev/null +++ b/modules/comalgs/src/VO_FaceDetectionAlgs.cpp @@ -0,0 +1,975 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_FacePart.h" +#include "VO_FaceKeyPoint.h" +#include "VO_FaceDetectionAlgs.h" + + +/** +* @author JIA Pei +* @version 2010-02-04 +* @brief Set detection configuration +* @param strfrontalface Input - XML for frontal face +* @param strprofileface Input - XML for profile face +* @param strlefteye Input - XML for left eye +* @param strrighteye Input - XML for right eye +* @param strnose Input - XML for nose +* @param strmouth Input - XML for mouth +* @param mtd Input - detection method +* @return facetype Input - face type, frontal or profile, etc. +*/ +void CFaceDetectionAlgs::SetConfiguration( const string& strfrontalface, + const string& strprofileface, + const string& strlefteye, + const string& strrighteye, + const string& strnose, + const string& strmouth, + unsigned int mtd, + unsigned int facetype) +{ + this->m_iDetectionMethod = mtd; + this->m_iFaceType = facetype; + switch(this->m_iDetectionMethod) + { + case VO_AdditiveStrongerClassifier::BAGGING: + { + if(strfrontalface!="") + { + this->m_sFile2BLoadFrontalFace = strfrontalface; + this->m_rtreeClassifierProfileFace.read( this->m_sFile2BLoadFrontalFace.c_str() ); + } + if(strprofileface!="") + { + this->m_sFile2BLoadProfileFace = strprofileface; + this->m_rtreeClassifierProfileFace.read( this->m_sFile2BLoadProfileFace.c_str() ); + } + if(strlefteye!="") + { + this->m_sFile2BLoadLeftEye = strlefteye; + this->m_rtreeClassifierLeftEye.read( this->m_sFile2BLoadLeftEye.c_str() ); + } + if(strrighteye!="") + { + this->m_sFile2BLoadRightEye = strrighteye; + this->m_rtreeClassifierRightEye.read( this->m_sFile2BLoadRightEye.c_str() ); + } + if(strnose!="") + { + this->m_sFile2BLoadNose = strnose; + this->m_rtreeClassifierNose.read( this->m_sFile2BLoadNose.c_str() ); + } + if(strmouth!="") + { + this->m_sFile2BLoadMouth = strmouth; + this->m_rtreeClassifierMouth.read( this->m_sFile2BLoadMouth.c_str() ); + } + } + break; + case VO_AdditiveStrongerClassifier::BOOSTING: + default: + { + if(strfrontalface!="") + { + this->m_sFile2BLoadFrontalFace = strfrontalface; + this->m_cascadeClassifierFrontalFace.load( this->m_sFile2BLoadFrontalFace ); + } + if(strprofileface!="") + { + this->m_sFile2BLoadProfileFace = strprofileface; + this->m_cascadeClassifierProfileFace.load( this->m_sFile2BLoadProfileFace ); + } + if(strlefteye!="") + { + this->m_sFile2BLoadLeftEye = strlefteye; + this->m_cascadeClassifierLeftEye.load( this->m_sFile2BLoadLeftEye ); + } + if(strrighteye!="") + { + this->m_sFile2BLoadRightEye = strrighteye; + this->m_cascadeClassifierRightEye.load( this->m_sFile2BLoadRightEye ); + } + if(strnose!="") + { + this->m_sFile2BLoadNose = strnose; + this->m_cascadeClassifierNose.load( this->m_sFile2BLoadNose ); + } + if(strmouth!="") + { + this->m_sFile2BLoadMouth = strmouth; + this->m_cascadeClassifierMouth.load( this->m_sFile2BLoadMouth ); + } + } + break; + } +} + + +/** +* @author JIA Pei +* @version 2010-02-04 +* @brief Face Detection -- refer to CDetectionAlgs::Detection +* @param iImg Input - image to be searched within +* @param confinedArea Input - only detecting the object within this confined area +* @param scale Input - scalar for img scaling +* @param sSize Input - detected object should be bigger than sSize +* @param bSize Input - detected object should be smaller than bSize +* @return double Return - the entire detection time +*/ +double CFaceDetectionAlgs::FaceDetection( const Mat& iImg, + const Rect* confinedArea, + const double scale, + Size sSize, + Size bSize) +{ + double res = (double)getTickCount(); + + switch(this->m_iDetectionMethod) + { + case VO_AdditiveStrongerClassifier::BAGGING: + CDetectionAlgs::BaggingDetection( + this->m_vDetectedFaceRects, + this->m_rtreeClassifierFrontalFace, + iImg, + confinedArea, + scale, + sSize, + bSize); + break; + case VO_AdditiveStrongerClassifier::BOOSTING: + CDetectionAlgs::BoostingDetection( + this->m_vDetectedFaceRects, + this->m_cascadeClassifierFrontalFace, + iImg, + confinedArea, + scale, + sSize, + bSize); + break; + } + + if ( this->m_vDetectedFaceRects.size() > 0 ) + { + this->m_bFaceDetected = true; + this->m_VOFaceComponents.SetObjectRect( this->m_vDetectedFaceRects[0] ); + this->m_VOFaceComponents0.SetObjectRect( this->m_vDetectedFaceRects[0] ); + this->m_CVDetectedFaceWindow2SM = + CFaceDetectionAlgs::VO_FaceRectFromDetection2SM( + iImg.size(), + this->m_VOFaceComponents0.m_rectObject ); + this->m_VOFaceComponents0.m_rectObject = + Rect( 0, + 0, + this->m_CVDetectedFaceWindow2SM.width, + this->m_CVDetectedFaceWindow2SM.height); + this->m_CVDetectedFaceImagePatch2SM = iImg(this->m_CVDetectedFaceWindow2SM ); + } + else + this->m_bFaceDetected = false; + + res = ((double)getTickCount() - res) + / ((double)cv::getTickFrequency()*1000.); + return res; +} + + +/** +* @author JIA Pei +* @version 2008-03-04 +* @brief Face Detection +* @param iImg Input - the input image, in which the face detection will be carried out +* @param confinedArea Input - only detecting the object within this confined area +* @param face Input - whether to detect face? always true! +* @param lefteye Input - whether to detect lefteye? +* @param righteye Input - whether to detect righteye? +* @param nose Input - whether to detect nose? +* @param mouth Input - whether to detect mouth? +* @param scale Input - always 1.0 +* @param sSize Input - smallest face +* @param bSize Input - biggest face +* @return double Return - the entire detection time +* @note This function defaultly detect face +*/ +double CFaceDetectionAlgs::FullFaceDetection ( const Mat& iImg, + const Rect* confinedArea, + bool lefteye, + bool righteye, + bool nose, + bool mouth, + const double scale, + Size sSize, + Size bSize ) +{ + double res = (double)cvGetTickCount(); + + // this is to detect the frontal face + this->FaceDetection(iImg, confinedArea, scale, sSize, bSize); + if ( this->m_vDetectedFaceRects.size() > 0 ) + this->m_bFaceDetected = true; + else + this->m_bFaceDetected = false; + + // if detected + if(this->m_bFaceDetected) + { + this->VO_FaceComponentsDetection( + this->m_CVDetectedFaceImagePatch2SM, + this->m_iFaceType, + lefteye, + righteye, + nose, + mouth); + } + else + { + this->m_bLeftEyeDetected = false; + this->m_bRightEyeDetected = false; + this->m_bNoseDetected = false; + this->m_bMouthDetected = false; + } + + res = ((double)cvGetTickCount() - res) + / ((double)cvGetTickFrequency()*1000.); + return res; +} + + +/** +* @author JIA Pei +* @version 2010-02-04 +* @brief Set detection configuration +* @param iImg Input - input image, a face +* @param faceOrient Input - face orientation +* @param lefteye Input - whether to detect left eye +* @param righteye Input - whether to detect right eye +* @param nose Input - whether to detect nose +* @param mouth Input - whether to detect mouth +*/ +double CFaceDetectionAlgs::VO_FaceComponentsDetection( const Mat& iImg, + int faceOrient, + bool lefteye, + bool righteye, + bool nose, + bool mouth) +{ + double res = (double)cvGetTickCount(); + + // Emphasized by JIA Pei, make sure you update m_CVDetectedFaceWindow finally! + // set the top left half to look for left eye + if ( lefteye ) + { + this->m_CVLeftEyePossibleWindow = CFaceDetectionAlgs::VO_SetDetectedFacePartsPossibleWindow(iImg.cols, iImg.rows, VO_FacePart::LEFTEYE, faceOrient); + this->m_bLeftEyeDetected = this->VO_FacePartDetection ( iImg, this->m_CVLeftEyePossibleWindow, this->m_VOFaceComponents0.m_rectLeftEye, VO_FacePart::LEFTEYE); + + if ( this->m_bLeftEyeDetected ) + { + this->m_VOFaceComponents.m_rectLeftEye.x = ( int ) ( this->m_VOFaceComponents0.m_rectLeftEye.x + this->m_CVLeftEyePossibleWindow.x + this->m_CVDetectedFaceWindow2SM.x ); + this->m_VOFaceComponents.m_rectLeftEye.y = ( int ) ( this->m_VOFaceComponents0.m_rectLeftEye.y + this->m_CVLeftEyePossibleWindow.y + this->m_CVDetectedFaceWindow2SM.y ); + this->m_VOFaceComponents.m_rectLeftEye.width = ( int ) ( this->m_VOFaceComponents0.m_rectLeftEye.width ); + this->m_VOFaceComponents.m_rectLeftEye.height = ( int ) ( this->m_VOFaceComponents0.m_rectLeftEye.height ); + this->m_VOFaceComponents0.m_rectLeftEye.x = ( int ) ( this->m_VOFaceComponents0.m_rectLeftEye.x + this->m_CVLeftEyePossibleWindow.x ); + this->m_VOFaceComponents0.m_rectLeftEye.y = ( int ) ( this->m_VOFaceComponents0.m_rectLeftEye.y + this->m_CVLeftEyePossibleWindow.y ); + + } + else + { + this->m_VOFaceComponents.m_rectLeftEye.x = ( int ) ( this->m_CVLeftEyePossibleWindow.x + this->m_CVDetectedFaceWindow2SM.x ); + this->m_VOFaceComponents.m_rectLeftEye.y = ( int ) ( this->m_CVLeftEyePossibleWindow.y + this->m_CVDetectedFaceWindow2SM.y ); + this->m_VOFaceComponents.m_rectLeftEye.width = ( int ) ( this->m_CVLeftEyePossibleWindow.width ); + this->m_VOFaceComponents.m_rectLeftEye.height = ( int ) ( this->m_CVLeftEyePossibleWindow.height); + this->m_VOFaceComponents0.m_rectLeftEye.x = ( int ) ( this->m_CVLeftEyePossibleWindow.x ); + this->m_VOFaceComponents0.m_rectLeftEye.y = ( int ) ( this->m_CVLeftEyePossibleWindow.y ); + + } + } + + // set the top right half to look for right eye + if ( righteye ) + { + this->m_CVRightEyePossibleWindow = CFaceDetectionAlgs::VO_SetDetectedFacePartsPossibleWindow(iImg.cols, iImg.rows, VO_FacePart::RIGHTEYE, faceOrient); + this->m_bRightEyeDetected = this->VO_FacePartDetection ( iImg, this->m_CVRightEyePossibleWindow, this->m_VOFaceComponents0.m_rectRightEye, VO_FacePart::RIGHTEYE ); + + if ( this->m_bRightEyeDetected ) + { + this->m_VOFaceComponents.m_rectRightEye.x = ( int ) ( this->m_VOFaceComponents0.m_rectRightEye.x + this->m_CVRightEyePossibleWindow.x + this->m_CVDetectedFaceWindow2SM.x ); + this->m_VOFaceComponents.m_rectRightEye.y = ( int ) ( this->m_VOFaceComponents0.m_rectRightEye.y + this->m_CVRightEyePossibleWindow.y + this->m_CVDetectedFaceWindow2SM.y ); + this->m_VOFaceComponents.m_rectRightEye.width = ( int ) ( this->m_VOFaceComponents0.m_rectRightEye.width ); + this->m_VOFaceComponents.m_rectRightEye.height = ( int ) ( this->m_VOFaceComponents0.m_rectRightEye.height); + this->m_VOFaceComponents0.m_rectRightEye.x = ( int ) ( this->m_VOFaceComponents0.m_rectRightEye.x + this->m_CVRightEyePossibleWindow.x ); + this->m_VOFaceComponents0.m_rectRightEye.y = ( int ) ( this->m_VOFaceComponents0.m_rectRightEye.y + this->m_CVRightEyePossibleWindow.y ); + + } + else + { + this->m_VOFaceComponents.m_rectRightEye.x = ( int ) ( this->m_CVRightEyePossibleWindow.x + this->m_CVDetectedFaceWindow2SM.x ); + this->m_VOFaceComponents.m_rectRightEye.y = ( int ) ( this->m_CVRightEyePossibleWindow.y + this->m_CVDetectedFaceWindow2SM.y ); + this->m_VOFaceComponents.m_rectRightEye.width = ( int ) ( this->m_CVRightEyePossibleWindow.width ); + this->m_VOFaceComponents.m_rectRightEye.height = ( int ) ( this->m_CVRightEyePossibleWindow.height ); + this->m_VOFaceComponents0.m_rectRightEye.x = ( int ) ( this->m_CVRightEyePossibleWindow.x ); + this->m_VOFaceComponents0.m_rectRightEye.y = ( int ) ( this->m_CVRightEyePossibleWindow.y ); + + } + } + + if ( nose ) + { + this->m_CVNosePossibleWindow = CFaceDetectionAlgs::VO_SetDetectedFacePartsPossibleWindow(iImg.cols, iImg.rows, VO_FacePart::NOSE, faceOrient); + this->m_bNoseDetected = this->VO_FacePartDetection ( iImg, this->m_CVNosePossibleWindow, this->m_VOFaceComponents0.m_rectNose, VO_FacePart::NOSE ); + + if ( this->m_bNoseDetected ) + { + this->m_VOFaceComponents.m_rectNose.x = ( int ) ( this->m_VOFaceComponents0.m_rectNose.x + this->m_CVNosePossibleWindow.x + this->m_CVDetectedFaceWindow2SM.x ); + this->m_VOFaceComponents.m_rectNose.y = ( int ) ( this->m_VOFaceComponents0.m_rectNose.y + this->m_CVNosePossibleWindow.y + this->m_CVDetectedFaceWindow2SM.y ); + this->m_VOFaceComponents.m_rectNose.width = ( int ) ( this->m_VOFaceComponents0.m_rectNose.width ); + this->m_VOFaceComponents.m_rectNose.height = ( int ) ( this->m_VOFaceComponents0.m_rectNose.height ); + this->m_VOFaceComponents0.m_rectNose.x = ( int ) ( this->m_VOFaceComponents0.m_rectNose.x + this->m_CVNosePossibleWindow.x ); + this->m_VOFaceComponents0.m_rectNose.y = ( int ) ( this->m_VOFaceComponents0.m_rectNose.y + this->m_CVNosePossibleWindow.y ); + + } + else + { + this->m_VOFaceComponents.m_rectNose.x = ( int ) ( this->m_CVNosePossibleWindow.x + this->m_CVDetectedFaceWindow2SM.x ); + this->m_VOFaceComponents.m_rectNose.y = ( int ) ( this->m_CVNosePossibleWindow.y + this->m_CVDetectedFaceWindow2SM.y ); + this->m_VOFaceComponents.m_rectNose.width = ( int ) ( this->m_CVNosePossibleWindow.width ); + this->m_VOFaceComponents.m_rectNose.height = ( int ) ( this->m_CVNosePossibleWindow.height ); + this->m_VOFaceComponents0.m_rectNose.x = ( int ) ( this->m_CVNosePossibleWindow.x ); + this->m_VOFaceComponents0.m_rectNose.y = ( int ) ( this->m_CVNosePossibleWindow.y ); + + } + } + + if ( mouth ) + { + this->m_CVMouthPossibleWindow = CFaceDetectionAlgs::VO_SetDetectedFacePartsPossibleWindow(iImg.cols, iImg.rows, VO_FacePart::LIPOUTERLINE, faceOrient); + this->m_bMouthDetected = this->VO_FacePartDetection ( iImg, this->m_CVMouthPossibleWindow, this->m_VOFaceComponents0.m_rectMouth, VO_FacePart::LIPOUTERLINE ); + + if ( this->m_bMouthDetected ) + { + this->m_VOFaceComponents0.m_rectMouth.x += static_cast(this->m_VOFaceComponents0.m_rectMouth.width*0.1); + // ensure this->m_VOFaceComponents.m_rectMouth.x + this->m_VOFaceComponents.m_rectMouth.width < iImg.cols-1 + while (this->m_VOFaceComponents0.m_rectMouth.x + this->m_VOFaceComponents0.m_rectMouth.width >= iImg.cols-1) + { + this->m_VOFaceComponents0.m_rectMouth.x--; + } + + this->m_VOFaceComponents.m_rectMouth.x = ( int ) ( this->m_VOFaceComponents0.m_rectMouth.x + this->m_CVMouthPossibleWindow.x + this->m_CVDetectedFaceWindow2SM.x ); + this->m_VOFaceComponents.m_rectMouth.y = ( int ) ( this->m_VOFaceComponents0.m_rectMouth.y + this->m_CVMouthPossibleWindow.y + this->m_CVDetectedFaceWindow2SM.y ); + this->m_VOFaceComponents.m_rectMouth.width = ( int ) ( this->m_VOFaceComponents0.m_rectMouth.width ); + this->m_VOFaceComponents.m_rectMouth.height = ( int ) ( this->m_VOFaceComponents0.m_rectMouth.height); + this->m_VOFaceComponents0.m_rectMouth.x = ( int ) ( this->m_VOFaceComponents0.m_rectMouth.x + this->m_CVMouthPossibleWindow.x ); + this->m_VOFaceComponents0.m_rectMouth.y = ( int ) ( this->m_VOFaceComponents0.m_rectMouth.y + this->m_CVMouthPossibleWindow.y ); + + // For mouth, we need a modification due to the reason that the mouth is not able to be well detected. + // We need small adjustment. + this->m_VOFaceComponents0.m_rectMouth.y -= static_cast(this->m_VOFaceComponents0.m_rectMouth.height*0.2); + this->m_VOFaceComponents.m_rectMouth.y -= static_cast(this->m_VOFaceComponents.m_rectMouth.height*0.2); + // ensure this->m_VOFaceComponents.m_rectMouth.y > 0 + while (this->m_VOFaceComponents0.m_rectMouth.y <= 0) + { + this->m_VOFaceComponents0.m_rectMouth.y++; + } + while (this->m_VOFaceComponents.m_rectMouth.y <= 0) + { + this->m_VOFaceComponents.m_rectMouth.y++; + } + } + else + { + this->m_VOFaceComponents.m_rectMouth.x = ( int ) ( this->m_CVMouthPossibleWindow.x + this->m_CVDetectedFaceWindow2SM.x ); + this->m_VOFaceComponents.m_rectMouth.y = ( int ) ( this->m_CVMouthPossibleWindow.y + this->m_CVDetectedFaceWindow2SM.y ); + this->m_VOFaceComponents.m_rectMouth.width = ( int ) ( this->m_CVMouthPossibleWindow.width ); + this->m_VOFaceComponents.m_rectMouth.height = ( int ) ( this->m_CVMouthPossibleWindow.height ); + this->m_VOFaceComponents0.m_rectMouth.x = ( int ) ( this->m_CVMouthPossibleWindow.x ); + this->m_VOFaceComponents0.m_rectMouth.y = ( int ) ( this->m_CVMouthPossibleWindow.y ); + + } + } + + // Default logical relationship among all the above 4 face parts + // According to my observation, the mouth some times goes wrong. + // the nose x direction goes wrong some times, but y direction keeps correct + if ( this->m_bNoseDetected && this->m_bMouthDetected ) // mouth must be lower than nose + { + if ( this->m_VOFaceComponents.m_rectMouth.y+1.0f*this->m_VOFaceComponents.m_rectMouth.height/2.0f <= ( this->m_VOFaceComponents.m_rectNose.y + this->m_VOFaceComponents.m_rectNose.height ) ) + this->m_VOFaceComponents.m_rectMouth.y += this->m_VOFaceComponents.m_rectMouth.height; + if ( this->m_VOFaceComponents0.m_rectMouth.y+1.0f*this->m_VOFaceComponents0.m_rectMouth.height/2.0f <= ( this->m_VOFaceComponents0.m_rectNose.y + this->m_VOFaceComponents0.m_rectNose.height ) ) + this->m_VOFaceComponents0.m_rectMouth.y += this->m_VOFaceComponents0.m_rectMouth.height; + } + + res = ((double)cvGetTickCount() - res) + / ((double)cvGetTickFrequency()*1000.); + return res; +} + + +/** +* @author JIA Pei +* @version 2010-02-04 +* @brief Face Component Detection +* @param iImg Input - the input image, in which the face detection will be carried out +* - Here, make sure iImg is gray level image +* @param iWindow Input - Possible input face part window, in which this face part is in +* @param oWindow Output - Output detected face part window +* @param facepart Input - Which face part is it to be detected in. + VO_FacePart::LEFTEYE, + VO_FacePart::RIGHTEYE, + VO_FacePart::NOSE, MOUTH +* @return bool Return - Whether the face component has been detected or not +*/ +bool CFaceDetectionAlgs::VO_FacePartDetection ( const Mat& iImg, + const Rect& iWindow, + Rect& oWindow, + unsigned int facepart) +{ + bool result = false; + Mat smallImgROI = iImg(iWindow); + + vector detectedfp; + switch ( facepart ) + { + case VO_FacePart::LEFTEYE: + { + switch(this->m_iDetectionMethod) + { + case VO_AdditiveStrongerClassifier::BAGGING: + CDetectionAlgs::BaggingDetection( + detectedfp, + this->m_rtreeClassifierLeftEye, + smallImgROI, + 0, + 1.0, + Size(iImg.cols/4, iImg.cols/8), + Size(iImg.cols, iImg.cols*2/3) ); + // Size(18, 12), + // Size(54, 36) ); + break; + case VO_AdditiveStrongerClassifier::BOOSTING: + CDetectionAlgs::BoostingDetection( + detectedfp, + this->m_cascadeClassifierLeftEye, + smallImgROI, + 0, + 1.0, + Size(iImg.cols/4, iImg.cols/8), + Size(iImg.cols, iImg.cols*2/3) ); + // Size(18, 12), + // Size(54, 36) ); + break; + } + } + break; + case VO_FacePart::RIGHTEYE: + { + switch(this->m_iDetectionMethod) + { + case VO_AdditiveStrongerClassifier::BAGGING: + CDetectionAlgs::BaggingDetection( + detectedfp, + this->m_rtreeClassifierRightEye, + smallImgROI, + 0, + 1.0, + Size(iImg.cols/4, iImg.cols/8), + Size(iImg.cols, iImg.cols*2/3) ); + // Size(18, 12), + // Size(54, 36) ); + break; + case VO_AdditiveStrongerClassifier::BOOSTING: + CDetectionAlgs::BoostingDetection( + detectedfp, + this->m_cascadeClassifierRightEye, + smallImgROI, + 0, + 1.0, + Size(iImg.cols/4, iImg.cols/8), + Size(iImg.cols, iImg.cols*2/3) ); + // Size(18, 12), + // Size(54, 36) ); + break; + } + } + break; + case VO_FacePart::NOSE: + { + switch(this->m_iDetectionMethod) + { + case VO_AdditiveStrongerClassifier::BAGGING: + CDetectionAlgs::BaggingDetection( + detectedfp, + this->m_rtreeClassifierNose, + smallImgROI, + 0, + 1.0, + Size(iImg.cols/6, iImg.rows/6), + Size(iImg.cols, iImg.rows) ); + // Size(18, 15), + // Size(54, 45) ); + break; + case VO_AdditiveStrongerClassifier::BOOSTING: + CDetectionAlgs::BoostingDetection( + detectedfp, + this->m_cascadeClassifierNose, + smallImgROI, + 0, + 1.0, + Size(iImg.cols/6, iImg.rows/6), + Size(iImg.cols, iImg.rows) ); + // Size(18, 15), + // Size(54, 45) ); + break; + } + } + break; + case VO_FacePart::LIPOUTERLINE: + { + switch(this->m_iDetectionMethod) + { + case VO_AdditiveStrongerClassifier::BAGGING: + CDetectionAlgs::BaggingDetection( + detectedfp, + this->m_rtreeClassifierMouth, + smallImgROI, + 0, + 1.0, + Size(iImg.cols/6, iImg.rows/6), + Size(iImg.cols, iImg.rows) ); + // Size(25, 15), + // Size(75, 45) ); + break; + case VO_AdditiveStrongerClassifier::BOOSTING: + CDetectionAlgs::BoostingDetection( + detectedfp, + this->m_cascadeClassifierMouth, + smallImgROI, + 0, + 1.0, + Size(iImg.cols/6, iImg.rows/6), + Size(iImg.cols, iImg.rows) ); + // Size(25, 15), + // Size(75, 45) ); + break; + } + } + break; + } + + // Within one specific face, there is only 1 face component. + // For instance, 1 lefteye, 1 righteye, 1 mouth, 1 nose. + if ( detectedfp.size() >= 1 ) + { + oWindow = detectedfp[0]; + result = true; + } + else + { + oWindow = iWindow; + result = false; + } + + return result; +} + + +/** +* @brief detect face directions +* @param iImg -- must be detected face image patch +* @param faceOrient -- frontal +*/ +int CFaceDetectionAlgs::VO_DetectFaceDirection(int faceOrient) +{ + if( this->m_bFaceDetected && + this->m_bLeftEyeDetected && + this->m_bRightEyeDetected && + this->m_bNoseDetected) + { + this->m_CVNoseCentralArea = + CFaceDetectionAlgs::VO_SetDetectedFacePartsPossibleWindow( + this->m_CVDetectedFaceImagePatch2SM.cols, + this->m_CVDetectedFaceImagePatch2SM.rows, + VO_FacePart::NOSECENTRALAREA, + faceOrient); + return CFaceDetectionAlgs::VO_DetectFaceDirection( + this->m_VOFaceComponents0, + this->m_CVNosePossibleWindow, + this->m_CVNoseCentralArea); + } + else + return CFaceDetectionAlgs::UNDETECTED; +} + + +int CFaceDetectionAlgs::VO_DetectFaceDirection( + const VO_FaceCompPos& facecomponents, + const Rect& possiblenose, + const Rect& nosecentralarea) +{ + int dir = CFaceDetectionAlgs::DIR_FRONTAL; + + int leftright = 0; + int updown = 0; + + vector centers = facecomponents.VO_CalcFaceComponentsCenters(); + float middleX = (centers[1].x + centers[2].x)/2.0f; + Point2f possiblenoseCenter; + possiblenoseCenter.x = possiblenose.x + possiblenose.width/2.0f; + possiblenoseCenter.y = possiblenose.y + possiblenose.height/2.0f; + + // centers[3] is the nose center + if(centers[3].x < nosecentralarea.x && centers[3].x < middleX) + leftright = -1; // from the user to the laptop screen, left; but actually right + if(centers[3].x > nosecentralarea.x+nosecentralarea.width && centers[3].x > middleX) + leftright = 1; // from the user to the laptop screen, right; but actually left + if(centers[3].y < nosecentralarea.y && centers[3].y < possiblenoseCenter.y) + updown = -1; // up + if(centers[3].y > nosecentralarea.y+nosecentralarea.height && centers[3].y > possiblenoseCenter.y) + updown = 1; // down + + if( leftright == 0 && updown == 1) dir = DIR_DOWNFRONTAL; + if( leftright == 1 && updown == 1) dir = DIR_DOWNLEFT; + if( leftright == -1 && updown == 1) dir = DIR_DOWNRIGHT; + if( leftright == 0 && updown == -1) dir = DIR_UPFRONTAL; + if( leftright == 1 && updown == -1) dir = DIR_UPLEFT; + if( leftright == -1 && updown == -1) dir = DIR_UPRIGHT; + if( leftright == 1 && updown == 0) dir = DIR_LEFT; + if( leftright == -1 && updown == 0) dir = DIR_RIGHT; + if( leftright == 0 && updown == 0) dir = DIR_FRONTAL; + + return dir; +} + + +/** +* @author JIA Pei +* @version 2008-03-04 +* @brief Draw face detection parts +* @param ioImg Input & Output - the input image, +* in which the face detection will be carried out +* @param face Input - whether to detect face? always true! +* @param lefteye Input - whether to detect lefteye? +* @param righteye Input - whether to detect righteye? +* @param nose Input - whether to detect nose? +* @param mouth Input - whether to detect mouth? +* @param color Input - line color +*/ +void CFaceDetectionAlgs::VO_DrawDetection ( Mat& ioImg, + bool face, + bool lefteye, + bool righteye, + bool nose, + bool mouth, + Scalar color) +{ + Point pt1, pt2, pt; + + // Face + if ( face ) + { + // pt1.x = this->m_VOFaceComponents.m_rectObject.x; + // pt1.y = this->m_VOFaceComponents.m_rectObject.y; + // pt2.x = this->m_VOFaceComponents.m_rectObject.x + this->m_VOFaceComponents.m_rectObject.width; + // pt2.y = this->m_VOFaceComponents.m_rectObject.y + this->m_VOFaceComponents.m_rectObject.height; + // cv::rectangle( ioImg, pt1, pt2, color, 2, 8, 0 ); + + pt1.x = this->m_CVDetectedFaceWindow2SM.x; + pt1.y = this->m_CVDetectedFaceWindow2SM.y; + pt2.x = this->m_CVDetectedFaceWindow2SM.x + this->m_CVDetectedFaceWindow2SM.width; + pt2.y = this->m_CVDetectedFaceWindow2SM.y + this->m_CVDetectedFaceWindow2SM.height; + cv::rectangle( ioImg, pt1, pt2, color, 2, 8, 0 ); + } + + // Left eye + if ( lefteye ) + { + pt1.x = this->m_VOFaceComponents.m_rectLeftEye.x; + pt1.y = this->m_VOFaceComponents.m_rectLeftEye.y; + pt2.x = this->m_VOFaceComponents.m_rectLeftEye.x + this->m_VOFaceComponents.m_rectLeftEye.width; + pt2.y = this->m_VOFaceComponents.m_rectLeftEye.y + this->m_VOFaceComponents.m_rectLeftEye.height; + cv::rectangle ( ioImg, pt1, pt2, colors[1], 1, 8, 0 ); + } + + // Right eye + if ( righteye ) + { + pt1.x = this->m_VOFaceComponents.m_rectRightEye.x; + pt1.y = this->m_VOFaceComponents.m_rectRightEye.y; + pt2.x = this->m_VOFaceComponents.m_rectRightEye.x + this->m_VOFaceComponents.m_rectRightEye.width; + pt2.y = this->m_VOFaceComponents.m_rectRightEye.y + this->m_VOFaceComponents.m_rectRightEye.height; + cv::rectangle ( ioImg, pt1, pt2, colors[2], 1, 8, 0 ); + } + + // Nose + if ( nose ) + { + pt1.x = this->m_VOFaceComponents.m_rectNose.x; + pt1.y = this->m_VOFaceComponents.m_rectNose.y; + pt2.x = this->m_VOFaceComponents.m_rectNose.x + this->m_VOFaceComponents.m_rectNose.width; + pt2.y = this->m_VOFaceComponents.m_rectNose.y + this->m_VOFaceComponents.m_rectNose.height; + cv::rectangle ( ioImg, pt1, pt2, colors[3], 1, 8, 0 ); + } + + // Mouth + if ( mouth ) + { + pt1.x = this->m_VOFaceComponents.m_rectMouth.x; + pt1.y = this->m_VOFaceComponents.m_rectMouth.y; + pt2.x = this->m_VOFaceComponents.m_rectMouth.x + this->m_VOFaceComponents.m_rectMouth.width; + pt2.y = this->m_VOFaceComponents.m_rectMouth.y + this->m_VOFaceComponents.m_rectMouth.height; + cv::rectangle ( ioImg, pt1, pt2, colors[4], 1, 8, 0 ); + } +} + + +/** +* @param ptType point type, which point is it? +* @return Point2f the point coordinates +*/ +void CFaceDetectionAlgs::CalcFaceKeyPoints() +{ + if(this->m_bFaceDetected) + { + this->m_CVDetectedCenterOfGravity.x = this->m_VOFaceComponents.m_rectObject.x + this->m_VOFaceComponents.m_rectObject.width/2.0f; + this->m_CVDetectedCenterOfGravity.y = this->m_VOFaceComponents.m_rectObject.y + this->m_VOFaceComponents.m_rectObject.height/2.0f; + + // No matter the face components have been detected or not, m_VOFaceComponents always have some values ! + // if(this->m_bLeftEyeDetected) + { + this->m_CVDetectedLeftEyeCenter.x = this->m_VOFaceComponents.m_rectLeftEye.x + this->m_VOFaceComponents.m_rectLeftEye.width/2.0f; + this->m_CVDetectedLeftEyeCenter.y = this->m_VOFaceComponents.m_rectLeftEye.y + this->m_VOFaceComponents.m_rectLeftEye.height/2.0f; + + //this->m_CVDetectedLeftEyeLeftCorner + //this->m_CVDetectedLeftEyeRightCorner + } + + // if(this->m_bRightEyeDetected) + { + this->m_CVDetectedRightEyeCenter.x = this->m_VOFaceComponents.m_rectRightEye.x + this->m_VOFaceComponents.m_rectRightEye.width/2.0f; + this->m_CVDetectedRightEyeCenter.y = this->m_VOFaceComponents.m_rectRightEye.y + this->m_VOFaceComponents.m_rectRightEye.height/2.0f; + + //this->m_CVDetectedLeftEyeLeftCorner + //this->m_CVDetectedLeftEyeRightCorner + } + + // if(this->m_bNoseDetected) + { + this->m_CVDetectedNoseCenter.x = this->m_VOFaceComponents.m_rectNose.x + this->m_VOFaceComponents.m_rectNose.width/2.0f; + this->m_CVDetectedNoseCenter.y = this->m_VOFaceComponents.m_rectNose.y + this->m_VOFaceComponents.m_rectNose.height/2.0f; + + //this->m_CVDetectedLeftEyeLeftCorner + //this->m_CVDetectedLeftEyeRightCorner + } + + // if(this->m_bMouthDetected) + { + this->m_CVDetectedMouthCenter.x = this->m_VOFaceComponents.m_rectMouth.x + this->m_VOFaceComponents.m_rectMouth.width/2.0f; + this->m_CVDetectedMouthCenter.y = this->m_VOFaceComponents.m_rectMouth.y + this->m_VOFaceComponents.m_rectMouth.height/2.0f; + + //this->m_CVDetectedLeftEyeLeftCorner + //this->m_CVDetectedLeftEyeRightCorner + } + } +} + + +/** +* @param ptType point type, which point is it? +* @return Point2f the point coordinates +*/ +Point2f CFaceDetectionAlgs::GetDetectedFaceKeyPoint(unsigned int ptType) const +{ + switch(ptType) + { + case VO_KeyPoint::CENTEROFGRAVITY: + return this->m_CVDetectedCenterOfGravity; + case VO_KeyPoint::LEFTEYELEFTCORNER: + return this->m_CVDetectedLeftEyeLeftCorner; + case VO_KeyPoint::LEFTEYERIGHTCORNER: + return this->m_CVDetectedLeftEyeRightCorner; + case VO_KeyPoint::LEFTEYECENTER: + return this->m_CVDetectedLeftEyeCenter; + case VO_KeyPoint::RIGHTEYELEFTCORNER: + return this->m_CVDetectedRightEyeLeftCorner; + case VO_KeyPoint::RIGHTEYERIGHTCORNER: + return this->m_CVDetectedRightEyeRightCorner; + case VO_KeyPoint::RIGHTEYECENTER: + return this->m_CVDetectedRightEyeCenter; + case VO_KeyPoint::NOSETIPKEY: + return this->m_CVDetectedNoseTip; + case VO_KeyPoint::NOSTRILLEFT: + return this->m_CVDetectedNostrilLeft; + case VO_KeyPoint::NOSTRILRIGHT: + return this->m_CVDetectedNostrilRight; + case VO_KeyPoint::NOSECENTER: + return this->m_CVDetectedNoseCenter; + case VO_KeyPoint::MOUTHLEFTCORNER: + return this->m_CVDetectedMouthLeftCorner; + case VO_KeyPoint::MOUTHRIGHTCORNER: + return this->m_CVDetectedMouthRightCorner; + case VO_KeyPoint::MOUTHCENTER: + return this->m_CVDetectedMouthCenter; + } + throw; //unreachable code. + return Point2f(); +} + + +/** +* @author Yao Wei +* @brief Basically, this function turns the detected face square to a +* rectangle of size 1*1.1 +* @param imgSize image size +* @param iFaceRect the detected face rectangle +* @return Rect the adjusted face rectangle +*/ +Rect CFaceDetectionAlgs::VO_FaceRectFromDetection2SM (const Size& imgSize, + const Rect& iFaceRect ) +{ + Rect res; + + // Note: copied from aamlibrary and asmlibrary + static const float CONF_VjHeightShift = 0.15f; // shift height down by 15% + static const float CONF_VjShrink = 0.80f; // shrink size of VJ box by 20% + + float xMin = static_cast(iFaceRect.x); + float yMin = static_cast(iFaceRect.y); + float xMax = static_cast(iFaceRect.x + iFaceRect.width); + float yMax = static_cast(iFaceRect.y + iFaceRect.height); + + float NewWidth = CONF_VjShrink * static_cast(iFaceRect.width); + float NewHeight = CONF_VjShrink * static_cast(iFaceRect.height); + + float yMean = ( yMin + yMax ) / 2; + yMean += CONF_VjHeightShift * NewHeight; // move face down + float xMean = ( xMin + xMax ) / 2; + + res.x = xMean - 0.5f * NewWidth > 0 ? + (int)(xMean - 0.5f * NewWidth) : 0; + res.y = yMean - 0.5f * NewHeight > 0 ? + (int)(yMean - 0.5f * NewHeight) : 0; + if(NewWidth + res.x < imgSize.width) + res.width = static_cast(NewWidth); + else + res.width = imgSize.width - res.x; + if(NewHeight*1.1f + res.y < imgSize.height) + res.height = static_cast(NewHeight*1.1f); + else + res.height = imgSize.height - res.y; + //res.width = NewWidth < imgSize.width ? (int)NewWidth : imgSize.width; + //res.height = (NewHeight*1.1f) < imgSize.height ? (int)(NewHeight*1.1f) : imgSize.height; + + return res; +} + + +/** +* @author Pei JIA +* @param imgWidth image size +* @param imgHeight the detected face rectangle +* @param facepart which face part is it? +* @param dir face directions +* @return Rect the possible rectangle for this specific face rectangle +*/ +Rect CFaceDetectionAlgs::VO_SetDetectedFacePartsPossibleWindow( + int imgWidth, + int imgHeight, + unsigned int facepart, + unsigned int dir) +{ + Rect rect; + switch(dir) + { + case LEFTPROFILE: + break; + case RIGHTPROFILE: + break; + case FRONTAL: + default: + { + switch(facepart) + { + case VO_FacePart::LEFTEYE: + rect = Rect ( ( int ) ( imgWidth/20.0f ), ( int ) ( imgHeight/10.0f ), ( int ) ( 9.0f*imgWidth/20.0f ), ( int ) ( 3*imgHeight/10.0f ) ); + break; + case VO_FacePart::RIGHTEYE: + rect = Rect ( ( int ) ( 1.0f*imgWidth/2.0f ), ( int ) ( imgHeight/10.0f ), ( int ) ( 9.0f*imgWidth/20.0f ), ( int ) ( 3*imgHeight/10.0f ) ); + break; + case VO_FacePart::NOSE: + rect = Rect ( ( int ) ( imgWidth/4.0f ), ( int ) ( imgHeight/5.0f ), ( int ) ( imgWidth/2.0f ), ( int ) ( 3.0*imgHeight/5.0f ) ); + break; + case VO_FacePart::LIPOUTERLINE: + rect = Rect ( ( int ) ( imgWidth/5.0f ), ( int ) ( 1.0f*imgHeight/2.0f ), ( int ) ( 3.0f*imgWidth/5.0f ), ( int ) ( 1.0f*imgHeight/2.0f ) ); + break; + case VO_FacePart::NOSECENTRALAREA: + rect = Rect ( ( int ) ( 0.48*imgWidth ), ( int ) ( 0.45*imgHeight ), ( int ) ( 0.05*imgWidth ), ( int ) ( 0.05*imgHeight ) ); + break; + default: + break; + } + } + break; + } + + return rect; +} + diff --git a/modules/comalgs/src/VO_HumanDetectionAlgs.cpp b/modules/comalgs/src/VO_HumanDetectionAlgs.cpp new file mode 100644 index 0000000..68cf854 --- /dev/null +++ b/modules/comalgs/src/VO_HumanDetectionAlgs.cpp @@ -0,0 +1,146 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_HumanDetectionAlgs.h" + + +/************************************************************************/ +/*@author JIA Pei */ +/*@version 2011-02-07 */ +/*@brief Human Detection -- refer to CDetectionAlgs::Detection */ +/*@param iImg Input - image to be searched in */ +/*@param confinedArea Input - only detect human in this */ +/* confined area */ +/*@param scale Input - scalar for img scaling */ +/*@param sSize Input - detected human must be bigger than sSize */ +/*@param bSize Input - detected human must be smaller than bSize */ +/*@return detection time cost */ +/************************************************************************/ +double CHumanDetectionAlgs::HumanDetection( const Mat& iImg, + const Rect* confinedArea, + const double scale, + Size sSize, + Size bSize) +{ + double res = (double)getTickCount(); + + vector found; + this->m_hog.detectMultiScale(iImg, found, 0, sSize, bSize, 1.05, 2); + + size_t i, j; + for( i = 0; i < found.size(); i++ ) + { + Rect r = found[i]; + for( j = 0; j < found.size(); j++ ) + if( j != i && (r & found[j]) == r) + break; + if( j == found.size() ) + this->m_vDetectedHumanRects.push_back(r); + } + + if ( this->m_vDetectedHumanRects.size() <= 0 ) + this->m_bHumanDetected = false; + else + { + this->m_bHumanDetected = true; + + for( i = 0; i < this->m_vDetectedHumanRects.size(); i++ ) + { + Rect r = this->m_vDetectedHumanRects[i]; + // the HOG detector returns slightly larger rectangles than the real objects. + // so we slightly shrink the rectangles to get a nicer output. + r.x += cvRound(r.width*0.1); + r.width = cvRound(r.width*0.8); + r.y += cvRound(r.height*0.07); + r.height = cvRound(r.height*0.8); + } + } + + res = ((double)cvGetTickCount() - res) / ((double)cvGetTickFrequency()*1000.); + return res; +} + + +/************************************************************************/ +/*@author JIA Pei */ +/*@version 2011-02-07 */ +/*@brief Draw human detection results */ +/*@param ioImg Input & Output - the input image, in which */ +/*@param the face detection will be carried out */ +/*@param color Input - line color */ +/*@return void */ +/************************************************************************/ +void CHumanDetectionAlgs::VO_DrawDetection (Mat& ioImg, Scalar color) +{ + Point pt1, pt2; + unsigned int NbOfDetectedHumans = this->m_vDetectedHumanRects.size(); + + for(unsigned int i = 0; i < NbOfDetectedHumans; i++) + { + pt1.x = this->m_vDetectedHumanRects[i].x; + pt1.y = this->m_vDetectedHumanRects[i].y; + pt2.x = this->m_vDetectedHumanRects[i].x + + this->m_vDetectedHumanRects[i].width; + pt2.y = this->m_vDetectedHumanRects[i].y + + this->m_vDetectedHumanRects[i].height; + cv::rectangle( ioImg, pt1, pt2, color, 2, 8, 0 ); + } +} diff --git a/modules/comalgs/src/VO_LocalizationAlgs.cpp b/modules/comalgs/src/VO_LocalizationAlgs.cpp new file mode 100644 index 0000000..f6f287b --- /dev/null +++ b/modules/comalgs/src/VO_LocalizationAlgs.cpp @@ -0,0 +1,183 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_LocalizationAlgs.h" + +using namespace cv; + + +/** +* @author JIA Pei +* @version 2009-10-04 +* @brief Object Localization +* @param img Input - image to be searched in +* @param sSize Input - localized obj must be bigger than sSize +* @param bSize Input - localized object must be smaller than bSize +* @return localization time cost +*/ +double CLocalizationAlgs::Localization( const Mat& img, + Size sSize, + Size bSize) +{ + return ( CLocalizationAlgs::Localization( + img, + this->m_detectionAlgs, + this->m_trackingAlgs, + this->m_bObjectLocalized, + this->m_CVLocalizedObjectRect, + sSize, + bSize) ); +} + + +/** +* @author JIA Pei +* @version 2009-10-04 +* @brief Object Detection +* @param img Input - image to be searched within +* @param detectAlg Input - detection algorithm +* @param trackAlg Input - tracking algorithm +* @param isLocalized Output - is the object localized? +* @param objPos Output - objects' positions +* @param sSize Input - detected object should be bigger than sSize +* @param bSize Input - detected object should be smaller than bSize +* @return localization time cost +*/ +double CLocalizationAlgs::Localization( const Mat& img, + CDetectionAlgs& detectAlg, + CTrackingAlgs& trackAlg, + bool& isLocalized, + Rect& objPos, + const Size& sSize, + const Size& bSize) +{ + double res = (double)cvGetTickCount(); + double scale = 1.0; + + // only detection is used + if ( trackAlg.m_iTrackingMethod == CTrackingAlgs::NONE ) + { + detectAlg.Detection(img, + NULL, + scale, + sSize, + bSize); + if( detectAlg.IsObjectDetected() ) + { + isLocalized = true; + objPos = detectAlg.GetDetectedObjectRects()[0]; + } + } + // tracking is used + else + { + // not localized yet + if( !isLocalized || + (objPos.x < 0 && + objPos.y < 0 && + objPos.width < 0 && + objPos.height < 0) ) + { + detectAlg.Detection(img, + NULL, + scale, + sSize, + bSize); + if( detectAlg.IsObjectDetected() ) + { + isLocalized = true; + objPos = detectAlg.GetDetectedObjectRects()[0]; + trackAlg.UpdateTracker(img, objPos); + } + } + // keep tracking now + else + { + trackAlg.Tracking( objPos, + img, + sSize, + bSize); + isLocalized = trackAlg.IsObjectTracked(); + } + } + + res = ((double)cvGetTickCount() - res) + / ((double)cvGetTickFrequency()*1000.); + return res; +} + + +void CLocalizationAlgs::VO_DrawLocalization(Mat& ioImg, Scalar color) +{ + Rect curRect; + Point lefttop, rightbottom; + + if ( this->m_bObjectLocalized ) + { + curRect = this->m_CVLocalizedObjectRect; + lefttop.x = cvRound(curRect.x); + lefttop.y = cvRound(curRect.y); + rightbottom.x = cvRound((curRect.x+curRect.width)); + rightbottom.y = cvRound((curRect.y+curRect.height)); + cv::rectangle(ioImg, lefttop, rightbottom, color, 2, 8, 0); + } +} + diff --git a/modules/comalgs/src/VO_RecognitionAlgs.cpp b/modules/comalgs/src/VO_RecognitionAlgs.cpp new file mode 100644 index 0000000..4ddd8d2 --- /dev/null +++ b/modules/comalgs/src/VO_RecognitionAlgs.cpp @@ -0,0 +1,934 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include +#include +#include +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_FaceKeyPoint.h" +#include "VO_RecognitionAlgs.h" + + + + +/** +* @param trackalg- input and output the track algorithm, + will record some information for every frame +* @param iImg - input input image +* @param iShape - input the current tracked shape +* @return bool whether the tracked shape is acceptable? +*/ +bool CRecognitionAlgs::EvaluateFaceTrackedByProbabilityImage( + CTrackingAlgs* trackalg, + const Mat& iImg, + const VO_Shape& iShape, + Size smallSize, + Size bigSize) +{ + double t = (double)cvGetTickCount(); + + Rect rect = iShape.GetShapeBoundRect(); + + trackalg->SetConfiguration( CTrackingAlgs::CAMSHIFT, + CTrackingAlgs::PROBABILITYIMAGE); + trackalg->Tracking( rect, + iImg, + smallSize, + bigSize ); + + bool res = false; + if( !trackalg->IsObjectTracked() ) + res = false; + else if ( ((double)rect.height/(double)rect.width <= 0.75) + || ((double)rect.height/(double)rect.width >= 2.5) ) + res = false; + else + res = true; + + t = ((double)cvGetTickCount() - t ) + / (cvGetTickFrequency()*1000.); + cout << "Camshift Tracking time cost: " << t << "millisec" << endl; + + return res; +} + + +/** +* @brief whether the tracked shape is really a face? +* If we can detect both eyes and mouth +* according to some prior knowledge due to its shape, +* we may regard this shape correctly describe a face. +* @param iImg - input input image +* @param iShape - input the current tracked shape +* @param iShapeInfo - input shape info +* @param iFaceParts - input face parts +* @return bool whether the tracked shape is acceptable? +*/ +bool CRecognitionAlgs::EvaluateFaceTrackedByCascadeDetection( + const CFaceDetectionAlgs* fd, + const Mat& iImg, + const VO_Shape& iShape, + const vector& iShapeInfo, + const VO_FaceParts& iFaceParts) +{ + double t = (double)cvGetTickCount(); + + unsigned int ImgWidth = iImg.cols; + unsigned int ImgHeight = iImg.rows; + + vector leftEyePoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::LEFTEYE).GetIndexes(); + vector rightEyePoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::RIGHTEYE).GetIndexes(); + vector lipOuterLinerPoints= iFaceParts.VO_GetOneFacePart(VO_FacePart::LIPOUTERLINE).GetIndexes(); + + VO_Shape leftEyeShape = iShape.GetSubShape(leftEyePoints); + VO_Shape rightEyeShape = iShape.GetSubShape(rightEyePoints); + VO_Shape lipOuterLinerShape = iShape.GetSubShape(lipOuterLinerPoints); + + float dolEye = 12.0f; + float dolMouth = 12.0f; + + unsigned int possibleLeftEyeMinX = 0.0f > (leftEyeShape.MinX() - dolEye) ? 0: (int)(leftEyeShape.MinX() - dolEye); + unsigned int possibleLeftEyeMinY = 0.0f > (leftEyeShape.MinY() - dolEye) ? 0: (int)(leftEyeShape.MinY() - dolEye); + unsigned int possibleLeftEyeMaxX = (leftEyeShape.MaxX() + dolEye) > ImgWidth ? ImgWidth : (int)(leftEyeShape.MaxX() + dolEye); + unsigned int possibleLeftEyeMaxY = (leftEyeShape.MaxY() + dolEye) > ImgHeight ? ImgHeight : (int)(leftEyeShape.MaxY() + dolEye); + unsigned int possibleLeftEyeWidth = possibleLeftEyeMaxX - possibleLeftEyeMinX; + unsigned int possibleLeftEyeHeight = possibleLeftEyeMaxY - possibleLeftEyeMinY; + unsigned int possibleRightEyeMinX = 0.0f > (rightEyeShape.MinX() - dolEye) ? 0: (int)(rightEyeShape.MinX() - dolEye); + unsigned int possibleRightEyeMinY = 0.0f > (rightEyeShape.MinY() - dolEye) ? 0: (int)(rightEyeShape.MinY() - dolEye); + unsigned int possibleRightEyeMaxX = (rightEyeShape.MaxX() + dolEye) > ImgWidth ? ImgWidth : (int)(rightEyeShape.MaxX() + dolEye); + unsigned int possibleRightEyeMaxY = (rightEyeShape.MaxY() + dolEye) > ImgHeight ? ImgHeight : (int)(rightEyeShape.MaxY() + dolEye); + unsigned int possibleRightEyeWidth = possibleRightEyeMaxX - possibleRightEyeMinX; + unsigned int possibleRightEyeHeight = possibleRightEyeMaxY - possibleRightEyeMinY; + unsigned int possibleMouthMinX = 0.0f > (lipOuterLinerShape.MinX() - dolMouth) ? 0: (int)(lipOuterLinerShape.MinX() - dolMouth); + unsigned int possibleMouthMinY = 0.0f > (lipOuterLinerShape.MinY() - dolMouth) ? 0: (int)(lipOuterLinerShape.MinY() - dolMouth); + unsigned int possibleMouthMaxX = (lipOuterLinerShape.MaxX() + dolMouth) > ImgWidth ? ImgWidth : (int)(lipOuterLinerShape.MaxX() + dolMouth); + unsigned int possibleMouthMaxY = (lipOuterLinerShape.MaxY() + dolMouth) > ImgHeight ? ImgHeight : (int)(lipOuterLinerShape.MaxY() + dolMouth); + unsigned int possibleMouthWidth = possibleMouthMaxX - possibleMouthMinX; + unsigned int possibleMouthHeight = possibleMouthMaxY - possibleMouthMinY; + + Rect LeftEyePossibleWindow = Rect( possibleLeftEyeMinX, possibleLeftEyeMinY, possibleLeftEyeWidth, possibleLeftEyeHeight ); + Rect RightEyePossibleWindow = Rect( possibleRightEyeMinX, possibleRightEyeMinY, possibleRightEyeWidth, possibleRightEyeHeight ); + Rect MouthPossibleWindow = Rect( possibleMouthMinX, possibleMouthMinY, possibleMouthWidth, possibleMouthHeight ); + Rect CurrentWindow = Rect( 0, 0, iImg.cols, iImg.rows ); + Rect DetectedLeftEyeWindow, DetectedRightEyeWindow, DetectedMouthWindow; + + bool LeftEyeDetected = const_cast(fd)->VO_FacePartDetection ( iImg, LeftEyePossibleWindow, DetectedLeftEyeWindow, VO_FacePart::LEFTEYE); + bool RightEyeDetected = const_cast(fd)->VO_FacePartDetection ( iImg, RightEyePossibleWindow, DetectedRightEyeWindow, VO_FacePart::RIGHTEYE ); + bool MouthDetected = const_cast(fd)->VO_FacePartDetection ( iImg, MouthPossibleWindow, DetectedMouthWindow, VO_FacePart::LIPOUTERLINE ); + + t = ((double)cvGetTickCount() - t ) + / (cvGetTickFrequency()*1000.0f); + cout << "Detection Confirmation time cost: " << t << "millisec" << endl; + + if(LeftEyeDetected && RightEyeDetected && MouthDetected) + return true; + else + return false; +} + + +/** +* @param shape1 - input shape1 +* @param shape2 - input shape2 +* @return the shape distance +*/ +float CRecognitionAlgs::ShapeDistance( const VO_Shape& shape1, + const VO_Shape& shape2) +{ + VO_Shape shapediff = const_cast(shape1) - shape2; + return shapediff.GetShapeNorm(); +} + + +/** +* @param avgSParam - input mean shape parameters +* @param icovSParam - input covariance matrix of shape parameters +* @param avgTParam - input mean texture parameters +* @param icovTParam - input covariance matrix of texture parameters +* @param iSParams - input the input shape parameter +* @param iTParams - input the input texture parameter +* @param ShapeDistMean - input mean texture parameters +* @param ShapeDistStddev - input covariance matrix of texture parameters +* @param TextureDistMean - input the input shape parameter +* @param TextureDistStddev- input the input texture parameter +* @param sDist - output shape distance +* @param tDist - output texture distance +* @param WeakFitting - input only shape parameter is used? +* @return whether the fitting is acceptable +*/ +bool CRecognitionAlgs::CalcFittingEffect4StaticImage( + const Mat_& avgSParam, + const Mat_& icovSParam, + const Mat_& avgTParam, + const Mat_& icovTParam, + const Mat_& iSParams, + const Mat_& iTParams, + const Scalar& ShapeDistMean, + const Scalar& ShapeDistStddev, + const Scalar& TextureDistMean, + const Scalar& TextureDistStddev, + float& sDist, + float& tDist, + bool WeakFitting ) +{ + sDist = safeDoubleToFloat(cv::Mahalanobis( iSParams, avgSParam, icovSParam )); + tDist = safeDoubleToFloat(cv::Mahalanobis( iTParams, avgTParam, icovTParam )); + + if(WeakFitting) + { + if( ( fabs( sDist - ShapeDistMean.val[0] ) + < 1.5f * ShapeDistStddev.val[0] ) ) + return true; + else + return false; + } + else + { + if( ( fabs( sDist - ShapeDistMean.val[0] ) + < 1.5f * ShapeDistStddev.val[0] ) && + ( fabs( tDist - TextureDistMean.val[0] ) + < 3.0f*TextureDistStddev.val[0] ) ) + return true; + else + return false; + } +} + + +/** +* @param avgSParam - input mean shape parameters +* @param icovSParam - input covariance matrix of shape parameters +* @param avgTParam - input mean texture parameters +* @param icovTParam - input covariance matrix of texture parameters +* @param iSParams - input the vector of multiple input shape parameters +* @param iTParams - input the vector of multiple input texture parameter +* @param ShapeDistMean - input mean texture parameters +* @param ShapeDistStddev - input covariance matrix of texture parameters +* @param TextureDistMean - input the input shape parameter +* @param TextureDistStddev - input the input texture parameter +* @param WeakFitting - input only shape parameter is used? +* @return whether the fitting is acceptable +*/ +bool CRecognitionAlgs::CalcFittingEffect4ImageSequence( + const Mat_& avgSParam, + const Mat_& icovSParam, + const Mat_& avgTParam, + const Mat_& icovTParam, + const Mat_& iSParams, + const Mat_& iTParams, + const Scalar& ShapeDistMean, + const Scalar& ShapeDistStddev, + const Scalar& TextureDistMean, + const Scalar& TextureDistStddev, + bool WeakFitting ) +{ + assert(iSParams.rows == iTParams.rows); + unsigned int NbOfSamples = iSParams.rows; + vector sDists, tDists; + sDists.resize(NbOfSamples); + tDists.resize(NbOfSamples); + + for(unsigned int i = 0; i < NbOfSamples; ++i) + { + CRecognitionAlgs::CalcFittingEffect4StaticImage( + avgSParam, + icovSParam, + avgTParam, + icovTParam, + iSParams.row(i), + iTParams.row(i), + ShapeDistMean, + ShapeDistStddev, + TextureDistMean, + TextureDistStddev, + sDists[i], + tDists[i], + WeakFitting ); + } + + unsigned int NbOfGood1 = 0; + unsigned int NbOfGood2 = 0; + + for(unsigned int i = 0; i < NbOfSamples; ++i) + { + if( ( fabs( sDists[i] - ShapeDistMean.val[0] ) + < 1.5f * ShapeDistStddev.val[0] ) ) + { + NbOfGood1++; + if( ( fabs( tDists[i] - TextureDistMean.val[0] ) + < 3.0f*TextureDistStddev.val[0] ) ) + { + NbOfGood2++; + } + } + } + + if(WeakFitting) + { + if(NbOfGood1 >= (unsigned int )(0.75*NbOfSamples) ) + return true; + else + return false; + } + else + { + if(NbOfGood2 >= (unsigned int )(0.75*NbOfGood1) ) + return true; + else + return false; + } +} + + +/** +* @brief Calculate face fitting effect +* @param refShape - input reference shape +* @param fittedShape - input fitting result +* @param deviation - output what is the deviation from refShape to fittedShape +* @param ptErrorFreq - output point error frequency +* @param nb - input how many evaluation levels that is to be used +* @return whether the fitting is acceptable +*/ +void CRecognitionAlgs::CalcShapeFittingEffect( const VO_Shape& refShape, + const VO_Shape& fittedShape, + float& deviation, + vector& ptErrorFreq, + int nb, + vector* ptErrPerPoint) +{ + assert(refShape.GetNbOfDim() == fittedShape.GetNbOfDim()); + assert(refShape.GetNbOfPoints() == fittedShape.GetNbOfPoints()); + unsigned int NbOfShapeDim = refShape.GetNbOfDim(); + unsigned int NbOfPoints = refShape.GetNbOfPoints(); + ptErrorFreq.resize(nb); + + vector ptDists(NbOfPoints, 0.0f); + + for(unsigned int i = 0; i < NbOfPoints; i++) + { + ptDists[i] = 0.0f; + for(unsigned int j = 0; j < NbOfShapeDim; j++) + { + ptDists[i] += pow(refShape.GetAShape(j*NbOfPoints+i) - fittedShape.GetAShape(j*NbOfPoints+i), 2.0f); + } + ptDists[i] = sqrt(ptDists[i]); + } + + ptErrorFreq.resize(nb); + for(int i = 0; i < nb; i++) + { + for (unsigned int j = 0; j < NbOfPoints; j++) + { + if (ptDists[j] < i) + { + ptErrorFreq[i]++; + } + } + ptErrorFreq[i] /= static_cast(NbOfPoints); + } + float sumPtDist = 0.0; + for(unsigned int i = 0; i& ptErrorFreq) +{ + string fn; + fn = fd + "/" + fnIdx + ".res"; + + fstream fp; + fp.open(fn.c_str (), ios::out); + + fp << "Total Deviation" << endl << deviation << endl; // deviation + fp << "Point Error -- Frequency" << endl; + for(unsigned int i = 0; i < ptErrorFreq.size(); i++) + { + fp << ptErrorFreq[i] << " "; + } + + fp << endl; + + fp.close();fp.clear(); +} + +/** + * @param fd - input folder name + * @param fnIdx - input fitting result + * @param deviation - input what is the deviation from refShape to fittedShape + * @param ptErrorFreq - input for curve to display frequency -- point distance + * @param fittedShape - input fitting result + * @return whether the fitting is acceptable + */ +void CRecognitionAlgs::SaveShapeResults( const string& fd, + const string& fnIdx, + float deviation, + vector& ptDists, + vector& ptErrorFreq, + const VO_Shape& fittedShape) +{ + string fn; + fn = fd + "/" + fnIdx + ".res"; + + fstream fp; + fp.open(fn.c_str (), ios::out); + + fp << "Error per point -- Distance from ground truth" << endl; + for(unsigned int i = 0; i < ptDists.size(); ++i){ + fp << ptDists[i] << endl; + } + fp << endl; + + fp << "Total landmark error" << endl; + float errSum = std::accumulate(ptDists.begin(),ptDists.end(),0.0f); + fp << errSum << endl; + fp <<"Average landmark distance" << endl; + fp << errSum / ptDists.size() << endl; + fp << endl; + + fp << "Total Deviation" << endl << deviation << endl; // deviation + fp << "Point Error -- Frequency" << endl; + for(unsigned int i = 0; i < ptErrorFreq.size(); i++) + { + fp << ptErrorFreq[i] << " "; + } + fp << endl; + fp << endl; + fp << "Fitted points" << endl; + //output actual points along with error frequency + unsigned int NbOfShapeDim = fittedShape.GetNbOfDim(); + unsigned int NbOfPoints = fittedShape.GetNbOfPoints(); + for(unsigned int i = 0; i < NbOfPoints; i++) + { + for(unsigned int j = 0; j < NbOfShapeDim; j++) + { + fp << fittedShape.GetAShape(j*NbOfPoints+i) << " "; + } + fp << endl; + } + fp << endl; + + fp.close();fp.clear(); +} + +/** + * @param fd - input folder name + * @param fnIdx - input fitting result + * @param deviation - input what is the deviation from refShape to fittedShape + * @param ptErrorFreq - input for curve to display frequency -- point distance + * @param fittedShape - input fitting result + * @param gt_cp - input ground truth canidate points + * @param t_cp - input tested canidate points (l eye, r eye, mouth) + * @return whether the fitting is acceptable + */ +void CRecognitionAlgs::SaveFittingResults( const string& fd, + const string& fnIdx, + float deviation, + vector& ptDists, + vector& ptErrorFreq, + const VO_Shape& fittedShape, + cv::Point2f* gt_cP, + cv::Point2f* t_cP, + float fitTime) +{ + string fn; + fn = fd + "/" + fnIdx + ".res"; + + fstream fp; + fp.open(fn.c_str (), ios::out); + + fp << "Error per point -- Distance from ground truth" << endl; + for(unsigned int i = 0; i < ptDists.size(); ++i){ + fp << ptDists[i] << endl; + } + fp << endl; + + fp << "Total landmark error" << endl; + float errSum = std::accumulate(ptDists.begin(),ptDists.end(),0.0f); + fp << errSum << endl; + fp << "Average landmark distance" << endl; + fp << errSum / ptDists.size() << endl; + fp << "Candidate point error (Left eye, Right eye, Mouth)" << endl; + //messy distance, too lazy + float le_dist = sqrt(pow(gt_cP[0].x - t_cP[0].x,2) + pow(gt_cP[0].y - t_cP[0].y,2)); + float re_dist = sqrt(pow(gt_cP[1].x - t_cP[1].x,2) + pow(gt_cP[1].y - t_cP[1].y,2)); + float m_dist = sqrt(pow(gt_cP[2].x - t_cP[2].x,2) + pow(gt_cP[2].y - t_cP[2].y,2)); + + fp << le_dist << endl; + fp << re_dist << endl; + fp << m_dist << endl; + fp << endl; + fp << "Fitting time" << endl; + fp << fitTime << endl; + fp << endl; + + fp << "Total deviation" << endl << deviation << endl; // deviation + fp << "Point error -- Frequency" << endl; + for(unsigned int i = 0; i < ptErrorFreq.size(); i++) + { + fp << ptErrorFreq[i] << " "; + } + fp << endl; + fp << endl; + fp << "Canidate points" << endl; + fp << t_cP[0].x << " " << t_cP[0].y << endl; + fp << t_cP[1].x << " " << t_cP[1].y << endl; + fp << t_cP[2].x << " " << t_cP[2].y << endl; + fp << "Fitted points" << endl; + //output actual points along with error frequency + unsigned int NbOfShapeDim = fittedShape.GetNbOfDim(); + unsigned int NbOfPoints = fittedShape.GetNbOfPoints(); + for(unsigned int i = 0; i < NbOfPoints; i++) + { + for(unsigned int j = 0; j < NbOfShapeDim; j++) + { + fp << fittedShape.GetAShape(j*NbOfPoints+i) << " "; + } + fp << endl; + } + fp << endl; + + fp.close();fp.clear(); +} + +// Estimate face absolute orientations +vector CRecognitionAlgs::CalcAbsoluteOrientations( + const VO_Shape& iShape2D, + const VO_Shape& iShape3D, + VO_Shape& oShape2D) +{ + assert (iShape2D.GetNbOfPoints() == iShape3D.GetNbOfPoints() ); + unsigned int NbOfPoints = iShape3D.GetNbOfPoints(); + Point3f pt3d; + Point2f pt2d; + float height1 = iShape2D.GetHeight(); + float height2 = iShape3D.GetHeight(); + VO_Shape tempShape2D = iShape2D; + tempShape2D.Scale(height2/height1); + + //Create the model points + std::vector modelPoints; + for(unsigned int i = 0; i < NbOfPoints; ++i) + { + pt3d = iShape3D.GetA3DPoint(i); + modelPoints.push_back(cvPoint3D32f(pt3d.x, pt3d.y, pt3d.z)); + } + + //Create the image points + std::vector srcImagePoints; + for(unsigned int i = 0; i < NbOfPoints; ++i) + { + pt2d = tempShape2D.GetA2DPoint(i); + srcImagePoints.push_back(cvPoint2D32f(pt2d.x, pt2d.y)); + } + + //Create the POSIT object with the model points + CvPOSITObject *positObject = cvCreatePOSITObject( &modelPoints[0], NbOfPoints ); + + //Estimate the pose + CvMatr32f rotation_matrix = new float[9]; + CvVect32f translation_vector = new float[3]; + CvTermCriteria criteria = cvTermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 100, 1.0e-4f); + cvPOSIT( positObject, &srcImagePoints[0], FOCAL_LENGTH, criteria, rotation_matrix, translation_vector ); + + //rotation_matrix to Euler angles, refer to VO_Shape::GetRotation + float sin_beta = -rotation_matrix[0 * 3 + 2]; + float tan_alpha = rotation_matrix[1 * 3 + 2] / rotation_matrix[2 * 3 + 2]; + float tan_gamma = rotation_matrix[0 * 3 + 1] / rotation_matrix[0 * 3 + 0]; + + //Project the model points with the estimated pose + oShape2D = tempShape2D; + for ( unsigned int i=0; i < NbOfPoints; ++i ) + { + pt3d.x = rotation_matrix[0] * modelPoints[i].x + + rotation_matrix[1] * modelPoints[i].y + + rotation_matrix[2] * modelPoints[i].z + + translation_vector[0]; + pt3d.y = rotation_matrix[3] * modelPoints[i].x + + rotation_matrix[4] * modelPoints[i].y + + rotation_matrix[5] * modelPoints[i].z + + translation_vector[1]; + pt3d.z = rotation_matrix[6] * modelPoints[i].x + + rotation_matrix[7] * modelPoints[i].y + + rotation_matrix[8] * modelPoints[i].z + + translation_vector[2]; + if ( pt3d.z != 0 ) + { + pt2d.x = FOCAL_LENGTH * pt3d.x / pt3d.z; + pt2d.y = FOCAL_LENGTH * pt3d.y / pt3d.z; + } + oShape2D.SetA2DPoint(pt2d, i); + } + + //return Euler angles + vector pos(3); + pos[0] = atan(tan_alpha); // yaw + pos[1] = asin(sin_beta); // pitch + pos[2] = atan(tan_gamma); // roll + return pos; +} + + +float CRecognitionAlgs::CalcFaceRoll(const vector& iLine) +{ + float roll = 0.0f; + + if(iLine[1] < FLT_MIN) roll = 0.0f; + else + { + float temp = atan ( iLine[0] / iLine[1] ) * safeDoubleToFloat(180.0 / CV_PI); + roll = temp >= 0.0f ? - (90.0f - temp ) : (90.0f + temp); + } + return roll; +} + + +float CRecognitionAlgs::CalcFaceYaw(const vector& iLine, + const VO_Shape& iShape, + const VO_FaceParts& iFaceParts) +{ + float yaw = 0.0f; + int dim = iShape.GetNbOfDim(); + + // Theoretically, using eye corner is correct, but it's not stable at all. Therefore, here we use COG_left and COG_right instead. + /////////////////////////////////////////////////////////////////////////////// + // float leftDist = 0.0f, rightDist = 0.0f; + // vector eyeCornerPoints = iFaceParts.GetEyeCornerPoints().GetIndexes(); + // Point2f leftmostEyeCorner = Point2f(FLT_MAX, 0.0f); + // Point2f rightmostEyeCorner = Point2f(0.0f, 0.0f); + // + // for(unsigned int i = 0; i < eyeCornerPoints.size(); ++i) + // { + // if(leftmostEyeCorner.x > iShape.GetAShape(dim*eyeCornerPoints[i]) ) + // { + // leftmostEyeCorner.x = iShape.GetAShape(dim*eyeCornerPoints[i]); + // leftmostEyeCorner.y = iShape.GetAShape(dim*eyeCornerPoints[i]+1); + // } + // if(rightmostEyeCorner.x < iShape.GetAShape(dim*eyeCornerPoints[i]) ) + // { + // rightmostEyeCorner.x = iShape.GetAShape(dim*eyeCornerPoints[i]); + // rightmostEyeCorner.y = iShape.GetAShape(dim*eyeCornerPoints[i]+1); + // } + // } + // leftDist = cvDistFromAPoint2ALine2D(leftmostEyeCorner, iLine); + // rightDist = cvDistFromAPoint2ALine2D(rightmostEyeCorner, iLine); + // float r = leftDist/rightDist; + // Refer to my PhD dissertation. Chapter 4 + // yaw = atan ( ( 0.65*(r-1) ) / ( 0.24 * (r+1) ) ) * 180.0f / CV_PI; + /////////////////////////////////////////////////////////////////////////////// + + float leftDist = 0.0f, rightDist = 0.0f; + vector leftSidePoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::LEFTSIDEPOINTS).GetIndexes(); + vector rightSidePoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::RIGHTSIDEPOINTS).GetIndexes(); + for(unsigned int i = 0; i < leftSidePoints.size(); ++i) + { + leftDist += cvDistFromAPoint2ALine2D(Point2f(iShape.GetAShape(dim*leftSidePoints[i]), iShape.GetAShape(dim*leftSidePoints[i]+1)), iLine); + } + for(unsigned int i = 0; i < rightSidePoints.size(); ++i) + { + rightDist += cvDistFromAPoint2ALine2D(Point2f(iShape.GetAShape(dim*rightSidePoints[i]), iShape.GetAShape(dim*rightSidePoints[i]+1)), iLine); + } + + float r = leftDist/rightDist; + // Refer to my PhD dissertation. Chapter 4 + // yaw = atan ( ( 0.65*(r-1) ) / ( 0.24 * (r+1) ) ) * 180.0f / CV_PI; + yaw = atan ( ( (r-1) ) / ((r+1) ) ) * safeDoubleToFloat(180.0 / CV_PI); + + return yaw; +} + + +// Refer to my PhD thesis, chapter 4 +float CRecognitionAlgs::CalcFacePitch( const VO_Shape& iShape, + const VO_FaceParts& iFaceParts) +{ + float pitch = 0.0f; + int dim = iShape.GetNbOfDim(); + float NNQ, ENQ, EQ, NO; + + // Theoretically, using eye corner is correct, but it's not quite stable at all. It's better we use two nostrils first if nostirl is defined in faceparts + /////////////////////////////////////////////////////////////////////////////// + // unsigned int nosetipBottom = 0; + // vector nosePoints = iFaceParts.GetNose().GetIndexes(); + // vector midlinePoints = iFaceParts.GetMidlinePoints().GetIndexes(); + // vector pitchAxisPoints = iFaceParts.GetPitchAxisLinePoints().GetIndexes(); + // VO_Shape nose, midLine, pitchAxis; + // nose.SetDim(dim); + // midLine.SetDim(dim); + // pitchAxis.SetDim(dim); + // nose.SetSize( nosePoints.size()*dim ); + // midLine.SetSize( midlinePoints.size()*dim ); + // pitchAxis.SetSize(pitchAxisPoints.size()*dim ); + // + // for(unsigned int i = 0; i < nosePoints.size(); ++i) + // { + // for(unsigned int j = 0; j < midlinePoints.size(); ++j) + // { + // if(nosePoints[i] == midlinePoints[j]) + // { + // nosetipBottom = nosePoints[i]; + // break; + // } + // } + // } + // + // Point2f ntPoint = Point2f(iShape.GetAShape(dim*nosetipBottom), iShape.GetAShape(dim*nosetipBottom+1)); + // Point2f paPoint1 = Point2f(iShape.GetAShape(dim*pitchAxisPoints[0]), iShape.GetAShape(dim*pitchAxisPoints[0]+1)); + // Point2f paPoint2 = Point2f(iShape.GetAShape(dim*pitchAxisPoints[1]), iShape.GetAShape(dim*pitchAxisPoints[1]+1)); + // + // float NNQ = ( (ntPoint.y - paPoint1.y) + (ntPoint.y - paPoint2.y) ) / 2.0f; + // float ENQ = fabs(ntPoint.x - paPoint1.x) > fabs(paPoint2.x - ntPoint.x) ? fabs(ntPoint.x - paPoint1.x) : fabs(paPoint2.x - ntPoint.x); + // float EQ = sqrt(ENQ*ENQ + NNQ*NNQ); + // float NO = sqrt(2.0f)/2.0f*EQ; + /////////////////////////////////////////////////////////////////////////////// + + vector nostrilPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::NOSTRIL).GetIndexes(); + if(nostrilPoints.size() != 0) + { + vector pitchAxisPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::PITCHAXISLINEPOINTS).GetIndexes(); + + Point2f ntPoint1 = Point2f(iShape.GetAShape(dim*nostrilPoints[0]), iShape.GetAShape(dim*nostrilPoints[0]+1)); + Point2f ntPoint2 = Point2f(iShape.GetAShape(dim*nostrilPoints[1]), iShape.GetAShape(dim*nostrilPoints[1]+1)); + Point2f paPoint1 = Point2f(iShape.GetAShape(dim*pitchAxisPoints[0]), iShape.GetAShape(dim*pitchAxisPoints[0]+1)); + Point2f paPoint2 = Point2f(iShape.GetAShape(dim*pitchAxisPoints[1]), iShape.GetAShape(dim*pitchAxisPoints[1]+1)); + + NNQ = ( (ntPoint1.y - paPoint1.y) + (ntPoint2.y - paPoint2.y) ) / 2.0f; + ENQ = fabs(ntPoint1.x - paPoint1.x) > fabs(paPoint2.x - ntPoint2.x) ? fabs(ntPoint1.x - paPoint1.x + (ntPoint2.x - ntPoint1.x) / 2.0f) : fabs(paPoint2.x - ntPoint2.x + (ntPoint2.x - ntPoint1.x) / 2.0f); + EQ = sqrt(ENQ*ENQ + NNQ*NNQ); + NO = sqrt(2.0f)/2.0f*EQ; + } + else + { + unsigned int nosetipBottom = 0; + vector nosePoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::NOSE).GetIndexes(); + vector midlinePoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::MIDLINEPOINTS).GetIndexes(); + vector pitchAxisPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::PITCHAXISLINEPOINTS).GetIndexes(); + + for(unsigned int i = 0; i < nosePoints.size(); ++i) + { + for(unsigned int j = 0; j < midlinePoints.size(); ++j) + { + if(nosePoints[i] == midlinePoints[j]) + { + nosetipBottom = nosePoints[i]; + break; + } + } + } + + Point2f ntPoint = Point2f(iShape.GetAShape(dim*nosetipBottom), iShape.GetAShape(dim*nosetipBottom+1)); + Point2f paPoint1 = Point2f(iShape.GetAShape(dim*pitchAxisPoints[0]), iShape.GetAShape(dim*pitchAxisPoints[0]+1)); + Point2f paPoint2 = Point2f(iShape.GetAShape(dim*pitchAxisPoints[1]), iShape.GetAShape(dim*pitchAxisPoints[1]+1)); + + NNQ = ( (ntPoint.y - paPoint1.y) + (ntPoint.y - paPoint2.y) ) / 2.0f; + ENQ = fabs(ntPoint.x - paPoint1.x) > fabs(paPoint2.x - ntPoint.x) ? fabs(ntPoint.x - paPoint1.x) : fabs(paPoint2.x - ntPoint.x); + EQ = sqrt(ENQ*ENQ + NNQ*NNQ); + NO = sqrt(2.0f)/2.0f*EQ; + } + + if( fabs(NNQ/NO) < 1.0f) + pitch = asin ( NNQ / NO ) * safeDoubleToFloat(180.0 / CV_PI); + else if (NNQ * NO < 0.0f) + pitch = -90.0f; + else + pitch = 90.0f; + + return pitch; +} + + +void CRecognitionAlgs::CalcFittedFaceAngle2D( vector& angles, + const VO_Shape& iShape, + const VO_FaceParts& iFaceParts) +{ + angles.resize(3); + + VO_Shape tempShape; + + // float facewidth = iShape.GetWidth(); + // int dim = iShape.GetDim(); + // vector eyeCornerPoints = iFaceParts.GetEyeCornerPoints().GetIndexes(); + // VO_Shape eyeCorner; + // eyeCorner.SetDim(dim); + // eyeCorner.SetSize( eyeCornerPoints.size()*dim ); + // for(unsigned int i = 0; i < eyeCornerPoints.size(); i = ++i) + // { + // for(unsigned int j = 0; j < dim; j++) + // eyeCorner.SetAShape( iShape.GetAShape(dim*eyeCornerPoints[i]+j), dim*i+j ); + // } + // //cout << iShape << endl; + // cout << iShape.GetAShape(eyeCornerPoints[0]*dim) << " " << iShape.GetAShape(eyeCornerPoints[0]*dim + 1) << endl; + // cout << iShape.GetAShape(eyeCornerPoints[1]*dim) << " " << iShape.GetAShape(eyeCornerPoints[1]*dim + 1) << endl; + // cout << iShape.GetAShape(eyeCornerPoints[2]*dim) << " " << iShape.GetAShape(eyeCornerPoints[2]*dim + 1) << endl; + // cout << iShape.GetAShape(eyeCornerPoints[3]*dim) << " " << iShape.GetAShape(eyeCornerPoints[3]*dim + 1) << endl; + // cout << eyeCorner << endl; + // float eyewidth = eyeCorner.GetWidth(); + // float ratio = eyewidth/facewidth; + + + vector midline, eyecorner; + VO_KeyPoint::CalcFaceKeyline(midline, iShape, iFaceParts, tempShape, VO_FacePart::MIDLINEPOINTS); + VO_KeyPoint::CalcFaceKeyline(eyecorner, iShape, iFaceParts, tempShape, VO_FacePart::EYECORNERPOINTS); + angles[2] = CRecognitionAlgs::CalcFaceRoll(midline); + angles[1] = CRecognitionAlgs::CalcFaceYaw(midline, iShape, iFaceParts); + angles[0] = CRecognitionAlgs::CalcFacePitch(iShape, iFaceParts); + + // cout << "Pitch = " << angles[0] << endl; + // cout << "Yaw = " << angles[1] << endl; + // cout << "Roll = " << angles[2] << endl; +} + + +void CRecognitionAlgs::CalcFittedFaceAngle3D( vector& angles, + const VO_Shape& iShape, + const VO_FaceParts& iFaceParts) +{ + // CvSeq* point_seq = cvCreateSeq( CV_32FC3, sizeof(CvSeq), sizeof(CvPoint3D32f), storage ); + // for(unsigned int i = 0; i < midlinePoints.size(); ++i) + // { + // cvSeqPush(point_seq, &cvPoint3D32f(iShape.GetAShape(dim*midlinePoints[i]), iShape.GetAShape(dim*midlinePoints[i]+1), iShape.GetAShape(dim*midlinePoints[i] + 2)) ); + // } + // // cvFitPlane( point_seq, CV_DIST_L2, 0, 0.001, 0.001, line ); + // cvClearSeq(point_seq); +} + + +// #include "opencv/cv.h" +// #include +// #include +// float myLinearity(CvSeq *); +// int main(void) +// { +// int i; +// double fx[] = {0.0, 0.301, 0.477, 0.602, 0.699, 0.778, 0.845, 0.903, 0.954, 1.0}; +// double fy[] = {3.874, 3.202, 2.781, 2.49, 2.274, 2.156, 1.934, 1.74, 1.653, 1.662}; +// float *line = new float[4]; +// float linearity=0.0f; +// //入れ物の確保 +// CvMemStorage* storage = cvCreateMemStorage(0); +// //3次元の場合はCV_32FC2がCV_32FC3に、Point2fがCvPoint3D32fになる +// CvSeq* point_seq = cvCreateSeq( CV_32FC2, sizeof(CvSeq), sizeof(Point2f), storage ); +// for (i=0; i<10; i++){ +// //値の追加はcvSeqPush +// cvSeqPush(point_seq, &Point2f(fx[i],fy[i])); +// } +// linearity = myLinearity(point_seq); +// cvFitLine(point_seq,CV_DIST_L2,0,0.01,0.01,line); +// fprintf(stdout,"v=(%f,%f),vy/vx=%f,(x,y)=(%f,%f), Linearity=%f\n",line[0],line[1],line[1]/line[0],line[2],line[3],linearity); +// cvClearSeq(point_seq); +// cvReleaseMemStorage(&storage); +// return 0; +// } +// //大津の直線度 +// float myLinearity(CvSeq *seq) +// { +// int i; +// Point2f *p; +// float *x = new float[seq->total]; +// float *y = new float[seq->total]; +// float x_bar=0.0, y_bar=0.0; +// float u11=0.0, u20=0.0, u02=0.0; +// float linearity=0.0; +// //吸い出し cvGetSeqElemでポインタを得るのでキャストして取得 +// for (i=0; i < seq->total; i++){ +// p=(Point2f*)cvGetSeqElem(seq,i); +// x[i]=p->x; +// y[i]=p->y; +// } +// //x_bar, y_bar +// for (i=0; i < seq->total; i++){ +// x_bar+=x[i]; +// y_bar+=y[i]; +// } +// x_bar/=seq->total; +// y_bar/=seq->total; +// //セントラルモーメント +// for (i=0; i < seq->total; i++){ +// u11+=((x[i]-x_bar)*(y[i]-y_bar)); +// u20+=pow(x[i]-x_bar,2.0f); +// u02+=pow(y[i]-y_bar,2.0f); +// } +// u11/=seq->total; +// u20/=seq->total; +// u02/=seq->total; +// //直線度の算出 +// linearity = sqrt(4*pow(u11,2.0f)+pow(u20-u02,2.0f))/(u20+u02); +// return linearity; +// } diff --git a/modules/comalgs/src/VO_TrackingAlgs.cpp b/modules/comalgs/src/VO_TrackingAlgs.cpp new file mode 100644 index 0000000..05282e2 --- /dev/null +++ b/modules/comalgs/src/VO_TrackingAlgs.cpp @@ -0,0 +1,353 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_TrackingAlgs.h" + + +//int CTrackingAlgs::histSize[] = {hbins, sbins}; // histSize +int CTrackingAlgs::histSize[] = {hbins}; // histSize +float CTrackingAlgs::hranges[] = { 0, 180 }; // hue varies from 0 to 179, see cvtColor +float CTrackingAlgs::sranges[] = { 0, 256 }; // saturation varies from 0 (black-gray-white) to 255 (pure spectrum color) +//const float* CTrackingAlgs::ranges[] = { hranges, sranges }; // ranges +const float* CTrackingAlgs::ranges[] = { hranges }; // ranges +int CTrackingAlgs::channels[] = {0}; + + +/** +* @brief Initialize tracker +*/ +void CTrackingAlgs::UpdateTracker(const Mat& img, const Rect& obj) +{ + switch(this->m_iTrackingMethod) + { + case CAMSHIFT: + { + this->m_bTrackerInitialized = + CTrackingAlgs::CamshiftUpdateTracker(img, obj, this->m_hist); + } + break; + case KALMANFILTER: + { + } + break; + case PARTICLEFILTER: + { + } + break; + case ASMAAM: + { + } + break; + case NONE: + default: + { + this->m_bTrackerInitialized = false; + } + break; + } +} + + +bool CTrackingAlgs::CamshiftUpdateTracker( const Mat& img, + const Rect& obj, + MatND& hist) +{ + Mat hsv, hue, mask, backproject; + cv::cvtColor( img, hsv, CV_BGR2HSV ); + + int _vmin = CTrackingAlgs::vmin, _vmax = CTrackingAlgs::vmax; + + cv::inRange( hsv, Scalar(0,CTrackingAlgs::smin,MIN(_vmin,_vmax),0), + Scalar(180,256,MAX(_vmin,_vmax),0), mask ); + vector vhsv(3); + cv::split( hsv, vhsv ); + vhsv[0].copyTo(hue); + + double max_val = 0.f; + Mat roi = hue( Range(obj.y, obj.y+obj.height), + Range(obj.x, obj.x+obj.width) ); + Mat roi_mask = mask( Range(obj.y, obj.y+obj.height), + Range(obj.x, obj.x+obj.width) ); + cv::calcHist( &roi, 1, CTrackingAlgs::channels, roi_mask, + hist, 1, CTrackingAlgs::histSize, CTrackingAlgs::ranges, + true, // the histogram is uniform + false ); + cv::minMaxLoc(hist, 0, &max_val, 0, 0); + hist.convertTo(hist, hist.type(), (max_val ? 255. / max_val : 0.), 0); + + return true; +} + + +/** +* @author JIA Pei +* @version 2009-10-04 +* @brief Object Tracking +* @param obj Input - object to be tracked +* @param img Input - image to be searched within +* @param smallSize Input - smallSize +* @param bigSize Input - bigSize +* @return detection time cost +*/ +double CTrackingAlgs::Tracking( Rect& obj, + const Mat& img, + Size smallSize, + Size bigSize) +{ + double res = (double)cvGetTickCount(); + + if(this->m_bTrackerInitialized) + { + switch(this->m_iTrackingMethod) + { + case CAMSHIFT: + { + CTrackingAlgs::CamshiftTracking(obj, + img, + this->m_hist, + this->m_bObjectTracked, + smallSize, + bigSize); + this->m_CVTrackedObjectRect = obj; + } + break; + case KALMANFILTER: + { + CTrackingAlgs::KalmanTracking( obj, + img, + this->m_bObjectTracked, + smallSize, + bigSize); + this->m_CVTrackedObjectRect = obj; + } + break; + case PARTICLEFILTER: + { + CTrackingAlgs::ParticleFilterTracking( + obj, + img, + this->m_bObjectTracked, + smallSize, + bigSize); + this->m_CVTrackedObjectRect = obj; + } + break; + case ASMAAM: + { + CTrackingAlgs::ASMAAMTracking( obj, + img, + this->m_bObjectTracked, + smallSize, + bigSize); + this->m_CVTrackedObjectRect = obj; + } + break; + case NONE: + default: + { + this->m_bObjectTracked = false; + this->m_bTrackerInitialized = false; + } + break; + } + } + else + { + this->UpdateTracker(img, obj); + } + + res = ((double)cvGetTickCount() - res) + / ((double)cvGetTickFrequency()*1000.); + return res; +} + + +/** +* @author JIA Pei +* @version 2010-02-02 +* @brief Camshift Tracking +* @param obj Input - object to be tracked +* @param img Input - image to be searched within +* @param isTracked output - is this obj tracked? +* @param smallSize Input - the smallest possible object size +* @param bigSize Input - the biggest possible object size +* @return detection time cost +*/ +double CTrackingAlgs::CamshiftTracking( Rect& obj, + const Mat& img, + MatND& hist, + bool& isTracked, + Size smallSize, + Size bigSize) +{ + double res = (double)cvGetTickCount(); + + if(obj.x <= 0) obj.x = 0; + if(obj.y <= 0) obj.y = 0; + if(obj.x + obj.width > img.cols) obj.width = img.cols - obj.x; + if(obj.y + obj.height > img.rows) obj.height = img.rows - obj.y; + + Rect trackwindow = obj; + Mat hsv, hue, mask, backproject; + cv::cvtColor( img, hsv, CV_BGR2HSV ); + + int _vmin = CTrackingAlgs::vmin, _vmax = CTrackingAlgs::vmax; + + cv::inRange( hsv, Scalar(0,CTrackingAlgs::smin,MIN(_vmin,_vmax),0), + Scalar(180,256,MAX(_vmin,_vmax),0), mask ); + vector vhsv(3); + cv::split( hsv, vhsv ); + vhsv[0].copyTo(hue); + + cv::calcBackProject( &hue, 1, CTrackingAlgs::channels, hist, backproject, + CTrackingAlgs::ranges); + cv::bitwise_and( backproject, mask, backproject ); + RotatedRect trackbox = CamShift( backproject, trackwindow, + TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, + 10, 1) ); + obj = trackwindow; + + // cv::ellipse(img, trackbox, CV_RGB(255,0,0), 3, CV_AA); + + Point pt1 = Point( (int)(obj.x), (int)(obj.y) ); + Point pt2 = Point( (int)(obj.x + obj.width), + (int)(obj.y + obj.height) ); + + // Judge whether it is losing the object or not... + if(obj.width >= bigSize.width + || obj.height >= bigSize.height + || obj.width <= smallSize.width + || obj.height <= smallSize.height + || pt1.x < FRAMEEDGE + || pt1.y < FRAMEEDGE + || (pt2.x > (img.cols - FRAMEEDGE)) + || (pt2.y > (img.rows - FRAMEEDGE))) + { + isTracked = false; + obj.x = obj.y = obj.width = obj.height = -1; + } + else + isTracked = true; + + res = ((double)cvGetTickCount() - res) + / ((double)cvGetTickFrequency()*1000.); + return res; +} + + +double CTrackingAlgs::KalmanTracking( Rect& obj, + const Mat& img, + bool& isTracked, + Size smallSize, + Size bigSize) +{ + double res = (double)cvGetTickCount(); + + res = ((double)cvGetTickCount() - res) + / ((double)cvGetTickFrequency()*1000.); + return res; +} + + +double CTrackingAlgs::ParticleFilterTracking( Rect& obj, + const Mat& img, + bool& isTracked, + Size smallSize, + Size bigSize) +{ + double res = (double)cvGetTickCount(); + + res = ((double)cvGetTickCount() - res) + / ((double)cvGetTickFrequency()*1000.); + return res; +} + + +double CTrackingAlgs::ASMAAMTracking( Rect& obj, + const Mat& img, + bool& isTracked, + Size smallSize, + Size bigSize) +{ + double res = (double)cvGetTickCount(); + + res = ((double)cvGetTickCount() - res) + / ((double)cvGetTickFrequency()*1000.); + return res; +} + + +void CTrackingAlgs::VO_DrawTracking(Mat& ioImg, Scalar color) +{ + Rect curRect; + Point lefttop, rightbottom; + + if ( this->m_bObjectTracked ) + { + curRect = this->m_CVTrackedObjectRect; + lefttop.x = cvRound(curRect.x); + lefttop.y = cvRound(curRect.y); + rightbottom.x = cvRound((curRect.x+curRect.width)); + rightbottom.y = cvRound((curRect.y+curRect.height)); + cv::rectangle(ioImg, lefttop, rightbottom, color, 2, 8, 0); + } +} + diff --git a/modules/common/CMakeLists.txt b/modules/common/CMakeLists.txt new file mode 100644 index 0000000..a8f4f6f --- /dev/null +++ b/modules/common/CMakeLists.txt @@ -0,0 +1,2 @@ +set(deps opencv_core opencv_imgproc opencv_highgui) +define_vosm_module(utils ${deps}) diff --git a/modules/common/common.vcxproj b/modules/common/common.vcxproj new file mode 100644 index 0000000..2b60b98 --- /dev/null +++ b/modules/common/common.vcxproj @@ -0,0 +1,83 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {6DC6813E-859A-4759-94CD-A1F1A6466B7B} + common + Svn + Svn + Svn + SubversionScc + + + + StaticLibrary + true + MultiByte + v110 + + + Application + false + true + MultiByte + v110 + + + + + + + + + + + + + + + Level3 + Disabled + ./include;../include;C:\boost\boost_1_53_0;%(AdditionalIncludeDirectories) + NOMINMAX;%(PreprocessorDefinitions) + + + true + + + true + + + + + Level3 + MaxSpeed + true + true + + + true + true + true + + + + + + + + + + + + + \ No newline at end of file diff --git a/modules/common/common.vcxproj.filters b/modules/common/common.vcxproj.filters new file mode 100644 index 0000000..429b597 --- /dev/null +++ b/modules/common/common.vcxproj.filters @@ -0,0 +1,30 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Header Files + + + Header Files + + + + + Source Files + + + \ No newline at end of file diff --git a/modules/common/include/VO_Common.h b/modules/common/include/VO_Common.h new file mode 100644 index 0000000..b10030a --- /dev/null +++ b/modules/common/include/VO_Common.h @@ -0,0 +1,750 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-03-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_COMMON_H__ +#define __VO_COMMON_H__ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +//#include +//#include +#include +using namespace boost::filesystem; + + +using namespace std; + +#define INF DBL_MAX + +#define ROW 1 +#define COL 2 + + +////////////////////////////////////////////////////////////////////////// +#define AVNONE 0 +#define AVAUDIO 1 +#define AVVIDEO 2 +#define AVAUDIOVIDEO 3 + +#define SYNCEVEN 1 +#define SYNCTIMESTAMP 2 + +#define INTER_NONSYNC 1 +#define INTER_SYNCAUDIO 2 +#define INTER_SYNCVIDEO 3 + +#define AUDIO_MFCC 1 +#define AUDIO_RASTA 2 +#define AUDIO_PLP 3 +#define AUDIO_TECC 4 // Teager Energy + + +#define SR_LINEAR 1 + +#define MFCC_FRAMEOVERLAPCUT 1 +#define MFCC_FRAMEOVERLAPACCUMULATE 2 + +#define VAD_FRAMEENERGYFREQUENCY 1 +#define VAD_SPECTRALENTROPYWITHENERGY 2 +#define VAD_AUTOCORRELATION 3 +#define VAD_WAVELET 4 +#define VAD_SPEEX 10 + +#define DTW_FULL 1 +#define DTW_GC_SakoeChibaBand 1 +#define DTW_GC_ItakuraParallelogram 2 + +#define LB_KEOGH 1 +#define LB_YI 2 +#define LB_KIM 3 + +#define CLAMP 1 +#define STRETCH 2 + +// For detected face or tracking face constrain +#define FACESMALLESTSIZE 80 // for both detection and tracking +#define FACEBIGGESTSIZE 240 // for both detection and tracking +#define FACEPARTSMALLESTSIZE 16 +#define FACEPARTBIGGESTSIZE 128 +#define FRAMEEDGE 5 // if too close the image boundary, look on as lost + +#define AAMADABOOSTDOWNPERCENTAGE 0.2 + +#define FACEHEIGHT2WIDTH 1.1 + +#define SHAPE 1 +#define TEXTURE 2 + +// a) Single image b) image sequence c) video d) webcam +#define NOTHINGLOADED 0 +#define SINGLEIMAGE 1 +#define IMAGESEQUENCE 2 +#define CAM 3 +#define AVI 4 + + +#define HORIZONTAL 1 +#define VERTICAL 2 +#define ANY 3 + +#define GRAYCHANNELS 1 +#define COLORCHANNELS 3 // R G B 3 channels + +#define DIRECTMAP 1 +#define LINEARIZEMAP 2 + + +/** video show type */ +#define ORIGINAL 0 +#define DETECTED 1 +#define FIT 2 + +#define MEAN0NORM1 1 +#define VARY01 2 + + +#define STRING_MAXSIZE 1024 + +#define FP_TOLERANCE 4.88e-04 +enum { + Audio_LChannel, + Audio_RChannel, + Audio_BChannel +}; + +/** Returns true if directory already exists, false if it creates the directory */ +static bool MakeDirectory(path dirPath) +{ + if (!boost::filesystem::is_directory(dirPath) ) + { + boost::filesystem::create_directory( dirPath ); + return false; + } + else + return true; +} + +static void SafeInputFileOpen(ifstream& iFile, string fname) +{ + iFile.open(fname.c_str(), ios::in); + if(!iFile.good()) + { + std::cerr << "File " << fname << " could not be opened for input." << std::endl; + exit(EXIT_FAILURE); + return; + } + return; +} +static void SafeOutputFileOpen(fstream& iFile, string fname) +{ + iFile.open(fname.c_str(), ios::out); + if(!iFile.good()) + { + std::cerr << "File " << fname << " could not be opened for input." << std::endl; + exit(EXIT_FAILURE); + return; + } + return; +} + +/** local (static) compare function for the qsort() call */ +static int str_compare( const void *arg1, const void *arg2 ) +{ + return strcmp ( ( * ( std::string* ) arg1 ).c_str (), + ( * ( std::string* ) arg2 ).c_str () ); +} + +template +float safeDoubleToFloat(const DblType &num){ + //check if float fits into double + float dest = static_cast(num); + double residual = num - static_cast(dest); + //essentially checking if less than machine precision? + if ( static_cast(residual) <= FP_TOLERANCE) + { + return static_cast(num); //safe to cast + } + else{ + //cast anyway, but print warning + cerr << "Value changed double to float:" << num << "(" << residual << ")" << endl; + return static_cast(num); + } + +} + +/** simple clamp function */ +template +static T2 clamp ( T1 input, T2 low, T2 up ) +{ + return ( input < low ? low : ( input > up ? up : ( T2 ) input ) ); +} + + +/** vector element-wise summation */ +template +static std::vector operator+ ( const std::vector& vect1, + const std::vector& vect2 ) +{ + std::vector res; + unsigned int size = vect1.size() < vect2.size() ? vect1.size() : vect2.size(); + res.resize ( size ); + for ( unsigned int i = 0; i < size; ++i ) + res[i] = vect1[i] + vect2[i]; + return res; +} + + +/** every element of a vector add the same value */ +template +static std::vector operator+ ( const std::vector& vect1, T1 value ) +{ + std::vector res; + unsigned int size = vect1.size(); + res.resize ( size ); + for ( unsigned int i = 0; i < size; ++i ) + res[i] = vect1[i] + value; + return res; +} + + +/** vector element-wise summation */ +template +static std::vector operator += ( std::vector& vect1, + const std::vector& vect2 ) +{ + unsigned int size = vect1.size() < vect2.size() ? vect1.size() : vect2.size(); + for ( unsigned int i = 0; i < size; ++i ) + vect1[i] += vect2[i]; + return vect1; +} + + +/** every element of a vector add the same value */ +template +static std::vector operator += ( std::vector& vect1, T1 value ) +{ + unsigned int size = vect1.size(); + for ( unsigned int i = 0; i < size; ++i ) + vect1[i] += value; + return vect1; +} + + +/** reverse sign */ +template +static std::vector operator- ( const std::vector& vect ) +{ + std::vector res; + unsigned int size = vect.size(); + res.resize ( size ); + for ( unsigned int i = 0; i < size; ++i ) + res[i] = -vect[i]; + return res; +} + + +/** vector element-wise subtraction */ +template +static std::vector operator- ( const std::vector& vect1, + const std::vector& vect2 ) +{ + std::vector res; + unsigned int size = vect1.size() < vect2.size() ? vect1.size() : vect2.size(); + res.resize ( size ); + for ( unsigned int i = 0; i < size; ++i ) + res[i] = vect1[i] - vect2[i]; + return res; +} + + +/** every element of a vector subtract the same value */ +template +static std::vector operator- ( const std::vector& vect1, T1 value ) +{ + std::vector res; + unsigned int size = vect1.size(); + res.resize ( size ); + for ( unsigned int i = 0; i < size; ++i ) + res[i] = vect1[i] - value; + return res; +} + + +/** vector element-wise subtraction */ +template +static std::vector operator -= ( std::vector& vect1, + const std::vector& vect2 ) +{ + unsigned int size = vect1.size() < vect2.size() ? vect1.size() : vect2.size(); + for ( unsigned int i = 0; i < size; ++i ) + vect1[i] -= vect2[i]; + return vect1; +} + + +/** every element of a vector subtract the same value */ +template +static std::vector operator -= ( std::vector& vect1, T1 value ) +{ + unsigned int size = vect1.size(); + for ( unsigned int i = 0; i < size; ++i ) + vect1[i] -= value; + return vect1; +} + + +/** vector element-wise multiplication */ +template +static std::vector operator* ( const std::vector& vect1, + const std::vector& vect2 ) +{ + std::vector res; + unsigned int size = vect1.size() < vect2.size() ? vect1.size() : vect2.size(); + res.resize ( size ); + for ( unsigned int i = 0; i < size; ++i ) + res[i] = vect1[i] * vect2[i]; + return res; +} + + +/** every element of a vector multiply the same value */ +template +static std::vector operator* ( const std::vector& vect1, T1 value ) +{ + std::vector res; + unsigned int size = vect1.size(); + res.resize ( size ); + for ( unsigned int i = 0; i < size; ++i ) + res[i] = vect1[i] * value; + return res; +} + + +/** vector element-wise multiplication */ +template +static std::vector operator *= ( std::vector& vect1, + const std::vector& vect2 ) +{ + unsigned int size = vect1.size() < vect2.size() ? vect1.size() : vect2.size(); + for ( unsigned int i = 0; i < size; ++i ) + vect1[i] *= vect2[i]; + return vect1; +} + + +/** every element of a vector multiply the same value */ +template +static std::vector operator *= ( std::vector& vect1, T1 value ) +{ + unsigned int size = vect1.size(); + for ( unsigned int i = 0; i < size; ++i ) + vect1[i] *= value; + return vect1; +} + + +/** vector element-wise division */ +template +static std::vector operator/ ( const std::vector& vect1, + const std::vector& vect2 ) +{ + std::vector res; + unsigned int size = vect1.size() < vect2.size() ? vect1.size() : vect2.size(); + res.resize ( size ); + for ( unsigned int i = 0; i < size; ++i ) + { + if (fabs((float)vect2[i]) < FLT_MIN) + { + cerr << "divide by 0 ! " << endl; + exit(EXIT_FAILURE); + } + res[i] = vect1[i] / vect2[i]; + } + return res; +} + + +/** every element of a vector multiply the same value */ +template +static std::vector operator/ ( const std::vector& vect1, T1 value ) +{ + if (fabs((float)value) < FLT_MIN) + { + cerr << "divide by 0 ! " << endl; + exit(EXIT_FAILURE); + } + std::vector res; + unsigned int size = vect1.size(); + res.resize ( size ); + for ( unsigned int i = 0; i < size; ++i ) + res[i] = vect1[i] / value; + return res; +} + + +/** vector element-wise division */ +template +static std::vector operator /= ( std::vector& vect1, + const std::vector& vect2 ) +{ + unsigned int size = vect1.size() < vect2.size() ? vect1.size() : vect2.size(); + for ( unsigned int i = 0; i < size; ++i ) + { + if (fabs((float)vect2[i]) < FLT_MIN) + { + cerr << "divide by 0 ! " << endl; + exit(EXIT_FAILURE); + } + vect1[i] /= vect2[i]; + } + return vect1; +} + + +/** every element of a vector multiply the same value */ +template +static std::vector operator /= ( std::vector& vect1, T1 value ) +{ + if (fabs((float)value) < FLT_MIN) + { + cerr << "divide by 0 ! " << endl; + exit(EXIT_FAILURE); + } + unsigned int size = vect1.size(); + for ( unsigned int i = 0; i < size; ++i ) + vect1[i] /= value; + return vect1; +} + + +///** output gsl_matrix */ +//static std::ostream& operator<< (std::ostream &os, const gsl_matrix* gslm) +//{ +// for (unsigned int i = 0; i < gslm->size1; ++i) +// { +// for(unsigned int j = 0; j < gslm->size2; ++j) +// { +// os << gsl_matrix_get (gslm, i, j) << " "; +// } +// os << std::endl; +// } +// return os; +//} +// +// +///** input gsl_matrix */ +//static std::istream& operator>> ( std::istream &is, gsl_matrix* gslm) +//{ +// for ( unsigned int i = 0; i < gslm->size1; ++i ) +// { +// for ( unsigned int j = 0; j < gslm->size2; ++j ) +// { +// is >> gslm->data[i * gslm->tda + j]; +// } +// } +// return is; +//} + + +template +static std::ostream& operator<< ( std::ostream &os, const std::vector& vec ) +{ + unsigned int size = vec.size(); + for ( unsigned int i = 0; i < size; i++ ) + { + os << vec[i] << std::endl; + } + return os; +} + + +template +static std::istream& operator>> ( std::istream &is, std::vector& vec ) +{ + unsigned int size = vec.size(); + for ( unsigned int i = 0; i < size; i++ ) + { + is >> vec[i]; + } + return is; +} + + +template +static std::ostream& operator<< ( std::ostream &os, + const std::vector< std::vector >& vec ) +{ + unsigned int row = vec.size(); + if(row !=0) + { + unsigned int col = vec[0].size(); + for ( unsigned int i = 0; i < row; i++ ) + { + for ( unsigned int j = 0; j < col; j++ ) + { + os << vec[i][j] << " "; + } + os << std::endl; + } + } + return os; +} + + +template +static std::istream& operator>> ( std::istream &is, + std::vector< std::vector >& vec ) +{ + unsigned int row = vec.size(); + if(row !=0) + { + unsigned int col = vec[0].size(); + for ( unsigned int i = 0; i < row; i++ ) + { + for ( unsigned int j = 0; j < col; j++ ) + { + is >> vec[i][j]; + } + } + } + return is; +} + + +/** static template function to judge whether the vector contains a value */ +template +static bool IsContaining ( std::vector v, T t ) +{ + unsigned int size = v.size(); + for ( unsigned int i = 0; i < size; i++ ) + { + if ( v[i] == t ) + return true; + } + + return false; +} + + +/** static template function to judge whether + the vector contains another vector */ +template +static bool IsContaining ( std::vector v, std::vector t ) +{ + unsigned int size = t.size(); + for ( unsigned int i = 0; i < size; i++ ) + { + if ( !IsContaining ( v, t[i] ) ) + return false; + } + + return true; +} + + +static int sign(double in) +{ + if (fabs(in) <= DBL_MIN ) return 0; + else if (in > DBL_MIN ) return 1; + else return -1; +} + + +static string StandardizeFN(unsigned int idx, + unsigned int totalNumber = 100000, + const string& prefix="", + const string& suffix=".jpg") +{ + string res = prefix; + stringstream ssidx, sstotal; + string stridx, strtotal; + ssidx << idx; + ssidx >> stridx; + sstotal << totalNumber; + sstotal >> strtotal; + + for(unsigned int i = 0; i < strtotal.length() - stridx.length(); ++i ) + { + res += "0"; + } + + res+=stridx; + res+=suffix; + + ssidx.clear(); + sstotal.clear(); + + return res; +} + + +static string getSystemTimeInString(bool year = true, + bool mon = true, + bool mday = true, + bool hour = false, + bool min = false, + bool sec = false) +{ + string res = ""; + stringstream ss; + string s_year, s_mon, s_mday, s_hour, s_min, s_sec; + + time_t rawtime; + struct tm * timeinfo = NULL; + time ( &rawtime ); + +#ifdef _WIN32 + localtime_s(timeinfo,&rawtime); +#else + timeinfo = localtime ( &rawtime ); +#endif + if(!timeinfo) + return ""; + + if(year) + { + ss << (timeinfo->tm_year+1900); + ss >> s_year; res += s_year; + ss.clear(); + } + if(mon) + { + ss << (timeinfo->tm_mon+1); ss >> s_mon; + if(timeinfo->tm_mon < 10) + res += "0" + s_mon; + else + res += s_mon; + ss.clear(); + } + if(mday) + { + ss << timeinfo->tm_mday; ss >> s_mday; + if(timeinfo->tm_mday < 10) + res += "0" + s_mday; + else + res += s_mday; + ss.clear(); + } + if(hour) + { + ss << timeinfo->tm_hour; + ss >> s_hour; + res += s_hour; + ss.clear(); + } + if(min) + { + ss << timeinfo->tm_min; + ss >> s_min; + res += s_min; + ss.clear(); + } + if(sec) + { + ss << timeinfo->tm_sec; + ss >> s_sec; + res += s_sec; + ss.clear(); + } + + return res; +} + + +static void SaveData( const double * d, + unsigned int datasize, + const string& fn) +{ + ofstream fp; + fp.open(fn.c_str ()); + fp << "# Coordinates" << endl; + for(unsigned int i = 0; i < datasize; ++i) + { + fp << i << " " << d[i] << endl; + } + fp.close(); +} + + +static void SaveData( const vector& d, + const string& fn) +{ + ofstream fp; + fp.open(fn.c_str ()); + fp << "# Coordinates" << endl; + for(unsigned int i = 0; i < d.size(); ++i) + { + fp << i << " " << d[i] << endl; + } + fp.close(); +} + +#endif // __VO_COMMON_H__ + diff --git a/modules/common/include/VO_ScanFilesInDir.h b/modules/common/include/VO_ScanFilesInDir.h new file mode 100644 index 0000000..f733946 --- /dev/null +++ b/modules/common/include/VO_ScanFilesInDir.h @@ -0,0 +1,102 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-03-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_SCANFILEINDIR_H__ +#define __VO_SCANFILEINDIR_H__ + +#include +#include // for std::cout +#include + +#include "VO_Common.h" + + + +/** + * @author JIA Pei + * @brief IO for file scanning. + */ +class VO_IO +{ +public: + + // scan all files with the "file_extension" under "dir_path" directory + // and sort them + static bool ScanDirectory( const path &dir_path, + const string& file_extension, + vector& files); + + // scan all annotation files + static vector ScanNSortAnnotationInDirectory(const string& dir_path); + + // scan all image files + static vector ScanNSortImagesInDirectory(const string& dir_path); + + // scale all YML files + static vector ScanNSortXMLYMLsInDirectory(const string& dir_path); + + // scale all text files + static vector ScanNSortTextFilesInDirectory(const string& dir_path); + +}; + +#endif // __VO_SCANFILEINDIR_H__ + diff --git a/modules/common/src/VO_ScanFilesInDir.cpp b/modules/common/src/VO_ScanFilesInDir.cpp new file mode 100644 index 0000000..c3a2b46 --- /dev/null +++ b/modules/common/src/VO_ScanFilesInDir.cpp @@ -0,0 +1,288 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-03-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include +#include +#include +#include + +#include "VO_ScanFilesInDir.h" + + + +/** + * @author JIA Pei + * @version 2007-08-08 + * @brief Scan a directory and sort all files with a specified extension + * @param dir_path input parameter, path to read from + * @param file_extension input parameter, file extension to search for + * @param files output parameter, file names found without any path + * @return bool fail to scan or not + * @note http://www.boost.org/libs/filesystem/doc/index.htm +*/ +bool VO_IO::ScanDirectory( const path &dir_path, + const string &file_extension, + vector& files) +{ + + if ( !exists( dir_path ) ) + { + return false; + } + + directory_iterator end_itr; + + for ( directory_iterator itr(dir_path); itr != end_itr; ++itr ) + { + if ( is_directory(itr->status()) ) + { + VO_IO::ScanDirectory(itr->path(), file_extension, files); + } + else if ( extension( itr->path() ) == file_extension ) + { + files.push_back( itr->path().string() ); + } + } + + return true; +} + + +/** + * @author JIA Pei + * @version 2007-08-08 + * @brief Scan a directory and sorts all file in format of .pts or .asf + * @param dir_path input parameter, path to read from + * @return vector A vector of all scanned file names +*/ +vector VO_IO::ScanNSortAnnotationInDirectory(const string &dir_path) +{ + vector annotationALL; + vector< vector > annotations; + annotations.resize(4); + for(unsigned int i = 0; i < annotations.size(); i++) + { + annotations[i].resize(0); + } + + path tempp = path(dir_path); + + // 3 types, pts, asf, obj, wrl + // Here, look on Lightwave .obj and VRML .wrl as annotations + VO_IO::ScanDirectory( tempp, ".pts", annotations[0]); + VO_IO::ScanDirectory( tempp, ".asf", annotations[1]); + VO_IO::ScanDirectory( tempp, ".obj", annotations[2]); + VO_IO::ScanDirectory( tempp, ".wrl", annotations[3]); + + for(unsigned int i = 0; i < annotations.size(); i++) + { + for(unsigned int j = 0; j < annotations[i].size(); j++) + { + annotationALL.push_back(annotations[i][j]); + } + } + + // sort the filenames + if(annotationALL.size() > 0) + { + qsort( (void *)&(annotationALL[0]), + (size_t)annotationALL.size(), + sizeof(string), + str_compare ); + } + + return annotationALL; +} + + +/** + * @author JIA Pei + * @version 2007-08-08 + * @brief Scan a directory and sort all files in format of .jpg .bmp .ppm .pgm .png .gif .tif + * @param dir_path input parameter, path to read from + * @return vector A vector of all scanned file names +*/ +vector VO_IO::ScanNSortImagesInDirectory(const string &dir_path) +{ + vector imgALL; + vector< vector > imgs; + imgs.resize(7); + for(unsigned int i = 0; i < imgs.size(); i++) + { + imgs[i].resize(0); + } + + path tempp = path(dir_path); + + VO_IO::ScanDirectory( tempp, ".jpg", imgs[0]); + VO_IO::ScanDirectory( tempp, ".bmp", imgs[1]); + VO_IO::ScanDirectory( tempp, ".ppm", imgs[2]); + VO_IO::ScanDirectory( tempp, ".pgm", imgs[3]); + VO_IO::ScanDirectory( tempp, ".png", imgs[4]); + VO_IO::ScanDirectory( tempp, ".gif", imgs[5]); + VO_IO::ScanDirectory( tempp, ".tif", imgs[6]); + + for(unsigned int i = 0; i < imgs.size(); i++) + { + for(unsigned int j = 0; j < imgs[i].size(); j++) + { + imgALL.push_back(imgs[i][j]); + } + } + + // sort the filenames + if(imgALL.size() > 0) + { + qsort( (void *)&(imgALL[0]), + (size_t)imgALL.size(), + sizeof(string), + str_compare ); + } + + return imgALL; +} + + +/** + * @author JIA Pei + * @version 2010-03-13 + * @brief Scan a directory and sort all files in format of .xml, .yml, .yaml + * @param dir_path input parameter, path to read from + * @return vector A vector of all scanned file names +*/ +vector VO_IO::ScanNSortXMLYMLsInDirectory(const string& dir_path) +{ + vector mlALL; + vector< vector > mls; + mls.resize(3); + for(unsigned int i = 0; i < mls.size(); i++) + { + mls[i].resize(0); + } + + path tempp = path(dir_path); + + VO_IO::ScanDirectory( tempp, ".xml", mls[0]); + VO_IO::ScanDirectory( tempp, ".yml", mls[1]); + VO_IO::ScanDirectory( tempp, ".yaml", mls[2]); + + for(unsigned int i = 0; i < mls.size(); i++) + { + for(unsigned int j = 0; j < mls[i].size(); j++) + { + mlALL.push_back(mls[i][j]); + } + } + + // sort the filenames + if(mlALL.size() > 0) + { + qsort( (void *)&(mlALL[0]), + (size_t)mlALL.size(), + sizeof(string), + str_compare ); + } + + return mlALL; +} + + +/** + * @author JIA Pei + * @version 2010-03-13 + * @brief Scan a directory and sort all files in format of .txt + * @param dir_path input parameter, path to read from + * @return vector A vector of all scanned file names +*/ +vector VO_IO::ScanNSortTextFilesInDirectory(const string& dir_path) +{ + vector textALL; + vector< vector > texts; + texts.resize(3); + for(unsigned int i = 0; i < texts.size(); i++) + { + texts[i].resize(0); + } + + path tempp = path(dir_path); + + VO_IO::ScanDirectory( tempp, ".txt", texts[0]); + + for(unsigned int i = 0; i < texts.size(); i++) + { + for(unsigned int j = 0; j < texts[i].size(); j++) + { + textALL.push_back(texts[i][j]); + } + } + + // sort the filenames + if(textALL.size() > 0) + { + qsort( (void *)&(textALL[0]), + (size_t)textALL.size(), + sizeof(string), + str_compare ); + } + + return textALL; +} + diff --git a/modules/cvcommon/CMakeLists.txt b/modules/cvcommon/CMakeLists.txt new file mode 100644 index 0000000..263b2b2 --- /dev/null +++ b/modules/cvcommon/CMakeLists.txt @@ -0,0 +1,3 @@ +include_directories("${CMAKE_CURRENT_SOURCE_DIR}/../common/include" "${CMAKE_CURRENT_SOURCE_DIR}/../smbuilding/include") +set(deps opencv_core opencv_imgproc opencv_highgui) +define_vosm_module(cvcommon vosm_common ${deps}) diff --git a/modules/cvcommon/cvcommon.vcxproj b/modules/cvcommon/cvcommon.vcxproj new file mode 100644 index 0000000..889d98b --- /dev/null +++ b/modules/cvcommon/cvcommon.vcxproj @@ -0,0 +1,85 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {4E713018-B49C-44EE-BF6A-4AAECBD0D148} + cvcommon + + + + StaticLibrary + true + MultiByte + v110 + + + Application + false + true + MultiByte + v110 + + + + + + + + + + + + + + + Level3 + Disabled + ./include;../include;../common/include;../smbuilding/include;C:\OpenCV2.4.5\opencv\build\include;C:\boost\boost_1_53_0;%(AdditionalIncludeDirectories) + NOMINMAX;_MBCS;%(PreprocessorDefinitions) + + + true + + + true + + + + + Level3 + MaxSpeed + true + true + + + true + true + true + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/modules/cvcommon/cvcommon.vcxproj.filters b/modules/cvcommon/cvcommon.vcxproj.filters new file mode 100644 index 0000000..fed1744 --- /dev/null +++ b/modules/cvcommon/cvcommon.vcxproj.filters @@ -0,0 +1,48 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + \ No newline at end of file diff --git a/modules/cvcommon/include/VO_AnnotationDBIO.h b/modules/cvcommon/include/VO_AnnotationDBIO.h new file mode 100644 index 0000000..85bc2e2 --- /dev/null +++ b/modules/cvcommon/include/VO_AnnotationDBIO.h @@ -0,0 +1,150 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_ANNOTATIONDBIO_H__ +#define __VO_ANNOTATIONDBIO_H__ + + +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_CVCommon.h" +//#include "yaml.h" +#include "VO_Shape.h" + + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief IO to read/write 2D annotation files from standard databases, + * as well as 3D .obj files from 3D face database USF. + */ +class CAnnotationDBIO +{ +public: + /** databases for statistical shape models */ + enum DB{ BFM3D = 0, + USF3D = 1, + XM2VTS3D = 2, + PUT = 3, + IMM = 4, + AGING = 5, + BIOID = 6, + XM2VTS = 7, + FRANCK = 8, + EMOUNT = 9, + JIAPEI = 10, + BOSPHORUS = 11}; + + /** Constructor */ + CAnnotationDBIO() {} + + /** Destructor */ + ~CAnnotationDBIO() {} + + /** read all annotations for .asf files */ + static void ReadASF(const string& filename, + VO_Shape& oAAMShape); + + /** read all annotations for .pts files */ + static void ReadPTS(const string& filename, + VO_Shape& oAAMShape); + + /** read all annotations for .obj files */ + static void ReadOBJ(const string& filename, + VO_Shape& oAAMShape); + + /** read all annotations for .wrl files */ + static void ReadWRL(const string& filename, + VO_Shape& oAAMShape); + + /** write all annotations for .asf files */ + static void WriteASF( const string& filename, + const VO_Shape& iAAMShape); + + /** write all annotations for .pts files */ + static void WritePTS( const string& filename, + const VO_Shape& iAAMShape); + + /** write all annotations for .obj files */ + static void WriteOBJ( const string& filename, + const VO_Shape& iAAMShape); + + /** write all annotations for .wrl files */ + static void WriteWRL( const string& filename, + const VO_Shape& iAAMShape); + + /** Load Training data for shape model */ + static void VO_LoadShapeTrainingData( + const vector& allLandmarkFiles4Training, + unsigned int database, + vector& oShapes); + + /** Load Training data for shape model */ + static void VO_LoadShapeTrainingData( + const vector& allLandmarkFiles4Training, + vector& oShapes); +}; + +#endif // __VO_ANNOTATIONDBIO_H__ + diff --git a/modules/cvcommon/include/VO_CVCommon.h b/modules/cvcommon/include/VO_CVCommon.h new file mode 100644 index 0000000..3affee0 --- /dev/null +++ b/modules/cvcommon/include/VO_CVCommon.h @@ -0,0 +1,1154 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-06 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_CVCOMMON_H__ +#define __VO_CVCOMMON_H__ + + +#include "opencv/cxcore.h" +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_Common.h" + + +using namespace std; +using namespace cv; + + +static Scalar colors[] = +{ + Scalar(0,0,255), // red + Scalar(255,0,0), // blue + Scalar(0,255,0), // green + Scalar(0,255,255), // yellow + Scalar(255,255,0), // cyan + Scalar(255,0,255), // magenta + Scalar(0,0,0), // black + Scalar(255,255,255), // white + Scalar(0,128,255), // + Scalar(255,128,0), // + Scalar(0,255,128) // +}; + + +static std::ostream& operator<< ( std::ostream& os, const Mat& mat ) +{ + unsigned int channels = mat.channels(); + unsigned int depth = mat.depth(); + if (!(channels == 1 || channels == 3)) + { + cerr << "Cannot deal with images with nChannels != 1 or 3." << endl; + exit(EXIT_FAILURE); + } + + if(channels == 1) + { + switch( depth ) + { + case CV_8U: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + os << mat.at(i, j) << " "; + } + os << endl; + } + } + break; + case CV_8S: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + os << mat.at(i, j) << " "; + } + os << endl; + } + } + break; + case CV_16U: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + os << mat.at(i, j) << " "; + } + os << endl; + } + } + break; + case CV_16S: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + os << mat.at(i, j) << " "; + } + os << endl; + } + } + break; + case CV_32S: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + os << mat.at(i, j) << " "; + } + os << endl; + } + } + break; + case CV_32F: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + os << mat.at(i, j) << " "; + } + os << endl; + } + } + break; + case CV_64F: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + os << mat.at(i, j) << " "; + } + os << endl; + } + } + break; + } + } + else if (channels == 3) + { + switch( depth ) + { + case CV_8U: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + os << mat.at(i, j)[0] << " " + << mat.at(i, j)[1] << " " + << mat.at(i, j)[2] << " "; + } + os << endl; + } + } + break; + case CV_16S: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + os << mat.at(i, j)[0] << " " + << mat.at(i, j)[1] << " " + << mat.at(i, j)[2] << " "; + } + os << endl; + } + } + break; + case CV_32S: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + os << mat.at(i, j)[0] << " " + << mat.at(i, j)[1] << " " + << mat.at(i, j)[2] << " "; + } + os << endl; + } + } + break; + case CV_32F: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + os << mat.at(i, j)[0] << " " + << mat.at(i, j)[1] << " " + << mat.at(i, j)[2] << " "; + } + os << endl; + } + } + break; + case CV_64F: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + os << mat.at(i, j)[0] << " " + << mat.at(i, j)[1] << " " + << mat.at(i, j)[2] << " "; + } + os << endl; + } + } + break; + } + } + + return os; +} + + +static std::istream& operator>> ( std::istream& is, Mat& mat ) +{ + unsigned int channels = mat.channels(); + unsigned int depth = mat.depth(); + if (!(channels == 1 || channels == 3)) + { + cerr << "Cannot deal with images with nChannels != 1 or 3." << endl; + exit(EXIT_FAILURE); + } + + if(channels == 1) + { + switch( depth ) + { + case CV_8U: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + is >> mat.at(i, j); + } + } + } + break; + case CV_8S: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + is >> mat.at(i, j); + } + } + } + break; + case CV_16U: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + is >> mat.at(i, j); + } + } + } + break; + case CV_16S: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + is >> mat.at(i, j); + } + } + } + break; + case CV_32S: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + is >> mat.at(i, j); + } + } + } + break; + case CV_32F: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + is >> mat.at(i, j); + } + } + } + break; + case CV_64F: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + is >> mat.at(i, j); + } + } + } + break; + } + } + + else if (channels == 3) + { + switch( mat.depth() ) + { + case CV_8U: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + is >> mat.at(i, j)[0] + >> mat.at(i, j)[1] + >> mat.at(i, j)[2]; + } + } + } + break; + case CV_16S: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + is >> mat.at(i, j)[0] + >> mat.at(i, j)[1] + >> mat.at(i, j)[2]; + } + } + } + break; + case CV_32S: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + is >> mat.at(i, j)[0] + >> mat.at(i, j)[1] + >> mat.at(i, j)[2]; + } + } + } + break; + case CV_32F: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + is >> mat.at(i, j)[0] + >> mat.at(i, j)[1] + >> mat.at(i, j)[2]; + } + } + } + break; + case CV_64F: + { + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + is >> mat.at(i, j)[0] + >> mat.at(i, j)[1] + >> mat.at(i, j)[2]; + } + } + } + break; + } + } + + return is; +} + + +template +static std::ostream& operator<< ( std::ostream& os, const Mat_& mat ) +{ + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + os << mat(i, j) << " "; + } + os << endl; + } + return os; +} + + +static std::ostream& operator<< ( std::ostream& os, const Rect& rect ) +{ + os << rect.x << " " << rect.y << " " << + rect.width << " " << rect.height << endl; + return os; +} + + +static std::ostream& operator<< ( std::ostream &os, const Point& pt ) +{ + os << pt.x << " " << pt.y << endl; + return os; +} + +static std::ostream& operator<< ( std::ostream &os, const Point2f& pt ) +{ + os << pt.x << " " << pt.y << endl; + return os; +} + + +template +static std::istream& operator>> ( std::istream& is, Mat_& mat ) +{ + for (int i = 0; i < mat.rows; i++ ) + { + for (int j = 0; j < mat.cols; j++ ) + { + is >> mat(i, j); + } + } + return is; +} + + +/** Size compare */ +static int cvSizeCompare( const void *arg1, const void *arg2 ) +{ + return ( (*(Size*)arg1).width * (*(Size*)arg1).height - + (*(Size*)arg2).width * (*(Size*)arg2).height ); +} + + +/** Rect compare */ +static int cvRectCompare( const void *arg1, const void *arg2 ) +{ + return ( (*(Size*)arg1).width * (*(Size*)arg1).height - + (*(Size*)arg2).width * (*(Size*)arg2).height ); +} + + + +/** Image clamp */ +template +static void clamp ( const Mat& iImg, Mat& oImg, T low, T up ) +{ + if(low < up) + { + cerr << "lower bound is smaller than upper bound!" << endl; + exit(1); + } + + unsigned int NbOfDepth = iImg.depth(); + unsigned int NbOfChannels = iImg.channels(); + + if(NbOfChannels == 1) + { + switch(NbOfDepth) + { + case CV_8U: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + if( iImg.at(y,x) < (uchar) low ) + oImg.at(y,x) = (uchar) low; + else if( iImg.at(y,x) > (uchar) up ) + oImg.at(y,x) = (uchar) up; + else + oImg.at(y,x) = iImg.at(y,x); + } + } + } + break; + case CV_8S: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + if( iImg.at(y,x) < (char) low ) + oImg.at(y,x) = (char) low; + else if( iImg.at(y,x) > (char) up ) + oImg.at(y,x) = (char) up; + else + oImg.at(y,x) = iImg.at(y,x); + } + } + } + break; + case CV_16U: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + if( iImg.at(y,x) < (unsigned short) low ) + oImg.at(y,x) = (unsigned short) low; + else if( iImg.at(y,x) > (unsigned short) up ) + oImg.at(y,x) = (unsigned short) up; + else + oImg.at(y,x) = iImg.at(y,x); + } + } + } + break; + case CV_16S: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + if( iImg.at(y,x) < (short) low ) + oImg.at(y,x) = (short) low; + else if( iImg.at(y,x) > (short) up ) + oImg.at(y,x) = (short) up; + else + oImg.at(y,x) = iImg.at(y,x); + } + } + } + break; + case CV_32S: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + if( iImg.at(y,x) < (int) low ) + oImg.at(y,x) = (int) low; + else if( iImg.at(y,x) > (int) up ) + oImg.at(y,x) = (int) up; + else + oImg.at(y,x) = iImg.at(y,x); + } + } + } + break; + case CV_32F: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + if( iImg.at(y,x) < (float) low ) + oImg.at(y,x) = (float) low; + else if( iImg.at(y,x) > (float) up ) + oImg.at(y,x) = (float) up; + else + oImg.at(y,x) = iImg.at(y,x); + } + } + } + break; + case CV_64F: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + if( iImg.at(y,x) < (double) low ) + oImg.at(y,x) = (double) low; + else if( iImg.at(y,x) > (double) up ) + oImg.at(y,x) = (double) up; + else + oImg.at(y,x) = iImg.at(y,x); + } + } + } + break; + default: + break; + } + } + else if(NbOfChannels == 3) + { + for(unsigned int chan = 0; chan < NbOfChannels; chan++) + { + switch(NbOfDepth) + { + case CV_8U: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + if( iImg.at(y,x)[chan] < (uchar) low ) + oImg.at(y,x)[chan] = (uchar) low; + else if( iImg.at(y,x)[chan] > (uchar) up ) + oImg.at(y,x)[chan] = (uchar) up; + else + oImg.at(y,x)[chan] = iImg.at(y,x)[chan]; + } + } + } + break; + //case CV_8S: + // { + // for(unsigned int x = 0; x < iImg.rows; x++) + // { + // for(unsigned int y = 0; y < iImg.cols; y++) + // { + // if( iImg.at(y,x)[chan] < (char) low ) + // oImg.at(y,x)[chan] = (char) low; + // else if( iImg.at(y,x)[chan] > (char) up ) + // oImg.at(y,x)[chan] = (char) up; + // else + // oImg.at(y,x)[chan] = iImg.at(y,x)[chan]; + // } + // } + // } + // break; + case CV_16U: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + if( iImg.at(y,x)[chan] < (unsigned short) low ) + oImg.at(y,x)[chan] = (unsigned short) low; + else if( iImg.at(y,x)[chan] > (unsigned short) up ) + oImg.at(y,x)[chan] = (unsigned short) up; + else + oImg.at(y,x)[chan] = iImg.at(y,x)[chan]; + } + } + } + break; + case CV_16S: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + if( iImg.at(y,x)[chan] < (short) low ) + oImg.at(y,x)[chan] = (short) low; + else if( iImg.at(y,x)[chan] > (short) up ) + oImg.at(y,x)[chan] = (short) up; + else + oImg.at(y,x)[chan] = iImg.at(y,x)[chan]; + } + } + } + break; + case CV_32S: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + if( iImg.at(y,x)[chan] < (int) low ) + oImg.at(y,x)[chan] = (int) low; + else if( iImg.at(y,x)[chan] > (int) up ) + oImg.at(y,x)[chan] = (int) up; + else + oImg.at(y,x)[chan] = iImg.at(y,x)[chan]; + } + } + } + break; + case CV_32F: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + if( iImg.at(y,x)[chan] < (float) low ) + oImg.at(y,x)[chan] = (float) low; + else if( iImg.at(y,x)[chan] > (float) up ) + oImg.at(y,x)[chan] = (float) up; + else + oImg.at(y,x)[chan] = iImg.at(y,x)[chan]; + } + } + } + break; + case CV_64F: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + if( iImg.at(y,x)[chan] < (double) low ) + oImg.at(y,x)[chan] = (double) low; + else if( iImg.at(y,x)[chan] > (double) up ) + oImg.at(y,x)[chan] = (double) up; + else + oImg.at(y,x)[chan] = iImg.at(y,x)[chan]; + } + } + } + break; + default: + break; + } + } + } +} + + +/** Image strech */ +template +static void strech ( const Mat& iImg, Mat& oImg, T low, T up ) +{ + if(low < up) + { + cerr << "lower bound is smaller than upper bound!" << endl; + exit(1); + } + + unsigned int NbOfDepth = iImg.depth(); + unsigned int NbOfChannels = iImg.channels(); + + if(NbOfChannels == 1) + { + double minVal = 0.0, maxVal = 0.0, diff = 0.0, temp = 0.0; + cv::minMaxLoc(iImg, &minVal, &maxVal); + diff = maxVal - minVal; + + if(diff <= FLT_EPSILON) + { + oImg = Mat::zeros(iImg.size(), iImg.type()); + return; + } + + switch(NbOfDepth) + { + case CV_8U: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + temp = (double)iImg.at(y,x); + oImg.at(y,x) = (uchar)( (temp - minVal)/ + diff*(double)(up-low)+(double)low ); + } + } + } + break; + case CV_8S: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + temp = (double)iImg.at(y,x); + oImg.at(y,x) = (char)( (temp - minVal)/ + diff*(double)(up-low)+(double)low ); + } + } + } + break; + case CV_16U: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + temp = (double)iImg.at(y,x); + oImg.at(y,x) = + (unsigned short)( (temp - minVal)/ + diff*(double)(up-low)+(double)low ); + } + } + } + break; + case CV_16S: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + temp = (double)iImg.at(y,x); + oImg.at(y,x) = (short)( (temp - minVal)/diff + *(double)(up-low)+(double)low ); + } + } + } + break; + case CV_32S: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + temp = (double)iImg.at(y,x); + oImg.at(y,x) = (int)( (temp - minVal)/diff + *(double)(up-low)+(double)low ); + } + } + } + break; + case CV_32F: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + temp = (double)iImg.at(y,x); + oImg.at(y,x) = (float)( (temp - minVal)/diff + *(double)(up-low)+(double)low ); + } + } + } + break; + case CV_64F: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + temp = (double)iImg.at(y,x); + oImg.at(y,x) = (double)( (temp - minVal)/diff + *(double)(up-low)+(double)low ); + } + } + } + break; + default: + break; + } + } + else if(NbOfChannels == 3) + { + double minVal_b = 0.0, maxVal_b = 0.0, + minVal_g = 0.0, maxVal_g = 0.0, + minVal_r = 0.0, maxVal_r = 0.0, + minVal = 0.0, maxVal = 0.0, + diff = 0.0, temp = 0.0; + vector bgr; + cv::split(iImg, bgr); + + cv::minMaxLoc(bgr[0], &minVal_b, &maxVal_b); + cv::minMaxLoc(bgr[1], &minVal_g, &maxVal_g); + cv::minMaxLoc(bgr[2], &minVal_r, &maxVal_r); + minVal = min(min(minVal_b, minVal_g), minVal_r); + maxVal = max(max(maxVal_b, maxVal_g), maxVal_r); + diff = maxVal - minVal; + + if(diff <= FLT_EPSILON) + { + oImg = Mat::zeros(iImg.size(), iImg.type()); + return; + } + + for(unsigned int chan = 0; chan < NbOfChannels; chan++) + { + switch(NbOfDepth) + { + case CV_8U: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + temp = (double)iImg.at(y,x)[chan]; + oImg.at(y,x)[chan] = (uchar)( (temp - minVal) + /diff*(double)(up-low)+(double)low ); + } + } + } + break; +// case CV_8S: +// { +// for(unsigned int x = 0; x < iImg.rows; x++) +// { +// for(unsigned int y = 0; y < iImg.cols; y++) +// { +// if( iImg.at(y,x)[chan] < (char) low ) +// oImg.at(y,x)[chan] = (char) low; +// else if( iImg.at(y,x)[chan] > (char) up ) +// oImg.at(y,x)[chan] = (char) up; +// else +// oImg.at(y,x)[chan] = iImg.at(y,x)[chan]; +// } +// } +// } +// break; + case CV_16U: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + temp = (double)iImg.at(y,x)[chan]; + oImg.at(y,x)[chan] = + (unsigned short)( (temp - minVal)/diff + *(double)(up-low)+(double)low ); + } + } + } + break; + case CV_16S: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + temp = (double)iImg.at(y,x)[chan]; + oImg.at(y,x)[chan] = + (short)( (temp - minVal)/diff + *(double)(up-low)+(double)low ); + } + } + } + break; + case CV_32S: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + temp = (double)iImg.at(y,x)[chan]; + oImg.at(y,x)[chan] = + (int)( (temp - minVal)/diff + *(double)(up-low)+(double)low ); + } + } + } + break; + case CV_32F: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + temp = (double)iImg.at(y,x)[chan]; + oImg.at(y,x)[chan] = + (float)( (temp - minVal)/diff + *(double)(up-low)+(double)low ); + } + } + } + break; + case CV_64F: + { + for(unsigned int x = 0; x < iImg.rows; x++) + { + for(unsigned int y = 0; y < iImg.cols; y++) + { + temp = (double)iImg.at(y,x)[chan]; + oImg.at(y,x)[chan] = + (double)( (temp - minVal)/diff + *(double)(up-low)+(double)low ); + } + } + } + break; + default: + break; + } + } + } +} + + +template +static Point2f operator / ( const Point2f& pt, T t) +{ + if (fabs((float)t) < FLT_MIN) + { + std::cerr << "divided by zero" << endl; + exit(0); + } + Point2f res; + res.x = pt.x / t; + res.y = pt.y / t; + return res; +} + + +static Point2f operator+ ( const Point2f& pt1, const Point2f& pt2 ) +{ + Point2f res; + res.x = pt1.x + pt2.x; + res.y = pt1.y + pt2.y; + return res; +} + + +static Point2f operator- ( const Point2f& pt1, const Point2f& pt2 ) +{ + Point2f res; + res.x = pt1.x - pt2.x; + res.y = pt1.y - pt2.y; + return res; +} + + +template +static Rect operator* ( const Rect& rect1, T value ) +{ + Rect res; + res.x = rect1.x * value; + res.y = rect1.y * value; + res.width = rect1.width * value; + res.height = rect1.height * value; + return res; +} + + +/** + * @Ref http://www.newton.dep.anl.gov/newton/askasci/1995/math/MATH061.HTM + * abs(Ar + Bs + C)/sqrt(A^2 + B^2) +*/ +static float cvDistFromAPoint2ALine2D( const Point2f& pt, + const vector& lp) +{ + return ( fabs(lp[0]*pt.x+lp[1]*pt.y+lp[2]) / + sqrt(lp[0]*lp[0]+lp[1]*lp[1]) ); +} + + +/** +* @brief Save all images in sequence in a specific folder +* @param imgs -- Input the source images +* @param fdname -- Input the specific folder +* @ref OpenCV 2.x documentation + */ +static void SaveSequentialImagesInFolder( const vector& imgs, + const string& fdname) +{ + MakeDirectory(fdname); + + stringstream ssi; + string stri; + string filename; + unsigned int NbOfImages = imgs.size(); + for(unsigned int i = 0; i < NbOfImages; i++) + { + ssi << i; + ssi >> stri; + if(stri.length() == 1) + filename = fdname + "/0" + stri + ".jpg"; + else if(stri.length() == 2) + filename = fdname + "/" + stri + ".jpg"; + imwrite(filename.c_str(), imgs[i] ); + ssi.clear(); + } +} + + +/** + * @brief convolve using DFT. + * @param A -- Input the source image + * @param B -- Input the filtering template + * @param C -- Output the convoluted image, must of the same size as A + * @ref OpenCV 2.x documentation + */ +static void convolveDFT(const Mat& A, const Mat& B, Mat& C) +{ + // reallocate the output array if needed + // Mat C(A.rows + B.rows - 1, A.cols + B.cols - 1, A.type()); + Size dftSize; + // compute the size of DFT transform + dftSize.width = getOptimalDFTSize(A.cols + B.cols - 1); + dftSize.height = getOptimalDFTSize(A.rows + B.rows - 1); + // allocate temporary buffers and initialize them with 0’s + Mat tempA(dftSize, A.type(), Scalar::all(0)); + Mat tempB(dftSize, B.type(), Scalar::all(0)); + // copy A and B to the top-left corners of tempA and tempB, respectively + Mat roiA(tempA, Rect(0,0,A.cols,A.rows)); + A.copyTo(roiA); + Mat roiB(tempB, Rect(0,0,B.cols,B.rows)); + B.copyTo(roiB); + // now transform the padded A & B in-place; + // use "nonzeroRows" hint for faster processing + cv::dft(tempA, tempA, 0, A.rows); + cv::dft(tempB, tempB, 0, B.rows); + // multiply the spectrums; + // the function handles packed spectrum representations well + cv::mulSpectrums(tempA, tempB, tempA, 0); + // transform the product back from the frequency domain. + // Even though all the result rows will be non-zero, + // we need only the first C.rows of them, and thus we + // pass nonzeroRows == C.rows + cv::dft(tempA, tempA, DFT_INVERSE + DFT_SCALE, A.rows + B.rows - 1); + // now copy the result back to C. + // tempA(Rect(0, 0, C.cols, C.rows)).copyTo(C); + tempA(Rect(B.cols/2, B.rows/2, A.cols, A.rows)).copyTo(C); + // all the temporary buffers will be deallocated automatically +} + +#endif // __VO_CVCOMMON_H__ + diff --git a/modules/cvcommon/include/VO_DetectionDBIO.h b/modules/cvcommon/include/VO_DetectionDBIO.h new file mode 100644 index 0000000..32643b5 --- /dev/null +++ b/modules/cvcommon/include/VO_DetectionDBIO.h @@ -0,0 +1,184 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-06 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_DETECTIONDBIO_H__ +#define __VO_DETECTIONDBIO_H__ + + +#include +#include + +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_CVCommon.h" +//#include "yaml.h" +#include "VO_FaceCompPos.h" + + +using namespace std; +using namespace cv; + + +struct PUTYMLRect { + string sx; + int x; + string sy; + int y; + string swidth; + int width; + string sheight; + int height; +}; + +struct PUTYMLdoc { + string name; + PUTYMLRect rect; +}; + + + +//void operator >> (const YAML::Node& node, vector& onePUTYML) { +// onePUTYML.resize(5); +// YAML::Node& face = node["Face"]; +// node["Face"] >> onePUTYML[0]; +// node["Left eye"] >> onePUTYML[1]; +// node["Right eye"] >> onePUTYML[2]; +// node["Lips"] >> onePUTYML[3]; +// node["Nose"] >> onePUTYML[4]; +//} + + +/** + * @author JIA Pei + * @brief IO to read/write 2D object region files, + * specifically for face database PUT. + */ +class CDetectionDBIO +{ +public: + /** databases for object detection, + * multiview face -- PUT, CASPEALR1 + * pedestrian -- MITCBCL, INRIAPerson + * car -- MITCAR, INRIACar */ + enum DB{PUT = 0, CASPEALR1 = 1, MITCBCL = 2, + INRIAPerson = 3, MITCAR = 4, INRIACar = 5}; + + /** for CASPEALR1, down, middle, up are separated */ + enum DMU{ DOWN = 0, MIDDLE = 1, UP = 2}; + + /** Constructor */ + CDetectionDBIO() {} + + /** Destructor */ + ~CDetectionDBIO() {} + + /** Read all files from PUT face database for MVFD training */ + static void ReadAllFilesFromPUT4MVFD( + const string& dir_path, + vector& faceComps); + + /** Read a single file from PUT face database for MVFD training */ + static void ReadOneFileFromPUT4MVFD( + const string& filename, + VO_FaceCompPos& oFaceComps); + + /** Read all files from CAS-PEAL-R1 face database for MVFD training */ + static void ReadAllFilesFromCASPEALR14MVFD( + const string& dir_path, + vector< vector >& faceComps); + + /** Read a single file from CAS-PEAL-R1 face database for MVFD training */ + static void ReadOneFileFromCASPEALR14MVFD( + const string& filename, + vector& oFaceComps); + + /** Analyze POSE filenames in CAS-PEAL-R1 face database for MVFD training */ + static void AnalyzeFileNameFromCASPEALR14MVFD( + const string& filename, + int& dmu, + int& degrees); + + /** Roughly estimate the face rectangle and eye rectangle of + CAS-PEAL-R1 face database for MVFD training */ + static void EstimateRects4MVFD( + const Point& leftEyeCenter, + const Point& rightEyeCenter, + int dmu, + int degrees, + Rect& faceRect, + Rect& leftEyeRect, + Rect& rightEyeRect, + Rect& mouthRect, + Rect& noseRect); + + /** Load Training data for shape model */ + static void VO_LoadFaceCompTrainingData( + const vector& allFaceCompFiles4Training, + unsigned int database, + vector& faceComps); + static void VO_LoadFaceCompTrainingData( + const vector& allFaceCompFiles4Training, + unsigned int database, + vector& objRects); +}; + +#endif // __VO_DETECTIONDBIO_H__ + diff --git a/modules/cvcommon/include/VO_FaceCompPos.h b/modules/cvcommon/include/VO_FaceCompPos.h new file mode 100644 index 0000000..cf105fa --- /dev/null +++ b/modules/cvcommon/include/VO_FaceCompPos.h @@ -0,0 +1,146 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-11 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_FACECOMPPOS_H__ +#define __VO_FACECOMPPOS_H__ + +#include +#include +#include + +#include "opencv/cxcore.h" +#include "opencv/cv.h" +#include "VO_CVCommon.h" + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Define some face components' positions, + * which are described as rectangles. + */ +class VO_FaceCompPos +{ +friend class CDetectionDBIO; +friend class CFaceDetectionAlgs; +protected: + /** Every VO_FaceCompPos is corresponding to one face image */ + string m_sCorrespondingImageFile; + + /** The object's location, including face location */ + Rect m_rectObject; + + /** Left eye's location*/ + Rect m_rectLeftEye; + + /** Right eye's location*/ + Rect m_rectRightEye; + + /** Nose's location*/ + Rect m_rectNose; + + /** Mouth's location*/ + Rect m_rectMouth; + +public: + /** Default constructor */ + VO_FaceCompPos(); + + /** Constructor */ + VO_FaceCompPos( const string& fn, + const Rect* rectObj, + const Rect* rectLeftEye = NULL, + const Rect* rectRightEye = NULL, + const Rect* rectNose = NULL, + const Rect* rectMouth = NULL ); + + /** Destructor */ + ~VO_FaceCompPos(); + + void Scale(double nWidth, double nHeight); + + /** Calculate face components' centers */ + vector VO_CalcFaceComponentsCenters() const; + + /** Draw all rectangles */ + void VO_DrawRects(Mat& oImg, Scalar color = colors[8]); + + /** Gets and Sets */ + string GetImageFileName() const {return this->m_sCorrespondingImageFile;} + Rect GetObjectRect() const {return this->m_rectObject;} + Rect GetLeftEyeRect() const {return this->m_rectLeftEye;} + Rect GetRightEyeRect() const {return this->m_rectRightEye;} + Rect GetMouthRect() const {return this->m_rectMouth;} + Rect GetNoseRect() const {return this->m_rectNose;} + + void SetImageFileName(const string& fn) {this->m_sCorrespondingImageFile = fn;} + void SetObjectRect(const Rect& iRect) {this->m_rectObject = iRect;} + void SetLeftEyeRect(const Rect& iRect) {this->m_rectLeftEye = iRect;} + void SetRightEyeRect(const Rect& iRect) {this->m_rectRightEye = iRect;} + void SetMouthRect(const Rect& iRect) {this->m_rectMouth = iRect;} + void SetNoseRect(const Rect& iRect) {this->m_rectNose = iRect;} + +}; + +#endif // __VO_FACECOMPPOS_H__ + diff --git a/modules/cvcommon/include/VO_FaceKeyPoint.h b/modules/cvcommon/include/VO_FaceKeyPoint.h new file mode 100644 index 0000000..9779029 --- /dev/null +++ b/modules/cvcommon/include/VO_FaceKeyPoint.h @@ -0,0 +1,121 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_FACEKEYPOINT_H__ +#define __VO_FACEKEYPOINT_H__ + + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_Shape.h" +#include "VO_FaceParts.h" + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Calculate some face key points. + */ +class VO_KeyPoint +{ +public: + enum { CENTEROFGRAVITY = 0, + LEFTEYELEFTCORNER = 1, + LEFTEYERIGHTCORNER = 2, + LEFTEYECENTER = 3, + RIGHTEYELEFTCORNER = 4, + RIGHTEYERIGHTCORNER = 5, + RIGHTEYECENTER = 6, + NOSETIPKEY = 7, + NOSTRILLEFT = 8, + NOSTRILRIGHT = 9, + NOSECENTER = 10, + MOUTHLEFTCORNER = 11, + MOUTHRIGHTCORNER = 12, + MOUTHCENTER = 13, + EARLOBELEFT = 14, + EARLOBERIGHT = 15 }; + + /** Constructor */ + explicit VO_KeyPoint() {} + + /** Destructor */ + ~VO_KeyPoint() {} + + static void CalcFaceKeyline( + vector& oLine, + const VO_Shape& iShape, + const VO_FaceParts& iFaceParts, + VO_Shape& oSubshape, + unsigned int partIdx = VO_FacePart::MIDLINEPOINTS); + + static void CalcFaceKeyPoint( + Point2f& oPoint, + const VO_Shape& iShape, + const VO_FaceParts& iFaceParts, + unsigned int ptType ); +}; + +#endif // __VO_FACEKEYPOINT_H__ + diff --git a/modules/cvcommon/src/VO_AnnotationDBIO.cpp b/modules/cvcommon/src/VO_AnnotationDBIO.cpp new file mode 100644 index 0000000..5841191 --- /dev/null +++ b/modules/cvcommon/src/VO_AnnotationDBIO.cpp @@ -0,0 +1,464 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_AnnotationDBIO.h" + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Read a file and obtain all annotation data in VO_Shape + * @param filename input parameter, + * which .asf annotation file to read + * @param oAAMShape output parameter, + * save annotation data to AAM shape data structure +*/ +void CAnnotationDBIO::ReadASF( const string &filename, + VO_Shape& oAAMShape ) +{ + oAAMShape.SetAnnotationFileName(filename); + + fstream fp; + fp.open(filename.c_str (), ios::in); + + stringstream ss; + string temp; + float tempFloat = 0.0f; + + // Just for the specific .asf + for(unsigned int i = 0; i < 10; i++) + //fp >> temp; + getline(fp, temp); + + unsigned int NbOfPoints = atoi(temp.c_str ()); + oAAMShape.Resize(2, NbOfPoints); + + // Just for the specific .asf + for(unsigned int i = 0; i < 6; i++) + //fp >> temp; + getline(fp, temp); + + for (unsigned int i = 0; i < NbOfPoints; i++) + { + fp >> temp >> temp >> temp; + // In DTU IMM , x means rows from left to right + ss << temp; + ss >> tempFloat; + ss.clear(); + oAAMShape(0, i) = tempFloat; + fp >> temp; + // In DTU IMM , y means cols from top to bottom + ss << temp; + ss >> tempFloat; + ss.clear(); + //fp >> temp; + getline(fp, temp); + // In sum, topleft is (0,0), right bottom is (640,480) + oAAMShape(1, i) = tempFloat; + } + + // Just for the specific .asf + for(unsigned int i = 0; i < 5; i++) + fp >> temp; + + fp.close (); + +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Read a file and obtain all annotation data in VO_Shape + * @param filename input parameter, which .pts annotation file to read + * @param oAAMShape output parameter, save annotation data to AAM shape data structure +*/ +void CAnnotationDBIO::ReadPTS( const string &filename, + VO_Shape& oAAMShape) +{ + oAAMShape.SetAnnotationFileName(filename); + + fstream fp; + fp.open(filename.c_str (), ios::in); + + string temp, oneLine; + stringstream ss; + float tempFloat = 0.0f; + + do + { + fp >> temp; + }while (temp!="n_points:"); + + + fp >> temp; + ss << temp; + unsigned int NbOfPoints; + ss >> NbOfPoints; + ss.clear(); + oAAMShape.Resize(2, NbOfPoints); + + fp >> temp; + + for (unsigned int i = 0; i < NbOfPoints; i++) + { + fp >> temp; + // x refers to a row from left to right + ss << temp; + ss >> tempFloat; + ss.clear(); + oAAMShape(0, i) = tempFloat; + fp >> temp; + // y refers to a col from top to bottom + ss << temp; + ss >> tempFloat; + ss.clear(); + // In sum, topleft is (0,0), right bottom is (720,576) + oAAMShape(1, i) = tempFloat; + } + + fp.close ();fp.clear(); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Read a file and obtain all annotation data in VO_Shape + * @param filename input parameter, which .asf annotation file to read + * @param oAAMShape output parameter, save annotation data to AAM shape data structure +*/ +void CAnnotationDBIO::ReadOBJ( const string &filename, + VO_Shape& oAAMShape ) +{ + oAAMShape.SetAnnotationFileName(filename); + + fstream fp; + fp.open(filename.c_str (), ios::in); + + stringstream ss; + string temp; + float tempFloat = 0.0f; + + // Just for the specific .obj produced by Freiburg + for(unsigned int i = 0; i < 4; i++) + fp >> temp; + + fp >> temp >> temp; + unsigned int NbOfPoints = atoi(temp.c_str ()); + fp >> temp; + + for (unsigned int i = 0; i < NbOfPoints; i++) + { + fp >> temp; // v + + for(unsigned int j = 0; j < 3; j++) + { + fp >> temp; + ss << temp; + ss >> tempFloat; + ss.clear(); + //oAAMShape.InsertShape(tempFloat); + } + } + + fp.close (); + +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Read a file and obtain all annotation data in VO_Shape + * @param filename input parameter, which .pts annotation file to read + * @param oAAMShape output parameter, save annotation data to AAM shape data structure +*/ +void CAnnotationDBIO::ReadWRL( const string &filename, + VO_Shape& oAAMShape) +{ + oAAMShape.SetAnnotationFileName(filename); + + fstream fp; + fp.open(filename.c_str (), ios::in); + + string temp; + stringstream ss; + Mat_ translation(3, 1); + float tempFloat = 0.0f; + + // Just for the specific .wrl produced by XM2VTS3D + // 2 children of the scene, find the 2nd translation + do + { + getline(fp, temp); + }while (temp.find("Transform") == string::npos); + do + { + getline(fp, temp); + }while (temp.find("Transform") == string::npos); + fp >> temp; + fp >> translation; + + do + { + getline(fp, temp); + }while (temp.find("point [ 0 0 0,") == string::npos); + + do + { + for(unsigned int i = 0; i < 3; i++) + { + fp >> tempFloat; + //oAAMShape.InsertShape( tempFloat + translation[i]); + //oAAMShape.InsertShape( tempFloat); + } + fp >> temp; + }while (temp != "]"); + + fp.close (); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Write all annotation data in VO_Shape to a file + * @param filename output parameter, which .pts annotation file to write + * @param iAAMShape input parameter, save annotation data from AAM shape data structure +*/ +void CAnnotationDBIO::WriteASF( const string &filename, + const VO_Shape& iAAMShape) +{ + +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Write all annotation data in VO_Shape to a file + * @param filename output parameter, which .pts annotation file to write + * @param iAAMShape input parameter, save annotation data from AAM shape data structure +*/ +void CAnnotationDBIO::WritePTS( const string &filename, + const VO_Shape& iAAMShape) +{ + fstream fp; + fp.open(filename.c_str (), ios::out); + + string temp, oneLine; + stringstream ss; + float tempFloat = 0.0f; + unsigned int NbOfPoints = iAAMShape.GetNbOfPoints(); + + fp << "version: 1" << endl + << "n_points: " << NbOfPoints << endl + << "{" << endl; + + for (unsigned int i = 0; i < NbOfPoints; i++) + { + fp << iAAMShape.GetA2DPoint(i).x << " " << iAAMShape.GetA2DPoint(i).y << endl; + } + + fp << "}" << endl << endl; + + fp.close (); +} + + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Write all annotation data in VO_Shape to a file + * @param filename output parameter, which .pts annotation file to write + * @param iAAMShape input parameter, save annotation data from AAM shape data structure +*/ +void CAnnotationDBIO::WriteOBJ( const string &filename, + const VO_Shape& iAAMShape) +{ + +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Write all annotation data in VO_Shape to a file + * @param filename output parameter, which .pts annotation file to write + * @param iAAMShape input parameter, save annotation data from AAM shape data structure +*/ +void CAnnotationDBIO::WriteWRL( const string &filename, + const VO_Shape& iAAMShape) +{ + +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Load Training data for shape model + * @param allLandmarkFiles4Training Input - all landmark files + * @param database Input - which database is it? + * @param oShapes Output - the loaded shape + * @return void +*/ +void CAnnotationDBIO::VO_LoadShapeTrainingData( + const vector& allLandmarkFiles4Training, + unsigned int database, + vector& oShapes ) +{ + oShapes.clear(); + + // Load the annotated information + switch ( database ) + { + case BFM3D: + case USF3D: + break; + case XM2VTS3D: + break; + case AGING: + case BIOID: + case XM2VTS: + case FRANCK: + case EMOUNT: + case JIAPEI: + { + // Read all PTS file data into shps + for ( unsigned int i = 0; i < allLandmarkFiles4Training.size(); i++ ) + { + VO_Shape tempAAMShape(2); + CAnnotationDBIO::ReadPTS ( + allLandmarkFiles4Training[i], + tempAAMShape ); + oShapes.push_back ( tempAAMShape ); + oShapes[i] = tempAAMShape; + } + } + break; + case IMM: + { + // Read all ASF file data into shps and scale back to the original size + for ( unsigned int i = 0; i < allLandmarkFiles4Training.size(); i++ ) + { + Mat_ scale = Mat_::zeros(2, 1); + scale(0, 0) = 640; + scale(1, 0) = 480; + VO_Shape tempAAMShape(2); + CAnnotationDBIO::ReadASF ( + allLandmarkFiles4Training[i], + tempAAMShape ); + tempAAMShape.Scale ( scale ); + oShapes.push_back ( tempAAMShape ); + } + } + default: + { + cerr << "Invalid database type specified" << endl; + exit(EXIT_FAILURE); + } + break; + } +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Load Training data for shape model + * @param allLandmarkFiles4Training Input - all landmark files + * @return void +*/ +void CAnnotationDBIO::VO_LoadShapeTrainingData( + const vector& allLandmarkFiles4Training, + vector& oShapes) +{ + unsigned int NbOfAnnotations = allLandmarkFiles4Training.size(); + string fileType = allLandmarkFiles4Training[0].substr + (allLandmarkFiles4Training[0].find_last_of("."), 3); + if(fileType == "pts") + { + // Read all PTS file data into shps + for ( unsigned int i = 0; i < NbOfAnnotations; i++ ) + { + VO_Shape tempAAMShape(2); + CAnnotationDBIO::ReadPTS ( allLandmarkFiles4Training[i], tempAAMShape ); + oShapes.push_back ( tempAAMShape ); + } + } + else if(fileType == "asf") + { + // Read all ASF file data into shps and scale back to the original size + for ( unsigned int i = 0; i < NbOfAnnotations; i++ ) + { + Mat_ scale = Mat_::zeros(2, 1); + scale(0, 0) = 640; + scale(1, 0) = 480; + VO_Shape tempAAMShape(2); + CAnnotationDBIO::ReadASF ( allLandmarkFiles4Training[i], tempAAMShape ); + tempAAMShape.Scale ( scale ); + oShapes.push_back ( tempAAMShape ); + } + } +} diff --git a/modules/cvcommon/src/VO_DetectionDBIO.cpp b/modules/cvcommon/src/VO_DetectionDBIO.cpp new file mode 100644 index 0000000..8331229 --- /dev/null +++ b/modules/cvcommon/src/VO_DetectionDBIO.cpp @@ -0,0 +1,471 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-06 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include +#include +#include "VO_DetectionDBIO.h" +#include "VO_ScanFilesInDir.h" + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Read all files from PUT face database 4 MVFD purpose + * @param dir_path Input -- The input directory + * @param faceComps Output -- face components + * @return void +*/ +void CDetectionDBIO::ReadAllFilesFromPUT4MVFD( + const string& dir_path, + vector& faceComps) +{ + vector labelFileNames = VO_IO::ScanNSortXMLYMLsInDirectory ( dir_path ); + int NbOfFiles = labelFileNames.size(); + faceComps.resize(NbOfFiles); + + for (int i = 0; i < NbOfFiles; i++ ) + { + CDetectionDBIO::ReadOneFileFromPUT4MVFD(labelFileNames[i], faceComps[i]); + } +} + + +//void operator >> (const YAML::Node& node, Rect& rect) { +void operator >> (const FileNode& node, Rect& rect) +{ + node["x"] >> rect.x; + node["y"] >> rect.y; + node["width"] >> rect.width; + node["height"] >> rect.height; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Read a single file from PUT face database 4 MVFD purpose + * @param filename Input -- The file name + * @param faceComps Output -- face components + * @return void +*/ +void CDetectionDBIO::ReadOneFileFromPUT4MVFD( + const string& filename, + VO_FaceCompPos& oFaceComps) +{ + vector allRects; + allRects.resize(5); + + //std::ifstream fin(filename.c_str()); + //YAML::Parser parser(fin); + //YAML::Node doc; + //parser.GetNextDocument(doc); + + //doc["Face"] >> oFaceComps.m_rectObject; + //// Explained by JIA Pei. for PUT database, lefteye and righteye should be reversed. + ////doc["Left eye"] >> oFaceComps.m_rectLeftEye; + ////doc["Right eye"] >> oFaceComps.m_rectRightEye; + //doc["Left eye"] >> oFaceComps.m_rectRightEye; + //doc["Right eye"] >> oFaceComps.m_rectLeftEye; + //doc["Mouth"] >> oFaceComps.m_rectMouth; + //doc["Nose"] >> oFaceComps.m_rectNose; + + FileStorage fs(filename, FileStorage::READ); + FileNode node = fs["Face"]; + + node >> oFaceComps.m_rectObject; + // Explained by JIA Pei. for PUT database, lefteye and righteye should be reversed. + //fs["Left eye"] >> oFaceComps.m_rectLeftEye; + //fs["Right eye"] >> oFaceComps.m_rectRightEye; + node = fs["Left eye"]; + node >> oFaceComps.m_rectRightEye; + node = fs["Right eye"]; + node >> oFaceComps.m_rectLeftEye; + node = fs["Mouth"]; + node >> oFaceComps.m_rectMouth; + node = fs["Nose"]; + node >> oFaceComps.m_rectNose; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Read all files from CAS-PEAL-R1 face databse for MVFD training + * @param dir_path Input -- The input directory + * @param faceComps Output -- face components + * @return void +*/ +void CDetectionDBIO::ReadAllFilesFromCASPEALR14MVFD( + const string& dir_path, + vector< vector >& faceComps) +{ + vector labelFileNames = VO_IO::ScanNSortTextFilesInDirectory ( dir_path ); + int NbOfFiles = labelFileNames.size(); + faceComps.resize(NbOfFiles); + + // bottom, middel, up -- 3 + // -67,-45,-22,0,22,45,67 -- 7 + // -45,-30,-15,0,15,30,45 -- 7 + for (int i = 0; i < NbOfFiles; i++ ) + { + CDetectionDBIO::ReadOneFileFromCASPEALR14MVFD(labelFileNames[i], faceComps[i]); + } +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Read a single file from CAS-PEAL-R1 face database for MVFD training + * @note a single faceFP file from CAS-PEAL-R1 contains all information of all 21 images for one individual + * @param filename Input -- The input file name + * @param faceComps Output -- face components corresponding to 1 person. should be a vector of 21 + * @return void +*/ +void CDetectionDBIO::ReadOneFileFromCASPEALR14MVFD( + const string& filename, + vector& oFaceComps) +{ + int dmu, degrees, idx; + Point leftEyeCenter, rightEyeCenter; + + Rect leftEyeRect, rightEyeRect, faceRect, noseRect, mouthRect; + + string fn; + ifstream ifs; + ifs.open(filename.c_str(), ifstream::in); + + oFaceComps.resize(21); + for(int i = 0; i < 21; i++) + { + ifs >> fn >> leftEyeCenter.x >> leftEyeCenter.y + >> rightEyeCenter.x >> rightEyeCenter.y; + + CDetectionDBIO::AnalyzeFileNameFromCASPEALR14MVFD(filename, dmu, degrees); + CDetectionDBIO::EstimateRects4MVFD( leftEyeCenter, + rightEyeCenter, + dmu, + degrees, + faceRect, + leftEyeRect, + rightEyeRect, + mouthRect, + noseRect); + + idx = dmu*7+(degrees/15+3); + oFaceComps[idx].SetImageFileName(filename); + oFaceComps[idx].SetLeftEyeRect(leftEyeRect); + oFaceComps[idx].SetRightEyeRect(rightEyeRect); + oFaceComps[idx].SetObjectRect(faceRect); + } + + ifs.close(); +} + + +/** + * @author JIA Pei + * @version 2010-04-07 + * @brief Read degree information from the file name + * @param filename Input -- filename hides the degrees + * @param dmu Output -- face directions, down, middle or up + * @param degrees Output -- face degrees, read from filename + * @return void +*/ +void CDetectionDBIO::AnalyzeFileNameFromCASPEALR14MVFD( + const string& filename, + int& dmu, + int& degrees) +{ + string s_dmu = filename.substr(17, 2); + if(s_dmu == "PD") dmu = DOWN; + else if(s_dmu == "PM") dmu = MIDDLE; + else if(s_dmu == "PU") dmu = UP; + + string s_degrees = filename.substr(20, 2); + stringstream ss; + ss << s_degrees; + ss >> degrees; +} + + +/** + * @author JIA Pei + * @version 2010-04-07 + * @brief Estimate face rectangle using a simple 3D face model + * @param leftEyeCenter Input -- left eye center + * @param rightEyeCenter Input -- right eye center + * @param dmu Input -- down, middle, up, supposing down -30 degrees, middle 0 degrees, up +30 degrees + * @param degrees Input -- face degrees, read from filename + * @param faceRect Output -- face rectangle + * @param leftEyeRect Output -- left eye rectangle + * @param rightEyeRect Output -- right eye rectangle + * @param mouthRect Output -- mouth rectangle + * @param noseRect Output -- nose rectangle + * @return void +*/ +void CDetectionDBIO::EstimateRects4MVFD( + const Point& leftEyeCenter, + const Point& rightEyeCenter, + int dmu, + int degrees, + Rect& faceRect, + Rect& leftEyeRect, + Rect& rightEyeRect, + Rect& mouthRect, + Rect& noseRect) +{ + switch( dmu*7+(degrees/15+3) ) + { + case 0: + { + + } + break; + case 1: + { + + } + break; + case 2: + { + + } + break; + case 3: + { + + } + break; + case 4: + { + + } + break; + case 5: + { + + } + break; + case 6: + { + + } + break; + case 7: + { + + } + break; + case 8: + { + + } + break; + case 9: + { + + } + break; + case 10: + { + + } + break; + case 11: + { + leftEyeRect.x = leftEyeCenter.x - 16; + leftEyeRect.y = leftEyeCenter.y - 16; + leftEyeRect.width = 32; + leftEyeRect.height = 32; + rightEyeRect.x = rightEyeCenter.x - 16; + rightEyeRect.y = rightEyeCenter.y - 16; + rightEyeRect.width = 32; + rightEyeRect.height = 32; + } + break; + case 12: + { + + } + break; + case 13: + { + + } + break; + case 14: + { + + } + break; + case 15: + { + + } + break; + case 16: + { + + } + break; + case 17: + { + + } + break; + case 18: + { + + } + break; + case 19: + { + + } + break; + case 20: + { + + } + break; + } +} + + +/** + * @author JIA Pei + * @version 2010-04-07 + * @brief Load face components from training files + * @param allFaceCompFiles4Training Input -- file names of all training face files + * @param database Input -- database, including PUT and CASPEALR1 + * @param faceComps Output -- a vector of all face components from all files + * @return void +*/ +void CDetectionDBIO::VO_LoadFaceCompTrainingData( + const vector& allFaceCompFiles4Training, + unsigned int database, + vector& faceComps) +{ + faceComps.clear(); + + // Load face database + switch ( database ) + { + case CASPEALR1: + { + int NbOfFiles = allFaceCompFiles4Training.size(); + vector< vector > tempFaceComps; + tempFaceComps.resize(NbOfFiles); + + for (int i = 0; i < NbOfFiles; i++ ) + { + CDetectionDBIO::ReadOneFileFromCASPEALR14MVFD(allFaceCompFiles4Training[i], tempFaceComps[i]); + for(unsigned int j = 0; j < tempFaceComps[i].size(); j++) + { + faceComps.push_back(tempFaceComps[i][j]); + } + } + } + break; + case PUT: + default: + { + int NbOfFiles = allFaceCompFiles4Training.size(); + faceComps.resize(NbOfFiles); + + for (int i = 0; i < NbOfFiles; i++ ) + { + CDetectionDBIO::ReadOneFileFromPUT4MVFD(allFaceCompFiles4Training[i], faceComps[i]); + } + } + break; + } +} + + +/** + * @author JIA Pei + * @version 2010-04-07 + * @brief Load object rectangles from training files + * @param allFaceCompFiles4Training Input -- file names of all training face files + * @param database Input -- database, including PUT and CASPEALR1 + * @param objRects Output -- a vector of all face components from all files + * @return void +*/ +void CDetectionDBIO::VO_LoadFaceCompTrainingData( + const vector& allFaceCompFiles4Training, + unsigned int database, + vector& objRects) +{ + vector faceComps; + CDetectionDBIO::VO_LoadFaceCompTrainingData(allFaceCompFiles4Training, + database, + faceComps); + objRects.clear(); + unsigned int NbOfObjRects = faceComps.size(); + + for(unsigned int i = 0; i < NbOfObjRects; i++) + { + objRects.push_back( faceComps[i].GetObjectRect() ); + } +} + diff --git a/modules/cvcommon/src/VO_FaceCompPos.cpp b/modules/cvcommon/src/VO_FaceCompPos.cpp new file mode 100644 index 0000000..163b59b --- /dev/null +++ b/modules/cvcommon/src/VO_FaceCompPos.cpp @@ -0,0 +1,192 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-11 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_FaceCompPos.h" + + + +VO_FaceCompPos::VO_FaceCompPos() +{ + this->m_sCorrespondingImageFile = ""; + this->m_rectObject = Rect(0, 0, 0, 0); + this->m_rectLeftEye = Rect(0, 0, 0, 0); + this->m_rectRightEye = Rect(0, 0, 0, 0); + this->m_rectNose = Rect(0, 0, 0, 0); + this->m_rectMouth = Rect(0, 0, 0, 0); +} + + +VO_FaceCompPos::VO_FaceCompPos( + const string& fn, + const Rect* rectObj, + const Rect* rectLeftEye, + const Rect* rectRightEye, + const Rect* rectNose, + const Rect* rectMouth) +{ + this->m_sCorrespondingImageFile = fn; + this->m_rectObject = *rectObj; + if(rectLeftEye) this->m_rectLeftEye = *rectLeftEye; + if(rectRightEye) this->m_rectRightEye = *rectRightEye; + if(rectNose) this->m_rectNose = *rectNose; + if(rectMouth) this->m_rectMouth = *rectMouth; +} + + +VO_FaceCompPos::~VO_FaceCompPos() +{ + +} + + +void VO_FaceCompPos::Scale(double nWidth, double nHeight) +{ + this->m_rectObject.x *= static_cast(nWidth); + this->m_rectObject.y *= static_cast(nHeight); + this->m_rectObject.width *= static_cast(nWidth); + this->m_rectObject.height *= static_cast(nHeight); + + this->m_rectLeftEye.x *= static_cast(nWidth); + this->m_rectLeftEye.y *= static_cast(nHeight); + this->m_rectLeftEye.width *= static_cast(nWidth); + this->m_rectLeftEye.height *= static_cast(nHeight); + + this->m_rectRightEye.x *= static_cast(nWidth); + this->m_rectRightEye.y *= static_cast(nHeight); + this->m_rectRightEye.width *= static_cast(nWidth); + this->m_rectRightEye.height *= static_cast(nHeight); + + this->m_rectMouth.x *= static_cast(nWidth); + this->m_rectMouth.y *= static_cast(nHeight); + this->m_rectMouth.width *= static_cast(nWidth); + this->m_rectMouth.height *= static_cast(nHeight); + + this->m_rectNose.x *= static_cast(nWidth); + this->m_rectNose.y *= static_cast(nHeight); + this->m_rectNose.width *= static_cast(nWidth); + this->m_rectNose.height *= static_cast(nHeight); +} + + +vector VO_FaceCompPos::VO_CalcFaceComponentsCenters() const +{ + vector res; + res.resize(5); + + res[0].x = this->m_rectObject.x + this->m_rectObject.width/2.0f; + res[0].y = this->m_rectObject.y + this->m_rectObject.height/2.0f; + + res[1].x = this->m_rectLeftEye.x + this->m_rectLeftEye.width/2.0f; + res[1].y = this->m_rectLeftEye.y + this->m_rectLeftEye.height/2.0f; + + res[2].x = this->m_rectRightEye.x + this->m_rectRightEye.width/2.0f; + res[2].y = this->m_rectRightEye.y + this->m_rectRightEye.height/2.0f; + + res[3].x = this->m_rectNose.x + this->m_rectNose.width/2.0f; + res[3].y = this->m_rectNose.y + this->m_rectNose.height/2.0f; + + res[4].x = this->m_rectMouth.x + this->m_rectMouth.width/2.0f; + res[4].y = this->m_rectMouth.y + this->m_rectMouth.height/2.0f; + + return res; +} + + +void VO_FaceCompPos::VO_DrawRects(Mat& oImg, Scalar color) +{ + Point lefttop, rightbottom; + + // for the face + lefttop.x = cvRound(this->m_rectObject.x); + lefttop.y = cvRound(this->m_rectObject.y); + rightbottom.x = cvRound(this->m_rectObject.x+this->m_rectObject.width); + rightbottom.y = cvRound(this->m_rectObject.y+this->m_rectObject.height); + cv::rectangle(oImg, lefttop, rightbottom, color, 2, 8, 0); + + // for the left eye + lefttop.x = cvRound(this->m_rectLeftEye.x); + lefttop.y = cvRound(this->m_rectLeftEye.y); + rightbottom.x = cvRound(this->m_rectLeftEye.x+this->m_rectLeftEye.width); + rightbottom.y = cvRound(this->m_rectLeftEye.y+this->m_rectLeftEye.height); + cv::rectangle(oImg, lefttop, rightbottom, color, 1, 8, 0); + + // for the right eye + lefttop.x = cvRound(this->m_rectRightEye.x); + lefttop.y = cvRound(this->m_rectRightEye.y); + rightbottom.x = cvRound(this->m_rectRightEye.x+this->m_rectRightEye.width); + rightbottom.y = cvRound(this->m_rectRightEye.y+this->m_rectRightEye.height); + cv::rectangle(oImg, lefttop, rightbottom, color, 1, 8, 0); + + // for the nose + lefttop.x = cvRound(this->m_rectNose.x); + lefttop.y = cvRound(this->m_rectNose.y); + rightbottom.x = cvRound(this->m_rectNose.x+this->m_rectNose.width); + rightbottom.y = cvRound(this->m_rectNose.y+this->m_rectNose.height); + cv::rectangle(oImg, lefttop, rightbottom, color, 1, 8, 0); + + // for the mouth + lefttop.x = cvRound(this->m_rectMouth.x); + lefttop.y = cvRound(this->m_rectMouth.y); + rightbottom.x = cvRound(this->m_rectMouth.x+this->m_rectMouth.width); + rightbottom.y = cvRound(this->m_rectMouth.y+this->m_rectMouth.height); + cv::rectangle(oImg, lefttop, rightbottom, color, 1, 8, 0); +} + diff --git a/modules/cvcommon/src/VO_FaceKeyPoint.cpp b/modules/cvcommon/src/VO_FaceKeyPoint.cpp new file mode 100644 index 0000000..26e1ba3 --- /dev/null +++ b/modules/cvcommon/src/VO_FaceKeyPoint.cpp @@ -0,0 +1,322 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_FaceKeyPoint.h" +#include "VO_Common.h" + +/** +* @brief Calculate some key lines on the face +* @param oLine Output output those lines +* @param iShape Input the known shape +* @param iFaceParts Input the faceparts +* @param oSubshape Output the output subshape, namely, the line is represented by a VO_Shape +* @param partIdx Input which part is it +* @return void + */ +void VO_KeyPoint::CalcFaceKeyline( + vector& oLine, + const VO_Shape& iShape, + const VO_FaceParts& iFaceParts, + VO_Shape& oSubshape, + unsigned int partIdx) +{ + oLine.resize(3); + int dim = iShape.GetNbOfDim(); + + Vec4f line; + vector linePoints; + + switch(partIdx) + { + case VO_FacePart::NOSTRIL: + linePoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::NOSTRIL).GetIndexes(); + break; + case VO_FacePart::MOUTHCORNERPOINTS: + linePoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::MOUTHCORNERPOINTS).GetIndexes(); + break; + case VO_FacePart::PITCHAXISLINEPOINTS: + linePoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::PITCHAXISLINEPOINTS).GetIndexes(); + break; + case VO_FacePart::EYECORNERPOINTS: + linePoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::EYECORNERPOINTS).GetIndexes(); + break; + case VO_FacePart::MIDLINEPOINTS: + default: + linePoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::EYECORNERPOINTS).GetIndexes(); + break; + } + + oSubshape = iShape.GetSubShape(linePoints); + + // Explained by JIA Pei, some times, there is no linePoints, which means the specified parts are not in one of the database + if(linePoints.size() >= 2 ) + { + cv::fitLine( oSubshape.GetTheShape(), line, CV_DIST_L2, 0, 0.001, 0.001 ); + + // Ax+By+C = 0 + oLine[0] = -line[1]; + oLine[1] = line[0]; + oLine[2] = line[1]*line[2]-line[0]*line[3]; + } +} + + +/** + * @brief + * @param oPoint output point list + * @param iShape input shape + * @param iFaceParts inut faceparts + * @param ptType input point type + * @return void + */ +void VO_KeyPoint::CalcFaceKeyPoint( Point2f& oPoint, + const VO_Shape& iShape, + const VO_FaceParts& iFaceParts, + unsigned int ptType) +{ + vector facePartsPoints; + VO_Shape subiShape; + // Very very very very important. + // Explained by JIA Pei. + // "resize()" is just for resize; + // it doesn't always set what's already inside the the vector to "0" + // Therefore, clear() is a must before resize(). + + switch(ptType) + { + case CENTEROFGRAVITY: + if (iShape.GetNbOfPoints() > 0) + oPoint = iShape.GetA2DPoint( VO_Shape::CENTER); + break; + case LEFTEYELEFTCORNER: + { + facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::LEFTEYE).GetIndexes(); + if (facePartsPoints.size() > 0) + { + subiShape = iShape.GetSubShape(facePartsPoints); + oPoint = subiShape.GetA2DPoint(VO_Shape::LEFTMOST); + } + } + break; + case LEFTEYERIGHTCORNER: + { + facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::LEFTEYE).GetIndexes(); + if (facePartsPoints.size() > 0) + { + subiShape = iShape.GetSubShape(facePartsPoints); + oPoint = subiShape.GetA2DPoint(VO_Shape::RIGHTMOST); + } + } + break; + case LEFTEYECENTER: + { + facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::LEFTEYE).GetIndexes(); + if (facePartsPoints.size() > 0) + { + subiShape = iShape.GetSubShape(facePartsPoints); + oPoint = subiShape.GetA2DPoint( VO_Shape::CENTER); + } + } + break; + case RIGHTEYELEFTCORNER: + { + facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::RIGHTEYE).GetIndexes(); + if (facePartsPoints.size() > 0) + { + subiShape = iShape.GetSubShape(facePartsPoints); + oPoint = subiShape.GetA2DPoint(VO_Shape::LEFTMOST); + } + } + break; + case RIGHTEYERIGHTCORNER: + { + facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::RIGHTEYE).GetIndexes(); + if (facePartsPoints.size() > 0) + { + subiShape = iShape.GetSubShape(facePartsPoints); + oPoint = subiShape.GetA2DPoint(VO_Shape::RIGHTMOST); + } + } + break; + case RIGHTEYECENTER: + { + facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::RIGHTEYE).GetIndexes(); + if (facePartsPoints.size() > 0) + { + subiShape = iShape.GetSubShape(facePartsPoints); + oPoint = subiShape.GetA2DPoint( VO_Shape::CENTER); + } + } + break; + case NOSETIPKEY: + { + facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::NOSETIP).GetIndexes(); // Just one point + if (facePartsPoints.size() == 1) + oPoint = iShape.GetA2DPoint(facePartsPoints[0]); + } + break; + case NOSTRILLEFT: + { + facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::NOSTRIL).GetIndexes(); + if (facePartsPoints.size() > 0) + { + subiShape = iShape.GetSubShape(facePartsPoints); + oPoint = subiShape.GetA2DPoint(VO_Shape::LEFTMOST); + } + } + break; + case NOSTRILRIGHT: + { + facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::NOSTRIL).GetIndexes(); + if (facePartsPoints.size() > 0) + { + subiShape = iShape.GetSubShape(facePartsPoints); + oPoint = subiShape.GetA2DPoint(VO_Shape::RIGHTMOST); + } + } + break; + case NOSECENTER: + { + facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::NOSE).GetIndexes(); + if (facePartsPoints.size() > 0) + { + subiShape = iShape.GetSubShape(facePartsPoints); + oPoint = subiShape.GetA2DPoint( VO_Shape::CENTER); + } + } + break; + case MOUTHLEFTCORNER: + { + facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::LIPOUTERLINE).GetIndexes(); + if (facePartsPoints.size() > 0) + { + subiShape = iShape.GetSubShape(facePartsPoints); + oPoint = subiShape.GetA2DPoint(VO_Shape::LEFTMOST); + } + } + break; + case MOUTHRIGHTCORNER: + { + facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::LIPOUTERLINE).GetIndexes(); + if (facePartsPoints.size() > 0) + { + subiShape = iShape.GetSubShape(facePartsPoints); + oPoint = subiShape.GetA2DPoint(VO_Shape::RIGHTMOST); + } + } + break; + case MOUTHCENTER: + { + facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::LIPOUTERLINE).GetIndexes(); + if (facePartsPoints.size() > 0) + { + subiShape = iShape.GetSubShape(facePartsPoints); + oPoint = subiShape.GetA2DPoint( VO_Shape::CENTER); + } + } + break; + case EARLOBELEFT: + { + facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::LEFTEAR).GetIndexes(); + if (facePartsPoints.size() > 0) + { + } + } + break; + case EARLOBERIGHT: + { + facePartsPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::RIGHTEAR).GetIndexes(); + if (facePartsPoints.size() > 0) + { + } + } + break; + } +} + + +// string stri; +// string keypointjpgstr, keypointtxtstr; +// /** image patches for key points */ +// for(unsigned int i = 0; i < this->m_iNbOfKeyPoints; ++i) +// { +// ssi << i; +// ssi >> stri; +// if(stri.length() == 1) +// { +// keypointjpgstr = fn + "/keypoint0" + stri + ".jpg"; +// keypointtxtstr = fn + "/keypoint0" + stri + ".txt"; +// } +// else +// { +// keypointjpgstr = fn + "/keypoint" + stri + ".jpg"; +// keypointtxtstr = fn + "/keypoint" + stri + ".txt"; +// } +// +// this->m_vLocalImagePatchAroundKeyPoints[i] = imread(keypointjpgstr.c_str(), CV_LOAD_IMAGE_ANYCOLOR ); +// +// this->m_vGSLWaveletParams4KeyPoints[i] = gsl_matrix_alloc (this->m_vLocalImagePatchAroundKeyPoints[i].cols, this->m_vLocalImagePatchAroundKeyPoints[i].rows); +// fp.open(keypointtxtstr.c_str (), ios::in); +// fp >> this->m_vGSLWaveletParams4KeyPoints[i]; +// fp.close();fp.clear(); +// +// ssi.clear(); +// } diff --git a/modules/ensembletraining/CMakeLists.txt b/modules/ensembletraining/CMakeLists.txt new file mode 100644 index 0000000..229378e --- /dev/null +++ b/modules/ensembletraining/CMakeLists.txt @@ -0,0 +1,3 @@ +include_directories("${CMAKE_CURRENT_SOURCE_DIR}/../common/include" "${CMAKE_CURRENT_SOURCE_DIR}/../featureextraction/include") +set(deps opencv_core opencv_imgproc opencv_highgui) +define_vosm_module(ensembletraining vosm_common vosm_featureextraction ${deps}) diff --git a/modules/ensembletraining/ensembletraining.vcxproj b/modules/ensembletraining/ensembletraining.vcxproj new file mode 100644 index 0000000..acccf4c --- /dev/null +++ b/modules/ensembletraining/ensembletraining.vcxproj @@ -0,0 +1,86 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {838E7724-4E9B-4489-B26C-B419547473D2} + ensembletraining + + + + StaticLibrary + true + MultiByte + v110 + + + Application + false + true + MultiByte + v110 + + + + + + + + + + + + + + + Level3 + Disabled + ./include;../include;../featureextraction/include;C:\OpenCV2.4.5\opencv\build\include;%(AdditionalIncludeDirectories) + NOMINMAX;_MBCS;%(PreprocessorDefinitions) + + + true + + + true + + + + + Level3 + MaxSpeed + true + true + + + true + true + true + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/modules/ensembletraining/ensembletraining.vcxproj.filters b/modules/ensembletraining/ensembletraining.vcxproj.filters new file mode 100644 index 0000000..f1a8cb1 --- /dev/null +++ b/modules/ensembletraining/ensembletraining.vcxproj.filters @@ -0,0 +1,51 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + \ No newline at end of file diff --git a/modules/ensembletraining/include/VO_AdditiveStrongerClassifier.h b/modules/ensembletraining/include/VO_AdditiveStrongerClassifier.h new file mode 100644 index 0000000..c56ea82 --- /dev/null +++ b/modules/ensembletraining/include/VO_AdditiveStrongerClassifier.h @@ -0,0 +1,106 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-16 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_ADDITIVESTRONGERCLASSIFIER_H__ +#define __VO_ADDITIVESTRONGERCLASSIFIER_H__ + +#include +#include + +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_WeakClassifier.h" + +using namespace std; +using namespace cv; + + +class VO_AdditiveStrongerClassifier +{ +protected: + VO_WeakClassifier m_VOWeakClassifier; + + /** Number of samples */ + unsigned int m_iNbOfSamples; + + /** Number of positive samples */ + unsigned int m_iNbOfPositiveSamples; + + /** Number of negative samples */ + unsigned int m_iNbOfNegativeSamples; + +public: + /** model type, such as additive model -- including boosting, bagging, etc. */ + enum { + UNDEFINED = 0, + BOOSTING = 1, + BAGGING = 2}; + + /** Constructor */ + VO_AdditiveStrongerClassifier(); + + /** Destructor */ + ~VO_AdditiveStrongerClassifier(); + +}; + +#endif diff --git a/modules/ensembletraining/include/VO_BaggingRandomForestClassifier.h b/modules/ensembletraining/include/VO_BaggingRandomForestClassifier.h new file mode 100644 index 0000000..d8f2528 --- /dev/null +++ b/modules/ensembletraining/include/VO_BaggingRandomForestClassifier.h @@ -0,0 +1,90 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-16 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_BAGGINGRANDOMFORESTCLASSIFIER_H__ +#define __VO_BAGGINGRANDOMFORESTCLASSIFIER_H__ + +#include +#include + +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_AdditiveStrongerClassifier.h" + +using namespace std; +using namespace cv; + + +class VO_BaggingRandomForestClassifier : public VO_AdditiveStrongerClassifier +{ +public: + /** Bagging Type, RANDOMFOREST, etc. */ + enum {RANDOMFOREST = 0}; + + /** Constructor */ + VO_BaggingRandomForestClassifier() {} + + /** Destructor */ + ~VO_BaggingRandomForestClassifier() {} +}; + +#endif // __VO_BAGGINGRANDOMFORESTCLASSIFIER_H__ diff --git a/modules/ensembletraining/include/VO_BoostingCascadeClassifier.h b/modules/ensembletraining/include/VO_BoostingCascadeClassifier.h new file mode 100644 index 0000000..a780be6 --- /dev/null +++ b/modules/ensembletraining/include/VO_BoostingCascadeClassifier.h @@ -0,0 +1,131 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-16 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_BOOSTINGCASCADECLASSIFIER_H__ +#define __VO_BOOSTINGCASCADECLASSIFIER_H__ + +#include +#include + +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_Features.h" +#include "VO_AdditiveStrongerClassifier.h" + +#include "VO_BoostingSingleEnsembleClassifier.h" + +using namespace std; +using namespace cv; + + + + +class VO_BoostingCascadeClassifier : public VO_AdditiveStrongerClassifier +{ +private: + /** Number of stages */ + unsigned int m_iNbOfStages; + + /** min true positive */ + float m_fMinTruePositive; + + /** max wrong classification */ + float m_fMaxWrongClassification; + + /** Used Features */ + /** Explained by JIA Pei. Abstract class can't be instantiated, so, use pointer */ + /** In addition, Dimashova, Maria's precious suggestion is "do not load all features once for all" + * http://opencv.willowgarage.com/wiki/Contributors */ + VO_Features* m_VOFeatures; + +public: + enum { + UNDEFINED = 0, + SINGLECASCADE = 1, + PARALLELCASCADE = 2, + CASCADEDECISIONTREE = 3, + BFSCASCADE = 4}; + + /** Constructor */ + VO_BoostingCascadeClassifier() + { + this->m_iNbOfStages = 0; + this->m_VOFeatures = NULL; + } + + /** Destructor */ + ~VO_BoostingCascadeClassifier() {} + + /** training the boosting cascade classifier */ + bool train( const string& _cascadeDirName, + const vector _posFilenames, + const vector _negFilenames, + int _precalcValBufSize, + int _precalcIdxBufSize, + int _numStages, + float _minTruePositive, + float _maxWrongClassification, + const VO_Features* _featureParams); +}; + +#endif + diff --git a/modules/ensembletraining/include/VO_BoostingSingleEnsembleClassifier.h b/modules/ensembletraining/include/VO_BoostingSingleEnsembleClassifier.h new file mode 100644 index 0000000..f2e5579 --- /dev/null +++ b/modules/ensembletraining/include/VO_BoostingSingleEnsembleClassifier.h @@ -0,0 +1,117 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-16 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_BOOSTINGSINGLEENSEMBLECLASSIFIER_H_ +#define __VO_BOOSTINGSINGLEENSEMBLECLASSIFIER_H_ + + +#include "opencv/ml.h" +#include "VO_WeakClassifier.h" + +using namespace std; +using namespace cv; + + +class VO_BoostingSingleEnsembleClassifier : public CvBoost +{ +protected: + /** true positive requirement when training */ + float m_fMinTruePositive; + + /** false when training */ + float m_fMaxWrongClassification; + +public: + enum {DISCRETE = 0, REAL = 1}; + enum {STEEPESTDESCENT = 0, CONJUGATEGRADIENT = 1, NEWTONRAPHSON = 2}; + /** Margin1 -- e^{−yhf(x)} Margin2 -- ln(1 + e^{−yhf(x))} */ + enum {MARGIN1 = 0, MARGIN2 =1}; + + /** Constructor */ + VO_BoostingSingleEnsembleClassifier(float minTurePositive = 0.95f, + float maxWrongClassification = 0.5f) + { + this->m_fMinTruePositive = minTurePositive; + this->m_fMaxWrongClassification = maxWrongClassification; + } + + /** Destructor */ + virtual ~VO_BoostingSingleEnsembleClassifier() {} + + /** train */ +// virtual bool train( const CvFeatureEvaluator* _featureEvaluator, +// int _numSamples, int _precalcValBufSize, int _precalcIdxBufSize, +// const CvCascadeBoostParams& _params=CvCascadeBoostParams() ); + + /** predict */ + virtual float predict( int sampleIdx, bool returnSum = false ) const; + + void write( FileStorage &fs, const Mat& featureMap ) const; +// bool read( const FileNode &node, const CvFeatureEvaluator* _featureEvaluator, +// const CvCascadeBoostParams& _params ); + void markUsedFeaturesInMap( Mat& featureMap ); +protected: + virtual bool set_params( const CvBoostParams& _params ); + virtual void update_weights( CvBoostTree* tree ); + virtual bool isErrDesired(); +}; + +#endif // __VO_BOOSTINGSINGLEENSEMBLECLASSIFIER_H_ diff --git a/modules/ensembletraining/include/VO_WeakClassifier.h b/modules/ensembletraining/include/VO_WeakClassifier.h new file mode 100644 index 0000000..8452cd2 --- /dev/null +++ b/modules/ensembletraining/include/VO_WeakClassifier.h @@ -0,0 +1,98 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-16 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_WEAKCLASSIFIER_H_ +#define __VO_WEAKCLASSIFIER_H_ + + +#include "opencv/ml.h" + +using namespace std; +using namespace cv; + +class VO_WeakClassifier : public CvBoostTree +{ +protected: + /** For every weak classifier, every single feature has a single threshold to do binary classification */ + float m_fThreshold; + +public: + /** stump type, STUMP, SVMSTUMP, etc. */ + enum { + UNDEFINED = 0, + STUMP = 1, + SVMSTUMP = 2, + SELFDEFINEDWEAKCLASSIFIER = 3}; + + /** Constructor */ + VO_WeakClassifier() { this->m_fThreshold = 0.0; } + + /** Destructor */ + ~VO_WeakClassifier(){} + + /** Split */ +// void split_node_data( CvDTreeNode* node ); + + float GetThreshold() const {return this->m_fThreshold;} + +}; + +#endif // __VO_WEAKCLASSIFIER_H_ diff --git a/modules/ensembletraining/src/VO_AdditiveStrongerClassifier.cpp b/modules/ensembletraining/src/VO_AdditiveStrongerClassifier.cpp new file mode 100644 index 0000000..4629e89 --- /dev/null +++ b/modules/ensembletraining/src/VO_AdditiveStrongerClassifier.cpp @@ -0,0 +1,78 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-16 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_AdditiveStrongerClassifier.h" +#include + + +VO_AdditiveStrongerClassifier::VO_AdditiveStrongerClassifier() +{ + this->m_iNbOfSamples = 0; + this->m_iNbOfPositiveSamples = 0; + this->m_iNbOfNegativeSamples = 0; +} + + +VO_AdditiveStrongerClassifier::~VO_AdditiveStrongerClassifier() +{ + +} + diff --git a/modules/ensembletraining/src/VO_BaggingRandomForestClassifier.cpp b/modules/ensembletraining/src/VO_BaggingRandomForestClassifier.cpp new file mode 100644 index 0000000..afc5b69 --- /dev/null +++ b/modules/ensembletraining/src/VO_BaggingRandomForestClassifier.cpp @@ -0,0 +1,62 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-16 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include "VO_BaggingRandomForestClassifier.h" +#include diff --git a/modules/ensembletraining/src/VO_BoostingCascadeClassifier.cpp b/modules/ensembletraining/src/VO_BoostingCascadeClassifier.cpp new file mode 100644 index 0000000..4b385b5 --- /dev/null +++ b/modules/ensembletraining/src/VO_BoostingCascadeClassifier.cpp @@ -0,0 +1,103 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-16 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_BoostingCascadeClassifier.h" +#include + + + +/** + * @brief Train a boosting cascade classifier + * @param _cascadeDirName -- Input where to store the cascade (could be half trained) + * @param _posFilenames -- Input all positive file names + * @param _negFilenames -- Input all negative file names + * @param _precalcValBufSize -- Input all positive file names + * @param _precalcIdxBufSize -- Input all negative file names + * @param _numStages -- Input number of stages + * @param _featureParams -- Input how to extract those features + * + */ +bool VO_BoostingCascadeClassifier::train( const string& _cascadeDirName, + const vector _posFilenames, + const vector _negFilenames, + int _precalcValBufSize, + int _precalcIdxBufSize, + int _numStages, + float _minTruePositive, + float _maxWrongClassification, + const VO_Features* _featureParams) +{ + this->m_iNbOfPositiveSamples = _posFilenames.size(); + this->m_iNbOfNegativeSamples = _negFilenames.size(); + this->m_iNbOfSamples = this->m_iNbOfPositiveSamples + + this->m_iNbOfNegativeSamples; + this->m_iNbOfStages = _numStages; + this->m_fMinTruePositive = _minTruePositive; + this->m_fMaxWrongClassification = _maxWrongClassification; + + string dirName; + if ( _cascadeDirName.find('/') != string::npos ) + dirName = _cascadeDirName + '/'; + else + dirName = _cascadeDirName + '\\'; + + return true; +} diff --git a/modules/ensembletraining/src/VO_BoostingSingleEnsembleClassifier.cpp b/modules/ensembletraining/src/VO_BoostingSingleEnsembleClassifier.cpp new file mode 100644 index 0000000..5c5a862 --- /dev/null +++ b/modules/ensembletraining/src/VO_BoostingSingleEnsembleClassifier.cpp @@ -0,0 +1,68 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-16 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_BoostingSingleEnsembleClassifier.h" +#include + + + + + diff --git a/modules/ensembletraining/src/VO_WeakClassifier.cpp b/modules/ensembletraining/src/VO_WeakClassifier.cpp new file mode 100644 index 0000000..45ecb57 --- /dev/null +++ b/modules/ensembletraining/src/VO_WeakClassifier.cpp @@ -0,0 +1,287 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-16 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_WeakClassifier.h" +#include + +//void VO_WeakClassifier::split_node_data( CvDTreeNode* node ) +//{ +// int vi, i, n = node->sample_count, nl, nr, scount = data->sample_count; +// char* dir = (char*)data->direction->data.ptr; +// CvDTreeNode *left = 0, *right = 0; +// int* new_idx = data->split_buf->data.i; +// int new_buf_idx = data->get_child_buf_idx( node ); +// int work_var_count = data->get_work_var_count(); +// CvMat* buf = data->buf; +// cv::AutoBuffer inn_buf(n*(3*sizeof(int) + sizeof(float))); +// int* temp_buf = (int*)(uchar*)inn_buf; +// bool split_input_data; +// +// complete_node_dir(node); +// +// for( i = nl = nr = 0; i < n; i++ ) +// { +// int d = dir[i]; +// // initialize new indices for splitting ordered variables +// new_idx[i] = (nl & (d-1)) | (nr & -d); // d ? ri : li +// nr += d; +// nl += d^1; +// } +// +// node->left = left = data->new_node( node, nl, new_buf_idx, node->offset ); +// node->right = right = data->new_node( node, nr, new_buf_idx, node->offset + nl ); +// +// split_input_data = node->depth + 1 < data->params.max_depth && +// (node->left->sample_count > data->params.min_sample_count || +// node->right->sample_count > data->params.min_sample_count); +// +// // split ordered variables, keep both halves sorted. +// for( vi = 0; vi < ((CvCascadeBoostTrainData*)data)->numPrecalcIdx; vi++ ) +// { +// int ci = data->get_var_type(vi); +// +// if( ci >= 0 || !split_input_data ) +// continue; +// +// int n1 = node->get_num_valid(vi); +// float* src_val_buf = (float*)(uchar*)(temp_buf + n); +// int *src_sorted_idx_buf = (int*)(src_val_buf + n); +// int *src_sample_idx_buf = src_sorted_idx_buf + n; +// const int* src_sorted_idx = 0; +// const float* src_val = 0; +// data->get_ord_var_data(node, vi, src_val_buf, src_sorted_idx_buf, &src_val, &src_sorted_idx, src_sample_idx_buf); +// +// for(i = 0; i < n; i++) +// temp_buf[i] = src_sorted_idx[i]; +// +// if (data->is_buf_16u) +// { +// unsigned short *ldst, *rdst, *ldst0, *rdst0; +// ldst0 = ldst = (unsigned short*)(buf->data.s + left->buf_idx*buf->cols + +// vi*scount + left->offset); +// rdst0 = rdst = (unsigned short*)(ldst + nl); +// +// // split sorted +// for( i = 0; i < n1; i++ ) +// { +// int idx = temp_buf[i]; +// int d = dir[idx]; +// idx = new_idx[idx]; +// if (d) +// { +// *rdst = (unsigned short)idx; +// rdst++; +// } +// else +// { +// *ldst = (unsigned short)idx; +// ldst++; +// } +// } +// assert( n1 == n); +// +// left->set_num_valid(vi, (int)(ldst - ldst0)); +// right->set_num_valid(vi, (int)(rdst - rdst0)); +// } +// else +// { +// int *ldst0, *ldst, *rdst0, *rdst; +// ldst0 = ldst = buf->data.i + left->buf_idx*buf->cols + +// vi*scount + left->offset; +// rdst0 = rdst = buf->data.i + right->buf_idx*buf->cols + +// vi*scount + right->offset; +// +// // split sorted +// for( i = 0; i < n1; i++ ) +// { +// int idx = temp_buf[i]; +// int d = dir[idx]; +// idx = new_idx[idx]; +// if (d) +// { +// *rdst = idx; +// rdst++; +// } +// else +// { +// *ldst = idx; +// ldst++; +// } +// } +// +// left->set_num_valid(vi, (int)(ldst - ldst0)); +// right->set_num_valid(vi, (int)(rdst - rdst0)); +// CV_Assert( n1 == n); +// } +// } +// +// // split cv_labels using new_idx relocation table +// int *src_lbls_buf = temp_buf + n; +// const int* src_lbls = data->get_cv_labels(node, src_lbls_buf); +// +// for(i = 0; i < n; i++) +// temp_buf[i] = src_lbls[i]; +// +// if (data->is_buf_16u) +// { +// unsigned short *ldst = (unsigned short *)(buf->data.s + left->buf_idx*buf->cols + +// (work_var_count-1)*scount + left->offset); +// unsigned short *rdst = (unsigned short *)(buf->data.s + right->buf_idx*buf->cols + +// (work_var_count-1)*scount + right->offset); +// +// for( i = 0; i < n; i++ ) +// { +// int idx = temp_buf[i]; +// if (dir[i]) +// { +// *rdst = (unsigned short)idx; +// rdst++; +// } +// else +// { +// *ldst = (unsigned short)idx; +// ldst++; +// } +// } +// +// } +// else +// { +// int *ldst = buf->data.i + left->buf_idx*buf->cols + +// (work_var_count-1)*scount + left->offset; +// int *rdst = buf->data.i + right->buf_idx*buf->cols + +// (work_var_count-1)*scount + right->offset; +// +// for( i = 0; i < n; i++ ) +// { +// int idx = temp_buf[i]; +// if (dir[i]) +// { +// *rdst = idx; +// rdst++; +// } +// else +// { +// *ldst = idx; +// ldst++; +// } +// } +// } +// for( vi = 0; vi < data->var_count; vi++ ) +// { +// left->set_num_valid(vi, (int)(nl)); +// right->set_num_valid(vi, (int)(nr)); +// } +// +// // split sample indices +// int *sample_idx_src_buf = temp_buf + n; +// const int* sample_idx_src = data->get_sample_indices(node, sample_idx_src_buf); +// +// for(i = 0; i < n; i++) +// temp_buf[i] = sample_idx_src[i]; +// +// if (data->is_buf_16u) +// { +// unsigned short* ldst = (unsigned short*)(buf->data.s + left->buf_idx*buf->cols + +// work_var_count*scount + left->offset); +// unsigned short* rdst = (unsigned short*)(buf->data.s + right->buf_idx*buf->cols + +// work_var_count*scount + right->offset); +// for (i = 0; i < n; i++) +// { +// unsigned short idx = (unsigned short)temp_buf[i]; +// if (dir[i]) +// { +// *rdst = idx; +// rdst++; +// } +// else +// { +// *ldst = idx; +// ldst++; +// } +// } +// } +// else +// { +// int* ldst = buf->data.i + left->buf_idx*buf->cols + +// work_var_count*scount + left->offset; +// int* rdst = buf->data.i + right->buf_idx*buf->cols + +// work_var_count*scount + right->offset; +// for (i = 0; i < n; i++) +// { +// int idx = temp_buf[i]; +// if (dir[i]) +// { +// *rdst = idx; +// rdst++; +// } +// else +// { +// *ldst = idx; +// ldst++; +// } +// } +// } +// +// // deallocate the parent node data that is not needed anymore +// data->free_node_data(node); +//} + diff --git a/modules/featureextraction/CMakeLists.txt b/modules/featureextraction/CMakeLists.txt new file mode 100644 index 0000000..7dc7104 --- /dev/null +++ b/modules/featureextraction/CMakeLists.txt @@ -0,0 +1,3 @@ +include_directories("${CMAKE_CURRENT_SOURCE_DIR}/../common/include" "${CMAKE_CURRENT_SOURCE_DIR}/../integraltransform/include") +set(deps opencv_core opencv_imgproc opencv_highgui) +define_vosm_module(featureextraction vosm_common vosm_integraltransform ${deps}) diff --git a/modules/featureextraction/featureextraction.vcxproj b/modules/featureextraction/featureextraction.vcxproj new file mode 100644 index 0000000..c8c7b63 --- /dev/null +++ b/modules/featureextraction/featureextraction.vcxproj @@ -0,0 +1,91 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {01A5E120-9D02-44A1-A7E1-F6190F212AA5} + featureextraction + + + + StaticLibrary + true + MultiByte + v110 + + + Application + false + true + MultiByte + v110 + + + + + + + + + + + + + + + Level3 + Disabled + ./include;../include;../common/include;../integraltransform/include;C:\OpenCV2.4.5\opencv\build\include;C:\boost\boost_1_53_0;%(AdditionalIncludeDirectories) + NOMINMAX;_MBCS;%(PreprocessorDefinitions) + + + true + + + %(AdditionalDependencies) + C:\OpenCV2.4.5\opencv\build\x86\vc11\staticlib + + + true + + + + + Level3 + MaxSpeed + true + true + + + true + true + true + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/modules/featureextraction/featureextraction.vcxproj.filters b/modules/featureextraction/featureextraction.vcxproj.filters new file mode 100644 index 0000000..4910334 --- /dev/null +++ b/modules/featureextraction/featureextraction.vcxproj.filters @@ -0,0 +1,54 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + \ No newline at end of file diff --git a/modules/featureextraction/include/VO_DaubechiesFeatures.h b/modules/featureextraction/include/VO_DaubechiesFeatures.h new file mode 100644 index 0000000..185ec82 --- /dev/null +++ b/modules/featureextraction/include/VO_DaubechiesFeatures.h @@ -0,0 +1,152 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-08 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef _VO_DAUBECHIESFEATURES_H_ +#define _VO_DAUBECHIESFEATURES_H_ + +#include "VO_Features.h" +#include "VO_Daubechies.h" + +#define CC_RECTS "rects" +#define CC_TILTED "tilted" + + +/** + * @author JIA Pei + * @brief Daubechies wavelet features. + */ +class VO_DaubechiesFeatures : public VO_Features +{ +protected: + /** which Haar mode is to be used? BASIC, CORE or ALL? */ + unsigned int m_iMode; + + class Feature + { + public: + Feature(); + Feature( int offset, int x, int y, int _block_w, int _block_h ); + float filter( const Mat_& iImg ); + void calc( const Mat_& iImg ); + void write( FileStorage &fs ) const; + + bool isFiltering; + Rect rect; + VO_Daubechies daub; + Mat_ freqs; + }; + + vector m_vAllFeatures; + + /** Initialization */ + void init(); + +public: + enum { + DAUBECHIES4 = 0, + DAUBECHIES6 = 1, + DAUBECHIES8 = 2, + DAUBECHIES10 = 3, + DAUBECHIES12 = 4, + DAUBECHIES14 = 5, + DAUBECHIES16 = 6, + DAUBECHIES18 = 7, + DAUBECHIES20 = 8 + }; + + /** default constructor */ + VO_DaubechiesFeatures () {this->m_iFeatureType = DAUBECHIES;} + + /** destructor */ + virtual ~VO_DaubechiesFeatures () {this->m_vAllFeatures.clear();} + + /** Generate all features with a specific mode */ + virtual void VO_GenerateAllFeatureInfo(const Size& size, unsigned int generatingMode = DAUBECHIES4); + virtual void VO_GenerateAllFeatures(const Mat& iImg, Point pt = Point(0,0)); + + /** Read and write */ + virtual void ReadFeatures( const FileStorage& fs, Mat_& featureMap ); + virtual void WriteFeatures( FileStorage& fs, const Mat_& featureMap ) const; +}; + + + +/** +* @brief Daubechies filtering +* @param iImg Input -- input image + */ +inline float VO_DaubechiesFeatures::Feature::filter( const Mat_& iImg ) +{ + return 0.0; +} + + +/** +* @brief calculate one feature +* @param iImg Input -- input image + */ +inline void VO_DaubechiesFeatures::Feature::calc(const Mat_& iImg) +{ + daub.VO_ForwardTransform(iImg, freqs); +} + +#endif // _VO_DAUBECHIESFEATURES_H_ + diff --git a/modules/featureextraction/include/VO_DirectFeatures.h b/modules/featureextraction/include/VO_DirectFeatures.h new file mode 100644 index 0000000..f869bf5 --- /dev/null +++ b/modules/featureextraction/include/VO_DirectFeatures.h @@ -0,0 +1,110 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-08 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef _VO_DIRECTFEATURES_H_ +#define _VO_DIRECTFEATURES_H_ + +#include "VO_Features.h" +#include "VO_Gabor.h" + +#define CC_RECTS "rects" +#define CC_TILTED "tilted" + + +#define NBOFLAMBDA 3 +#define UNDIRTHETA 4 +#define DIRTHETA 8 +#define SIGMA 4 +#define NBOFGAMMA 3 + + +#define HFP_NAME "gaborFeatureParams" + + +/** + * @author JIA Pei + * @brief Direct image intensities. + */ +class VO_DirectFeatures : public VO_Features +{ +protected: + /** Intialization */ + void init(); + +public: + /** default constructor */ + VO_DirectFeatures () {this->m_iFeatureType = DIRECT;} + + /** destructor */ + virtual ~VO_DirectFeatures () {} + + /** Generate all features with a specific mode */ + virtual void VO_GenerateAllFeatureInfo(const Size& size, unsigned int generatingMode = 0); + virtual void VO_GenerateAllFeatures(const Mat& iImg, Point pt = Point(0,0)); + + /** Read and write */ + virtual void ReadFeatures( const FileStorage& fs, Mat_& featureMap ); + virtual void WriteFeatures( FileStorage& fs, const Mat_& featureMap ) const; +}; + + + +#endif // _VO_DIRECTFEATURES_H_ + diff --git a/modules/featureextraction/include/VO_Features.h b/modules/featureextraction/include/VO_Features.h new file mode 100644 index 0000000..cffe1e2 --- /dev/null +++ b/modules/featureextraction/include/VO_Features.h @@ -0,0 +1,260 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-08 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef _VO_FEATURES_H_ +#define _VO_FEATURES_H_ + +#include "opencv/cxcore.h" +#include "opencv/cv.h" +#include "opencv/ml.h" + + +using namespace std; +using namespace cv; + +#define CC_FEATURES FEATURES +#define CC_FEATURE_PARAMS "featureParams" +#define CC_MAX_CAT_COUNT "maxCatCount" +#define CC_RECTS "rects" +#define CC_RECT "rect" + +#define FEATURES "features" + +#define CV_SUM_OFFSETS( p0, p1, p2, p3, rect, step ) \ + /* (x, y) */ \ + (p0) = (rect).x + (step) * (rect).y; \ + /* (x + w, y) */ \ + (p1) = (rect).x + (rect).width + (step) * (rect).y; \ + /* (x, y + h) */ \ + (p2) = (rect).x + (step) * ((rect).y + (rect).height); \ + /* (x + w, y + h) */ \ + (p3) = (rect).x + (rect).width + (step) * ((rect).y + (rect).height); + +#define CV_TILTED_OFFSETS( p0, p1, p2, p3, rect, step ) \ + /* (x, y) */ \ + (p0) = (rect).x + (step) * (rect).y; \ + /* (x + w, y + w) */ \ + (p1) = (rect).x + (rect).width + (step) * ((rect).y + (rect).width); \ + /* (x - h, y + h) */ \ + (p2) = (rect).x - (rect).height + (step) * ((rect).y + (rect).height);\ + /* (x + w - h, y + w + h) */ \ + (p3) = (rect).x + (rect).width - (rect).height \ + + (step) * ((rect).y + (rect).width + (rect).height); + + +template +void _writeFeatures(const vector features, + FileStorage &fs, + const Mat& featureMap ) +{ + fs << FEATURES << "["; + const Mat_& featureMap_ = (const Mat_&)featureMap; + for ( int fi = 0; fi < featureMap.cols; fi++ ) + if ( featureMap_(0, fi) >= 0 ) + { + fs << "{"; + features[fi].write( fs ); + fs << "}"; + } + fs << "]"; +} + + +//template +//void _readFeatures( vector features, const FileStorage &fs, const Mat& featureMap ) +//{ +// +// fs >> FEATURES >> "["; +// const Mat_& featureMap_ = (const Mat_&)featureMap; +// for ( int fi = 0; fi < featureMap.cols; fi++ ) +// if ( featureMap_(0, fi) >= 0 ) +// { +// fs << "{"; +// features[fi].write( fs ); +// fs << "}"; +// } +// fs << "]"; +//} +// + + +/** + * @author JIA Pei + * @version 2010-03-13 + * @brief Generalized class for feature evaluator, namely, extracted features + */ +class VO_Features +{ +friend class VO_ASMLTCs; +friend class VO_FittingASMLTCs; +protected: + /** All features are stored as a row vector */ + Mat_ m_MatFeatures; + + /** Feature type -- which type of the feature is it? LBP, Haar, Gabor, etc. */ + unsigned int m_iFeatureType; + + /** Total number of feature categories. Some times, + * we have thousands of features, but they can be categorized into 16 categories. + * -- 0 in case of numerical features */ + unsigned int m_iNbOfFeatureCategories; + + /** Total number of features */ + unsigned int m_iNbOfFeatures; + + /** Total number of adopted features */ + unsigned int m_iNbOfAdoptedFeatures; + + /** Adopted feature indexes */ + vector m_vAdoptedFeatureIndexes; + + /** All features are extracted from an image patch of such a size */ + Size m_CVSize; + + /** Integral image */ + Mat m_MatIntegralImage; + + /** Square image */ + Mat m_MatSquareImage; + + /** Tilted integral image */ + Mat m_MatTiltedIntegralImage; + + /** Normalization factor */ + Mat m_MatNormFactor; + + /** Initialization */ + void init() + { + this->m_MatFeatures.release(); + this->m_iFeatureType = UNDEFINED; + this->m_CVSize = Size(0, 0); + this->m_MatIntegralImage.release(); + this->m_MatSquareImage.release(); + this->m_MatTiltedIntegralImage.release(); + this->m_MatNormFactor.release(); + } + +public: + enum { + UNDEFINED = 0, + DIRECT = 1, + HISTOGRAMEQUALIZED = 2, + LAPLACE = 3, + GABOR = 4, + HAAR = 5, + LBP = 6, + DAUBECHIES = 7, + COIFLETS = 8, + SEGMENTATION = 9, + HARRISCORNER = 10, + SELFDEFINED = 11 + }; + + /** Default constructor */ + VO_Features() {this->init();} + + /** Constructor */ + VO_Features(unsigned int type, Size size) + { + this->init(); + this->m_iFeatureType = type; + this->m_CVSize = size; + } + + /** Destructor */ + virtual ~VO_Features() + { + this->m_MatFeatures.release(); + this->m_MatIntegralImage.release(); + this->m_MatSquareImage.release(); + this->m_MatTiltedIntegralImage.release(); + this->m_MatNormFactor.release(); + } + + /** Generate all features within such a size of window */ + virtual void VO_GenerateAllFeatureInfo(const Size& size, unsigned int generatingMode = 0) = 0; + virtual void VO_GenerateAllFeatures(const Mat& iImg, Point pt = Point(0,0)) = 0; + + /** Read and write */ + virtual void ReadFeatures( const FileStorage& fs, Mat_& featureMap ) = 0; + virtual void WriteFeatures( FileStorage& fs, const Mat_& featureMap ) const = 0; + + // Gets and Sets + unsigned int GetFeatureType() const { return this->m_iFeatureType; } + unsigned int GetNbOfFeatureCategories() const { return this->m_iNbOfFeatureCategories; } + unsigned int GetNbOfFeatures() const { return this->m_iNbOfFeatures; } + unsigned int GetNbOfAdoptedFeatures() const {return this->m_iNbOfAdoptedFeatures; } + vector GetAdoptedFeatureIndexes() const { return this->m_vAdoptedFeatureIndexes; } + Mat_ GetFeatures() const { return this->m_MatFeatures; } + Mat_ GetAdoptedFeatures() const + { + Mat_ res = Mat_::zeros(1, this->m_iNbOfAdoptedFeatures); + for(unsigned int i = 0; i < this->m_iNbOfAdoptedFeatures; i++) + { + res(0, i) = this->m_MatFeatures(0, this->m_vAdoptedFeatureIndexes[i]); + } + return res; + } + Size GetSize() const { return this->m_CVSize; } +}; + +#endif // _VO_FEATURES_H_ diff --git a/modules/featureextraction/include/VO_GaborFeatures.h b/modules/featureextraction/include/VO_GaborFeatures.h new file mode 100644 index 0000000..3f47d07 --- /dev/null +++ b/modules/featureextraction/include/VO_GaborFeatures.h @@ -0,0 +1,161 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-08 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef _VO_GABORFEATURES_H_ +#define _VO_GABORFEATURES_H_ + +#include "VO_Features.h" +#include "VO_Gabor.h" + +#define CC_RECTS "rects" +#define CC_TILTED "tilted" + + +#define NBOFLAMBDA 3 +#define UNDIRTHETA 4 +#define DIRTHETA 8 +#define SIGMA 4 +#define NBOFGAMMA 3 + + + +/** + * @author JIA Pei + * @brief Gabor features. + */ +class VO_GaborFeatures : public VO_Features +{ +protected: + /** which Haar mode is to be used? ISOTROPY, ANISOTROPY or ALL? */ + unsigned int m_iMode; + + class Feature + { + public: + Feature(); + Feature(int offset, + int x, + int y, + bool filtering, + unsigned int nstds, + float lambda, + float theta, + float psi, + float sigma, + float gamma ); + float filter( const Mat_& iImg ); + void calc( const Mat_& iImg ); + void write( FileStorage &fs ) const; + + bool isFiltering; + Rect rect; + VO_Gabor gabor; + Mat_ freqs; + }; + + vector m_vAllFeatures; + + /** Initialization */ + void init(); + +public: + /* 0 - ISOTROPY + * 1 - ANISOTROPY + * 2 - ALL = including 45 degrees */ + enum { ISOTROPY = 0, ANISOTROPY = 1, ALL = 2 }; + + /** default constructor */ + VO_GaborFeatures () {this->m_iFeatureType = GABOR;} + + /** destructor */ + virtual ~VO_GaborFeatures () {this->m_vAllFeatures.clear();} + + /** Generate all features with a specific mode */ + virtual void VO_GenerateAllFeatureInfo(const Size& size, unsigned int generatingMode = ISOTROPY); + virtual void VO_GenerateAllFeatures(const Mat& iImg, Point pt = Point(0,0)); + + /** Read and write */ + virtual void ReadFeatures( const FileStorage& fs, Mat_& featureMap ); + virtual void WriteFeatures( FileStorage& fs, const Mat_& featureMap ) const; +}; + + +/** + * @brief Gabor filtering + * @param iImg Input -- input image + */ +inline float VO_GaborFeatures::Feature::filter( const Mat_& iImg ) +{ + return gabor.VO_GaborFiltering(iImg); +} + + +/** + * @brief calculate one feature + * @param iImg Input -- input image + */ +inline void VO_GaborFeatures::Feature::calc(const Mat_& iImg) +{ + gabor.VO_ForwardTransform(iImg, freqs); +} + +#endif // _VO_GABORFEATURES_H_ + diff --git a/modules/featureextraction/include/VO_HaarFeatures.h b/modules/featureextraction/include/VO_HaarFeatures.h new file mode 100644 index 0000000..ba79ec1 --- /dev/null +++ b/modules/featureextraction/include/VO_HaarFeatures.h @@ -0,0 +1,149 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-08 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef _VO_HAARFEATURES_H_ +#define _VO_HAARFEATURES_H_ + +#include "VO_Features.h" + +#define CC_HAAR "HAAR" +#define CC_MODE "mode" +#define CC_MODE_BASIC "BASIC" +#define CC_MODE_CORE "CORE" +#define CC_MODE_ALL "ALL" +//#define CC_RECTS "rects" +#define CC_TILTED "tilted" +#define CV_HAAR_FEATURE_MAX 3 + + + +/** + * @author JIA Pei + * @brief Haar wavelet features. + */ +class VO_HaarFeatures : public VO_Features +{ +protected: + /** which Haar mode is to be used? BASIC, CORE or ALL? */ + unsigned int m_iMode; + + class Feature + { + public: + Feature(); + Feature( int offset, bool _tilted, + int x0, int y0, int w0, int h0, float wt0, + int x1, int y1, int w1, int h1, float wt1, + int x2 = 0, int y2 = 0, int w2 = 0, int h2 = 0, float wt2 = 0.0F ); + float calc( const Mat &sum, const Mat &tilted) const; + void write( FileStorage &fs ) const; + + bool tilted; + struct + { + Rect r; + float weight; + } rect[CV_HAAR_FEATURE_MAX]; + + struct + { + int p0, p1, p2, p3; + } fastRect[CV_HAAR_FEATURE_MAX]; + }; + + vector m_vAllFeatures; + + /** Initialization */ + void init(); + +public: + /* 0 - BASIC = Viola + * 1 - CORE = All upright + * 2 - ALL = All features */ + enum { BASIC = 0, CORE = 1, ALL = 2 }; + + /** default constructor */ + VO_HaarFeatures () {this->m_iFeatureType = HAAR;} + + /** destructor */ + virtual ~VO_HaarFeatures () {this->m_vAllFeatures.clear();} + + /** Generate all features with a specific mode */ + virtual void VO_GenerateAllFeatureInfo(const Size& size, unsigned int generatingMode = 0); + virtual void VO_GenerateAllFeatures(const Mat& iImg, Point pt = Point(0,0)); + + /** Read and write */ + virtual void ReadFeatures( const FileStorage& fs, Mat_& featureMap ); + virtual void WriteFeatures( FileStorage& fs, const Mat_& featureMap ) const; +}; + + +inline float VO_HaarFeatures::Feature::calc( const Mat &_sum, const Mat &_tilted) const +{ + const int* img = tilted ? _tilted.ptr(0) : _sum.ptr(0); + float ret = rect[0].weight * (img[fastRect[0].p0] - img[fastRect[0].p1] - img[fastRect[0].p2] + img[fastRect[0].p3] ) + + rect[1].weight * (img[fastRect[1].p0] - img[fastRect[1].p1] - img[fastRect[1].p2] + img[fastRect[1].p3] ); + if( rect[2].weight != 0.0f ) + ret += rect[2].weight * (img[fastRect[2].p0] - img[fastRect[2].p1] - img[fastRect[2].p2] + img[fastRect[2].p3] ); + return ret; +} + +#endif // _VO_HAARFEATURES_H_ diff --git a/modules/featureextraction/include/VO_LBPFeatures.h b/modules/featureextraction/include/VO_LBPFeatures.h new file mode 100644 index 0000000..2cc4fa6 --- /dev/null +++ b/modules/featureextraction/include/VO_LBPFeatures.h @@ -0,0 +1,133 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-08 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef _VO_LBPFEATURES_H_ +#define _VO_LBPFEATURES_H_ + +#include "VO_Features.h" + +#define CC_LBP "LBP" +//#define CC_RECT "rect" + +#define LBPF_NAME "lbpFeatureParams" + + +/** + * @author JIA Pei + * @brief LBP features. + */ +class VO_LBPFeatures : public VO_Features +{ +protected: + class Feature + { + public: + Feature(); + Feature( int offset, int x, int y, int _block_w, int _block_h ); + uchar calc( const Mat& _sum ) const; + void write( FileStorage &fs ) const; + + Rect rect; + int p[16]; + }; + + vector m_vAllFeatures; + + /** Initialization */ + void init(); + +public: + /** default constructor */ + VO_LBPFeatures () {this->m_iFeatureType = LBP;} + + /** destructor */ + virtual ~VO_LBPFeatures() {this->m_vAllFeatures.clear();} + + /** Generate all features with a specific mode */ + virtual void VO_GenerateAllFeatureInfo(const Size& size, unsigned int generatingMode = 0); + virtual void VO_GenerateAllFeatures(const Mat& iImg, Point pt = Point(0,0)); + + /** Read and write */ + virtual void ReadFeatures( const FileStorage& fs, Mat_& featureMap ); + virtual void WriteFeatures( FileStorage &fs, const Mat_& featureMap ) const; +}; + + +/** + * @brief calculate one feature + * @param _sum Input -- integral image + */ +inline uchar VO_LBPFeatures::Feature::calc(const Mat &_sum) const +{ + const int* sum = _sum.ptr(0); + int cval = sum[p[5]] - sum[p[6]] - sum[p[9]] + sum[p[10]]; + + return (uchar)( (sum[p[0]] - sum[p[1]] - sum[p[4]] + sum[p[5]] >= cval ? 128 : 0) | // 0 + (sum[p[1]] - sum[p[2]] - sum[p[5]] + sum[p[6]] >= cval ? 64 : 0) | // 1 + (sum[p[2]] - sum[p[3]] - sum[p[6]] + sum[p[7]] >= cval ? 32 : 0) | // 2 + (sum[p[6]] - sum[p[7]] - sum[p[10]] + sum[p[11]] >= cval ? 16 : 0) | // 5 + (sum[p[10]] - sum[p[11]] - sum[p[14]] + sum[p[15]] >= cval ? 8 : 0) | // 8 + (sum[p[9]] - sum[p[10]] - sum[p[13]] + sum[p[14]] >= cval ? 4 : 0) | // 7 + (sum[p[8]] - sum[p[9]] - sum[p[12]] + sum[p[13]] >= cval ? 2 : 0) | // 6 + (sum[p[4]] - sum[p[5]] - sum[p[8]] + sum[p[9]] >= cval ? 1 : 0)); // 3 +} + +#endif diff --git a/modules/featureextraction/src/VO_DaubechiesFeatures.cpp b/modules/featureextraction/src/VO_DaubechiesFeatures.cpp new file mode 100644 index 0000000..e98b8bb --- /dev/null +++ b/modules/featureextraction/src/VO_DaubechiesFeatures.cpp @@ -0,0 +1,157 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-08 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_DaubechiesFeatures.h" + + +void VO_DaubechiesFeatures::init( ) +{ + VO_Features::init(); + this->m_iFeatureType = DAUBECHIES; +} + + +void VO_DaubechiesFeatures::WriteFeatures( FileStorage &fs, const Mat_& featureMap ) const +{ + _writeFeatures( m_vAllFeatures, fs, featureMap ); +} + + +void VO_DaubechiesFeatures::ReadFeatures( const FileStorage& fs, Mat_& featureMap ) +{ + +} + + +/** +* @brief Generating all possible feature rectangles +* @param size Input -- the concerned size +* @param mode Input -- mode, for LBP, no use +* @return void + */ + void VO_DaubechiesFeatures::VO_GenerateAllFeatureInfo(const Size& size, unsigned int mode) + { + this->m_CVSize = size; + + int offset = this->m_CVSize.width + 1; + for( int x = 0; x < this->m_CVSize.width; x++ ) + { + for( int y = 0; y < this->m_CVSize.height; y++ ) + { + + } + } + this->m_iNbOfFeatures = this->m_vAllFeatures.size(); +} + + +/** +* @brief Generating all features within the specified rectangle from the input image +* @param iImg Input -- the input image +* @param *rect Input -- the concerned rectangle +* @param mode Input -- mode, BASIC, CORE or ALL +* @return void + */ +void VO_DaubechiesFeatures::VO_GenerateAllFeatures(const Mat& iImg, Point pt) +{ + if( this->m_CVSize.width > iImg.cols || + this->m_CVSize.height > iImg.rows || + pt.x <0 || + pt.y < 0 || + this->m_CVSize.width + pt.x > iImg.cols || + this->m_CVSize.height + pt.y > iImg.rows ) + { + cerr << "Feature rectangles are out of the image" << endl; + } + + Rect rect(pt.x, pt.y, this->m_CVSize.width, this->m_CVSize.height); + Mat rectImg = iImg(rect); + + this->m_MatFeatures = Mat_(1, this->m_iNbOfFeatures); + for(unsigned int i = 0; i < this->m_iNbOfFeatures; i++) + { + + } +} + + +/** Default constructor */ +VO_DaubechiesFeatures::Feature::Feature() +{ + rect = cvRect(0, 0, 0, 0); +} + + +/** + * @brief constructor to create all LBP features + * @param offset Input -- width+1 + */ +VO_DaubechiesFeatures::Feature::Feature( int offset, int x, int y, int _blockWidth, int _blockHeight ) +{ + Rect tr = rect = Rect(x, y, _blockWidth, _blockHeight); +} + + +void VO_DaubechiesFeatures::Feature::write(FileStorage &fs) const +{ + fs << CC_RECT << "[:" << rect.x << rect.y << rect.width << rect.height << "]"; +} + diff --git a/modules/featureextraction/src/VO_DirectFeatures.cpp b/modules/featureextraction/src/VO_DirectFeatures.cpp new file mode 100644 index 0000000..23adbd1 --- /dev/null +++ b/modules/featureextraction/src/VO_DirectFeatures.cpp @@ -0,0 +1,129 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-08 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_DirectFeatures.h" + + +void VO_DirectFeatures::init( ) +{ + VO_Features::init(); + this->m_iFeatureType = DIRECT; +} + + +void VO_DirectFeatures::WriteFeatures( FileStorage &fs, const Mat_& featureMap ) const +{ + +} + + +void VO_DirectFeatures::ReadFeatures( const FileStorage& fs, Mat_& featureMap ) +{ + +} + + +/** + * @brief Generating all possible feature rectangles + * @param size Input -- the concerned size + * @param mode Input -- mode, for LBP, no use + * @return void + */ + void VO_DirectFeatures::VO_GenerateAllFeatureInfo(const Size& size, unsigned int mode) + { + this->m_CVSize = size; + this->m_iNbOfFeatures = this->m_CVSize.width*this->m_CVSize.height; +} + + +/** + * @brief Generating all features within the specified rectangle from the input image + * @param iImg Input -- the input image + * @param pt Input -- start point at the top left corner + * @return void + */ +void VO_DirectFeatures::VO_GenerateAllFeatures(const Mat& iImg, Point pt) +{ + if( this->m_CVSize.width > iImg.cols || + this->m_CVSize.height > iImg.rows || + pt.x <0 || + pt.y < 0 || + this->m_CVSize.width + pt.x > iImg.cols || + this->m_CVSize.height + pt.y > iImg.rows ) + { + cerr << "Feature rectangles are out of the image" << endl; + } + + Rect rect(pt.x, pt.y, this->m_CVSize.width, this->m_CVSize.height); + Mat rectImg = iImg(rect); + + + this->m_MatFeatures = Mat_(1, this->m_iNbOfFeatures); + for(int i = 0; i < this->m_CVSize.height; i++) + { + for(int j = 0; j < this->m_CVSize.width; j++) + { + this->m_MatFeatures(0, i*this->m_CVSize.width+j) = (float)rectImg.at(i, j); + } + } +} + + diff --git a/modules/featureextraction/src/VO_GaborFeatures.cpp b/modules/featureextraction/src/VO_GaborFeatures.cpp new file mode 100644 index 0000000..24ac1aa --- /dev/null +++ b/modules/featureextraction/src/VO_GaborFeatures.cpp @@ -0,0 +1,219 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-08 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_GaborFeatures.h" + +#include "VO_Common.h" + +void VO_GaborFeatures::init( ) +{ + VO_Features::init(); + this->m_iFeatureType = GABOR; +} + + +void VO_GaborFeatures::WriteFeatures( FileStorage &fs, const Mat_& featureMap ) const +{ + _writeFeatures( m_vAllFeatures, fs, featureMap ); +} + + +void VO_GaborFeatures::ReadFeatures( const FileStorage& fs, Mat_& featureMap ) +{ + +} + + +/** + * @brief Generating all possible feature rectangles + * @param size Input -- the concerned size + * @param mode Input -- mode, for LBP, no use + * @return void + */ + void VO_GaborFeatures::VO_GenerateAllFeatureInfo(const Size& size, unsigned int mode) + { + this->m_CVSize = size; + int NbOfTheta = mode == ALL ? DIRTHETA : UNDIRTHETA; + + const unsigned int nstds = 4; + float sigma_x, sigma_y, lambda, theta, gamma; + int xmax, ymax; + int offset = this->m_CVSize.width + 1; + for( int x = 0; x < this->m_CVSize.width; x++ ) + { + for( int y = 0; y < this->m_CVSize.height; y++ ) + { +// for(int nstds = 1; nstds <= NBOFSTDS; nstds++) +// { + for(int nlambda = 0; nlambda < NBOFLAMBDA; nlambda++) + { + for(int ntheta = 0; ntheta < NbOfTheta; ntheta++) + { + for(int psi = 0; psi <=0; psi++) + { + for(int sigma = 1; sigma <= SIGMA; sigma++) + { + for(int ngamma = 0; ngamma < NBOFGAMMA; ngamma++) + { + lambda = safeDoubleToFloat(pow(2.0, (double)(nlambda-NBOFLAMBDA/2.0))); + gamma = safeDoubleToFloat(pow(2.0, (double)(ngamma-NBOFGAMMA/2.0))); + theta = 2.0f*safeDoubleToFloat(CV_PI)*(float)ntheta/(float)NbOfTheta; + sigma_x = (float)sigma; + sigma_y = (float)sigma/(float)gamma; + xmax = static_cast(ceil( max(1.0f, max(fabs((float)nstds*sigma_x*cos((float)theta)),fabs((float)nstds*sigma_y*sin((float)theta))) ) ) ); + ymax = static_cast(ceil( max(1.0f, max(fabs((float)nstds*sigma_x*sin((float)theta)),fabs((float)nstds*sigma_y*cos((float)theta))) ) ) ); + + if( (x+2*xmax+1 <= m_CVSize.width) && (y+2*ymax+1 <= this->m_CVSize.height) ) + this->m_vAllFeatures.push_back( Feature(offset, x, y, true, + nstds, + lambda, + theta, + static_cast(psi), + static_cast(sigma), + gamma) ); + } + } + } + } + } +// } + } + } + + this->m_iMode = mode; + this->m_iNbOfFeatures = this->m_vAllFeatures.size(); +} + + +/** + * @brief Generating all features within the specified rectangle from the input image + * @param iImg Input -- the input image + * @param pt Input -- start point at the top left corner + * @return void + */ +void VO_GaborFeatures::VO_GenerateAllFeatures(const Mat& iImg, Point pt) +{ + if( this->m_CVSize.width > iImg.cols || + this->m_CVSize.height > iImg.rows || + pt.x <0 || + pt.y < 0 || + this->m_CVSize.width + pt.x > iImg.cols || + this->m_CVSize.height + pt.y > iImg.rows ) + { + cerr << "Feature rectangles are out of the image" << endl; + } + + Rect rect(pt.x, pt.y, this->m_CVSize.width, this->m_CVSize.height); + Mat_ rectImg = iImg(rect); + + + this->m_MatFeatures = Mat_(1, this->m_iNbOfFeatures); + for(unsigned int i = 0; i < this->m_iNbOfFeatures; i++) + { + if(this->m_vAllFeatures[i].isFiltering) + this->m_MatFeatures(0, i) = m_vAllFeatures[i].filter(rectImg); + else + this->m_vAllFeatures[i].calc(rectImg); + } +} + + +/** Default constructor */ +VO_GaborFeatures::Feature::Feature() +{ + isFiltering = true; + rect = Rect(0, 0, 0, 0); +} + + +/** + * @brief constructor to create all LBP features + * @param offset Input -- width+1 + */ +VO_GaborFeatures::Feature::Feature( int offset, + int x, + int y, + bool filtering, + unsigned int nstds, + float lambda, + float theta, + float psi, + float sigma, + float gamma) +{ + isFiltering = filtering; + gabor.VO_PrepareGaborKernel(nstds, + lambda, + theta, + psi, + sigma, + gamma); + rect.x = x; + rect.y = y; + rect.width = gabor.GetWindowFunc()->GetWindowSize().width; + rect.height = gabor.GetWindowFunc()->GetWindowSize().width; +} + + +void VO_GaborFeatures::Feature::write(FileStorage &fs) const +{ + fs << CC_RECT << "[:" << rect.x << rect.y << rect.width << rect.height << "]"; +} diff --git a/modules/featureextraction/src/VO_HaarFeatures.cpp b/modules/featureextraction/src/VO_HaarFeatures.cpp new file mode 100644 index 0000000..8a976d8 --- /dev/null +++ b/modules/featureextraction/src/VO_HaarFeatures.cpp @@ -0,0 +1,322 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-08 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_HaarFeatures.h" + + +void VO_HaarFeatures::init( ) +{ + VO_Features::init(); + this->m_iFeatureType = HAAR; +} + + +void VO_HaarFeatures::WriteFeatures( FileStorage &fs, const Mat_& featureMap ) const +{ + _writeFeatures( m_vAllFeatures, fs, featureMap ); +} + + +void VO_HaarFeatures::ReadFeatures( const FileStorage& fs, Mat_& featureMap ) +{ + +} + + +/** + * @brief Generating all possible feature rectangles + * @param size Input -- the concerned size + * @param mode Input -- mode, BASIC, CORE or ALL + * @return void + */ + void VO_HaarFeatures::VO_GenerateAllFeatureInfo(const Size& size, unsigned int mode) + { + this->m_CVSize = size; + + int offset = this->m_CVSize.width + 1; // integral image is (m_CVSize.height+1)*(m_CVSize.width+1) + for( int x = 0; x < this->m_CVSize.width; x++ ) + { + for( int y = 0; y < this->m_CVSize.height; y++ ) + { + for( int dx = 1; dx <= this->m_CVSize.width; dx++ ) + { + for( int dy = 1; dy <= this->m_CVSize.height; dy++ ) + { + // haar_x2 + if ( (x+dx*2 <= this->m_CVSize.width) && (y+dy <= this->m_CVSize.height) ) + { + m_vAllFeatures.push_back( Feature( offset, false, + x, y, dx, dy, -1, + x+dx, y, dx, dy, +1 ) ); + } + // haar_y2 + if ( (x+dx <= this->m_CVSize.width) && (y+dy*2 <= this->m_CVSize.height) ) + { + m_vAllFeatures.push_back( Feature( offset, false, + x, y, dx, dy, -1, + x, y+dy, dx, dy, +1 ) ); + } + // haar_x3 + if ( (x+dx*3 <= this->m_CVSize.width) && (y+dy <= this->m_CVSize.height) ) + { + m_vAllFeatures.push_back( Feature( offset, false, + x, y, dx*3, dy, -1, + x+dx, y, dx , dy, +3 ) ); + } + // haar_y3 + if ( (x+dx <= this->m_CVSize.width) && (y+dy*3 <= this->m_CVSize.height) ) + { + m_vAllFeatures.push_back( Feature( offset, false, + x, y, dx, dy*3, -1, + x, y+dy, dx, dy, +3 ) ); + } + if( mode != VO_HaarFeatures::BASIC ) + { + // haar_x4 + if ( (x+dx*4 <= this->m_CVSize.width) && (y+dy <= this->m_CVSize.height) ) + { + m_vAllFeatures.push_back( Feature( offset, false, + x, y, dx*4, dy, -1, + x+dx, y, dx*2, dy, +2 ) ); + } + // haar_y4 + if ( (x+dx <= this->m_CVSize.width) && (y+dy*4 <= this->m_CVSize.height ) ) + { + m_vAllFeatures.push_back( Feature( offset, false, + x, y, dx, dy*4, -1, + x, y+dy, dx, dy*2, +2 ) ); + } + } + // x2_y2 + if ( (x+dx*2 <= this->m_CVSize.width) && (y+dy*2 <= this->m_CVSize.height) ) + { + m_vAllFeatures.push_back( Feature( offset, false, + x, y, dx*2, dy*2, -1, + x, y, dx, dy, +2, + x+dx, y+dy, dx, dy, +2 ) ); + } + if (mode != VO_HaarFeatures::BASIC) + { + // x3_y3 + if ( (x+dx*3 <= this->m_CVSize.width) && (y+dy*3 <= this->m_CVSize.height) ) + { + m_vAllFeatures.push_back( Feature( offset, false, + x , y , dx*3, dy*3, -1, + x+dx, y+dy, dx , dy , +9) ); + } + } + if (mode == VO_HaarFeatures::ALL) + { + // tilted haar_x2 + if ( (x+2*dx <= this->m_CVSize.width) && (y+2*dx+dy <= this->m_CVSize.height) && (x-dy>= 0) ) + { + m_vAllFeatures.push_back( Feature( offset, true, + x, y, dx, dy, -1, + x+dx, y, dx, dy, +1 ) ); + } + // tilted haar_y2 + if ( (x+dx <= this->m_CVSize.width) && (y+dx+2*dy <= this->m_CVSize.height) && (x-2*dy>= 0) ) + { + m_vAllFeatures.push_back( Feature( offset, true, + x, y, dx, dy, -1, + x, y+dy, dx, dy, +1 ) ); + } + // tilted haar_x3 + if ( (x+3*dx <= this->m_CVSize.width) && (y+3*dx+dy <= this->m_CVSize.height) && (x-dy>= 0) ) + { + m_vAllFeatures.push_back( Feature( offset, true, + x, y, dx*3, dy, -1, + x+dx, y, dx, dy, +3 ) ); + } + // tilted haar_y3 + if ( (x+dx <= this->m_CVSize.width) && (y+dx+3*dy <= this->m_CVSize.height) && (x-3*dy>= 0) ) + { + m_vAllFeatures.push_back( Feature( offset, true, + x, y, dx, dy*3, -1, + x, y+dy, dx, dy, +3 ) ); + } + // tilted haar_x4 + if ( (x+4*dx <= this->m_CVSize.width) && (y+4*dx+dy <= this->m_CVSize.height) && (x-dy>= 0) ) + { + m_vAllFeatures.push_back( Feature( offset, true, + x, y, dx*4, dy, -1, + x+dx, y, dx*2, dy, +2 ) ); + } + // tilted haar_y4 + if ( (x+dx <= this->m_CVSize.width) && (y+dx+4*dy <= this->m_CVSize.height) && (x-4*dy>= 0) ) + { + m_vAllFeatures.push_back( Feature( offset, true, + x, y, dx, dy*4, -1, + x, y+dy, dx, dy*2, +2 ) ); + } + } + } + } + } + } + + this->m_iMode = mode; + this->m_iNbOfFeatures = this->m_vAllFeatures.size(); +} + + +/** + * @brief Generating all features from the input image + * @param iImg Input -- the input image + * @param pt Input -- start point at the top left corner + * @return void + */ +void VO_HaarFeatures::VO_GenerateAllFeatures(const Mat& iImg, Point pt) +{ + if( this->m_CVSize.width > iImg.cols || + this->m_CVSize.height > iImg.rows || + pt.x <0 || + pt.y < 0 || + this->m_CVSize.width + pt.x > iImg.cols || + this->m_CVSize.height + pt.y > iImg.rows ) + { + cerr << "Feature rectangles are out of the image" << endl; + } + + Rect rect(pt.x, pt.y, this->m_CVSize.width, this->m_CVSize.height); + Mat rectImg = iImg(rect); + + // Calculate the integral image + cv::integral(rectImg, this->m_MatIntegralImage, this->m_MatSquareImage, this->m_MatTiltedIntegralImage); + + this->m_MatFeatures = Mat_(1, this->m_iNbOfFeatures); + for(unsigned int i = 0; i < this->m_iNbOfFeatures; i++) + { + this->m_MatFeatures(0, i) = this->m_vAllFeatures[i].calc(this->m_MatIntegralImage, this->m_MatTiltedIntegralImage); + } +} + + +/** Default constructor */ +VO_HaarFeatures::Feature::Feature() +{ + tilted = false; + rect[0].r = rect[1].r = rect[2].r = Rect(0,0,0,0); + rect[0].weight = rect[1].weight = rect[2].weight = 0; +} + + +/** +* @brief constructor to create all Haar features, Haar features an be computed from at most 3 rectangles. +* @param offset Input -- width+1 +* @param _tilted Input -- wether the image is tilted or not + */ +VO_HaarFeatures::Feature::Feature( int offset, bool _tilted, + int x0, int y0, int w0, int h0, float wt0, + int x1, int y1, int w1, int h1, float wt1, + int x2, int y2, int w2, int h2, float wt2 ) +{ + tilted = _tilted; + + rect[0].r.x = x0; + rect[0].r.y = y0; + rect[0].r.width = w0; + rect[0].r.height = h0; + rect[0].weight = wt0; + + rect[1].r.x = x1; + rect[1].r.y = y1; + rect[1].r.width = w1; + rect[1].r.height = h1; + rect[1].weight = wt1; + + rect[2].r.x = x2; + rect[2].r.y = y2; + rect[2].r.width = w2; + rect[2].r.height = h2; + rect[2].weight = wt2; + + if( !tilted ) + { + for( int j = 0; j < CV_HAAR_FEATURE_MAX; j++ ) + { + if( rect[j].weight == 0.0F ) + break; + CV_SUM_OFFSETS( fastRect[j].p0, fastRect[j].p1, fastRect[j].p2, fastRect[j].p3, rect[j].r, offset ) + } + } + else + { + for( int j = 0; j < CV_HAAR_FEATURE_MAX; j++ ) + { + if( rect[j].weight == 0.0F ) + break; + CV_TILTED_OFFSETS( fastRect[j].p0, fastRect[j].p1, fastRect[j].p2, fastRect[j].p3, rect[j].r, offset ) + } + } +} + + +void VO_HaarFeatures::Feature::write( FileStorage &fs ) const +{ + fs << CC_RECTS << "["; + for( int ri = 0; ri < CV_HAAR_FEATURE_MAX && rect[ri].r.width != 0; ++ri ) + { + fs << "[:" << rect[ri].r.x << rect[ri].r.y << + rect[ri].r.width << rect[ri].r.height << rect[ri].weight << "]"; + } + fs << "]" << CC_TILTED << tilted; +} diff --git a/modules/featureextraction/src/VO_LBPFeatures.cpp b/modules/featureextraction/src/VO_LBPFeatures.cpp new file mode 100644 index 0000000..249a582 --- /dev/null +++ b/modules/featureextraction/src/VO_LBPFeatures.cpp @@ -0,0 +1,164 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-08 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_LBPFeatures.h" + + +void VO_LBPFeatures::init( ) +{ + VO_Features::init(); + this->m_iFeatureType = LBP; +} + + +void VO_LBPFeatures::WriteFeatures( FileStorage &fs, const Mat_& featureMap ) const +{ + _writeFeatures( m_vAllFeatures, fs, featureMap ); +} + + +void VO_LBPFeatures::ReadFeatures( const FileStorage& fs, Mat_& featureMap ) +{ + +} + + +/** + * @brief Generating all possible feature rectangles + * @param size Input -- the concerned size + * @param mode Input -- mode, for LBP, no use + * @return void + */ + void VO_LBPFeatures::VO_GenerateAllFeatureInfo(const Size& size, unsigned int mode) + { + this->m_CVSize = size; + + int offset = this->m_CVSize.width + 1; + for( int x = 0; x < this->m_CVSize.width; x++ ) + for( int y = 0; y < this->m_CVSize.height; y++ ) + for( int w = 1; w <= this->m_CVSize.width / 3; w++ ) + for( int h = 1; h <= this->m_CVSize.height / 3; h++ ) + if ( (x+3*w <= this->m_CVSize.width) && (y+3*h <= this->m_CVSize.height) ) + this->m_vAllFeatures.push_back( Feature(offset, x, y, w, h ) ); + + this->m_iNbOfFeatures = this->m_vAllFeatures.size(); +} + + +/** + * @brief Generating all features from the input image + * @param iImg Input -- the input image + * @param pt Input -- start point at the top left corner + * @return void + */ +void VO_LBPFeatures::VO_GenerateAllFeatures(const Mat& iImg, Point pt) +{ + if( this->m_CVSize.width > iImg.cols || + this->m_CVSize.height > iImg.rows || + pt.x <0 || + pt.y < 0 || + this->m_CVSize.width + pt.x > iImg.cols || + this->m_CVSize.height + pt.y > iImg.rows ) + { + cerr << "Feature rectangles are out of the image" << endl; + } + + Rect rect(pt.x, pt.y, this->m_CVSize.width, this->m_CVSize.height); + Mat rectImg = iImg(rect); + + // Calculate the integral image + cv::integral(rectImg, this->m_MatIntegralImage, this->m_MatSquareImage, this->m_MatTiltedIntegralImage); + + this->m_MatFeatures = Mat_(1, this->m_iNbOfFeatures); + for(unsigned int i = 0; i < this->m_iNbOfFeatures; i++) + { + this->m_MatFeatures(0, i) = (float) this->m_vAllFeatures[i].calc(this->m_MatIntegralImage); + } +} + + +VO_LBPFeatures::Feature::Feature() +{ + rect = Rect(0, 0, 0, 0); +} + + +/** + * @brief constructor to create all LBP features + * @param offset Input -- width+1 + */ +VO_LBPFeatures::Feature::Feature( int offset, int x, int y, int _blockWidth, int _blockHeight ) +{ + Rect tr = rect = Rect(x, y, _blockWidth, _blockHeight); + CV_SUM_OFFSETS( p[0], p[1], p[4], p[5], tr, offset ) + tr.x += 2*rect.width; + CV_SUM_OFFSETS( p[2], p[3], p[6], p[7], tr, offset ) + tr.y +=2*rect.height; + CV_SUM_OFFSETS( p[10], p[11], p[14], p[15], tr, offset ) + tr.x -= 2*rect.width; + CV_SUM_OFFSETS( p[8], p[9], p[12], p[13], tr, offset ) +} + + +void VO_LBPFeatures::Feature::write(FileStorage &fs) const +{ + fs << CC_RECT << "[:" << rect.x << rect.y << rect.width << rect.height << "]"; +} diff --git a/modules/integraltransform/CMakeLists.txt b/modules/integraltransform/CMakeLists.txt new file mode 100644 index 0000000..182b888 --- /dev/null +++ b/modules/integraltransform/CMakeLists.txt @@ -0,0 +1,2 @@ +set(deps opencv_core opencv_imgproc opencv_highgui) +define_vosm_module(integraltransform ${deps}) diff --git a/modules/integraltransform/include/VO_CohenDaubechiesFeauveau.h b/modules/integraltransform/include/VO_CohenDaubechiesFeauveau.h new file mode 100644 index 0000000..542533c --- /dev/null +++ b/modules/integraltransform/include/VO_CohenDaubechiesFeauveau.h @@ -0,0 +1,100 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_COHENDAUBECHIESFEAUVEAU_H__ +#define __VO_COHENDAUBECHIESFEAUVEAU_H__ + +#include +#include +#include +#include "VO_DiscreteWavelet.h" + + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Cohen Daubechies Feauveau discrete wavelet transform. + */ +class VO_CohenDaubechiesFeauveau : public VO_DiscreteWavelet +{ +protected: + /** Initialization */ + void init(); + +public: + VO_CohenDaubechiesFeauveau() { this->init(); } + + virtual ~VO_CohenDaubechiesFeauveau() {} + + /** prepare the CohenDaubechiesFeauveau mother wavelet */ + virtual void VO_PrepareKernel(unsigned int CDFN); + + /** CohenDaubechiesFeauveau wavelet transform, must be Mat_ */ + virtual void VO_ForwardTransform(const Mat_& iImg, Mat_& oImg) const; + + /** Inverse CohenDaubechiesFeauveau wavelet transform, must be Mat_ */ + virtual void VO_BackwardTransform(const Mat_& iImg, Mat_& oImg) const; +}; + +#endif // __VO_COHENDAUBECHIESFEAUVEAU_H__ diff --git a/modules/integraltransform/include/VO_Coiflets.h b/modules/integraltransform/include/VO_Coiflets.h new file mode 100644 index 0000000..ef606b5 --- /dev/null +++ b/modules/integraltransform/include/VO_Coiflets.h @@ -0,0 +1,106 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_COIFLETS_H__ +#define __VO_COIFLETS_H__ + +#include +#include +#include +#include "VO_DiscreteWavelet.h" + + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Coiflets discrete wavelet transform. + */ +class VO_Coiflets : public VO_DiscreteWavelet +{ +protected: + /** Daubechies coefficients */ + static float coif[5][30]; + + /** Which Daub? */ + unsigned int m_iCoifN; + + /** Initialization */ + void init(); + +public: + VO_Coiflets() { this->init(); } + + virtual ~VO_Coiflets() {} + + /** prepare the Coiflets mother wavelet */ + void VO_PrepareKernel(unsigned int coifN); + + /** Coiflets wavelet transform, must be Mat_ */ + virtual void VO_ForwardTransform(const Mat_& iImg, Mat_& oImg) const; + + /** Inverse Coiflets wavelet transform, must be Mat_ */ + virtual void VO_BackwardTransform(const Mat_& iImg, Mat_& oImg) const; +}; + +#endif // __VO_COIFLETS_H__ diff --git a/modules/integraltransform/include/VO_Daubechies.h b/modules/integraltransform/include/VO_Daubechies.h new file mode 100644 index 0000000..cb61b84 --- /dev/null +++ b/modules/integraltransform/include/VO_Daubechies.h @@ -0,0 +1,107 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_DAUBECHIES_H__ +#define __VO_DAUBECHIES_H__ + +#include +#include +#include +#include +#include "VO_DiscreteWavelet.h" + + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Daubechies discrete wavelet transform. + */ +class VO_Daubechies : public VO_DiscreteWavelet +{ +protected: + /** Daubechies coefficients */ + static float daub[10][20]; + + /** Which Daub? */ + unsigned int m_iDaubN; + + /** Initialization */ + void init(); + +public: + VO_Daubechies() { this->init();} + + virtual ~VO_Daubechies(){} + + /** prepare the Daubechies mother wavelet */ + virtual void VO_PrepareKernel(unsigned int daubN); + + /** Daubechies wavelet transform, must be Mat_ */ + virtual void VO_ForwardTransform(const Mat_& iImg, Mat_& oImg) const; + + /** Inverse Daubechies wavelet transform, must be Mat_ */ + virtual void VO_BackwardTransform(const Mat_& iImg, Mat_& oImg) const; +}; + +#endif // __VO_DAUBECHIES_H__ diff --git a/modules/integraltransform/include/VO_DiscreteFIRMeyer.h b/modules/integraltransform/include/VO_DiscreteFIRMeyer.h new file mode 100644 index 0000000..230f308 --- /dev/null +++ b/modules/integraltransform/include/VO_DiscreteFIRMeyer.h @@ -0,0 +1,71 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_DISCRETEFIRMEYER_H__ +#define __VO_DISCRETEFIRMEYER_H__ + +/** + * @author JIA Pei + * @brief FIR Meyer discrete wavelet transform. + */ + +#endif // __VO_DISCRETEFIRMEYER_H__ + diff --git a/modules/integraltransform/include/VO_DiscreteWavelet.h b/modules/integraltransform/include/VO_DiscreteWavelet.h new file mode 100644 index 0000000..b047dc2 --- /dev/null +++ b/modules/integraltransform/include/VO_DiscreteWavelet.h @@ -0,0 +1,141 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_DISCRETEWAVELET_H__ +#define __VO_DISCRETEWAVELET_H__ + +#include +#include +#include +#include "VO_IntegralTransform.h" + + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Generalized class for discrete wavelet transform. + */ +class VO_DiscreteWavelet : public VO_IntegralTransform +{ +protected: + /** This is used to record wavelet types. */ + int m_iWaveletType; + + /** invertible? */ + bool m_bInvertible; + + /** orthogonal */ + bool m_bOrthogonal; + + /** biorthogonal */ + bool m_bBiorthogonal; + + /** Initialization */ + void init(); + +public: + /** Note: HAAR is just DAUBECHIES2 */ + enum { + HAAR = 0, + DAUBECHIES2 = 1, + DAUBECHIES4 = 2, + DAUBECHIES6 = 3, + DAUBECHIES8 = 4, + DAUBECHIES10 = 5, + DAUBECHIES12 = 6, + DAUBECHIES14 = 7, + DAUBECHIES16 = 8, + DAUBECHIES18 = 9, + DAUBECHIES20 = 10, + COIFLET6 = 11, + COIFLET12 = 12, + COIFLET18 = 13, + COIFLET24 = 14, + COIFLET30 = 15, + COHENDAUBECHIESFEAUVEAU = 16 + }; + + /** constructor */ + VO_DiscreteWavelet(Size windowSize = Size(0, 0)) + { + this->m_iWaveletType = HAAR; + this->m_bInvertible = true; + this->m_bOrthogonal = true; + this->m_bBiorthogonal = false; + } + + /** destructor */ + virtual ~VO_DiscreteWavelet() {} + + ///** Prepare the generalized mother wavelet */ + //virtual void VO_PrepareWavelet(unsigned int waveN) = 0; + + ///** Generalized wavelet transform, must be Mat_ */ + //virtual void VO_ForwardTransform(const Mat_& iImg, Mat_& oImg) = 0; + + ///** Inverse generalized wavelet transform, must be Mat_ */ + //virtual void VO_BackwardTransform(const Mat_& iImg, Mat_& oImg) = 0; + +}; + +#endif // __VO_DISCRETEWAVELET_H__ diff --git a/modules/integraltransform/include/VO_Gabor.h b/modules/integraltransform/include/VO_Gabor.h new file mode 100644 index 0000000..44501d0 --- /dev/null +++ b/modules/integraltransform/include/VO_Gabor.h @@ -0,0 +1,118 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_GABOR_H__ +#define __VO_GABOR_H__ + +#include +#include +#include +#include "VO_STFT.h" + + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Gabor transform, a kind of short time Fourier transform. + */ +class VO_Gabor : public VO_STFT +{ +protected: + /** how many (n) standard deviations? */ + float m_fNStds; + + /** the wavelength of the cosine factor */ + float m_fLamda; + + /** the orientation of the normal to the parallel stripes of a Gabor function */ + float m_fTheta; + + /** the phase offset */ + float m_fPSI; + + /** the sigma of the Gaussian envelope */ + float m_fSigma; + + /** the spatial aspect ratio, and specifies the ellipticity of the support of the Gabor function */ + float m_fGamma; + + /** Initialization */ + void init(); + +public: + VO_Gabor(); + virtual ~VO_Gabor(); + + /** prepare the Gabor Kernel */ + void VO_PrepareGaborKernel( unsigned int nstds = 4, + float lambda = 1.0f, + float theta = 0.0f, + float psi = 0.0f, + float sigma = 1.0f, + float gamma = 1.0f ); + + /** Gabor Filtering */ + float VO_GaborFiltering(const Mat_& iImg); +}; + +#endif // __VO_GABOR_H__ diff --git a/modules/integraltransform/include/VO_Gauss.h b/modules/integraltransform/include/VO_Gauss.h new file mode 100644 index 0000000..16c9463 --- /dev/null +++ b/modules/integraltransform/include/VO_Gauss.h @@ -0,0 +1,107 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_GAUSS_H__ +#define __VO_GAUSS_H__ + +#include +#include +#include +#include "VO_STFT.h" + + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Gauss transform, a kind of short time Fourier transform. + */ +class VO_Gauss : public VO_STFT +{ +protected: + /** how many (n) standard deviations? */ + float m_fNStds; + + /** the orientation of the normal to the parallel stripes of a Gauss function */ + float m_fTheta; + + /** the sigma of the Gaussian envelope */ + float m_fSigma; + + /** the spatial aspect ratio, and specifies the ellipticity of the support of the Gauss function */ + float m_fGamma; + + /** Initialization */ + void init(); + +public: + VO_Gauss(); + virtual ~VO_Gauss(); + + /** prepare the Gauss Kernel */ + void VO_PrepareGaussKernel( unsigned int nstds, + float theta, + float sigma, + float gamma); +}; + +#endif // __VO_GAUSS_H__ diff --git a/modules/integraltransform/include/VO_Haar.h b/modules/integraltransform/include/VO_Haar.h new file mode 100644 index 0000000..9ffc0db --- /dev/null +++ b/modules/integraltransform/include/VO_Haar.h @@ -0,0 +1,105 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_HAAR_H__ +#define __VO_HAAR_H__ + +#include +#include +#include +#include "VO_DiscreteWavelet.h" + + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Haar discrete wavelet transform. + */ +class VO_Haar : public VO_DiscreteWavelet +{ +protected: + /** Initialization */ + void init() + { + this->m_iWaveletType = HAAR; + this->m_bInvertible = true; + this->m_bOrthogonal = true; + this->m_bBiorthogonal = true; + } + +public: + VO_Haar() {this->init();} + + ~VO_Haar() {} + + /** prepare the Haar mother wavelet */ + virtual void VO_PrepareKernel(unsigned int haarN = 2); + + /** Haar wavelet transform, must be Mat_ */ + virtual void VO_ForwardTransform(const Mat_& iImg, Mat_& oImg) const; + + /** Inverse Haar wavelet transform, must be Mat_ */ + virtual void VO_BackwardTransform(const Mat_& iImg, Mat_& oImg) const; +}; + +#endif // __VO_HAAR_H__ diff --git a/modules/integraltransform/include/VO_IntegralTransform.h b/modules/integraltransform/include/VO_IntegralTransform.h new file mode 100644 index 0000000..772acba --- /dev/null +++ b/modules/integraltransform/include/VO_IntegralTransform.h @@ -0,0 +1,108 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_INTEGRALTRANSFORM_H__ +#define __VO_INTEGRALTRANSFORM_H__ + +#include +#include +#include + + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Generalized class for integral transform. + */ +class VO_IntegralTransform +{ +protected: + /** The integral transform kernel, must be a M*N rectangle matrix */ + Mat_ m_MatKernel; + +public: + /** constructor */ + VO_IntegralTransform(Size windowSize = Size(0, 0)) + { + //this->m_MatKernel.release(); + this->m_MatKernel = Mat_::zeros(windowSize); + } + + /** destructor */ + virtual ~VO_IntegralTransform() + { + this->m_MatKernel.release(); + } + + /** Prepare the generalized integral transform kernel */ + virtual void VO_PrepareKernel(unsigned int kernel = 0) = 0; + + /** Generalized integral transform, must be Mat_ */ + virtual void VO_ForwardTransform(const Mat_& iImg, Mat_& oImg) const = 0; + + /** Generalized inverse integral transform, must be Mat_ */ + virtual void VO_BackwardTransform(const Mat_& iImg, Mat_& oImg) const = 0; + +}; + +#endif // __VO_INTEGRALTRANSFORM_H__ diff --git a/modules/integraltransform/include/VO_ReverseBiorthogonal.h b/modules/integraltransform/include/VO_ReverseBiorthogonal.h new file mode 100644 index 0000000..1a66f7e --- /dev/null +++ b/modules/integraltransform/include/VO_ReverseBiorthogonal.h @@ -0,0 +1,69 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_REVERSEBIORTHOGONAL_H__ +#define __VO_REVERSEBIORTHOGONAL_H__ + +/** + * @author JIA Pei + * @brief Reverse Biorthogonal discrete wavelet transform. + */ + +#endif // __VO_REVERSEBIORTHOGONAL_H__ diff --git a/modules/integraltransform/include/VO_STFT.h b/modules/integraltransform/include/VO_STFT.h new file mode 100644 index 0000000..88d0255 --- /dev/null +++ b/modules/integraltransform/include/VO_STFT.h @@ -0,0 +1,121 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_STFT_H__ +#define __VO_STFT_H__ + +#include +#include +#include +#include "VO_WindowFunc.h" +#include "VO_IntegralTransform.h" + + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Generalized class for short-time Fourier transform. + */ +class VO_STFT : public VO_IntegralTransform +{ +protected: + /** This is used to record wavelet types. */ + VO_WindowFunc* m_VOWindowFunc; + + /** Initialization */ + void init() + { + this->m_VOWindowFunc = NULL; + } + +public: + /** Constructor */ + VO_STFT(Size windowSize = Size(0, 0), unsigned int windowFunc = 0) + { + this->init(); + this->m_VOWindowFunc = new VO_WindowFunc(windowSize, windowFunc); + } + + /** Destructor */ + ~VO_STFT() + { + if(this->m_VOWindowFunc) delete this->m_VOWindowFunc; + } + + /** prepare the STFT kernel */ + void VO_PrepareKernel(unsigned int kernel = 0); + + /** Forward transform, must be Mat_ */ + void VO_ForwardTransform(const Mat_& iImg, Mat_& oImg) const; + + /** Backward transform, must be Mat_ */ + void VO_BackwardTransform(const Mat_& iImg, Mat_& oImg) const; + + /** Forward transform */ + void VO_ForwardTransform(const Mat_& iImg, Point pt, Mat_& oImg) const; + + /** Get the window function */ + VO_WindowFunc* GetWindowFunc() const {return this->m_VOWindowFunc;} +}; + +#endif // __VO_STFT_H__ diff --git a/modules/integraltransform/include/VO_Symlets.h b/modules/integraltransform/include/VO_Symlets.h new file mode 100644 index 0000000..4b3d5b9 --- /dev/null +++ b/modules/integraltransform/include/VO_Symlets.h @@ -0,0 +1,99 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_SYMLETS_H__ +#define __VO_SYMLETS_H__ + +#include +#include +#include +#include "VO_DiscreteWavelet.h" + + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Symlets discrete wavelet transform. + */ +class VO_Symlets : public VO_DiscreteWavelet +{ +protected: + /** Initialization */ + void init(); + +public: + VO_Symlets() {this->init();} + + virtual ~VO_Symlets() {} + + /** prepare the Symlets mother wavelet */ + virtual void VO_PrepareKernel(unsigned int symletsN); + + /** Symlets wavelet transform, must be Mat_ */ + virtual void VO_ForwardTransform(const Mat_& iImg, Mat_& oImg) const; + + /** Inverse Symlets wavelet transform, must be Mat_ */ + virtual void VO_BackwardTransform(const Mat_& iImg, Mat_& oImg) const; +}; + +#endif // __VO_SYMLETS_H__ diff --git a/modules/integraltransform/include/VO_WindowFunc.h b/modules/integraltransform/include/VO_WindowFunc.h new file mode 100644 index 0000000..f0ae503 --- /dev/null +++ b/modules/integraltransform/include/VO_WindowFunc.h @@ -0,0 +1,138 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_WINDOWFUNC_H__ +#define __VO_WINDOWFUNC_H__ + +#include +#include +#include + + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief This is a class to generate 2D discrete window functions. + * If it only contains 1 single row or col, it's a 1D window function. + */ +class VO_WindowFunc +{ +friend class VO_STFT; +friend class VO_Gabor; +friend class VO_Gauss; +protected: + /** This is used to record window function size in 2D, could be 1D. */ + Size m_iWindowSize; + + /** This is used to record window function type */ + unsigned int m_iWindowFunc; + + /** The Window Function kernel */ + Mat_ m_MatWindowedKernel; + + /** Initialization */ + void init(); + +public: + enum { + RECTANGULAR = 0, + HAMMING = 1, + HANN = 2, + TUKEY = 3, + COSINE = 4, + LANCZOS = 5, + BARTLETT = 6, + TRIANGULAR = 7, + GAUSS = 8, + BARTLETTHANN = 9, + BLACKMAN = 10, + KAISER = 11, + NUTTALL = 12, + BLACKMANHARRIS = 13, + BLACKMANNUTTALL = 14, + FLATTOP = 15, + BESSEL = 16, + DOLPHCHEBYSHEV = 17, + EXPONENTIAL = 18, + RIFEVINCENT = 19, + GABOR = 20 + }; + + /** Constructor */ + VO_WindowFunc(Size windowSize, unsigned int windowFunc = RECTANGULAR); + + /** Destructor */ + ~VO_WindowFunc(); + + /** Generate the Window Function */ + Mat_ VO_GenerateWindowFunc(Size windowSize, unsigned int windowFunc); + + /** Display the Window Function */ + void VO_DisplayWindowFuncKernel(const string& fn); + + /** Gets and Sets */ + Size GetWindowSize() const { return this->m_iWindowSize;} + Mat_ GetWindowKernel() const { return this->m_MatWindowedKernel; } +}; + +#endif // __VO_WINDOWFUNC_H__ diff --git a/modules/integraltransform/integraltransform.vcxproj b/modules/integraltransform/integraltransform.vcxproj new file mode 100644 index 0000000..8ef21c6 --- /dev/null +++ b/modules/integraltransform/integraltransform.vcxproj @@ -0,0 +1,102 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {900F9166-CD10-49D8-A339-23E0A035F3E9} + integraltransform + + + + StaticLibrary + true + MultiByte + v110 + + + Application + false + true + MultiByte + v110 + + + + + + + + + + + + + $(SourcePath) + + + + Level3 + Disabled + ./include;../include;../common/include;C:\OpenCV2.4.5\opencv\build\include;C:\boost\boost_1_53_0;%(AdditionalIncludeDirectories) + NOMINMAX;%(PreprocessorDefinitions) + + + true + + + true + + + + + Level3 + MaxSpeed + true + true + + + true + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/modules/integraltransform/integraltransform.vcxproj.filters b/modules/integraltransform/integraltransform.vcxproj.filters new file mode 100644 index 0000000..1990eb2 --- /dev/null +++ b/modules/integraltransform/integraltransform.vcxproj.filters @@ -0,0 +1,93 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + \ No newline at end of file diff --git a/modules/integraltransform/src/VO_CohenDaubechiesFeauveau.cpp b/modules/integraltransform/src/VO_CohenDaubechiesFeauveau.cpp new file mode 100644 index 0000000..5c527ee --- /dev/null +++ b/modules/integraltransform/src/VO_CohenDaubechiesFeauveau.cpp @@ -0,0 +1,110 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include "VO_CohenDaubechiesFeauveau.h" + + +void VO_CohenDaubechiesFeauveau::init() +{ + this->m_iWaveletType = COHENDAUBECHIESFEAUVEAU; + this->m_bInvertible = true; + this->m_bOrthogonal = false; + this->m_bBiorthogonal = true; +} + + +/** + * @author JIA Pei + * @version 2010-03-05 + * @brief Prepare a CohenDaubechiesFeauveau Kernel + * @return void + */ +void VO_CohenDaubechiesFeauveau::VO_PrepareKernel(unsigned int CDFN) +{ + +} + + +/** + * @author JIA Pei + * @version 2010-03-05 + * @brief Forward CohenDaubechiesFeauveau Transform + * @return void + */ +void VO_CohenDaubechiesFeauveau::VO_ForwardTransform(const Mat_& iImg, Mat_& oImg) const +{ + +} + + +/** + * @author JIA Pei + * @version 2010-03-05 + * @brief Backward CohenDaubechiesFeauveau Transform + * @return void + */ +void VO_CohenDaubechiesFeauveau::VO_BackwardTransform(const Mat_& iImg, Mat_& oImg) const +{ + +} + + + + diff --git a/modules/integraltransform/src/VO_Coiflets.cpp b/modules/integraltransform/src/VO_Coiflets.cpp new file mode 100644 index 0000000..3224429 --- /dev/null +++ b/modules/integraltransform/src/VO_Coiflets.cpp @@ -0,0 +1,156 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include "VO_Coiflets.h" + + +// Explained by JIA Pei, we can only deal with Daubechies until D20 +float VO_Coiflets::coif[5][30] = { + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-0.1028594569415370, 0.4778594569415370, 1.2057189138830700, 0.5442810861169260, -0.1028594569415370, -0.0221405430584631}, + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0.0231751934774337, -0.0586402759669371, -0.0952791806220162, 0.5460420930695330, 1.1493647877137300, 0.5897343873912380, -0.1081712141834230, -0.0840529609215432, 0.0334888203265590, 0.0079357672259240, -0.0025784067122813, -0.0010190107982153}, + {0,0,0,0,0,0,0,0,0,0,0,0,-0.0053648373418441, 0.0110062534156628, 0.0331671209583407, -0.0930155289574539, -0.0864415271204239, 0.5730066705472950, 1.1225705137406600, 0.6059671435456480, -0.1015402815097780, -0.1163925015231710, 0.0488681886423339, 0.0224584819240757, -0.0127392020220977, -0.0036409178311325, 0.0015804102019152, 0.0006593303475864, -0.0001003855491065, -0.0000489314685106}, + {0,0,0,0,0,0,0.0012619224228619, -0.0023044502875399, -0.0103890503269406, 0.0227249229665297, 0.0377344771391261, -0.1149284838038540, -0.0793053059248983, 0.5873348100322010, 1.1062529100791000, 0.6143146193357710, -0.0942254750477914, -0.1360762293560410, 0.0556272739169390, 0.0354716628454062, -0.0215126323101745, -0.0080020216899011, 0.0053053298270610, 0.0017911878553906, -0.0008330003901883, -0.0003676592334273, 0.0000881604532320, 0.0000441656938246, -0.0000046098383254, -0.0000025243583600}, + {-0.0002999290456692, 0.0005071055047161, 0.0030805734519904, -0.0058821563280714, -0.0143282246988201, 0.0331043666129858, 0.0398380343959686, -0.1299967565094460, -0.0736051069489375, 0.5961918029174380, 1.0950165427080700, 0.6194005181568410, -0.0877346296564723, -0.1492888402656790, 0.0583893855505615, 0.0462091445541337, -0.0279425853727641, -0.0129534995030117, 0.0095622335982613, 0.0034387669687710, -0.0023498958688271, -0.0009016444801393, 0.0004268915950172, 0.0001984938227975, -0.0000582936877724, -0.0000300806359640, 0.0000052336193200, 0.0000029150058427, -0.0000002296399300, -0.0000001358212135} +}; + + +void VO_Coiflets::init() +{ +// this->m_iWaveletType = COIF; + this->m_bInvertible = true; + this->m_bOrthogonal = true; + this->m_bBiorthogonal = false; +} + + +/** + * @author JIA Pei + * @brief Create Coiflets wavelet + * @ref http://en.wikipedia.org/wiki/Coiflet + * B_k = ( − 1)^k *C_{N − 1 − k} + */ +void VO_Coiflets::VO_PrepareKernel(unsigned int coifN) +{ + if( ! (coifN % 6) ) + { + cerr << "VO_Coiflets coifN should be able to be devided by 6" << endl; + exit(1); + } + if( coifN < 6 || coifN > 30 ) + { + cerr << "VO_Coiflets coifN should be between 6 and 30" << endl; + exit(1); + } + + unsigned int idx = coifN/6 - 1; + this->m_MatKernel = Mat_::zeros(coifN, coifN); + for(unsigned int i = 0; i < coifN; i++) + { + for(unsigned int j = 0; j < coifN; j++) + { + this->m_MatKernel(i, j) = VO_Coiflets::coif[idx][coifN-1-j]*(j%2?-1:1); + } + } + + switch(coifN) + { + case 6: + this->m_iWaveletType = COIFLET6; + break; + case 12: + this->m_iWaveletType = COIFLET12; + break; + case 18: + this->m_iWaveletType = COIFLET18; + break; + case 24: + this->m_iWaveletType = COIFLET24; + break; + case 30: + this->m_iWaveletType = COIFLET30; + break; + } + this->m_iCoifN = coifN; +} + + +/** + * @author JIA Pei + * @version 2010-03-05 + * @brief Forward Coiflets Transform + * @return void + */ +void VO_Coiflets::VO_ForwardTransform(const Mat_& iImg, Mat_& oImg) const +{ + +} + + +/** + * @author JIA Pei + * @version 2010-03-05 + * @brief Backward Coiflets Transform + * @return void + */ +void VO_Coiflets::VO_BackwardTransform(const Mat_& iImg, Mat_& oImg) const +{ + +} + diff --git a/modules/integraltransform/src/VO_Daubechies.cpp b/modules/integraltransform/src/VO_Daubechies.cpp new file mode 100644 index 0000000..9241273 --- /dev/null +++ b/modules/integraltransform/src/VO_Daubechies.cpp @@ -0,0 +1,179 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include "VO_Daubechies.h" +#include + +using namespace std; + +// Explained by JIA Pei, we can only deal with Daubechies until D20 +float VO_Daubechies::daub[10][20] = { + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1, 1}, + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0.6830127, 1.1830127, 0.3169873, -0.1830127}, + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0.47046721, 1.14111692, 0.650365, -0.19093442, -0.12083221, 0.0498175}, + {0,0,0,0,0,0,0,0,0,0,0,0,0.32580343, 1.01094572, 0.8922014, -0.03957503, -0.26450717, 0.0436163, 0.0465036, -0.01498699}, + {0,0,0,0,0,0,0,0,0,0,0.22641898, 0.85394354, 1.02432694, 0.19576696, -0.34265671, -0.04560113, 0.10970265, -0.00882680, -0.01779187, 4.71742793e-3}, + {0,0,0,0,0,0,0,0,0.15774243, 0.69950381, 1.06226376, 0.44583132, -0.31998660, -0.18351806, 0.13788809, 0.03892321, -0.04466375, 7.83251152e-4, 6.75606236e-3, -1.52353381e-3}, + {0,0,0,0,0,0,0.11009943, 0.56079128, 1.03114849, 0.66437248, -0.20351382, -0.31683501, 0.1008467, 0.11400345, -0.05378245, -0.02343994, 0.01774979, 6.07514995e-4, -2.54790472e-3, 5.00226853e-4}, + {0,0,0,0,0.07695562, 0.44246725, 0.95548615, 0.82781653, -0.02238574, -0.40165863, 6.68194092e-4, 0.18207636, -0.02456390, -0.06235021, 0.01977216, 0.01236884, -6.88771926e-3, -5.54004549e-4, 9.55229711e-4, -1.66137261e-4}, + {0,0,0.05385035, 0.34483430, 0.85534906, 0.92954571, 0.18836955, -0.41475176, -0.13695355, 0.21006834, 0.043452675, -0.09564726, 3.54892813e-4, 0.03162417, -6.67962023e-3, -6.05496058e-3, 2.61296728e-3, 3.25814671e-4, -3.56329759e-4, 5.5645514e-5}, + {0.03771716, 0.26612218, 0.74557507, 0.97362811, 0.39763774, -0.35333620, -0.27710988, 0.18012745, 0.13160299, -0.10096657, -0.04165925, 0.04696981, 5.10043697e-3, -0.01517900, 1.97332536e-3, 2.81768659e-3, -9.69947840e-4, -1.64709006e-4, 1.32354367e-4, -1.875841e-5} +}; + + +void VO_Daubechies::init() +{ + this->m_iDaubN = 0; + //this->m_iWaveletType = DAUB; + this->m_bInvertible = true; + this->m_bOrthogonal = true; + this->m_bBiorthogonal = false; +} + + +/** + * @author JIA Pei + * @brief Create Daubechies wavelet + * @ref http://en.wikipedia.org/wiki/Daubechies_wavelet + * b_k = ( − 1)^k *a_{N − 1 − k} + */ +void VO_Daubechies::VO_PrepareKernel(unsigned int daubN) +{ + if( ! (daubN % 2) ) + { + cerr << "VO_Daubechies daubN should be an even number" << endl; + exit(1); + } + if( daubN < 2 || daubN > 20 ) + { + cerr << "VO_Daubechies daubN should be between 2 and 20" << endl; + exit(1); + } + + unsigned int idx = daubN/2 - 1; + this->m_MatKernel = Mat_::zeros(daubN, daubN); + for(unsigned int i = 0; i < daubN; i++) + { + for(unsigned int j = 0; j < daubN; j++) + { + this->m_MatKernel(i, j) = VO_Daubechies::daub[idx][daubN-1-j]*(j%2?-1:1); + } + } + + switch(daubN) + { + case 2: + this->m_iWaveletType = DAUBECHIES2; + break; + case 4: + this->m_iWaveletType = DAUBECHIES4; + break; + case 6: + this->m_iWaveletType = DAUBECHIES6; + break; + case 8: + this->m_iWaveletType = DAUBECHIES8; + break; + case 10: + this->m_iWaveletType = DAUBECHIES10; + break; + case 12: + this->m_iWaveletType = DAUBECHIES12; + break; + case 14: + this->m_iWaveletType = DAUBECHIES14; + break; + case 16: + this->m_iWaveletType = DAUBECHIES16; + break; + case 18: + this->m_iWaveletType = DAUBECHIES18; + break; + case 20: + this->m_iWaveletType = DAUBECHIES20; + break; + } + this->m_iDaubN = daubN; +} + + +/** + * @author JIA Pei + * @version 2010-03-05 + * @brief Forward Daubechies Transform + * @return void + */ +void VO_Daubechies::VO_ForwardTransform(const Mat_& iImg, Mat_& oImg) const +{ + +} + + +/** + * @author JIA Pei + * @version 2010-03-05 + * @brief Backward Daubechies Transform + * @return void + */ +void VO_Daubechies::VO_BackwardTransform(const Mat_& iImg, Mat_& oImg) const +{ + +} + diff --git a/modules/integraltransform/src/VO_DiscreteFIRMeyer.cpp b/modules/integraltransform/src/VO_DiscreteFIRMeyer.cpp new file mode 100644 index 0000000..a86b1e8 --- /dev/null +++ b/modules/integraltransform/src/VO_DiscreteFIRMeyer.cpp @@ -0,0 +1,61 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include "VO_DiscreteFIRMeyer.h" diff --git a/modules/integraltransform/src/VO_Gabor.cpp b/modules/integraltransform/src/VO_Gabor.cpp new file mode 100644 index 0000000..deaad2f --- /dev/null +++ b/modules/integraltransform/src/VO_Gabor.cpp @@ -0,0 +1,143 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include "VO_Gabor.h" + +#include "VO_Common.h" + +VO_Gabor::VO_Gabor() +{ + this->init(); +} + + +/** Initialization */ +void VO_Gabor::init() +{ +} + + +VO_Gabor::~VO_Gabor() +{ +} + + +/** + * @author JIA Pei + * @version 2010-03-05 + * @brief Prepare a Gabor Kernel + * @param nstds Input -- how many (n) standard deviations? + * @param lamda Input -- the wavelength of the cosine factor + * @param theta Input -- the orientation of the normal to the parallel stripes of a Gabor function + * @param psi Input -- the phase offset + * @param sigma Input -- the sigma of the Gaussian envelope + * @param gamma Input -- the spatial aspect ratio, and specifies the ellipticity of the support of the Gabor function + * @return void + * @ref http://en.wikipedia.org/wiki/Gabor_filter +*/ +void VO_Gabor::VO_PrepareGaborKernel( unsigned int nstds, + float lambda, + float theta, + float psi, + float sigma, + float gamma) +{ + float sigma_x = sigma; + float sigma_y = sigma/gamma; + float x_theta, y_theta; + + // Bounding box -- compute the kernel size + int xmax = static_cast(ceil( max(1.0f, max(fabs(static_cast(nstds)*sigma_x*cos(theta)),fabs(static_cast(nstds)*sigma_y*sin(theta))) ) ) ); + int ymax = static_cast(ceil( max(1.0f, max(fabs(static_cast(nstds)*sigma_x*sin(theta)),fabs(static_cast(nstds)*sigma_y*cos(theta))) ) ) ); + int xmin = -xmax; + int ymin = -ymax; + + this->m_VOWindowFunc->m_MatWindowedKernel = Mat_::zeros(2*ymax+1, 2*xmax+1); + + for(int y = ymin; y <= ymax; y++) + { + for(int x = xmin; x <= xmax; x++) + { + x_theta=x*cos(theta)+y*sin(theta); + y_theta=-x*sin(theta)+y*cos(theta); + this->m_VOWindowFunc->m_MatWindowedKernel(y - ymin, x - xmin) + = safeDoubleToFloat (exp(-.5f*(pow(x_theta,2.0f)/pow(sigma_x,2.0f)+pow(y_theta,2.0f)/pow(sigma_y,2.0f))) + *cos(2.0*CV_PI/lambda*x_theta+psi) ); + } + } + + // take record finally. + this->m_fNStds = static_cast(nstds); + this->m_fLamda = lambda; + this->m_fTheta = theta; + this->m_fPSI = psi; + this->m_fSigma = sigma; + this->m_fGamma = gamma; +} + + +/** + * @brief Gabor filtering + * @ref http://en.wikipedia.org/wiki/Gabor_filter + */ +float VO_Gabor::VO_GaborFiltering(const Mat_& iImg) +{ + return safeDoubleToFloat(iImg.dot(this->m_VOWindowFunc->m_MatWindowedKernel)); +} \ No newline at end of file diff --git a/modules/integraltransform/src/VO_Gauss.cpp b/modules/integraltransform/src/VO_Gauss.cpp new file mode 100644 index 0000000..2036c9b --- /dev/null +++ b/modules/integraltransform/src/VO_Gauss.cpp @@ -0,0 +1,136 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +//1 0.682689492137 +//2 0.954499736104 +//3 0.997300203937 +//4 0.999936657516 +//5 0.999999426697 +//6 0.999999998027 + + +#include "VO_Gauss.h" + +#include "VO_Common.h" + +VO_Gauss::VO_Gauss() +{ + this->init(); +} + + +/** Initialization */ +void VO_Gauss::init() +{ +} + + +VO_Gauss::~VO_Gauss() +{ +} + + +/** + * @author JIA Pei + * @version 2010-03-05 + * @brief Prepare a Gabor Kernel + * @param nstds Input -- how many (n) standard deviations? + * @param theta Input -- the orientation of the normal to the parallel stripes of a Gabor function + * @param sigma Input -- the sigma of the Gaussian envelope + * @param gamma Input -- the spatial aspect ratio, and specifies the ellipticity of the support of the Gabor function + * @return void + * @ref http://en.wikipedia.org/wiki/Normal_distribution +*/ +void VO_Gauss::VO_PrepareGaussKernel( unsigned int nstds, + float theta, + float sigma, + float gamma) +{ + float sigma_x = sigma; + float sigma_y = sigma/gamma; + float x_theta, y_theta; + + // Bounding box -- compute the kernel size + int xmax = static_cast(ceil( max(1.0f, max(fabs((float)nstds*sigma_x*cos(theta)),fabs((float)nstds*sigma_y*sin(theta))) ) ) ); + int ymax = static_cast(ceil( max(1.0f, max(fabs((float)nstds*sigma_x*sin(theta)),fabs((float)nstds*sigma_y*cos(theta))) ) ) ); + int xmin = -xmax; + int ymin = -ymax; + + this->m_VOWindowFunc->m_MatWindowedKernel = Mat_::zeros(2*ymax+1, 2*xmax+1); + + for(int y = ymin; y <= ymax; y++) + { + for(int x = xmin; x <= xmax; x++) + { + x_theta=x*cos(theta)+y*sin(theta); + y_theta=-x*sin(theta)+y*cos(theta); + this->m_VOWindowFunc->m_MatWindowedKernel(y - ymin, x - xmin) + = exp(-.5f*(pow(x_theta,2.0f)/pow(sigma_x,2.0f)+pow(y_theta,2.0f)/pow(sigma_y,2.0f))); + } + } + + // take record finally. + this->m_fNStds = static_cast(nstds); + this->m_fTheta = theta; + this->m_fSigma = sigma; + this->m_fGamma = gamma; +} + + diff --git a/modules/integraltransform/src/VO_Haar.cpp b/modules/integraltransform/src/VO_Haar.cpp new file mode 100644 index 0000000..8f2925c --- /dev/null +++ b/modules/integraltransform/src/VO_Haar.cpp @@ -0,0 +1,103 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include "VO_Haar.h" + + +/** + * @author JIA Pei + * @version 2010-03-05 + * @param haarN -- Input, not used at all. + * @brief Prepare a Haar Kernel + * @return void + */ +void VO_Haar::VO_PrepareKernel(unsigned int haarN) +{ + this->m_MatKernel = Mat_::ones(2, 2); + this->m_MatKernel(0, 0) = 1; + this->m_MatKernel(0, 1) = 1; + this->m_MatKernel(1, 0) = 1; + this->m_MatKernel(1, 1) = -1; +} + + +/** + * @author JIA Pei + * @version 2010-03-05 + * @brief Forward Haar Transform + * @return void + */ +void VO_Haar::VO_ForwardTransform(const Mat_& iImg, Mat_& oImg) const +{ + +} + + +/** + * @author JIA Pei + * @version 2010-03-05 + * @brief Backward Haar Transform + * @return void + */ +void VO_Haar::VO_BackwardTransform(const Mat_& iImg, Mat_& oImg) const +{ + +} + diff --git a/modules/integraltransform/src/VO_ReverseBiorthogonal.cpp b/modules/integraltransform/src/VO_ReverseBiorthogonal.cpp new file mode 100644 index 0000000..68d24fe --- /dev/null +++ b/modules/integraltransform/src/VO_ReverseBiorthogonal.cpp @@ -0,0 +1,61 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include "VO_ReverseBiorthogonal.h" diff --git a/modules/integraltransform/src/VO_STFT.cpp b/modules/integraltransform/src/VO_STFT.cpp new file mode 100644 index 0000000..393807c --- /dev/null +++ b/modules/integraltransform/src/VO_STFT.cpp @@ -0,0 +1,159 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include "VO_STFT.h" + + +/** + * @author JIA Pei + * @version 2010-03-05 + * @param kernel -- Input, not used at all. + * @brief Prepare a STFT Kernel + * @return void + */ +void VO_STFT::VO_PrepareKernel(unsigned int kernel) +{ + +} + + +/** + * @brief Window Function is just an element-wise product, a type of weighted staff. + * Absolutely not a convolution !!! + * @param iImg Input -- the original signal + * @param oImg Output -- the transformed signal + */ +void VO_STFT::VO_ForwardTransform( const Mat_& iImg, + Mat_& oImg) const +{ + if( (iImg.cols != this->m_VOWindowFunc->m_MatWindowedKernel.cols) || + (iImg.rows != this->m_VOWindowFunc->m_MatWindowedKernel.rows) ) + { + cerr << "STFT: signal should have the same size as Window kernel" << endl; + exit(1); + } + // Explained by JIA Pei. for STFT, it's element-wise weighting, rather than convolution!! + oImg = iImg.mul(this->m_VOWindowFunc->m_MatWindowedKernel); + cv::dft(oImg, oImg); +} + + +/** +* @brief Window Function is just an element-wise product, a type of weighted staff. +* Absolutely not a convolution !!! +* @param iImg Input -- the transformed signal (image) +* @param oImg Output -- the original signal + */ +void VO_STFT::VO_BackwardTransform(const Mat_& iImg, Mat_& oImg) const +{ + cv::idft(iImg, oImg, DFT_SCALE); + // Explained by JIA Pei. for STFT, it's element-wise weighting, rather than convolution!! + oImg /= this->m_VOWindowFunc->m_MatWindowedKernel; +} + + +/** +* @brief Window Function is just an element-wise product, a type of weighted staff. +* Absolutely not a convolution !!! +* @param iImg Input -- the original signal (image) +* @param pt Input -- the concern point, at the center of the rectangle +* @param oImg Output -- the transformed signal + */ +void VO_STFT::VO_ForwardTransform(const Mat_& iImg, Point pt, Mat_& oImg) const +{ + Mat_ tmpImg = Mat_::zeros( this->m_VOWindowFunc->m_MatWindowedKernel.size() ); + int y, x, yy, xx; + + if (pt.y-tmpImg.rows/2 >= 0 ) + { + y = pt.y-tmpImg.rows/2; + yy = 0; + } + else + { + y = 0; + yy = abs(pt.y-tmpImg.rows/2); + } + + for( ; + y <= (pt.y+tmpImg.rows/2 < oImg.rows ? pt.y+tmpImg.rows/2 : oImg.rows-1); + y++, yy++ ) + { + if (pt.x-tmpImg.cols/2 >= 0 ) + { + x = pt.x-tmpImg.cols/2; + xx = 0; + } + else + { + x = 0; + xx = abs(pt.x-tmpImg.cols/2); + } + for( ; + x <= (pt.x+tmpImg.cols/2 < oImg.cols ? pt.x+tmpImg.cols/2 : oImg.cols-1); + x++, xx++ ) + { + tmpImg(yy, xx) = iImg(y, x); + } + } + + this->VO_ForwardTransform(tmpImg, oImg); +} + diff --git a/modules/integraltransform/src/VO_Symlets.cpp b/modules/integraltransform/src/VO_Symlets.cpp new file mode 100644 index 0000000..f9908a0 --- /dev/null +++ b/modules/integraltransform/src/VO_Symlets.cpp @@ -0,0 +1,109 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include "VO_Symlets.h" + + +void VO_Symlets::init() +{ + //this->m_iWaveletType = HAAR; + this->m_bInvertible = true; + this->m_bOrthogonal = true; + this->m_bBiorthogonal = true; +} + + +/** + * @author JIA Pei + * @brief Create Symlets wavelet + * @ref http://en.wikipedia.org/wiki/Daubechies_wavelet + * b_k = ( − 1)^k *a_{N − 1 − k} + */ +void VO_Symlets::VO_PrepareKernel(unsigned int symletsN) +{ + +} + + +/** + * @author JIA Pei + * @version 2010-03-05 + * @brief Forward Symlets Transform + * @return void + */ +void VO_Symlets::VO_ForwardTransform(const Mat_& iImg, Mat_& oImg) const +{ + +} + + +/** + * @author JIA Pei + * @version 2010-03-05 + * @brief Backward Symlets Transform + * @return void + */ +void VO_Symlets::VO_BackwardTransform(const Mat_& iImg, Mat_& oImg) const +{ + +} + + + diff --git a/modules/integraltransform/src/VO_WindowFunc.cpp b/modules/integraltransform/src/VO_WindowFunc.cpp new file mode 100644 index 0000000..e0e7212 --- /dev/null +++ b/modules/integraltransform/src/VO_WindowFunc.cpp @@ -0,0 +1,220 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include "VO_WindowFunc.h" + +#include "VO_Common.h" + +VO_WindowFunc::VO_WindowFunc(Size windowSize, unsigned int windowFunc) +{ + this->init(); + this->m_iWindowSize = windowSize; + this->m_iWindowFunc = windowFunc; + this->m_MatWindowedKernel = this->VO_GenerateWindowFunc(this->m_iWindowSize, this->m_iWindowFunc); +} + + +void VO_WindowFunc::init() +{ + this->m_iWindowSize = Size(0, 0); + this->m_iWindowFunc = 0; + this->m_MatWindowedKernel.release(); +} + + +VO_WindowFunc::~VO_WindowFunc() +{ + this->m_MatWindowedKernel.release(); +} + + +/** + * @author JIA Pei + * @version 2010-02-22 + * @brief Generate all types of window functions + * @param windowSize Input -- the window size + * @param windowFunc Output -- the window function + * @return Mat_ return + */ +Mat_ VO_WindowFunc::VO_GenerateWindowFunc(Size windowSize, unsigned int windowFunc) +{ + Mat_ resMat = Mat_::ones(windowSize); + Mat_ rowVec = Mat_::ones(1, windowSize.width); + Mat_ colVec = Mat_::ones(windowSize.height, 1); + + switch(windowFunc) + { + case HAMMING: + { + double a = 2.0 * CV_PI / (windowSize.width - 1.0); + for(int i = 0; i < windowSize.width; ++i) + rowVec(0, i) = safeDoubleToFloat(0.53836 - 0.46164 * cos(a*i)); + double b = 2.0 * CV_PI / (windowSize.height - 1.0); + for(int i = 0; i < windowSize.height; ++i) + colVec(i, 0) = safeDoubleToFloat(0.53836 - 0.46164 * cos(b*i)); + resMat = colVec*rowVec; + } + break; + case HANN: + { + double a = 2.0 * CV_PI / (windowSize.width - 1.0); + for(int i = 0; i < windowSize.width; ++i) + rowVec(0, i) = safeDoubleToFloat(0.5 * (1.0 - cos(a*i))); + double b = 2.0 * CV_PI / (windowSize.height - 1.0); + for(int i = 0; i < windowSize.height; ++i) + colVec(i, 0) = safeDoubleToFloat(0.5 * (1.0 - cos(b*i))); + resMat = colVec*rowVec; + } + break; + case TUKEY: + break; + case COSINE: + { + double a = CV_PI / (windowSize.width - 1.0); + for(int i = 0; i < windowSize.width; ++i) + rowVec(0, i) = safeDoubleToFloat(sin(a*i)); + double b = CV_PI / (windowSize.height - 1.0); + for(int i = 0; i < windowSize.height; ++i) + colVec(i, 0) = safeDoubleToFloat(sin(b*i)); + resMat = colVec*rowVec; + } + break; + case LANCZOS: + break; + case BARTLETT: + break; + case TRIANGULAR: + break; + case GAUSS: + { + } + break; + case BARTLETTHANN: + break; + case BLACKMAN: + break; + case KAISER: + break; + case NUTTALL: + break; + case BLACKMANHARRIS: + break; + case BLACKMANNUTTALL: + break; + case FLATTOP: + break; + case BESSEL: + break; + case DOLPHCHEBYSHEV: + break; + case EXPONENTIAL: + break; + case RIFEVINCENT: + break; + case GABOR: + break; + case RECTANGULAR: + default: + { + for(int i = 0; i < windowSize.width; ++i) + for(int j = 0; j < windowSize.height; ++j) + resMat(i, j) = 1.0f; + } + break; + } + + return resMat; +} + + +/** + * @author JIA Pei + * @version 2010-03-05 + * @brief Display the Window Function Kernel using STRETCH + * @param fn Input -- file name to store an Gabor Kernel image + * @return void + */ +void VO_WindowFunc::VO_DisplayWindowFuncKernel(const string& fn) +{ + Mat img = Mat::ones(Size(this->m_MatWindowedKernel.rows, this->m_MatWindowedKernel.cols), CV_8UC1); + + double minVal = 0.0; + double maxVal = 0.0; + cv::minMaxLoc(this->m_MatWindowedKernel, &minVal, &maxVal); + double stretchLen = maxVal - minVal; + + if(fabs(stretchLen) < FLT_EPSILON) + { + img = img*(unsigned char) stretchLen; + } + else + { + for(int i = 0; i < this->m_MatWindowedKernel.rows; i++) + { + for(int j = 0; j < this->m_MatWindowedKernel.cols; j++) + { + img.at(i, j) = (unsigned char) ( (this->m_MatWindowedKernel(i,j) - minVal) / (maxVal - minVal) * 255.0 ); + } + } + } + imwrite(fn, img); +} + + diff --git a/modules/smbuilding/CMakeLists.txt b/modules/smbuilding/CMakeLists.txt new file mode 100644 index 0000000..814c935 --- /dev/null +++ b/modules/smbuilding/CMakeLists.txt @@ -0,0 +1,5 @@ +include_directories("${CMAKE_CURRENT_SOURCE_DIR}/../common/include" "${CMAKE_CURRENT_SOURCE_DIR}/../cvcommon/include" +"${CMAKE_CURRENT_SOURCE_DIR}/../ensembletraining/include" "${CMAKE_CURRENT_SOURCE_DIR}/../integraltransform/include" +"${CMAKE_CURRENT_SOURCE_DIR}/../featureextraction/include") +set(deps opencv_core opencv_imgproc opencv_highgui opencv_legacy) +define_vosm_module(smbuilding vosm_common vosm_cvcommon vosm_featureextraction vosm_integraltransform ${deps}) diff --git a/modules/smbuilding/include/VO_AAMBasic.h b/modules/smbuilding/include/VO_AAMBasic.h new file mode 100644 index 0000000..797e07d --- /dev/null +++ b/modules/smbuilding/include/VO_AAMBasic.h @@ -0,0 +1,322 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + + +#ifndef __VO_AAMBASIC_H__ +#define __VO_AAMBASIC_H__ + + +#include + +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_AXM.h" + + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Concatenated appearance Model -- + * a direct combination of shape model and texture model. + * @ref http://www2.imm.dtu.dk/~aam/main/ + */ +class VO_AAMBasic : public VO_AXM +{ +friend class VO_Fitting2DSM; +friend class VO_FittingAAMBasic; +friend class VO_FittingAAMForwardIA; +friend class VO_FittingAAMInverseIA; +friend class VO_FittingASMLTCs; +friend class VO_FittingASMNDProfiles; +friend class VO_FittingAFM; +protected: + /** PCA transform for appearance, including eigenvectors, eigenvalues, and mean */ + PCA m_PCAAppearance; + + /** Shape weights, for scaling to texture scale. 15*15 */ + Mat_ m_MatWeightsScaleShape2Texture; + + /** Original appearance project to truncated space. For IMM, 60*12 */ + Mat_ m_MatAppearanceProject2Truncated; + + /** The shape part of the appearance model eigenvectors, 12*15, refer to equation (5.7) + * Cootes "Statistical Model of Appearance for Computer Vision" */ + Mat_ m_MatPcs; + + /** The texture part of the appearance model eigenvectors, 12*36, refer to equation (5.7) + * Cootes "Statistical Model of Appearance for Computer Vision" */ + Mat_ m_MatPcg; + + /** For shape, 116*12, refer to equation (5.9) + * Cootes "Statistical Model of Appearance for Computer Vision" */ + Mat_ m_MatQs; + + /** For texture, 80259*12, refer to equation (5.9) + * Cootes "Statistical Model of Appearance for Computer Vision" */ + Mat_ m_MatQg; + + /** In face, m_MatRc and m_MatRt are just the Hessian Matrics!!! */ + /** For shape, 12*80259, Multivariate Linear Regression Matrix, + refer to Stegmann's AAM-API Equation (7.4) */ + Mat_ m_MatRc; + + /** For shape, 4*80259, Multivariate Linear Regression Matrix, + refer to Stegmann's AAM-API Equation (7.4) */ + /** JIA Pei declare that this variable is absolutely useless, + and if using it, it makes everything unreasonable and illogical */ + Mat_ m_MatRt; + + /** Totally, n=m(4(k+4))=60*(4*(12+4))=3840 displacements. 4*12 */ + vector< Mat_ > m_vvCDisps; + + /** Totally, n=m(4(k+4))=60*(4*(12+4))=3840 displacements. 4*4, + refer to AAM-API page 3 of 10 */ + vector< Mat_ > m_vvPoseDisps; + + /** Stegmann: Gradient Matrix 80259*12 */ + Mat_ m_MatCParamGradientMatrix; + + /** Stegmann: Pose Gradient Matrix 80259*4 we may ignore this */ + Mat_ m_MatPoseGradientMatrix; + + /** Number of Appearance m_iNbOfAppearance = m_iNbOfShapeEigens + + m_iNbOfTextureEigens. For IMM: 15+36=51 */ + unsigned int m_iNbOfAppearance; + + /** Most possible appearance model eigens before PCA. For IMM: min (51, 60) = 51 */ + unsigned int m_iNbOfEigenAppearanceAtMost; + + /** Number of appearance model eigens. For IMM: 12 */ + unsigned int m_iNbOfAppearanceEigens; + + /** Truncate Percentage for appearance PCA. Normally, 0.95 */ + float m_fTruncatedPercent_Appearance; + + /** Initialization */ + void init() + { + this->m_iMethod = VO_AXM::AAM_BASIC; // AAM_DIRECT + this->m_iNbOfAppearance = 0; + this->m_iNbOfEigenAppearanceAtMost = 0; + this->m_iNbOfAppearanceEigens = 0; + this->m_fTruncatedPercent_Appearance = 0.95f; + } + +public: + /** Default constructor to create a VO_AAMBasic object */ + VO_AAMBasic() + { + this->init(); + } + + /** Destructor */ + ~VO_AAMBasic() + { + this->m_vvCDisps.clear(); + this->m_vvPoseDisps.clear(); + } + + //////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////Regression////////////////////////////// + /** Unfinished yet.... */ + /** Off-line build prediction matrix for fitting. */ + /** Please refer to http://www2.imm.dtu.dk/~aam/main/node16.html; revised from AAM-API */ + void VO_CalcRegressionMatrices(); + + /** Carry out C displacement experiments */ + void VO_DoCParamExperiments(); + + /** Carry out pose displacement experiments */ + void VO_DoPoseExperiments(); + + /** Carry out multi variate linear regression experiments */ + void VO_DoRegression(); + //////////////////////////////////////////////////////////////////////////// + + //////////////////////////////////////////////////////////////////////////// + //////Gradient -- This is what's happening in Stegmann's code/////////////// + /** Build gradient matrices */ + void VO_CalcGradientMatrices(); + + /** Build gradient matrices in terms of C parameters */ + void VO_EstCParamGradientMatrix(Mat_& oCParamGM); + + /** Build gradient matrices in terms of pose */ + void VO_EstPoseGradientMatrix(Mat_& oPoseGM); + //////////////////////////////////////////////////////////////////////////////////////////// + + /** Appearance parameters constraints */ + void VO_AppearanceParameterConstraint(Mat_& ioC, + float nSigma = 4.0f); + + /** Shape and texture project to shape parameters and texture parameters, and then concatenated */ + void VO_ShapeTexture2Appearance( VO_Shape iShape, + VO_Texture iTexture, + Mat_& app ) const; + + /** Appearance projected to appearance parameters */ + void VO_AppearanceProjectToCParam( const Mat_& app, + Mat_& outC) const; + + /** Shape parameters and texture parameters projected to concatenated parameters */ + void VO_SParamTParamProjectToCParam( const Mat_& inS, + const Mat_& inT, + Mat_& outC) const; + + /** Appearance parameters back projected to appearance */ + void VO_CParamBackProjectToAppearance( const Mat_& inC, + Mat_& app) const; + + /** Concatenated parameters back projected to shape parameters and texture parameters */ + void VO_CParamBackProjectToSParamTParam( const Mat_& inC, + Mat_& outS, + Mat_& outT) const; + + /** Concatenated parameters back projected to aligned shape */ + void VO_CParamBackProjectToAlignedShape( const Mat_& inC, + VO_Shape& oShape, + int dim = 2) const; + + /** Concatenated parameters back projected to normalized texture */ + void VO_CParamBackProjectToNormalizedTexture(const Mat_& inC, + VO_Texture& oTexture, + int tr = 3) const; + + /** Build displacement sets */ + void VO_CreateDisplacementSets(); + + /** Build displacement sets for C parameters */ + static vector< Mat_ > VO_CalcCParamDisplacementVectors( const vector& vStdDisp, + const Mat_& cVectors); + + /** Build displacement sets for Pose parameters */ + static vector< Mat_ > VO_CalcPoseDisplacementVectors( const vector &vScaleDisp, + const vector& vRotDisp, + const vector& vXDisp, + const vector& vYDisp); + + /** Build Appearance model */ + void VO_BuildAppearanceModel(const vector& allLandmarkFiles4Training, + const vector& allImgFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + unsigned int channels = 3, + unsigned int levels = 1, + int trm = VO_Features::DIRECT, + float TPShape = 0.95f, + float TPTexture = 0.95f, + float TPConcatenated = 0.95f, + bool useKnownTriangles = false); + + /** Save Appearance Model, to a specified folder */ + void VO_Save(const string& fd); + + /** Load all parameters */ + void VO_Load(const string& fd); + + /** Load Parameters for fitting */ + void VO_LoadParameters4Fitting(const string& fd); + + /** Gets and Sets */ + Mat_ GetAppearanceMean() const {return this->m_PCAAppearance.mean;} + Mat_ GetAppearanceEigenValues() const {return this->m_PCAAppearance.eigenvalues;} + Mat_ GetAppearanceEigenVectors() const {return this->m_PCAAppearance.eigenvectors;} + Mat_ GetWeightsScaleShape2Texture() const {return this->m_MatWeightsScaleShape2Texture;} + Mat_ GetAppearanceProject2Truncated() const {return this->m_MatAppearanceProject2Truncated;} + Mat_ GetPcs() const {return this->m_MatPcs;} + Mat_ GetPcg() const {return this->m_MatPcg;} + Mat_ GetQs() const {return this->m_MatQs;} + Mat_ GetQg() const {return this->m_MatQg;} + Mat_ GetRc() const {return this->m_MatRc;} + Mat_ GetRt() const {return this->m_MatRt;} + vector< Mat_ > GetCDisps() const {return this->m_vvCDisps;} + vector< Mat_ > GetPoseDisps() const {return this->m_vvPoseDisps;} + Mat_ GetCParamGradientMatrix() const {return this->m_MatCParamGradientMatrix;} + Mat_ GetPoseGradientMatrix() const {return this->m_MatPoseGradientMatrix;} + unsigned int GetNbOfAppearance() const {return this->m_iNbOfAppearance;} + unsigned int GetNbOfEigenAppearanceAtMost() const {return this->m_iNbOfEigenAppearanceAtMost;} + unsigned int GetNbOfAppearanceEigens() const {return this->m_iNbOfAppearanceEigens;} + float GetTruncatedPercent_Concatenated() const {return this->m_fTruncatedPercent_Appearance;} + + // void SetWeightsScaleShape2Texture(const Mat_& inWeightsScaleShape2Texture) + // {inWeightsScaleShape2Texture.copyTo(this->m_MatWeightsScaleShape2Texture);} + // void SetPcs(const Mat_& inPcs) {inPcs.copyTo(this->m_MatPcs);} + // void SetPcg(const Mat_& inPcg) {inPcg.copyTo(this->m_MatPcg);} + // void SetQs(const Mat_& inQs) {inQs.copyTo(this->m_MatQs);} + // void SetQg(const Mat_& inQg) {inQg.copyTo(this->m_MatQg);} + // void SetRc(const Mat_& inRc) {inRc.copyTo(this->m_MatRc);} + // void SetRt(const Mat_& inRt) {inRt.copyTo(this->m_MatRt);} + // void SetCDisps(const vector >& inCDisps) {this->m_vvCDisps = inCDisps;} + // void SetPoseDisps(const vector >& inPoseDisps) {this->m_vvPoseDisps = inPoseDisps;} + // void SetCParamGradientMatrix(const Mat_& inCParamGradientMatrix) + // {inCParamGradientMatrix.copyTo(this->m_MatCParamGradientMatrix);} + // void SetPoseParamGradientMatrix(const Mat_& inPoseGradientMatrix) + // {inPoseGradientMatrix.copyTo(this->m_MatPoseGradientMatrix);} + // void SetNbOfConcatenated(unsigned int inNbOfConcatenated) {this->m_iNbOfAppearance = inNbOfConcatenated;} + // void SetTruncatedPercent_Concatenated(float inTruncatedPercent_Concatenated) + // {this->m_fTruncatedPercent_Appearance = inTruncatedPercent_Concatenated;} + }; + + +#endif // __VO_AAMBASIC_H__ diff --git a/modules/smbuilding/include/VO_AAMForwardIA.h b/modules/smbuilding/include/VO_AAMForwardIA.h new file mode 100644 index 0000000..a7fdd21 --- /dev/null +++ b/modules/smbuilding/include/VO_AAMForwardIA.h @@ -0,0 +1,132 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_AAMFORWARDIA_H__ +#define __VO_AAMFORWARDIA_H__ + + +#include + +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_AXM.h" + + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Forward image alignment Model. + * @note FIXME + */ +class VO_AAMForwardIA : public VO_AXM +{ +friend class VO_Fitting2DSM; +friend class VO_FittingAAMBasic; +friend class VO_FittingAAMForwardIA; +friend class VO_FittingAAMInverseIA; +friend class VO_FittingASMLTCs; +friend class VO_FittingASMNDProfiles; +friend class VO_FittingAFM; +private: +protected: + /** Initialization */ + void init() + { + this->m_iMethod = VO_AXM::AAM_FAIA; + } + +public: + /** Default constructor to create a VO_AAMForwardIA object */ + VO_AAMForwardIA() {this->init();} + + /** Destructor */ + ~VO_AAMForwardIA() {} + + /** Build ICIA AAM model */ + void VO_BuildAAMFCIA(const vector& allLandmarkFiles4Training, + const vector& allImgFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + unsigned int levels = 3, + unsigned int channels = 1, + int trm = VO_Features::DIRECT, + float TPShape = 0.95f, + float TPTexture = 0.95f, + bool useKnownTriangles = false); + + /** Save AAMICIA, to a specified folder */ + void VO_Save(const string& fd); + + /** Load all parameters */ + void VO_Load(const string& fd); + + /** Load Parameters for fitting */ + void VO_LoadParameters4Fitting(const string& fd); + +}; + + +#endif // __VO_AAMFORWARDIA_H__ + diff --git a/modules/smbuilding/include/VO_AAMInverseIA.h b/modules/smbuilding/include/VO_AAMInverseIA.h new file mode 100644 index 0000000..2f579a5 --- /dev/null +++ b/modules/smbuilding/include/VO_AAMInverseIA.h @@ -0,0 +1,168 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_AAMINVERSEIA_H__ +#define __VO_AAMINVERSEIA_H__ + + +#include + +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_AXM.h" + + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Inverse image alignment Model. + */ +class VO_AAMInverseIA : public VO_AXM +{ +friend class VO_Fitting2DSM; +friend class VO_FittingAAMBasic; +friend class VO_FittingAAMForwardIA; +friend class VO_FittingAAMInverseIA; +friend class VO_FittingASMLTCs; +friend class VO_FittingASMNDProfiles; +friend class VO_FittingAFM; +private: + Mat m_IplImageTempFaceX; + Mat m_IplImageTempFaceY; + Mat m_IplImageTempFace; +protected: + /** "Revisited" P26-27, 4*116 */ + Mat_ m_MatSimilarityTransform; + + /** Steepest Descent Images for each point, 90396*20 */ + Mat_ m_MatSteepestDescentImages4ShapeModel; + + /** Steepest Descent Images for global shape normalization, 90396*4 */ + Mat_ m_MatSteepestDescentImages4GlobalShapeNormalization; + + /** Combined Steepest Descent Images, 90396*24 */ + Mat_ m_MatSteepestDescentImages; + + /** Combined Modified Steepest Descent Images, 90396*24 */ + Mat_ m_MatModifiedSteepestDescentImages; + + /** Hessian Matrix, actually, Hessian matrix is summed up from all the pixels in the image, 20*20, or 24*24 */ + Mat_ m_MatHessianMatrixInverse; + + /** Pre computed matrix 24*90396 */ + Mat_ m_MatICIAPreMatrix; + + /** Initialization */ + void init() + { + this->m_iMethod = VO_AXM::AAM_IAIA; // AAM_CMUICIA + } + +public: + /** Default constructor to create a VO_AAMInverseIA object */ + VO_AAMInverseIA() {this->init();} + + /** Destructor */ + ~VO_AAMInverseIA() {} + + /** Pre-computation for Inverse Compositional Image Alignment AAM Fitting */ + + /** Calculate gradients in both X and Y directions for template face */ + void VO_CalcTemplateFaceGradients(); + + /** Calculate steepest descent image for template face */ + void VO_CalcSDI(); + + /** Calculate modified steepest descent image for template face */ + void VO_CalcModifiedSDI(); + + /** Calculate inverse Hessian matrix for template face */ + void VO_CalcInverseHessian(); + + /** Calculate Hessian matrix * MSDI^T */ + void VO_CalcICIAPreMatrix(); + + /** Build ICIA AAM model */ + void VO_BuildAAMICIA(const vector& allLandmarkFiles4Training, + const vector& allImgFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + unsigned int channels = 3, + unsigned int levels = 1, + int trm = VO_Features::DIRECT, + float TPShape = 0.95f, + float TPTexture = 0.95f, + bool useKnownTriangles = false); + + /** Save AAMICIA, to a specified folder */ + void VO_Save(const string& fd); + + /** Load all parameters */ + void VO_Load(const string& fd); + + /** Load Parameters for fitting */ + void VO_LoadParameters4Fitting(const string& fd); +}; + +#endif // __VO_AAMINVERSEIA_H__ + diff --git a/modules/smbuilding/include/VO_AFM.h b/modules/smbuilding/include/VO_AFM.h new file mode 100644 index 0000000..5022fd2 --- /dev/null +++ b/modules/smbuilding/include/VO_AFM.h @@ -0,0 +1,177 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_AFM_H__ +#define __VO_AFM_H__ + + +#include + +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_DiscreteWavelet.h" +#include "VO_AXM.h" + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief Active feature model. +*/ +class VO_AFM : public VO_AXM +{ +friend class VO_Fitting2DSM; +friend class VO_FittingAAMBasic; +friend class VO_FittingAAMForwardIA; +friend class VO_FittingAAMInverseIA; +friend class VO_FittingASMLTCs; +friend class VO_FittingASMNDProfiles; +friend class VO_FittingAFM; +protected: + /** PCA transform for concatenated local features, including eigenvectors, eigenvalues, and mean */ + PCA m_PCANormalizedFeatures; + + /** Local feature extraction method */ + unsigned int m_iLocalFeatureExtractionMethod; + + /** length of concatenated local features in format of f1f2f3f4f1f2f3f4...f1f2f3f4. * . For IMM, */ + unsigned int m_iNbOfFeatures; + + /** Most possible feature eigens before PCA. For IMM: min (, 240) = 240 */ + unsigned int m_iNbOfEigenFeaturesAtMost; + + /** Number of AFM model eigens. For IMM: 127 */ + unsigned int m_iNbOfFeaturesEigens; + + /** The template LTC average standard deviation : 12364.1 */ + float m_fAverageFeatureStandardDeviation; + + /** Truncate Percentage for LTC model PCA. Normally, 0.95 */ + float m_fTruncatedPercent_Feature; + + /** All loaded LTCs in a vector format. For IMM, 240*90396 */ + vector m_vLTCs; + + /** All normalized LTCs in a vector format. For IMM, 240*90396 */ + vector m_vNormalizedLTCs; + + /** image size for wavelet transform */ + Size m_localImageSize; + + /** Initialization */ + void init() + { + this->m_iMethod = VO_AXM::AFM; // CLM + this->m_iLocalFeatureExtractionMethod = VO_DiscreteWavelet::HAAR; + } + +public: + /** Default constructor to create a VO_AFM object */ + VO_AFM() {this->init();} + + /** Destructor */ + ~VO_AFM() {} + + /** Feature parameters constraints */ + void VO_FeatureParameterConstraint(Mat_& ioF, float nSigma = 3.0f); + + /** Build ASMLTCs model */ + void VO_BuildFeatureModel ( const vector& allLandmarkFiles4Training, + const vector& allImgFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + unsigned int channels = 1, + unsigned int levels = 3, + int trm = VO_Features::DIRECT, + float TPShape = 0.95f, + bool useKnownTriangles = false, + unsigned int ltcMtd = VO_DiscreteWavelet::HAAR, + Size imgSize = Size(16, 16) ); + + /** wavelet localization */ + static Rect VO_CalcImagePatchRect( const Mat& iImg, + const Point2f& pt, + Size imgSize); + + /** Load Training data for LTC model */ + void VO_LoadFeatureTrainingData(unsigned int mtd); + + /** load LTC around 1 single point for 1 single channel */ + Mat_ VO_LoadLTC4OneAnnotatedPoint( const Mat& iImg, + const VO_Shape& theShape, + unsigned int ptIdx, + Size imgSize, + unsigned int mtd); + + /** Save ASM LTCs, to a specified folder */ + void VO_Save(const string& fd); + + /** Load all parameters */ + void VO_Load(const string& fd); + + /** Load parameters for fitting */ + void VO_LoadParameters4Fitting(const string& fd); +}; + +#endif // __VO_AFM_H__ + diff --git a/modules/smbuilding/include/VO_ASMLTCs.h b/modules/smbuilding/include/VO_ASMLTCs.h new file mode 100644 index 0000000..2c3cb16 --- /dev/null +++ b/modules/smbuilding/include/VO_ASMLTCs.h @@ -0,0 +1,180 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_ASMLTCS_H__ +#define __VO_ASMLTCS_H__ + + +#include + +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_DiscreteWavelet.h" +#include "VO_AXM.h" +#include "VO_Features.h" + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief Local texture constrained ASM. + * @note Current VO_ASMLTCs only deal with 1 channel gray image + */ +class VO_ASMLTCs : public VO_AXM +{ +friend class VO_Fitting2DSM; +friend class VO_FittingAAMBasic; +friend class VO_FittingAAMForwardIA; +friend class VO_FittingAAMInverseIA; +friend class VO_FittingASMLTCs; +friend class VO_FittingASMNDProfiles; +friend class VO_FittingAFM; +protected: + /** */ + VO_Features* m_pVOfeatures; + + /** Local texture constrained methods */ + unsigned int m_iLTCMethod; + + /** length of LTC vector in format of rr...rrgg...ggbb...bb . */ + unsigned int m_iNbOfLTC4PerPoint; + + /** All loaded LTCs in a vector format. m_iNbOfSamples*m_iNbOfPyramidLevels*(m_iNbOfPoints*m_iNbOfLTC4PerPoint) */ + vector< vector< Mat_ > > m_vvLTCs; + + /** All normalized LTCs in a vector format. m_iNbOfSamples*m_iNbOfPyramidLevels*(m_iNbOfPoints*m_iNbOfLTC4PerPoint) */ + vector< vector< Mat_ > > m_vvNormalizedLTCs; + + /** mean LTC for all concerned points. m_iNbOfPyramidLevels*m_iNbOfPoints*(1*m_iNbOfLTC4PerPoint) */ + vector< vector< Mat_ > > m_vvLTCMeans; + + /** LTC covariance for all concerned points. m_iNbOfPyramidLevels*m_iNbOfPoints*(m_iNbOfLTC4PerPoint*m_iNbOfLTC4PerPoint) */ + vector< vector< Mat_ > > m_vvCVMInverseOfLTCCov; + + /** image size for wavelet transform */ + Size m_localImageSize; + + /** Initialization */ + void init() + { + this->m_iMethod = VO_AXM::ASM_LTC; + this->m_iLTCMethod = VO_Features::DIRECT; + this->m_iNbOfLTC4PerPoint = 0; + this->m_pVOfeatures = NULL; + } + +public: + /** Default constructor to create a VO_ASMLTCs object */ + VO_ASMLTCs() {this->init();} + + /** Destructor */ + ~VO_ASMLTCs() {} + + /** Build ASMLTCs model */ + void VO_BuildASMLTCs ( const vector& allLandmarkFiles4Training, + const vector& allImgFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + unsigned int channels = 1, + unsigned int levels = 4, + int trm = VO_Features::DIRECT, + float TPShape = 0.95f, + bool useKnownTriangles = false, + unsigned int ltcMtd = VO_Features::DIRECT, + Size imgSize = Size(16, 16) ); + + /** wavelet localization */ + static Rect VO_CalcImagePatchRect(const Mat& iImg, const Point2f& pt, Size imgSize); + + /** Load Training data for LTC model */ + void VO_LoadLTCTrainingData(); + + /** load LTC around 1 single point for 1 single channel */ + static void VO_LoadLTC4OneAnnotatedPoint( const Mat& iImg, + const VO_Shape& theShape, + unsigned int ptIdx, + Size imgSize, + VO_Features* vofeatures, + int shiftX = 0, + int shiftY = 0); + + /** calculate statistics */ + void VO_CalcStatistics4AllLTCs(); + + /** Output one wavelet transformed image, for the purpose of demonstration */ + void VO_HardSaveWaveletSingleChannelImage( const string& fn, + Size imgSize, + unsigned int displayMtd = STRETCH); + + /** Save ASM LTCs, to a specified folder */ + void VO_Save(const string& fd); + + /** Load all parameters */ + void VO_Load(const string& fd); + + /** Load parameters for fitting */ + void VO_LoadParameters4Fitting(const string& fd); +}; + +#endif // __VO_ASMLTCS_H__ + diff --git a/modules/smbuilding/include/VO_ASMNDProfiles.h b/modules/smbuilding/include/VO_ASMNDProfiles.h new file mode 100644 index 0000000..0e8a0e7 --- /dev/null +++ b/modules/smbuilding/include/VO_ASMNDProfiles.h @@ -0,0 +1,166 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_ASMNDPROFILES_H__ +#define __VO_ASMNDPROFILES_H__ + + +#include + +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_Profile.h" +#include "VO_AXM.h" + + +using namespace std; +using namespace cv; + +/** +* @author JIA Pei +* @brief ND Profile ASM. +* @note Current VO_ASMNDProfiles only deal with 1 channel gray image +*/ +class VO_ASMNDProfiles : public VO_AXM +{ +friend class VO_Fitting2DSM; +friend class VO_FittingAAMBasic; +friend class VO_FittingAAMForwardIA; +friend class VO_FittingAAMInverseIA; +friend class VO_FittingASMLTCs; +friend class VO_FittingASMNDProfiles; +friend class VO_FittingAFM; +protected: + /** Number of profile pixels refer to Cootes "Statistical Models of Appearance for Computer Vision" page 38 */ + /** rgb should have 3 times profiles than gray */ + vector m_iNbOfProfilesPerPixelAtLevels; + + /** Number of dimensions of profiles */ + unsigned int m_iNbOfProfileDim; + + /** All normalized profiles m_iNbOfSamples* m_iNbOfPyramidLevels*m_iNbOfPoints */ + vector< vector< vector< VO_Profile > > > m_vvvNormalizedProfiles; + + /** Average normalized profile m_iNbOfPyramidLevels*m_iNbOfPoints */ + vector< vector< VO_Profile > > m_vvMeanNormalizedProfile; + + /** inverse of Sg, m_iNbOfPyramidLevels*m_iNbOfPoints*m_iNbOfProfileDim*(m_iNbOfProfilesPerPixelAtLevels*m_iNbOfProfilesPerPixelAtLevels) refer to Cootes "Statistics Models of Appearance for Computer Vision", page 38 (7.2) */ + vector< vector< vector< Mat_ > > > m_vvvCVMInverseOfSg; + + /** Initialization */ + void init() + { + this->m_iMethod = VO_AXM::ASM_PROFILEND; + this->m_iNbOfProfilesPerPixelAtLevels.clear(); + this->m_vvvNormalizedProfiles.clear(); + this->m_vvMeanNormalizedProfile.clear(); + this->m_vvvCVMInverseOfSg.clear(); + } + +public: + enum { SAME = 0, PYRAMID = 1, DESCENDING = 2}; + + /** Default constructor to create a VO_ASMNDProfiles object */ + VO_ASMNDProfiles() {this->init();} + + /** Destructor */ + ~VO_ASMNDProfiles() + { + this->m_iNbOfProfilesPerPixelAtLevels.clear(); + this->m_vvvNormalizedProfiles.clear(); + this->m_vvMeanNormalizedProfile.clear(); + this->m_vvvCVMInverseOfSg.clear(); + } + + /** Build ASM ND Profile model */ + void VO_BuildASMNDProfiles( const vector& allLandmarkFiles4Training, + const vector& allImgFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + unsigned int channels = 1, + unsigned int levels = 4, + unsigned int profdim = 2, + unsigned int kk = 8, + int trm = VO_Features::DIRECT, + float TPShape = 0.95f, + bool useKnownTriangles = false); + + /** profiles calculation, refer to Cootes "Statistical Models of Appearance for Computer Vision" page 38 */ + void VO_LoadProfileTrainingData(int loadFlag); + + /** profiles calculation, refer to Cootes "Statistical Models of Appearance for Computer Vision" page 38 */ + void VO_CalcStatistics4AllProfiles(); + + /** This function determines how many elements are used to compose a single profile vector */ + static void VO_ProduceLevelProfileNumbers(vector& nbOfProfilesPerPixelAtLevels, unsigned int iLevel, unsigned int NbOfLevel0 = 17, unsigned int ProduceMethod = SAME); + + /** Save ASM ND Profile model, to a specified folder */ + void VO_Save(const string& fd); + + /** Load all parameters */ + void VO_Load(const string& fd); + + /** Load parameters for fitting */ + void VO_LoadParameters4Fitting(const string& fd); +}; + +#endif // __VO_ASMNDPROFILES_H__ + diff --git a/modules/smbuilding/include/VO_AXM.h b/modules/smbuilding/include/VO_AXM.h new file mode 100644 index 0000000..1617ea1 --- /dev/null +++ b/modules/smbuilding/include/VO_AXM.h @@ -0,0 +1,138 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_AXM_H__ +#define __VO_AXM_H__ + + +#include + +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_TextureModel.h" + + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief Generalized class for various models. +*/ +class VO_AXM : public VO_TextureModel +{ +friend class VO_Fitting2DSM; +friend class VO_FittingAAMBasic; +friend class VO_FittingAAMForwardIA; +friend class VO_FittingAAMInverseIA; +friend class VO_FittingASMLTCs; +friend class VO_FittingASMNDProfiles; +friend class VO_FittingAFM; +protected: + /** Number of Levels */ + unsigned int m_iNbOfPyramidLevels; + + /** objective functions -- ASMNDProfile, ASMLTC, AAMBasic, AAMICIA, CLM, AFM, etc */ + unsigned int m_iMethod; + + /** Initialization */ + void init(unsigned int method = ASM_PROFILEND, unsigned int levels = 4) + { + this->m_iMethod = method; + this->m_iNbOfPyramidLevels = levels; + } + +public: + /** The following classification is correct. CLM belongs to AFM */ + enum TYPE { ASM_PROFILEND = 1, + ASM_LTC = 2, + CLM = 8, + AFM = 9, + AAM_BASIC = 10, + AAM_DIRECT=11, + AAM_FAIA = 12, + AAM_CMUICIA = 13, + AAM_IAIA = 14, + MM3D = 20}; + + /** Default constructor to create a VO_AXM object */ + VO_AXM(unsigned int method = ASM_PROFILEND, unsigned int levels = 1) + { + this->init(method, levels); + } + + /** Destructor */ + ~VO_AXM() {} + + /** Save ASM, to a specified folder */ + void VO_Save(const string& fd); + + /** Load all parameters */ + void VO_Load(const string& fd); + + /** Load parameters for fitting */ + void VO_LoadParameters4Fitting(const string& fd); + + unsigned int GetNbOfPyramidLevels(){ return this->m_iNbOfPyramidLevels; } +}; + +#endif // __VO_AXM_H__ + diff --git a/modules/smbuilding/include/VO_Edge.h b/modules/smbuilding/include/VO_Edge.h new file mode 100644 index 0000000..ed9bc7f --- /dev/null +++ b/modules/smbuilding/include/VO_Edge.h @@ -0,0 +1,125 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_EDGE_H__ +#define __VO_EDGE_H__ + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Edge - indicating two point indexes. + */ +class VO_Edge +{ +friend ostream& operator<<(ostream& os, const VO_Edge& aamedge); +friend istream& operator>>(istream& is, VO_Edge& aamedge); +private: + /** First vertex' index of this edge */ + unsigned int index1; + + /** Second vertex' index of this edge */ + unsigned int index2; + +public: + /** Default constructor to create a VO_Edge object */ + VO_Edge() {} + + /** Constructor to create a VO_Edge object with two vertexes' indexes */ + VO_Edge(unsigned int p1, unsigned int p2) + { + this->index1 = p1; + this->index2 = p2; + } + + /** Destructor */ + virtual ~VO_Edge() {} + + /** Default constructor to create a VO_Edge object */ + bool operator==(const VO_Edge& e) const + { + if ( ( this->index1 == e.GetIndex1() && this->index2 == e.GetIndex2() ) || + ( this->index1 == e.GetIndex2() && this->index2 == e.GetIndex1() ) ) + return true; + else return false; + } + + /** Get first index of the first edge vertex */ + unsigned int GetIndex1() const { return this->index1;} + + /** Get second index of the first edge vertex */ + unsigned int GetIndex2() const { return this->index2;} + + /** Set the first index of the first edge vertex */ + void SetIndex1(unsigned int idx1) { this->index1 = idx1;} + + /** Set the second index of the second edge vertex */ + void SetIndex2(unsigned int idx2) { this->index2 = idx2;} + +}; + +#endif // __VO_EDGE_H__ + diff --git a/modules/smbuilding/include/VO_Ellipse.h b/modules/smbuilding/include/VO_Ellipse.h new file mode 100644 index 0000000..892836d --- /dev/null +++ b/modules/smbuilding/include/VO_Ellipse.h @@ -0,0 +1,228 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_ELLIPSE_H__ +#define __VO_ELLIPSE_H__ + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_Shape.h" +#include "VO_Common.h" + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief Ellipse - used to describe 2D Gaussian distribution. +*/ +class VO_Ellipse +{ +friend ostream& operator<<(ostream& os, const VO_Ellipse& ellipse); +friend istream& operator>>(istream& is, VO_Ellipse& ellipse); +protected: + float m_fPhimin; // Minimum angle (degrees) + float m_fPhimax; // Maximum angle (degrees) + float m_fAxisXHalfLen; // first radius + float m_fAxisYHalfLen; // second radius + float m_fTheta; // Rotation angle (degrees) + Point2f m_Center; // ellipse center + + void CopyData(const VO_Ellipse& ellipse) + { + this->m_fPhimax = ellipse.m_fPhimax; + this->m_fPhimin = ellipse.m_fPhimin; + this->m_fAxisXHalfLen = ellipse.m_fAxisXHalfLen; + this->m_fAxisYHalfLen = ellipse.m_fAxisYHalfLen; + this->m_fTheta = ellipse.m_fTheta; + this->m_Center.x = ellipse.m_Center.x; + this->m_Center.y = ellipse.m_Center.y; + } + +public: + /** Default constructor to create a VO_Ellipse object */ + VO_Ellipse() + { + this->m_fPhimax = 0.0f; + this->m_fPhimin = 0.0f; + this->m_fAxisXHalfLen = 0.0f; + this->m_fAxisYHalfLen = 0.0f; + this->m_fTheta = 0.0f; + this->m_Center.x = 0.0f; + this->m_Center.y = 0.0f; + } + + /** copy constructor */ + VO_Ellipse(const VO_Ellipse& ellipse) + { + this->CopyData(ellipse); + } + + /** Constructor to create a VO_Ellipse object with three coordinate vertexes in vector format */ + VO_Ellipse(Point2f cog, float l, float s, float phimin = 0, float phimax = 360, float theta = 0) + { + this->m_fPhimin = phimin; + this->m_fPhimax = phimax; + this->m_fAxisXHalfLen = l; + this->m_fAxisYHalfLen = s; + this->m_fTheta = theta; + this->m_Center = cog; + } + + /** Destructor */ + virtual ~VO_Ellipse() {} + + /** operator= overloading, similar to copy constructor */ + VO_Ellipse& operator=(const VO_Ellipse& ellipse); + VO_Ellipse operator*(float value); + VO_Ellipse& operator*=(float value); + + /** Translation */ + void Translate( const Mat_& translation ); + + /** Scale the Position */ + void ScaleCenter(float value); + + /** Calculate area of ellipse */ + float CalcArea( ) const {return safeDoubleToFloat(CV_PI)*this->m_fAxisXHalfLen*this->m_fAxisYHalfLen;} + + /** Calculate perimeter of ellipse: L=2πb+4(a-b) */ + float CalcPerimeter( ) + { + float longAxis = this->GetLongAxis(); + float shortAxis = this->GetShortAxis(); + return 2.0f*safeDoubleToFloat(CV_PI)*shortAxis + 4.0f*(longAxis - shortAxis); + } + + /** Evaluate whether the point is within the ellipse */ + bool IsPointWithinEllipse(const Point2f& pt, bool includeBoundary = true) + { + if(includeBoundary) + { + if( pow( ( pt.x * cos(this->m_fTheta) - pt.y * sin(this->m_fTheta) - this->m_Center.x ) / this->m_fAxisXHalfLen, 2.0f) + + pow( ( pt.x * sin(this->m_fTheta) + pt.y * cos(this->m_fTheta) - this->m_Center.y ) / this->m_fAxisYHalfLen, 2.0f) <= 1.0 ) + return true; + else + return false; + } + else + { + if( pow( ( pt.x * cos(this->m_fTheta) - pt.y * sin(this->m_fTheta) - this->m_Center.x ) / this->m_fAxisXHalfLen, 2.0f) + + pow( ( pt.x * sin(this->m_fTheta) + pt.y * cos(this->m_fTheta) - this->m_Center.y ) / this->m_fAxisYHalfLen, 2.0f) < 1.0 ) + return true; + else + return false; + } + } + + /** Evaluate whether the point is on the boundary of the ellipse */ + bool IsPointOnEllipseBoundary(const Point2f& pt) + { + if( fabs( pow( ( pt.x * cos(this->m_fTheta) - pt.y * sin(this->m_fTheta) - this->m_Center.x ) / this->m_fAxisXHalfLen, 2.0f) + + pow( ( pt.x * sin(this->m_fTheta) + pt.y * cos(this->m_fTheta) - this->m_Center.y ) / this->m_fAxisYHalfLen, 2.0f) - 1.0 ) < FLT_EPSILON ) + return true; + else + return false; + } + + /** Find the upright bounding rectangle for an ellipse */ + Rect CalcBoundingRect() const + { + Size2f size = Size2f(this->m_fAxisXHalfLen*2.0f, this->m_fAxisYHalfLen*2.0f); + RotatedRect rRect = RotatedRect(this->m_Center, size, this->m_fTheta); + return rRect.boundingRect(); + } + + /** Find the nearest point on the boundary to the input point + * leave here, tough. needs optimization */ + Point2f FindNearestPointOnBoundary(const Point2f& pt); + + /** Find the intersect point on the boundary connecting from the COG and the input point */ + Point2f FindIntersectPointOnBoundary(const Point2f& pt); + + /** Obtain point list within the ellipse according to its range */ + vector VO_PointsInEllipse( const VO_Ellipse& ellipse); + + /** Find bounding rectangle for multiple ellipses, this rectangle should cover all ellipses */ + static Rect VO_CalcBoundingRect4MultipleEllipses(const vector& ellipses); + + // Gets and sets + float GetLongAxis() const + { + return this->m_fAxisXHalfLen >= this->m_fAxisYHalfLen ? + this->m_fAxisXHalfLen : this->m_fAxisYHalfLen; + } + float GetShortAxis() const + { + return this->m_fAxisXHalfLen <= this->m_fAxisYHalfLen ? + this->m_fAxisXHalfLen : this->m_fAxisYHalfLen; + } + Point2f GetCOG() const {return this->m_Center;} + float GetStartAngle() const {return this->m_fPhimin;} + float GetEndAngle() const {return this->m_fPhimax;} + float GetAxisXHalfLen() const {return this->m_fAxisXHalfLen;} + float GetAxisYHalfLen() const {return this->m_fAxisYHalfLen;} + float GetAngle() const {return this->m_fTheta;} +}; + +#endif // __VO_ELLIPSE_H__ + diff --git a/modules/smbuilding/include/VO_FacePart.h b/modules/smbuilding/include/VO_FacePart.h new file mode 100644 index 0000000..562d21b --- /dev/null +++ b/modules/smbuilding/include/VO_FacePart.h @@ -0,0 +1,168 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_FACEPART_H__ +#define __VO_FACEPART_H__ + +#include +#include +#include + + +using namespace std; + + +/** +* @author JIA Pei +* @brief Ellipse - used to describe 2D Gaussian distribution. +*/ +class VO_FacePart +{ +friend ostream& operator<<(ostream& os, const VO_FacePart& facepart); +friend istream& operator>>(istream& is, VO_FacePart& facepart); +friend class VO_FaceParts; +private: + /** type */ + unsigned int m_iType; + + /** open or closed */ + bool m_bClosed; + + /** index vector */ + vector m_vIndexes; + + void init() + { + this->m_iType = 0; // actually, type 0 is just a default value, which is not used at all + this->m_bClosed = false; + this->m_vIndexes.clear(); + } + + void CopyData(const VO_FacePart &iFacePart) + { + this->m_iType = iFacePart.GetType(); + this->m_bClosed = iFacePart.IsClosedOrNot(); + this->m_vIndexes.resize( iFacePart.GetIndexes().size() ); + this->m_vIndexes = iFacePart.GetIndexes(); + } + +public: + enum { + WHOLEFACE = 0, + CHIN = 1, + LEFTEYEBROW = 2, + RIGHTEYEBROW = 3, + LEFTEYE = 4, + RIGHTEYE = 5, + NOSE = 6, + NOSTRIL = 7, + NOSETIP = 8, + LIPOUTERLINE = 9, + LIPINNERLINE = 10, + LIPUPPEROUTERLINE = 11, + LIPUPPERINNERLINE = 12, + LIPDOWNEROUTERLINE = 13, + LIPDOWNERINNERLINE = 14, + LEFTSIDEPOINTS = 15, + RIGHTSIDEPOINTS = 16, + MIDLINEPOINTS = 17, + EYECORNERPOINTS = 18, + MOUTHCORNERPOINTS = 19, + PITCHAXISLINEPOINTS = 20, + LEFTIRIS = 21, + RIGHTIRIS = 22, + LEFTEAR = 23, + RIGHTEAR = 24, + WHOLEMOUTH = 25, + NOSECENTRALAREA = 26}; + + /** Default constructor to create a VO_FaceParts object */ + VO_FacePart() { this->init(); } + + /** Copy constructor */ + VO_FacePart( const VO_FacePart &iFacePart ) + { + this->CopyData(iFacePart); + } + + /** Destructor */ + virtual ~VO_FacePart() { this->m_vIndexes.clear(); } + + /** operator= overloading, similar to copy constructor */ + VO_FacePart& operator=(const VO_FacePart &iFacePart) + { + this->CopyData(iFacePart); + return *this; + } + + void clear() { this->init();} + + unsigned int GetType() const {return this->m_iType;} + bool IsClosedOrNot() const {return this->m_bClosed;} + vector GetIndexes() const {return this->m_vIndexes;} + + void SetType(unsigned int type) {this->m_iType = type;} + void SetClosedOrNot(bool close) {this->m_bClosed = close;} + void SetIndexes(vector indexes) {this->m_vIndexes = indexes;} + +}; + +#endif // __VO_FACEPART_H__ + diff --git a/modules/smbuilding/include/VO_FaceParts.h b/modules/smbuilding/include/VO_FaceParts.h new file mode 100644 index 0000000..73b203d --- /dev/null +++ b/modules/smbuilding/include/VO_FaceParts.h @@ -0,0 +1,244 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_FACEPARTS_H__ +#define __VO_FACEPARTS_H__ + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_FacePart.h" + + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief Face Parts, pay attention to file "ShapeInfo.txt" +*/ +class VO_FaceParts +{ +friend ostream& operator<<(ostream& os, const VO_FaceParts& faceparts); +friend istream& operator>>(istream& is, VO_FaceParts& faceparts); +private: + /** Chin indexes */ + VO_FacePart m_vChin; + + /** LeftEyebrow indexes */ + VO_FacePart m_vLeftEyebrow; + + /** RightEyebrow indexes */ + VO_FacePart m_vRightEyebrow; + + /** LeftEye indexes */ + VO_FacePart m_vLeftEye; + + /** RightEye indexes */ + VO_FacePart m_vRightEye; + + /** Nose indexes */ + VO_FacePart m_vNose; + + /** Nostril indexes */ + VO_FacePart m_vNostril; + + /** Nose Tip indexes */ + VO_FacePart m_vNoseTip; + + /** Lip outerline indexes */ + VO_FacePart m_vLipOuterLine; + + /** Lip innerline indexes */ + VO_FacePart m_vLipInnerLine; + + /** Lip upper outer bound indexes */ + VO_FacePart m_vLipUpperOuterLine; + + /** Lip upper inner bound indexes */ + VO_FacePart m_vLipUpperInnerLine; + + /** Lip downer outer bound indexes */ + VO_FacePart m_vLipLowerOuterLine; + + /** Lip downer inner bound indexes */ + VO_FacePart m_vLipLowerInnerLine; + + /** Left Side Points indexes */ + VO_FacePart m_vLeftSidePoints; + + /** Right Side Points indexes */ + VO_FacePart m_vRightSidePoints; + + /** Midline Points indexes */ + VO_FacePart m_vMidlinePoints; + + /** Eyes' Corners' Points indexes */ + VO_FacePart m_vEyeCornerPoints; + + /** Mouth Corners' Points indexes */ + VO_FacePart m_vMouthCornerPoints; + + /** Pitch Axis Points indexes */ + VO_FacePart m_vPitchAxisLinePoints; + + /** Left Iris indexes */ + VO_FacePart m_vLeftIris; + + /** Right Iris indexes */ + VO_FacePart m_vRightIris; + + /** Left Ear indexes */ + VO_FacePart m_vLeftEar; + + /** Right Ear indexes */ + VO_FacePart m_vRightEar; + + /** Triangles' indexes. NbOfTriangles*3*/ + vector< vector < unsigned int > > m_vvTriangleIndexes; + + + void init() + { + this->m_vChin.clear(); + this->m_vLeftEyebrow.clear(); + this->m_vRightEyebrow.clear(); + this->m_vLeftEye.clear(); + this->m_vRightEye.clear(); + this->m_vNose.clear(); + this->m_vNostril.clear(); + this->m_vNoseTip.clear(); + this->m_vLipOuterLine.clear(); + this->m_vLipInnerLine.clear(); + this->m_vLipUpperOuterLine.clear(); + this->m_vLipUpperInnerLine.clear(); + this->m_vLipLowerOuterLine.clear(); + this->m_vLipLowerInnerLine.clear(); + this->m_vLeftSidePoints.clear(); + this->m_vRightSidePoints.clear(); + this->m_vMidlinePoints.clear(); + this->m_vEyeCornerPoints.clear(); + this->m_vMouthCornerPoints.clear(); + this->m_vPitchAxisLinePoints.clear(); + this->m_vLeftIris.clear(); + this->m_vRightIris.clear(); + this->m_vLeftEar.clear(); + this->m_vRightEar.clear(); + this->m_vvTriangleIndexes.clear(); + } + + void CopyData(const VO_FaceParts &iFaceParts) + { + this->m_vChin = iFaceParts.VO_GetOneFacePart(VO_FacePart::CHIN); + this->m_vLeftEyebrow = iFaceParts.VO_GetOneFacePart(VO_FacePart::LEFTEYEBROW); + this->m_vRightEyebrow = iFaceParts.VO_GetOneFacePart(VO_FacePart::RIGHTEYEBROW); + this->m_vLeftEye = iFaceParts.VO_GetOneFacePart(VO_FacePart::LEFTEYE); + this->m_vRightEye = iFaceParts.VO_GetOneFacePart(VO_FacePart::RIGHTEYE); + this->m_vNose = iFaceParts.VO_GetOneFacePart(VO_FacePart::NOSE); + this->m_vNostril = iFaceParts.VO_GetOneFacePart(VO_FacePart::NOSTRIL); + this->m_vNoseTip = iFaceParts.VO_GetOneFacePart(VO_FacePart::NOSETIP); + this->m_vLipOuterLine = iFaceParts.VO_GetOneFacePart(VO_FacePart::LIPOUTERLINE); + this->m_vLipInnerLine = iFaceParts.VO_GetOneFacePart(VO_FacePart::LIPINNERLINE); + this->m_vLipUpperOuterLine = iFaceParts.VO_GetOneFacePart(VO_FacePart::LIPUPPEROUTERLINE); + this->m_vLipUpperInnerLine = iFaceParts.VO_GetOneFacePart(VO_FacePart::LIPUPPERINNERLINE); + this->m_vLipLowerOuterLine = iFaceParts.VO_GetOneFacePart(VO_FacePart::LIPDOWNEROUTERLINE); + this->m_vLipLowerInnerLine = iFaceParts.VO_GetOneFacePart(VO_FacePart::LIPDOWNERINNERLINE); + this->m_vLeftSidePoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::LEFTSIDEPOINTS); + this->m_vRightSidePoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::RIGHTSIDEPOINTS); + this->m_vMidlinePoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::MIDLINEPOINTS); + this->m_vEyeCornerPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::EYECORNERPOINTS); + this->m_vMouthCornerPoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::MOUTHCORNERPOINTS); + this->m_vPitchAxisLinePoints = iFaceParts.VO_GetOneFacePart(VO_FacePart::PITCHAXISLINEPOINTS); + this->m_vLeftIris = iFaceParts.VO_GetOneFacePart(VO_FacePart::LEFTIRIS); + this->m_vRightIris = iFaceParts.VO_GetOneFacePart(VO_FacePart::RIGHTIRIS); + this->m_vLeftEar = iFaceParts.VO_GetOneFacePart(VO_FacePart::LEFTEAR); + this->m_vRightEar = iFaceParts.VO_GetOneFacePart(VO_FacePart::RIGHTEAR); + this->m_vvTriangleIndexes = iFaceParts.GetTriangleIndexes(); + } + +public: + /** Default constructor to create a VO_FaceParts object */ + VO_FaceParts() { this->init(); } + + /** Copy constructor */ + VO_FaceParts( const VO_FaceParts &iFaceParts ) { this->CopyData(iFaceParts); } + + /** Destructor */ + virtual ~VO_FaceParts() { this->clear(); } + + /** operator= overloading, similar to copy constructor */ + VO_FaceParts& operator=(const VO_FaceParts &iFaceParts) + { + this->CopyData(iFaceParts); + return *this; + } + + void clear() { this->init(); } + + VO_FacePart VO_GetOneFacePart( unsigned int partIdx) const; + void VO_SetOneFacePart(const VO_FacePart& iFacePart); + vector< vector < unsigned int > > GetTriangleIndexes() const {return this->m_vvTriangleIndexes;} + void SetTriangleIndexes(const vector< vector < unsigned int > >& iTriangleIndexes) { this->m_vvTriangleIndexes = iTriangleIndexes ;} +}; + +#endif // __VO_FACEPARTS_H__ + diff --git a/modules/smbuilding/include/VO_FeatureModel.h b/modules/smbuilding/include/VO_FeatureModel.h new file mode 100644 index 0000000..089aff4 --- /dev/null +++ b/modules/smbuilding/include/VO_FeatureModel.h @@ -0,0 +1,160 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_FEATUREMODEL_H__ +#define __VO_FEATUREMODEL_H__ + + +#include + +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_AXM.h" + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief Statistical feature model +*/ +class VO_FeatureModel : public VO_AXM +{ +friend class VO_Fitting2DSM; +friend class VO_FittingAAMBasic; +friend class VO_FittingAAMForwardIA; +friend class VO_FittingAAMInverseIA; +friend class VO_FittingASMLTCs; +friend class VO_FittingASMNDProfiles; +protected: + /** PCA transform for appearance, including eigenvectors, eigenvalues, and mean */ + PCA m_PCAFeatures; + + /** Features matrix. m_iNbOfSamples*m_iNbOfTotalFeatures */ + Mat_ m_MatFeatures; + + /** Number of features -- a) 2D shape, 2 features b) rgb textures, 3 features c) something more */ + unsigned int m_iNbOfTotalFeatures; + + /** Most possible feature eigens before PCA. For IMM: min (90396, 240) = 240 */ + unsigned int m_iNbOfEigenFeatureAtMost; + + /** Number of feature model eigens. For IMM: 127 */ + unsigned int m_iNbOfFeatureEigens; + + /** Truncate Percentage for feature model PCA. Normally, 0.95 */ + float m_fTruncatedPercent_Feature; + + /** All loaded features in a vector format. For IMM, 240*m_iNbOfTotalFeatures */ + vector< Mat_ > m_vFeatures; + + /** All normalized features in a vector format. For IMM, 240*m_iNbOfTotalFeatures */ + vector< Mat_ > m_vNormalizedFeatures; + + /** Initialization */ + void init(); + +public: + /** Default constructor to create a VO_FeatureModel object */ + VO_FeatureModel(); + + /** Destructor */ + ~VO_FeatureModel(); + + /** Load features for one face */ + static bool VO_LoadFeatures4OneFace(const VO_Shape& iShape, + const Mat& img, + const vector& templateTriangles, + const vector& warpInfo, + Mat_& oFeature, + int trm); + + /** Load Training data for feature model */ + bool VO_LoadFeatureTrainingData( const vector& allLandmarkFiles4Training, + const vector& allImgFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + unsigned int channels); + + /** Build Feature Model */ + void VO_BuildFeatureModel( const vector& allLandmarkFiles4Training, + const vector& allImgFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + unsigned int channels = 3, + int trm = VO_Features::DIRECT, + float TPShape = 0.95f, + float TPTexture = 0.95f, + bool useKnownTriangles = false); + + /** Save Feature Model, to a specified folder */ + void VO_Save(const string& fd); + + /** Load all parameters */ + void VO_Load(const string& fd); + + /** Load Parameters for fitting */ + void VO_LoadParameters4Fitting(const string& fd); + }; + + +#endif // __VO_FEATUREMODEL_H__ + diff --git a/modules/smbuilding/include/VO_Point2DDistributionModel.h b/modules/smbuilding/include/VO_Point2DDistributionModel.h new file mode 100644 index 0000000..e0573ba --- /dev/null +++ b/modules/smbuilding/include/VO_Point2DDistributionModel.h @@ -0,0 +1,131 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_POINT2DDISTRIBUTIONMODEL_H__ +#define __VO_POINT2DDISTRIBUTIONMODEL_H__ + + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_Shape.h" +#include "VO_Ellipse.h" + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief 2D point distribution model +*/ +class VO_Point2DDistributionModel +{ +friend class VO_Fitting2DSM; +friend class VO_FittingAAMBasic; +friend class VO_FittingAAMForwardIA; +friend class VO_FittingAAMInverseIA; +friend class VO_FittingAFM; +friend class VO_FittingASMLTCs; +friend class VO_FittingASMNDProfiles; +protected: + /** 2D normal distribution can be represented by rotated ellipse */ + vector m_VONormalizedEllipses; + + /** Initialization */ + void init() {this->m_VONormalizedEllipses.clear();} + +public: + /** Default constructor to create a VO_Point2DDistributionModel object */ + VO_Point2DDistributionModel() {this->init();} + + /** Destructor */ + virtual ~VO_Point2DDistributionModel() {this->m_VONormalizedEllipses.clear();} + + /** Build Point Model */ + void VO_BuildPointDistributionModel( const vector& allAlignedShapes ); + + /** Constrain respective single point according to its PDM */ + static void VO_ConstrainSinglePoint(Point2f& pt, const VO_Ellipse& ellipse); + + /** Scale all ellipses in this PDM */ + static void VO_ScalePDMEllipses(const vector& iEllipses, float scale, vector& oEllipses); + + /** Constrain a shape into Point Distribution Models */ + void VO_ConstrainAllPoints(VO_Shape& ioShape); + + /** Save Point Model, to a specified folder */ + void VO_Save(const string& fd); + + /** Load all parameters */ + void VO_Load(const string& fd); + + /** Load parameters for fitting */ + void VO_LoadParameters4Fitting(const string& fd); + + // Gets and Sets + vector GetPDMEllipses() const { return this->m_VONormalizedEllipses; } + +}; + +#endif // __VO_POINT2DDISTRIBUTIONMODEL_H__ + diff --git a/modules/smbuilding/include/VO_Profile.h b/modules/smbuilding/include/VO_Profile.h new file mode 100644 index 0000000..c446b98 --- /dev/null +++ b/modules/smbuilding/include/VO_Profile.h @@ -0,0 +1,173 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_PROFILE_H__ +#define __VO_PROFILE_H__ + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_Shape.h" +#include "VO_Shape2DInfo.h" +#include "VO_TextureModel.h" + + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief Profile used to build ND profile model or for ND profile model fitting +*/ +class VO_Profile +{ +friend class VO_ASMNDProfiles; +friend class VO_FittingASMNDProfiles; +friend ostream& operator<<(ostream& os, const VO_Profile& profile); +friend istream& operator>>(istream& is, VO_Profile& profile); +private: + /** the profile features, every single profile vector is a column. Multiple columns means multi-dim profiles */ + Mat_ m_MatProf; + + void CopyData(const VO_Profile& iProfile) {iProfile.m_MatProf.copyTo(this->m_MatProf);} +public: + // Constructors and Destructor + VO_Profile() { this->m_MatProf.release(); } + VO_Profile(unsigned int length, unsigned int dim = 1) {this->m_MatProf = Mat_::zeros(length, dim);} + VO_Profile(const VO_Profile& iProf) {this->CopyData (iProf);} + VO_Profile(const Mat_& iProf) {iProf.copyTo(this->m_MatProf);} + virtual ~VO_Profile() { this->m_MatProf.release(); } + const float operator()(unsigned int i) const { return this->m_MatProf(i, 0); } + + // operators + VO_Profile& operator=(const VO_Profile& iProfile); + VO_Profile& operator=(const Mat_& iProfile); + VO_Profile& operator=(float value); + VO_Profile operator+(float value); + VO_Profile& operator+=(float value); + VO_Profile operator+(const VO_Profile& iProfile); + VO_Profile& operator+=(const VO_Profile& iProfile); + VO_Profile operator-(float value); + VO_Profile& operator-=(float value); + VO_Profile operator-(const VO_Profile& iProfile); + VO_Profile& operator-=(const VO_Profile& iProfile); + VO_Profile operator*(float value); + VO_Profile& operator*=(float value); + VO_Profile operator*(const VO_Profile& iProfile); + VO_Profile& operator*=(const VO_Profile& iProfile); + VO_Profile operator/(float value); + VO_Profile& operator/=(float value); + VO_Profile operator/(const VO_Profile& iProfile); + VO_Profile& operator/=(const VO_Profile& iProfile); + float operator[](unsigned int idx) { return this->m_MatProf(idx, 0); } + float& operator() (unsigned row, unsigned col); + float operator() (unsigned row, unsigned col) const; + float dot(const VO_Profile& iProfile); + + void Resize(unsigned int rows, unsigned int cols); + + static void BoundPoint(Point2f& toBound,const int rows,const int cols); + + /** Key function to extract ND profile vector around one landmark */ + static void VO_Get1DProfileInMat4OneLandmark ( const Mat& iImg, + const Point2f& ThisPoint, + VO_Profile& oProf, + const float normX, + const float normY, + const unsigned int NbOfProfilesPerPixel = 17); + + static void VO_Get2DProfileInMat4OneLandmark ( const Mat& iImg, + const Point2f& ThisPoint, + VO_Profile& oProf, + const float normX, + const float normY, + const unsigned int NbOfProfilesPerPixel = 17); + + /** Key function to extract ND profile vector around one landmark */ + static void VO_GetNDProfiles4OneLandmark ( const Mat& iImg, + const VO_Shape& iShape, + const vector& iShapeInfo, + unsigned int ptIdx, + VO_Profile& oProf, + unsigned int dim = 1, + unsigned int channels = 1, + unsigned int NbOfProfilesPerPixel = 17, + float* pDeltaX = NULL, + float* pDeltaY = NULL); + /* Normalization for every dim */ + void Normalize(); + + unsigned int GetProfileLength() const { return this->m_MatProf.rows; } + unsigned int GetProfileDim() const { return this->m_MatProf.cols; } + Mat_ GetTheProfile() const { return this->m_MatProf; } + Mat_ GetSubProfile(int start, unsigned int length, unsigned int dimIdx) const; + Mat_ Get1DimProfile(unsigned int dimIdx) const {return this->m_MatProf.col(dimIdx);} + void SetProfile(const VO_Profile& iProf ) { iProf.m_MatProf.copyTo(this->m_MatProf); } + void SetProfile(const Mat_& iProf ) { iProf.copyTo(this->m_MatProf); } + void Set1DimProfile(const Mat_& iOneDimProf, unsigned int dimIdx ); + void Set1DimProfile(const VO_Profile& iOneDimProf, unsigned int dimIdx ); + void Set2DimProfile(const Mat_& iOneDimProf, unsigned int dimIdx ); + void Set2DimProfile(const VO_Profile& iOneDimProf, unsigned int dimIdx ); +}; + +#endif // __VO_PROFILE_H__ + diff --git a/modules/smbuilding/include/VO_Profiles.h b/modules/smbuilding/include/VO_Profiles.h new file mode 100644 index 0000000..2ee57e7 --- /dev/null +++ b/modules/smbuilding/include/VO_Profiles.h @@ -0,0 +1,100 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_PROFILES_H__ +#define __VO_PROFILES_H__ + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_Profile.h" + + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief Multiple profiles for a single face. +*/ +class VO_Profiles +{ +private: + /** the profile features, a col vector */ + vector m_vProfiles; + +public: + // Constructors and Destructor + VO_Profiles() { this->m_vProfiles.clear(); } + virtual ~VO_Profiles() { this->m_vProfiles.clear(); } + + // operators + VO_Profile operator[](unsigned int idx) { return this->m_vProfiles[idx]; } + + + unsigned int GetProfileDim() const { return this->m_vProfiles.size(); } + VO_Profile GetOneProfile(unsigned int idx) const { return this->m_vProfiles[idx];} + +}; + +#endif // __VO_PROFILES_H__ + diff --git a/modules/smbuilding/include/VO_Shape.h b/modules/smbuilding/include/VO_Shape.h new file mode 100644 index 0000000..aeff2da --- /dev/null +++ b/modules/smbuilding/include/VO_Shape.h @@ -0,0 +1,378 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_SHAPE_H__ +#define __VO_SHAPE_H__ + + +#include +#include +#include +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" + +//#include "VO_Triangle2DStructure.h" // always bear in mind, this is absolutely wrong!! +class VO_Triangle2DStructure; + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief Generalized class for shape. +*/ +class VO_Shape +{ +friend class VO_ShapeModel; +friend class VO_TextureModel; +friend class VO_AAMBasic; +friend class VO_AAMForwardIA; +friend class VO_AAMInverseIA; +friend class VO_AFM; +friend class VO_ASMLTCs; +friend class VO_ASMNDProfiles; +friend class VO_AXM; +friend ostream& operator<<(ostream& os, const VO_Shape& shape); +friend istream& operator>>(istream& is, VO_Shape& shape); +protected: + /** Every VO_Shape is corresponding to an annotation file */ + string m_sAnnotationFileName; + + /** the shape model, NbOfDim * NbOfAnnotatedPoints */ + Mat_ m_MatShape; + + /** whether this point in 3D coordinates is able to be seen on 2D screen - dealing with occlusion. NbOfAnnotatedPoints */ + vector m_MatSeeable; + + void CopyData(const VO_Shape& iShape) + { + // copy Mat_ data (i.e. the point coordinates) + iShape.m_MatShape.copyTo(this->m_MatShape); + if(iShape.m_MatSeeable.size() > 0) + this->m_MatSeeable = iShape.m_MatSeeable; + //copy everything. + this->m_sAnnotationFileName = iShape.m_sAnnotationFileName; + + } + +public: + enum { + LEFTMOST = -1, + RIGHTMOST = -2, + TOPMOST = -3, + BOTTOMMOST = -4, + CENTER = -5, + OUTERMOST = -6, + INNERMOST = -7}; + + /** Default constructor */ + VO_Shape(unsigned int dim = 0, unsigned int pts = 0) {this->m_MatShape.zeros(dim, pts);} + + /** Copy constructor */ + VO_Shape(const VO_Shape& iShape) {this->CopyData(iShape);} + + /** Constructor to create a VO_Shape object with a Mat_ of float values */ + VO_Shape(const Mat_& iShape) {iShape.copyTo(this->m_MatShape);} + VO_Shape(const Mat_& iShape, unsigned int dim) {this->SetTheShape(iShape, dim);} + VO_Shape(const vector& iShape) + { + unsigned int NbOfPoints = iShape.size(); + this->m_MatShape = Mat_::zeros(2, NbOfPoints); + for(unsigned int j = 0; j < NbOfPoints; ++j) + { + this->m_MatShape(0,j) = iShape[j].x; + this->m_MatShape(1,j) = iShape[j].y; + } + } + VO_Shape(const vector& iShape) + { + unsigned int NbOfPoints = iShape.size(); + this->m_MatShape = Mat_::zeros(3, NbOfPoints); + for(unsigned int j = 0; j < NbOfPoints; ++j) + { + + this->m_MatShape(0,j) = iShape[j].x; + this->m_MatShape(1,j) = iShape[j].y; + this->m_MatShape(2,j) = iShape[j].z; + } + } + + /** Destructor */ + virtual ~VO_Shape() {this->m_MatShape.release(); this->m_MatSeeable.clear();} + + /** Clone */ + void clone(const VO_Shape& iShape) {this->CopyData(iShape);} + + // operators + VO_Shape& operator=(const VO_Shape& iShape); + VO_Shape& operator=(const Mat_& iShape); + VO_Shape& operator=(float value); + VO_Shape operator+(float value); + VO_Shape& operator+=(float value); + VO_Shape operator+(const VO_Shape& iShape); + VO_Shape& operator+=(const VO_Shape& iShape); + VO_Shape operator-(float value); + VO_Shape& operator-=(float value); + VO_Shape operator-(const VO_Shape& iShape); + VO_Shape& operator-=(const VO_Shape& iShape); + VO_Shape operator*(float value); + VO_Shape& operator*=(float value); + VO_Shape operator*(const VO_Shape& iShape); + VO_Shape& operator*=(const VO_Shape& iShape); + VO_Shape operator/(float value); + VO_Shape& operator/=(float value); + VO_Shape operator/(const VO_Shape& iShape); + VO_Shape& operator/=(const VO_Shape& iShape); + Mat_ operator[](unsigned int idx) { return this->m_MatShape.col(idx); } + float& operator() (unsigned row, unsigned col); + float operator() (unsigned row, unsigned col) const; + float dot(const VO_Shape& iShape); + + void Resize(unsigned int rows, unsigned int cols); + Mat_ CenterOfGravity() const; + // Transformations + void Centralize(); + void Translate( const Mat_& translation ); + void Scale( float s); + void Scale( const Mat_& svec); + void ScaleX( float sX); + void ScaleY( float sY); + void ScaleZ( float sZ); + void Rotate( const vector& angles); + void Normalize(); + void Transform(const Mat_& t); + void Transform(float scale, vector rotateAngles, Mat_ translation); + float GetCentralizedShapeSize() const; + float GetShapeNorm() const; + /** This function is still to be evaluated !! */ + vector GetRotation( const VO_Shape& ref ) const; + + void ConstrainShapeInSize(const Size& isize); + void ConstrainShapeInImage(const Mat& iImg); + + // Align the shapes with respect to position, scale and orientation. + void AlignTo( const VO_Shape& ref, float* scale = NULL, vector* angles = NULL, Mat_* translation = NULL); + void Affine2D(const Mat_& affineMat); + void AlignTransformation( const VO_Shape& ref, float& scale, vector& angles, Mat_& translation ) const; + void ProcrustesAnalysis( const VO_Shape& ref, float& norm, vector& angles, Mat_& translation ); + void InverseProcrustesAnalysis( const float& norm, const vector& angles, const Mat_& translation ); + static void GlobalShapeNormalization2D(const VO_Shape& iShape, VO_Shape& oShape, const Mat_& q); + void GlobalShapeNormalization2D(const Mat_& q); + static void GlobalShapeNormalization2D(const VO_Shape& iShape, VO_Shape& oShape, float scale, const vector& angles, const Mat_& translation); + void GlobalShapeNormalization2D(float scale, const vector& angles, const Mat_& translation); + static void SimilarityTrans2GlobalShapeNormalization(float scale, const vector& angles, const Mat_& translation, Mat_& q); + static void GlobalShapeNormalization2SimilarityTrans(const Mat_& q, float& scale, vector& angles, Mat_& translation ); + + Mat_ Min() const; + float MinX() const; + float MinY() const; + float MinZ() const; + Mat_ Max() const; + float MaxX() const; + float MaxY() const; + float MaxZ() const; + Mat_ Mean() const; + float MeanX() const; + float MeanY() const; + float MeanZ() const; + void MinMaxX(double* minX, double* maxX) const; + void MinMaxY(double* minY, double* maxY) const; + void MinMaxZ(double* minZ, double* maxZ) const; + Point GetLeftTop() const; + Point GetRightBottom() const; + Point GetLeftBottom() const; + Point GetRightTop() const; +// Rect_ GetShapeRect() const {Rect_ res; res.x = MinX(); res.y = MinY(); res.width = GetWidth(); res.height = GetHeight(); return res;} + Rect_ GetShapeRect() const + { + double minX, maxX, minY, maxY; + cv::minMaxLoc(this->m_MatShape.row(0), &minX, &maxX, 0, 0); + cv::minMaxLoc(this->m_MatShape.row(1), &minY, &maxY, 0, 0); + Rect_ res; + res.x = (float)minX; + res.y = (float)minY; + res.width = (float)(maxX - minX); + res.height = (float)(maxY - minY); + return res; + } + Rect GetShapeBoundRect() const; + Mat ToPointList() const + { + Mat res(1, m_MatShape.cols, CV_32FC2); + for(int i = 0; i < m_MatShape.cols; ++i) + { + res.at(0, i) = Point2f(m_MatShape(0, i), m_MatShape(1, i)); + } + return res; + } + + double HausDorffDist(const VO_Shape& iShape); + + /** Is the current point "pt" in current shape? */ + int IsPointInShape(const Point2f& pt, const vector& triangles) const; + + /** Get VO_Triangle2DStructure specific for this VO_Shape */ + vector GetTriangle2DStructure(const vector triangles) const; + + /** Get shape width */ + float GetWidth() const { return this->MaxX() - this->MinX(); } + + /** Get shape height */ + float GetHeight() const { return this->MaxY() - this->MinY(); } + + /** Get shape height */ + float GetDepth() const { return this->MaxZ() - this->MinZ(); } + + /** Get area (number of pixels) inside the shape */ + unsigned int GetArea() const; + + /** Get shape dimension */ + unsigned int GetNbOfDim() const { return this->m_MatShape.rows; } + + /** Get Number of points */ + unsigned int GetNbOfPoints() const { return this->m_MatShape.cols; } + + /** Get the shape Mat_ */ + Mat_ GetTheShape() const { return this->m_MatShape; } + + /** Get a shape */ + float GetAShape(unsigned int idx) const + { + unsigned int row = idx / this->m_MatShape.cols; + unsigned int col = idx % this->m_MatShape.cols; + return this->m_MatShape(row, col); + } + float GetAShape(unsigned int row, unsigned int col) const + { + return this->m_MatShape(row, col); + } + + /** Get the shape Mat_ in a row, x1x2x3...y1y2y3...z1z2z3... */ + Mat_ GetTheShapeInARow() const + { + return this->m_MatShape.reshape(0, 1); + } + + /** Get a single col in Mat_ */ + Mat_ GetACol(unsigned int idx) const {return this->m_MatShape.col(idx);} + + /** Get a 2D point */ + Point2f GetA2DPoint(unsigned int idx) const; + + /** Get a 3D point */ + Point3f GetA3DPoint(unsigned int idx) const; + + /** Seeable point list */ + vector GetSeeable() const { return this->m_MatSeeable; } + + /** Get a sub shape from shape */ + VO_Shape GetSubShape(const vector& iPtIdx) const; + + /** Combine two shapes */ + static VO_Shape Combine2Shapes(const VO_Shape& shape1, const VO_Shape& shape2); + + /** Set a 2D point */ + void SetA2DPoint(const Point2f& pt, unsigned int idx) + { + assert (m_MatShape.rows == 2); + this->m_MatShape(0, idx) = pt.x; + this->m_MatShape(1, idx) = pt.y; + } + + /** Set a 3D point */ + void SetA3DPoint(const Point3f& pt, unsigned int idx) + { + assert (m_MatShape.rows == 3); + this->m_MatShape(0, idx) = pt.x; + this->m_MatShape(1, idx) = pt.y; + this->m_MatShape(2, idx) = pt.z; + } + + /** Set The shape */ + void SetTheShape(const Mat_& iShape) { iShape.copyTo(this->m_MatShape); } + + /** Set The shape in ND. iShape is of size 1*cols */ + void SetTheShape(const Mat_& iShape, unsigned int dim) + { + assert (iShape.rows == 1 && iShape.cols%dim == 0); + this->m_MatShape = iShape.reshape(0, dim); + } + /** Set a specific shape at some position */ + void SetAShape(float iS, unsigned int row, unsigned int col) + { + this->m_MatShape(row, col) = iS; + } + + /** Set seeable */ + void SetSeeable(const vector& iSeeable) { this->m_MatSeeable = iSeeable; } + + /** Set Annotation file name */ + void SetAnnotationFileName(const string& fn) { this->m_sAnnotationFileName = fn; } + + /** Export shape coordinates sequentially */ + static void ExportShape(const string& fn, const VO_Shape& shape); + static void ExportShape(const string& fn, const Mat_& shapeMat); +}; + +#endif // __VO_SHAPE_H__ + diff --git a/modules/smbuilding/include/VO_Shape2DInfo.h b/modules/smbuilding/include/VO_Shape2DInfo.h new file mode 100644 index 0000000..4b229ed --- /dev/null +++ b/modules/smbuilding/include/VO_Shape2DInfo.h @@ -0,0 +1,172 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_SHAPE2DINFO_H__ +#define __VO_SHAPE2DINFO_H__ + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_FaceParts.h" + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief Shape information, refer to "ShapeInfo.txt". +*/ +class VO_Shape2DInfo +{ +friend class VO_ShapeModel; +friend class VO_TextureModel; +friend ostream& operator<<(ostream& os, const VO_Shape2DInfo& shapeinfo); +friend istream& operator>>(istream& is, VO_Shape2DInfo& shapeinfo); + +protected: + /** Which path is this point in, refer to IMM face database */ + unsigned int m_iPath; + + /** Which type does this point belong to - closed or open */ + unsigned int m_iType; + + /** Index of this point, in the model vertex sequence */ + unsigned int m_iIndex; + + /** In the path, which point does this point connected from? */ + unsigned int m_iFrom; + + /** In the path, which point does this point connected to? */ + unsigned int m_iTo; + + void CopyData(const VO_Shape2DInfo& iShapeInfo) + { + this->m_iPath = iShapeInfo.GetPath(); + this->m_iType = iShapeInfo.GetType(); + this->m_iIndex = iShapeInfo.GetIndex(); + this->m_iFrom = iShapeInfo.GetFrom(); + this->m_iTo = iShapeInfo.GetTo(); + } + +public: + /** Default constructor to create a VO_Shape2DInfo object */ + VO_Shape2DInfo() + { + this->m_iPath = 0; + this->m_iType = 0; + this->m_iIndex = 0; + this->m_iFrom = 0; + this->m_iTo = 0; + } + + /** Copy constructor */ + VO_Shape2DInfo( const VO_Shape2DInfo &iShapeInfo ) + { + this->CopyData(iShapeInfo); + } + + /** Destructor */ + virtual ~VO_Shape2DInfo() {} + + /** operator= overloading, similar to copy constructor */ + VO_Shape2DInfo& operator=(const VO_Shape2DInfo &iShapeInfo) + { + this->CopyData(iShapeInfo); + return *this; + } + + /** Get path which this point is on */ + unsigned int GetPath() const { return this->m_iPath;} + + /** Get point type */ + unsigned int GetType() const { return this->m_iType;} + + /** Get point index, in the AAM model vertex sequence */ + unsigned int GetIndex() const { return this->m_iIndex;} + + /** Get the point index in the path, which this point connects from */ + unsigned int GetFrom() const { return this->m_iFrom;} + + /** Get the point index in the path, which this point connects to */ + unsigned int GetTo() const { return this->m_iTo;} + + /** Set path the point go through */ + void SetPath(unsigned int iPath) { this->m_iPath = iPath;} + + /** Set point type */ + void SetType(unsigned int iType) { this->m_iType = iType;} + + /** Set point index, in the AAM model vertex sequence */ + void SetIndex(unsigned int iIndex) { this->m_iIndex = iIndex;} + + /** Set the point index, which this point connected from in path */ + void SetFrom(unsigned int iFrom) { this->m_iFrom = iFrom;} + + /** Set the point index, which this point connected to in path */ + void SetTo(unsigned int iTo) { this->m_iTo = iTo;} + + // read VO_Shape2DInfo + static void ReadShape2DInfo(const string& filename, vector& oShapeInfo, VO_FaceParts& faceparts ); +}; + +#endif // __VO_SHAPE2DINFO_H__ + diff --git a/modules/smbuilding/include/VO_ShapeFace.h b/modules/smbuilding/include/VO_ShapeFace.h new file mode 100644 index 0000000..b2f8322 --- /dev/null +++ b/modules/smbuilding/include/VO_ShapeFace.h @@ -0,0 +1,106 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_SHAPEFACE_H__ +#define __VO_SHAPEFACE_H__ + + +#include +#include +#include +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_FaceCompPos.h" +#include "VO_Shape.h" +#include "VO_FaceParts.h" + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief A shape specifically to describe a face. +*/ +class VO_ShapeFace : public VO_Shape +{ +private: + VO_FaceCompPos m_VOFaceCompPos; + +public: + /** Default constructor */ + VO_ShapeFace(unsigned int dim = 0, unsigned int pts = 0):VO_Shape(dim, pts) {} + VO_ShapeFace(VO_Shape iShape):VO_Shape(iShape) {} + + /** desctructor */ + virtual ~VO_ShapeFace() {} + + /** calculate all rectangles */ + VO_FaceCompPos CalcFaceCompRects(const VO_FaceParts& faceparts); + + /** Gets and Sets */ + VO_FaceCompPos GetFaceCompPos() const {return this->m_VOFaceCompPos;} +}; + +#endif // __VO_SHAPEFACE_H__ + diff --git a/modules/smbuilding/include/VO_ShapeModel.h b/modules/smbuilding/include/VO_ShapeModel.h new file mode 100644 index 0000000..990b781 --- /dev/null +++ b/modules/smbuilding/include/VO_ShapeModel.h @@ -0,0 +1,296 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_SHAPEMODEL_H__ +#define __VO_SHAPEMODEL_H__ + + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "opencv/cvaux.h" + +#include "VO_Edge.h" +#include "VO_Shape.h" +#include "VO_Triangle2DStructure.h" +#include "VO_Shape2DInfo.h" +#include "VO_FaceParts.h" +#include "VO_Point2DDistributionModel.h" + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief Statistical shape model. +*/ +class VO_ShapeModel +{ +friend class VO_Fitting2DSM; +friend class VO_FittingAAMBasic; +friend class VO_FittingAAMForwardIA; +friend class VO_FittingAAMInverseIA; +friend class VO_FittingASMLTCs; +friend class VO_FittingASMNDProfiles; +protected: + /** PCA transform for shape, including eigenvectors, eigenvalues, and mean */ + PCA m_PCAAlignedShape; + + /** Number of training samples. For IMM: 240 */ + unsigned int m_iNbOfSamples; + + /** 2D or 3D, 116/58 = 2 */ + unsigned int m_iNbOfShapeDim; + + /** Number of points to describe per shape. For IMM: 58 */ + unsigned int m_iNbOfPoints; + + /** length of shape vector in format of xxx...yyy.... For IMM: 58*2=116 */ + unsigned int m_iNbOfShapes; + + /** Shape eigens at most before PCA. For IMM: min (116, 240) = 116 */ + unsigned int m_iNbOfEigenShapesAtMost; + + /** Number of shape model eigens. For IMM: 21 */ + unsigned int m_iNbOfShapeEigens; + + /** Number of edges */ + unsigned int m_iNbOfEdges; + + /** Number of triangles */ + unsigned int m_iNbOfTriangles; + + /** Reference shape which is of size close to "1", but not guaranteed */ + VO_Shape m_VOAlignedMeanShape; + + /** Reference shape which is scaled back to the original size and translated to (left top most = origin) */ + VO_Shape m_VOReferenceShape; + + /** The reference shape average size : 582.425 */ + float m_fAverageShapeSize; + + /** Truncate Percentage for shape model PCA. Normally, 0.95 */ + float m_fTruncatedPercent_Shape; + + /** All loaded shapes in a vector format. For IMM, 240*(58*2) */ + vector m_vShapes; + + /** All aligned shapes in a vector format. For IMM, 240*(58*2) */ + vector m_vAlignedShapes; + + /** shape information */ + vector m_vShape2DInfo; + + /** face parts information, seems to have the duplicate information as m_vShape2DInfo, but not! */ + VO_FaceParts m_FaceParts; + + /** Edges in the template shape, only contain the index pairs. For IMM, 152 */ + vector m_vEdge; + + /** Unnormalized triangles in the template shape. For IMM, 95 */ + vector m_vTemplateTriangle2D; + + /** Normalized triangles in the template shape - just replace the vertexes in m_vTemplateTriangle2D by corresponding vertexes of m_VOAlignedMeanShape. For IMM, 95 */ + vector m_vNormalizedTriangle2D; + + /** Normalized Point distribution model */ + VO_Point2DDistributionModel m_VOPDM; + + /** Initialization */ + void init(); + +public: + /** Default constructor to create a VO_ShapeModel object */ + VO_ShapeModel(); + + /** Destructor */ + virtual ~VO_ShapeModel(); + + /** Align all shapes */ + static float VO_AlignAllShapes(const vector& vShapes, vector& alignedShapes); + + /** Rescale all aligned shapes */ + static void VO_RescaleAllAlignedShapes(const VO_Shape& meanAlignedShape, vector& alignedShapes); + + /** Rescale one aligned shape */ + static void VO_RescaleOneAlignedShape(const VO_Shape& meanAlignedShape, VO_Shape& alignedShape); + + /** Returns the mean shape of all shapes */ + static void VO_CalcMeanShape(const vector& vShapes, VO_Shape& meanShape); + + /** Judge whether the point "pt" is in convex hull "ch" */ + static bool VO_IsPointInConvexHull(const Point2f pt, const Mat_& ch, bool includingBoundary); + + /** Judge whether edge indexed by (ind1+ind2) is already in the vector of "edges" */ + static bool VO_EdgeHasBeenCounted(const vector& edges, unsigned int ind1, unsigned int ind2); + + /** Judge whether triangle t is already in the vector of "triangles" */ + static bool VO_TriangleHasBeenCounted(const vector& triangles, const vector& t); + + /** Build Edges */ + static unsigned int VO_BuildEdges(const VO_Shape& iShape, const CvSubdiv2D* Subdiv, vector& outEdges); + + /** Build triangles */ + static unsigned int VO_BuildTriangles(const VO_Shape& iShape, const vector& edges, vector& outTriangles); + + /** Build Delaunay triangulation mesh */ + static void VO_BuildTemplateMesh(const VO_Shape& iShape, vector& edges, vector& triangles); + + /** Build triangulation mesh according to shapeInfo */ + static void VO_BuildTemplateMesh(const VO_Shape& iShape, const VO_FaceParts& iFaceParts, vector& edges, vector& triangles); + + /** Judge whether the shape is inside an image */ + static bool VO_IsShapeInsideImage(const VO_Shape& iShape, const Mat& img); + + /** Reference shape inShape back to aligned outShape */ + static void VO_ReferenceShapeBack2Aligned(const VO_Shape& inShape, float shapeSD,VO_Shape& outShape); + + /** Calculate the bounding rectangle from a list of rectangles */ + static Rect VO_CalcBoundingRectFromTriangles(const vector & triangles); + + /** Shape parameters constraints */ + void VO_ShapeParameterConstraint(Mat_& ioP, float nSigma = 4.0f); + + /** Shape projected to shape parameters*/ + void VO_AlignedShapeProjectToSParam(const VO_Shape& iShape, Mat_& outP) const; + + /** Shape parameters back projected to shape */ + void VO_SParamBackProjectToAlignedShape(const Mat_& inP, VO_Shape& oShape, int dim = 2) const; + void VO_SParamBackProjectToAlignedShape(const Mat_& inP, Mat_& oShapeMat) const; + + /** shape -> Procrustes analysis -> project to shape parameters */ + void VO_CalcAllParams4AnyShapeWithConstrain(const Mat_& iShape, Mat_& oShape, Mat_& outP, float& norm, vector& angles, Mat_& translation ); + void VO_CalcAllParams4AnyShapeWithConstrain(VO_Shape& ioShape, Mat_& outP, float& norm, vector& angles, Mat_& translation ); + //void VO_CalcAllParams4AnyShapeWithConstrain(const Mat_& iShape, Mat_& oShape, Mat_& outP, Mat_& outQ ); + //void VO_CalcAllParams4AnyShapeWithConstrain(VO_Shape& ioShape, Mat_& outP, Mat_& outQ ); + //void VO_BuildUpShapeFromRigidNonRigidParams(const Mat_& inP, const Mat_& inQ, VO_Shape& oShape ); + + /** Build Shape Model */ + void VO_BuildShapeModel( const vector& allLandmarkFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + float TPShape = 0.95, + bool useKnownTriangles = false); + + /** Save Shape Model, to a specified folder */ + void VO_Save(const string& fd); + + /** Load all parameters */ + void VO_Load(const string& fd); + + /** Load parameters for fitting */ + void VO_LoadParameters4Fitting(const string& fd); + + /** Gets and Sets */ + Mat_ GetAlignedShapesMean() const {return this->m_PCAAlignedShape.mean;} + Mat_ GetAlignedShapesEigenValues() const {return this->m_PCAAlignedShape.eigenvalues;} + Mat_ GetAlignedShapesEigenVectors() const {return this->m_PCAAlignedShape.eigenvectors;} + unsigned int GetNbOfSamples() const {return this->m_iNbOfSamples;} + unsigned int GetNbOfShapeDim() const {return this->m_iNbOfShapeDim;} + unsigned int GetNbOfPoints() const {return this->m_iNbOfPoints;} + unsigned int GetNbOfShapes() const {return this->m_iNbOfShapes;} + unsigned int GetNbOfEigenShapesAtMost() const {return this->m_iNbOfEigenShapesAtMost;} + unsigned int GetNbOfShapeEigens() const {return this->m_iNbOfShapeEigens;} + unsigned int GetNbOfEdges() const {return this->m_iNbOfEdges;} + unsigned int GetNbOfTriangles() const {return this->m_iNbOfTriangles;} + VO_Shape GetAlignedMeanShape() const { return this->m_VOAlignedMeanShape;} + VO_Shape GetReferenceShape() const {return this->m_VOReferenceShape;} + float GetAverageShapeSize() const {return this->m_fAverageShapeSize;} + float GetTruncatedPercent_Shape() const {return this->m_fTruncatedPercent_Shape;} + vector GetShapes() const {return this->m_vShapes;} + vector GetAlignedShapes() const {return this->m_vAlignedShapes;} + vector GetShapeInfo() const {return this->m_vShape2DInfo;} + VO_FaceParts GetFaceParts() const {return this->m_FaceParts;} + vector GetEdge() const {return this->m_vEdge;} + vector GetTriangle2D() const {return this->m_vTemplateTriangle2D;} + vector GetNormalizedTriangle2D() const {return this->m_vNormalizedTriangle2D;} + +// void SetAlignedShapesEigenVectors(const Mat_& inAlignedShapesEigenVectors) +// {inAlignedShapesEigenVectors.copyTo(this->m_PCAAlignedShape.eigenvectors);} +// void SetAlignedShapesEigenValues(const Mat_& inAlignedShapesEigenValues) +// {inAlignedShapesEigenValues.copyTo(this->m_PCAAlignedShape.eigenvalues);} +// void SetNbOfSamples(unsigned int iNbOfSamples) {this->m_iNbOfSamples = iNbOfSamples;} +// void SetNbOfPoints(unsigned int iNbOfPoints) {this->m_iNbOfPoints = iNbOfPoints;} +// void SetNbOfShapes(unsigned int iNbOfShapes) {this->m_iNbOfShapes = iNbOfShapes;} +// void SetNbOfShapeDim(unsigned int iNbOfShapeDim) {this->m_iNbOfShapeDim = iNbOfShapeDim;} +// void SetNbOfTruncatedShapes(unsigned int iNbOfTruncatedShapes) +// {this->m_iNbOfShapeEigens = iNbOfTruncatedShapes;} +// void SetNbOfEigenShapesAtMost(unsigned int iNbOfEigenShapesAtMost) +// {this->m_iNbOfEigenShapesAtMost = iNbOfEigenShapesAtMost;} +// void SetReferenceShape(const VO_Shape& iReferenceShape) +// {this->m_VOReferenceShape = iReferenceShape;} +// void SetAverageShapeSize(float inAverageShapeSize) {this->m_fAverageShapeSize = inAverageShapeSize;} +// void SetTruncatedPercent_Shape(float inTP) {this->m_fTruncatedPercent_Shape = inTP;} +// void SetShapes(const vector& iShapes) {this->m_vShapes = iShapes;} +// void SetAlignedShapes(const vector& iAlignedShapes) +// {this->m_vAlignedShapes = iAlignedShapes;} +// void SetShapeInfo(const vector& iShape2DInfo) +// {this->m_vShape2DInfo = iShape2DInfo;} +// void SetEdge(const vector& iEdge) {this->m_vEdge = iEdge;} +// void SetTriangle2D(const vector& iTriangle2D) +// {this->m_vTemplateTriangle2D = iTriangle2D;} +// void SetNormalizedTriangle2D(const vector& iNormalizedTriangle2D) +// {this->m_vNormalizedTriangle2D = iNormalizedTriangle2D;} +}; + +#endif // __VO_SHAPEMODEL_H__ + diff --git a/modules/smbuilding/include/VO_Texture.h b/modules/smbuilding/include/VO_Texture.h new file mode 100644 index 0000000..5384d02 --- /dev/null +++ b/modules/smbuilding/include/VO_Texture.h @@ -0,0 +1,208 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_TEXTURE_H__ +#define __VO_TEXTURE_H__ + + +#include +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief Generalized class for texture. +*/ +class VO_Texture +{ +friend class VO_ShapeModel; +friend class VO_TextureModel; +friend class VO_AAMBasic; +friend class VO_AAMForwardIA; +friend class VO_AAMInverseIA; +friend class VO_AFM; +friend class VO_ASMLTCs; +friend class VO_ASMNDProfiles; +friend class VO_AXM; +friend ostream& operator<<(ostream& os, const VO_Texture& texture); +friend istream& operator>>(istream& is, VO_Texture& texture); +private: + /** Every VO_Texture is corresponding to an image file */ + string m_sImageFileName; + + /** Generalized texture representation, the texture model. NbOfTextureRepresentation * NbOfPixels */ + Mat_ m_MatTexture; + + void CopyData( const VO_Texture& iTexture ) + { + iTexture.m_MatTexture.copyTo(this->m_MatTexture); + } + +public: + /** Default constructor to create a VO_Texture object */ + VO_Texture(unsigned int chan = 0, unsigned int pxs = 0) {this->m_MatTexture.zeros(chan, pxs);} + + /** Copy constructor */ + VO_Texture( const VO_Texture& iTexture ) {this->CopyData(iTexture); } + + /** Constructor to create a VO_Texture object with a vector of float values */ + VO_Texture( const Mat_& iTexture ) {iTexture.copyTo(this->m_MatTexture); } + + /** Destructor */ + virtual ~VO_Texture() {this->m_MatTexture.release(); } + + /** Clone */ + void clone(const VO_Texture& iTexture) {this->CopyData(iTexture);} + + // operators + VO_Texture& operator=(const VO_Texture& iTexture); + VO_Texture& operator=(const Mat_& iTexture); + VO_Texture& operator=(float value); + VO_Texture operator+(float value); + VO_Texture& operator+=(float value); + VO_Texture operator+(const VO_Texture& iTexture); + VO_Texture& operator+=(const VO_Texture& iTexture); + VO_Texture operator-(float value); + VO_Texture& operator-=(float value); + VO_Texture operator-(const VO_Texture& iTexture); + VO_Texture& operator-=(const VO_Texture& iTexture); + VO_Texture operator*(float value); + VO_Texture& operator*=(float value); + VO_Texture operator*(const VO_Texture& iTexture); + VO_Texture& operator*=(const VO_Texture& iTexture); + VO_Texture operator/(float value); + VO_Texture& operator/=(float value); + VO_Texture operator/(const VO_Texture& iShape); + VO_Texture& operator/=(const VO_Texture& iShape); + Mat_ operator[](unsigned int idx) { return this->m_MatTexture.col(idx); } + float& operator() (unsigned row, unsigned col); + float operator() (unsigned row, unsigned col) const; + float dot(const VO_Texture& iTexture); + + void Resize(unsigned int rows, unsigned int cols); + void Centralize(); + void Scale( float s); + void Shift( float s); + void Normalize(); + float GetStandardizedTextureNorm() const; + float GetTextureNorm() const; + + /** Clamp the texture, constrain between infimum and supremum */ + void Clamp(float inf, float sup); + + /** Get texture size */ + unsigned int GetNbOfTextureRepresentation() const { return this->m_MatTexture.rows; } + + /** Get Number of pixels */ + unsigned int GetNbOfPixels() const { return this->m_MatTexture.cols; } + + /** Get the texture vector */ + Mat_ GetTheTexture() const { return this->m_MatTexture;} + + /** Get a texture */ + float GetATexture(unsigned int idx) const + { + unsigned int row = idx / this->m_MatTexture.cols; + unsigned int col = idx % this->m_MatTexture.cols; + return this->m_MatTexture(row, col); + } + float GetATexture(unsigned int row, unsigned int col) const + { + return this->m_MatTexture(row, col); + } + + /** Get the texture Mat_ in a row, b1b2b3...g1g2g3...r1r2r3... */ + Mat_ GetTheTextureInARow() const + { + return this->m_MatTexture.reshape(0, 1); + } + + /** Get the texture value at texture vector position i */ + Mat_ GetACol(unsigned int idx) const { return this->m_MatTexture.col(idx); } + + /** Get the texture value at texture vector position i */ + Mat_ GetAPixel(unsigned int idx) const { return this->m_MatTexture.col(idx); } + + /** Set the texture. Not a hardcopy, but copy the header and data position */ + void SetTheTexture(const Mat_& iTexture) { this->m_MatTexture = iTexture;} + + /** Set The texture in ND. iTexture is of size 1*cols. Not a hardcopy, but copy the header and data position */ + void SetTheTexture(const Mat_& iTexture, unsigned int textureRep) + { + assert (iTexture.rows == 1 && iTexture.cols%textureRep == 0); + this->m_MatTexture = iTexture.reshape(0, textureRep); + } + + void SetAPixel(const Mat_& iCol, int idx) + { + assert (iCol.rows == this->m_MatTexture.rows && iCol.cols == 1); + Mat tmpCol = this->m_MatTexture.col(idx); + iCol.copyTo(tmpCol); + } + +}; + +#endif // __VO_TEXTURE_H__ + diff --git a/modules/smbuilding/include/VO_TextureModel.h b/modules/smbuilding/include/VO_TextureModel.h new file mode 100644 index 0000000..4516a60 --- /dev/null +++ b/modules/smbuilding/include/VO_TextureModel.h @@ -0,0 +1,315 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_TEXTUREMODEL_H__ +#define __VO_TEXTUREMODEL_H__ + + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_Texture.h" +#include "VO_WarpingPoint.h" +#include "VO_ShapeModel.h" +#include "VO_Features.h" + +#define AVERAGEFACETEXTURE 128 + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief Statistical texture model. +*/ +class VO_TextureModel : public VO_ShapeModel +{ +friend class VO_Fitting2DSM; +friend class VO_FittingAAMBasic; +friend class VO_FittingAAMForwardIA; +friend class VO_FittingAAMInverseIA; +friend class VO_FittingASMLTCs; +friend class VO_FittingASMNDProfiles; +protected: + /** PCA transform for texture, including eigenvectors, eigenvalues, and mean */ + PCA m_PCANormalizedTexture; + + /** Texture representation method - DIRECTTEXTURE, LAPLACETEXTURE, HARRISCORNERTEXTURE, HISTOGRAMEQUALIZED, GABOR, SEGMENTATION, etc. */ + unsigned int m_iTextureRepresentationMethod; + + /** Number of texture representations could be more or less than m_iNbOfChannels */ + unsigned int m_iNbOfTextureRepresentations; + + /** COLOR or Gray-level - 3 - COLOR; 1 - Gray-level */ + unsigned int m_iNbOfChannels; + + /** Number of pixels in template face convex hull or concave hull. For IMM, 30132 */ + unsigned int m_iNbOfPixels; + + /** length of texture vector in format of b1b2b3...g1g2g3...r1r2r3.... m_iNbOfTextureRepresentations*m_iNbOfPixels. For IMM, 30132*3=90396 */ + unsigned int m_iNbOfTextures; + + /** Most possible texture eigens before PCA. For IMM: min (90396, 240) = 240 */ + unsigned int m_iNbOfEigenTexturesAtMost; + + /** Number of texture model eigens. For IMM: 127 */ + unsigned int m_iNbOfTextureEigens; + + /** Reference texture which is of size close to "1", but not guaranteed */ + VO_Texture m_VONormalizedMeanTexture; + + /** Reference texture which is scaled back to the original gray/color intensities */ + VO_Texture m_VOReferenceTexture; + + /** The template texture average standard deviation : 12364.1 */ + float m_fAverageTextureStandardDeviation; + + /** Truncate Percentage for texture model PCA. Normally, 0.95 */ + float m_fTruncatedPercent_Texture; + + /** All loaded textures in a vector format. For IMM, 240*90396 */ + vector m_vTextures; + + /** All normalized textures in a vector format. For IMM, 240*90396 */ + vector m_vNormalizedTextures; + + /** Template face image */ + Mat m_ImageTemplateFace; + + /** Image of edges */ + Mat m_ImageEdges; + + /** Image of 2D normal distribution elllipses */ + Mat m_ImageEllipses; + + /** Unnormalized point warping information. For IMM, 30132 */ + vector m_vTemplatePointWarpInfo; + + /** Normalized point warping information. For IMM, 30132 */ + vector m_vNormalizedPointWarpInfo; + + /** We need these image file names for later image loading */ + vector m_vStringTrainingImageNames; + + /** Initialization */ + void init(); + +public: + /** Default constructor to create a VO_TextureModel object */ + VO_TextureModel(); + + /** Destructor */ + virtual ~VO_TextureModel(); + + /**Calculate point warping information */ + static unsigned int VO_CalcPointWarpingInfo(const vector& templateTriangles, vector& warpInfo); + + /** Load a texture vector from the image "img", in terms of "iShape", with a texture building method */ + static bool VO_LoadOneTextureFromShape( const VO_Shape& iShape, + const Mat& img, + const vector& templateTriangles, + const vector& warpInfo, + VO_Texture& oTexture, + int trm = VO_Features::DIRECT); + + /** Normalize all textures */ + static float VO_NormalizeAllTextures(const vector& vTextures, vector& normalizedTextures); + + /** Rescale all normalized textures */ + static void VO_RescaleAllNormalizedTextures(const VO_Texture& meanNormalizeTexture, vector& normalizedTextures); + + /** Rescale one normalized texture */ + static void VO_RescaleOneNormalizedTexture(const VO_Texture& meanNormalizeTexture, VO_Texture& normalizedTexture); + + /** Returns the mean texture of all textures */ + static void VO_CalcMeanTexture(const vector& vTextures, VO_Texture& meanTexture); + + /** Put one texture to the template face. Just for display! */ + static void VO_PutOneTextureToTemplateShape(const VO_Texture& texture, const vector& triangles, Mat& oImg); + + /** Warp form one shape to another */ + static unsigned int VO_WarpFromOneShapeToAnother(const VO_Shape& iShape, const VO_Shape& oShape, const vector& triangles, const Mat& iImg, Mat& oImg); + + /** Morphing */ + static void VO_Morphing(const VO_Shape& iShape1, const VO_Shape& iShape2, vector& oShapes, const vector& triangles, const Mat& iImg1, const Mat& iImg2, vector& oImgs, float step = 0.2); + + /** Put one texture to whatever shape, just for display */ + static void VO_PutOneTextureToOneShape(const VO_Texture& texture, const VO_Shape& oShape, const vector& triangles, Mat& oImg); + + /** Warp form one shape to another */ + static vector VO_CalcSubPixelTexture(float x, float y, const Mat& image); + static void VO_CalcSubPixelTexture(float x, float y, const Mat& image, float* gray); + static void VO_CalcSubPixelTexture(float x, float y, const Mat& image, float* g, float* d); + static void VO_CalcSubPixelTexture(float x, float y, const Mat& image, float* b, float* g, float* r); + + /** Get an intensity vector of size 1 or 3 (rgb), at point (x,y) in image */ + static void VO_CalcPixelRGB(int x, int y, const Mat& image, float& B, float& G, float& R); + + /** Change the normalized texture inTexture to the reference one outTexture */ + static void VO_NormalizedTexture2ReferenceScale(const VO_Texture& inTexture, float textureSD, VO_Texture& outTexture); + + /** Normalize inTexture to outTexture */ + static void VO_ReferenceTextureBack2Normalize(const VO_Texture& inTexture, float textureSD, VO_Texture& outTexture); + + /** Put edges on template face */ + static void VO_PutEdgesOnTemplateFace(const vector& edges, const VO_Shape& templateShape, const Mat& iImg, Mat& oImg); + + /** Put every single triangle onto iImg and obtaining oImg */ + static void VO_PutTrianglesOnTemplateFace(const vector& triangles, const Mat& iImg, vector& oImgs); + + /** Put every single ellipse onto iImg and obtaining oImg */ + static void VO_PutPDMEllipsesOnTemplateFace(const vector& ellipses, const Mat& iImg, Mat& oImg); + + /** Put a shape onto iImg and obtaining oImg, without the edge information */ + static void VO_PutShapeOnTemplateFace(const VO_Shape& iShape, const Mat& iImg, Mat& oImg); + + /** Texture parameters constraints */ + void VO_TextureParameterConstraint(Mat_& ioT, float nSigma = 4.0f); + + /** Texture projected to texture parameters*/ + void VO_NormalizedTextureProjectToTParam(const VO_Texture& iTexture, Mat_& outT) const; + + /** Texture parameters back projected to texture parameters */ + void VO_TParamBackProjectToNormalizedTexture(const Mat_& inT, VO_Texture& oTexture, int tr = 3) const; + void VO_TParamBackProjectToNormalizedTexture(const Mat_& inT, Mat_& oTextureMat) const; + + /** texture -> normalized -> project to texture parameters */ + void VO_CalcAllParams4AnyTexture(const Mat_& iTexture, Mat_& oTexture, Mat_& outT); + void VO_CalcAllParams4AnyTexture(VO_Texture& ioTexture, Mat_& outT); + + /** Load Training data for texture model */ + bool VO_LoadTextureTrainingData( const vector& allImgFiles4Training, + unsigned int channels, + int trm = VO_Features::DIRECT); + + /** Build Texture Model */ + void VO_BuildTextureModel( const vector& allLandmarkFiles4Training, + const vector& allImgFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + unsigned int channels = 3, + int trm = VO_Features::DIRECT, + float TPShape = 0.95f, + float TPTexture = 0.95f, + bool useKnownTriangles = false); + + /** Save Texture Model, to a specified folder */ + void VO_Save(const string& fd); + + /** Load all parameters */ + void VO_Load(const string& fd); + + /** Load Parameters for fitting */ + void VO_LoadParameters4Fitting(const string& fd); + + /** Gets and Sets */ + Mat_ GetNormalizedTextureMean() const {return this->m_PCANormalizedTexture.mean;} + Mat_ GetNormalizedTextureEigenValues() const {return this->m_PCANormalizedTexture.eigenvalues;} + Mat_ GetNormalizedTextureEigenVectors() const {return this->m_PCANormalizedTexture.eigenvectors;} + unsigned int GetTextureExtractionMethod() const {return this->m_iTextureRepresentationMethod;} + unsigned int GetNbOfTextureRepresentations() const {return this->m_iNbOfTextureRepresentations;} + unsigned int GetNbOfChannels() const {return this->m_iNbOfChannels;} + unsigned int GetNbOfPixels() const {return this->m_iNbOfPixels;} + unsigned int GetNbOfTextures() const {return this->m_iNbOfTextures;} + unsigned int GetNbOfEigenTexturesAtMost() const {return this->m_iNbOfEigenTexturesAtMost;} + unsigned int GetNbOfTextureEigens() const {return this->m_iNbOfTextureEigens;} + VO_Texture GetNormalizedMeanTexture() const {return this->m_VONormalizedMeanTexture;} + VO_Texture GetAAMReferenceTexture() const {return this->m_VOReferenceTexture;} + float GetAverageTextureStandardDeviation() const {return this->m_fAverageTextureStandardDeviation;} + float GetTruncatedPercent_Texture() const {return this->m_fTruncatedPercent_Texture;} + vector GetTextures() const {return this->m_vTextures;} + vector GetNormalizedTextures() const {return this->m_vNormalizedTextures;} + Mat GetIplTemplateFace() const {return this->m_ImageTemplateFace;} + Mat GetIplEdges() const {return this->m_ImageEdges;} + vector GetTemplatePointWarpInfo() const {return this->m_vTemplatePointWarpInfo;} + vector GetNormalizedPointWarpInfo() const {return this->m_vNormalizedPointWarpInfo;} + vector GetStringTrainingImageNames() const {return this->m_vStringTrainingImageNames;} + + void SetTextureExtractionMethod(unsigned int tem) {this->m_iTextureRepresentationMethod = tem;} + void SetNbOfNbOfChannels(unsigned int inNbOfNbOfChannels) {this->m_iNbOfTextureRepresentations = inNbOfNbOfChannels;} + void SetNbOfChannels(unsigned int inNbOfChannels) {this->m_iNbOfChannels = inNbOfChannels;} + void SetNbOfPixels(unsigned int inNbOfPixels) {this->m_iNbOfPixels = inNbOfPixels;} + void SetNbOfTextures(unsigned int inNbOfTextures) {this->m_iNbOfTextures = inNbOfTextures;} + void SetNbOfTruncatedTextures(unsigned int inNbOfTruncatedTextures) + {this->m_iNbOfTextureEigens = inNbOfTruncatedTextures;} + void SetNbOfEigenTexturesAtMost(unsigned int inNbOfEigenTexturesAtMost) + {this->m_iNbOfEigenTexturesAtMost = inNbOfEigenTexturesAtMost;} + void SetAAMReferenceTexture(const VO_Texture& inAAMReferenceTexture) + {this->m_VOReferenceTexture = inAAMReferenceTexture;} + void SetAverageTextureStandardDeviation(float inAverageTextureStandardDeviation) + {this->m_fAverageTextureStandardDeviation = inAverageTextureStandardDeviation;} + void SetTruncatedPercent_Texture(float inTP) {this->m_fTruncatedPercent_Texture = inTP;} + void SetIplTemplateFace(const Mat& inIplTF) + {inIplTF.copyTo(this->m_ImageTemplateFace);} + void SetIplEdges(const Mat& inIplEdges) + {inIplEdges.copyTo(this->m_ImageEdges);} + void SetNormalizedMeanTexture(const VO_Texture& inNormalizedMeanTexture) + {this->m_VONormalizedMeanTexture = inNormalizedMeanTexture;} + void SetPointWarpInfo(const vector& inPointWarpInfo) + {this->m_vNormalizedPointWarpInfo = inPointWarpInfo;} + +}; + + +#endif // __VO_TEXTUREMODEL_H__ + diff --git a/modules/smbuilding/include/VO_Triangle2D.h b/modules/smbuilding/include/VO_Triangle2D.h new file mode 100644 index 0000000..db13e65 --- /dev/null +++ b/modules/smbuilding/include/VO_Triangle2D.h @@ -0,0 +1,163 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_TRIANGLE2D_H__ +#define __VO_TRIANGLE2D_H__ + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_Shape.h" +#include "VO_Common.h" + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief A shape specifically to describe a 2D triangle. +*/ +class VO_Triangle2D : public VO_Shape +{ +protected: + /** Cached denominator for later Jacobian calculation */ + float m_dD; + + /** private function to calculate cached denominator */ + float Calc_dD() + { + if(this->m_MatShape.cols != 3) + cerr << "This is not a triangle!" << endl + << "Please ensure the shape has 3 columns" << endl; + if(this->m_MatShape.rows != 2) + cerr << "This triangle is not in 2D!" << endl; + + float x1, x2, x3, y1, y2, y3; + + x1 = this->m_MatShape(0,0); + x2 = this->m_MatShape(0,1); + x3 = this->m_MatShape(0,2); + y1 = this->m_MatShape(1,0); + y2 = this->m_MatShape(1,1); + y3 = this->m_MatShape(1,2); + + return (+x2*y3-x2*y1-x1*y3-x3*y2+x3*y1+x1*y2); + } + +public: + enum { COUNTER_CLOCKWISE = 0, CLOCKWISE = 1}; + + /** Default constructor to create a VO_Triangle2D object */ + VO_Triangle2D() {this->m_MatShape = Mat_::zeros(2,3); this->m_dD = 0.0f;} + + /** Constructor to create a VO_Triangle2D object with three coordinate vertexes in vector format */ + VO_Triangle2D(const vector& iVertexes):VO_Shape(iVertexes) + { + if(this->m_MatShape.cols !=3) + cerr << "This is not a triangle!" << endl; + if(this->m_MatShape.rows !=2) + cerr << "This triangle is not 2D!" << endl; + this->m_dD = this->Calc_dD (); + } + + /** Constructor to create a VO_Triangle2D object with three coordinate vertexes in Mat_ format */ + VO_Triangle2D(const Mat_& iVertexes):VO_Shape(iVertexes) + { + if(this->m_MatShape.cols !=3) + cerr << "This is not a triangle!" << endl; + if(this->m_MatShape.rows !=2) + cerr << "This triangle is not 2D!" << endl; + this->m_dD = this->Calc_dD (); + } + + /** Destructor */ + virtual ~VO_Triangle2D() {} + + /** operator= overloading, similar to copy constructor */ + VO_Triangle2D& operator=(const VO_Triangle2D& s) + { + this->CopyData(s); + this->m_dD = s.GetdD(); + return (*this); + } + + /** whether the triangle vertexes are list in clockwise or counter-clockwise */ + int determinant( ) + { + float determ = (this->m_MatShape(0,1) - this->m_MatShape(0,0)) + * (this->m_MatShape(1,2) - this->m_MatShape(1,0)) + - (this->m_MatShape(0,2) - this->m_MatShape(0,0)) + * (this->m_MatShape(1,1) - this->m_MatShape(1,0)); + if (determ <= 0.0) + return COUNTER_CLOCKWISE; + else + return CLOCKWISE; + } + + /** Get cached denominator of this triangle */ + float GetdD() const { return this->m_dD;} + + /** Set cached denominator for this triangle */ + void SetdD(float idD) { this->m_dD = idD;} +}; + +#endif // __VO_TRIANGLE2D_H__ + diff --git a/modules/smbuilding/include/VO_Triangle2DStructure.h b/modules/smbuilding/include/VO_Triangle2DStructure.h new file mode 100644 index 0000000..37a8108 --- /dev/null +++ b/modules/smbuilding/include/VO_Triangle2DStructure.h @@ -0,0 +1,178 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + + +#ifndef __VO_TRIANGLE2DSTRUCTURE_H__ +#define __VO_TRIANGLE2DSTRUCTURE_H__ + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_Triangle2D.h" + + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief A triangle specifically used to build statistical shape model. +*/ +class VO_Triangle2DStructure : public VO_Triangle2D +{ +friend class VO_ShapeModel; +friend class VO_TextureModel; +friend class VO_WarpingPoint; +friend ostream& operator<<(ostream& os, const VO_Triangle2DStructure& aamtriangle2d); +friend istream& operator>>(istream& is, VO_Triangle2DStructure& aamtriangle2d); +protected: + /** Indexes of 3 triangle vertexes in the built AAM model vertex sequence */ + vector m_vVertexIndexes; + + /** Indexes of this triangle in the built AAM model triangle sequence */ + unsigned int m_iTriangleIndex; + +public: + /** Default constructor to create a VO_Triangle2DStructure object */ + VO_Triangle2DStructure() {this->m_vVertexIndexes.resize(3);} + + /** Constructor to create a VO_Triangle2DStructure object with three coordinate vertexes in vector format */ + VO_Triangle2DStructure(const vector& iVertexes):VO_Triangle2D(iVertexes) + { + this->m_vVertexIndexes.resize(3); + } + + /** Constructor to create a VO_Triangle2DStructure object with three coordinate vertexes in a row in Mat_ format */ + VO_Triangle2DStructure(const Mat_& iVertexes):VO_Triangle2D(iVertexes) + { + this->m_vVertexIndexes.resize(3); + } + + /** Constructor to create a VO_Triangle2DStructure object with three coordinate vertexes and + three corresponding vertex indexes in vector format */ + VO_Triangle2DStructure( const vector& iVertexes, const vector& iVertexIndexes ) + :VO_Triangle2D(iVertexes) + { + assert (iVertexIndexes.size () == 3); + this->m_vVertexIndexes = iVertexIndexes; + } + + /** Constructor to create a VO_Triangle2DStructure object with three coordinate vertexes + and three corresponding vertex indexes in Mat_ format */ + VO_Triangle2DStructure( const Mat_& iVertexes, const vector& iVertexIndexes ) + :VO_Triangle2D(iVertexes) + { + assert (iVertexIndexes.size () == 3); + this->m_vVertexIndexes = iVertexIndexes; + } + + /** Destructor */ + virtual ~VO_Triangle2DStructure() {this->m_vVertexIndexes.clear ();} + + /** operator= overloading, similar to copy constructor */ + VO_Triangle2DStructure& operator=(const VO_Triangle2DStructure& s) + { + this->m_vVertexIndexes = s.m_vVertexIndexes; + this->m_iTriangleIndex = s.m_iTriangleIndex; + this->m_MatShape = s.m_MatShape; + this->SetdD( s.GetdD() ); + return (*this); + } + + /** Judge whether this triangle has a vertex of index "iIndex" */ + bool HasNode(unsigned int iIndex) const + { + if( (this->m_vVertexIndexes[0] == iIndex) + || (this->m_vVertexIndexes[1] == iIndex) + || (this->m_vVertexIndexes[2] == iIndex) ) + return true; + else + return false; + } + + /** VO_Triangle2DStructure to Mat_ */ + static VO_Shape Triangle2D2Shape(const vector & triangles); + + /** Adjust vertex sequence, to COUNTER_CLOCKWISE or CLOCKWISE */ + void AdjustVertexSequence(); + + /** Is point in triangles? If so, which triangle is it in? */ + static int IsPointInTriangles(const Point2f& pt, const vector& triangles); + + /** Get three indexes of three vertexes as a vector */ + vector GetVertexIndexes() const { return this->m_vVertexIndexes;} + + /** Get one index of one vertex, in the built AAM model vertex sequence. Apparently, ptIdx could only be 0,1,2 */ + unsigned int GetVertexIndex(unsigned int ptIdx) const { return this->m_vVertexIndexes[ptIdx];} + + /** Get the index of this triangle, in the built AAM model triangle sequence */ + unsigned int GetTriangleIndex() const { return this->m_iTriangleIndex;} + + /** Set the indexes of three vertexes of this triangle */ + void SetVertexIndexes(const vector& iVertexes) { this->m_vVertexIndexes = iVertexes;} + + /** Set the index of this triangle, in the built AAM model triangle sequence */ + void SetTriangleIndex(unsigned int iTriangleIndex) { this->m_iTriangleIndex = iTriangleIndex;} +}; + +#endif // __VO_TRIANGLE2DSTRUCTURE_H__ + diff --git a/modules/smbuilding/include/VO_WarpingPoint.h b/modules/smbuilding/include/VO_WarpingPoint.h new file mode 100644 index 0000000..11c1421 --- /dev/null +++ b/modules/smbuilding/include/VO_WarpingPoint.h @@ -0,0 +1,248 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#ifndef __VO_WARPINGPOINT_H__ +#define __VO_WARPINGPOINT_H__ + + +#include +#include + +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_Triangle2DStructure.h" + +using namespace std; +using namespace cv; + + +/** +* @author JIA Pei +* @brief Define every single warping point used in building statistical texture model +*/ +class VO_WarpingPoint +{ +friend class VO_ShapeModel; +friend class VO_TextureModel; +friend ostream& operator<<(ostream& os, const VO_WarpingPoint& warpingpoint); +friend istream& operator>>(istream& is, VO_WarpingPoint& warpingpoint); +private: + + /** Warping point coordinates */ + Point2f m_CVPosition; + + /** Which triangle is this point in, in the AAM model triangle vector? */ + unsigned int m_iTriangleIndex; + + /** What's the position in the AAM model template face point vector? */ + unsigned int m_iPointIndex; + + /** The triangle that this warping point belong to */ + VO_Triangle2DStructure m_VOTriangle2DStructure; + + /** Gradients in 3 channels (blue, green, red) for this warping point */ + vector< vector > m_Gradients; + + /** The Jacobian - 1st part , refer to http://www.visionopen.com/cv/aam.php */ + vector m_Jacobian_One; + + /** The Jacobian Matrix in terms of shape parameters 2*15 */ + vector > m_JacobianMatrix4ShapeModel; + + /** The Jacobian Matrix in terms of global shape normalizing transform 2*4 */ + vector > m_JacobianMatrix4GlobalShapeNorm; + + /** Steepest Descent Images 3 (blue, green, red) * 15 (p parameters). */ + vector< vector > m_SteepestDescentImages4ShapeModel; + + /** Steepest Descent Images for Global Shape Transform 3 (blue, green, red) * 4. */ + vector< vector > m_SteepestDescentImages4GlobalShapeNorm; + +public: + /** Default constructor to create a VO_WarpingPoint object */ + VO_WarpingPoint() + { + this->m_CVPosition = Point2f(0.0, 0.0); + this->m_iTriangleIndex = 0; + this->m_iPointIndex = 0; + //this->m_VOTriangle2DStructure; + this->m_Gradients.clear(); + this->m_Jacobian_One.clear(); + this->m_JacobianMatrix4ShapeModel.clear(); + this->m_JacobianMatrix4GlobalShapeNorm.clear(); + this->m_SteepestDescentImages4ShapeModel.clear(); + this->m_SteepestDescentImages4GlobalShapeNorm.clear(); + } + + /** Copy constructor */ + VO_WarpingPoint( const VO_WarpingPoint &t ) + { + this->m_CVPosition = t.GetPosition(); + this->m_iTriangleIndex = t.GetTriangleIndex(); + this->m_iPointIndex = t.GetPointIndex(); + this->m_VOTriangle2DStructure = t.GetAAMTriangle2d(); + this->CalcJacobianOne (); + this->m_JacobianMatrix4ShapeModel = t.GetJacobianMatrix(); + this->m_JacobianMatrix4GlobalShapeNorm = t.GetJacobianMatrix4GlobalShapeNorm(); + this->m_Gradients = t.GetGradients(); + this->m_SteepestDescentImages4ShapeModel = t.GetSteepestDescentImages4ShapeModel(); + this->m_SteepestDescentImages4GlobalShapeNorm = t.GetSteepestDescentImages4GlobalShapeNorm(); + } + + /** Destructor */ + ~VO_WarpingPoint() {} + + /** Calculate Jacobian first part for later Jacobian matrix calculation */ + void CalcJacobianOne(); + + /** Calculate Jacobian matrix for later Steepest Descent Image Calculation */ + void CalcJacobianMatrix4ShapeModel(const Mat_& TruncatedAlignedShapesEigenVectors); + + /** Calculate Jacobian matrix for global shape normalization for later Steepest Descent Image Calculation */ + void CalcJacobianMatrix4GlobalShapeNorm(const Mat_& SimilarityTransformMatrix); + + /** Calculate Steepest Descent Image */ + void CalcSteepestDescentImages4ShapeModel(unsigned int inChannels); + + /** Calculate Steepest Descent Image Calculation for global shape normalization */ + void CalcSteepestDescentImages4GlobalShapeNorm(unsigned int inChannels); + + /** A static function for general point warping */ + static void WarpOnePoint( const Point2f& iPt, const VO_Triangle2DStructure& its, + Point2f& oPt, const Point2f& ov1, const Point2f& ov2, const Point2f& ov3); + + /** Get this warping point coordinates */ + Point2f GetPosition() const {return this->m_CVPosition;} + + /** Get this warping point triangle index in AAM model triangle vector */ + int GetTriangleIndex() const {return this->m_iTriangleIndex;} + + /** Get this warping point index in AAM model point vector */ + int GetPointIndex() const {return this->m_iPointIndex;} + + /** Get the triangle that this warping point belong to */ + VO_Triangle2DStructure GetAAMTriangle2d() const {return this->m_VOTriangle2DStructure;} + + /** Get Jacobian one 1*3 vector - alpha, beta, gamma */ + vector GetJacobianOne() const { return this->m_Jacobian_One;} + + /** Get Jacobian matrix of this warping point */ + vector > GetJacobianMatrix() const { return this->m_JacobianMatrix4ShapeModel;} + + /** Get Jacobian matrix for global shape normalization of this warping point */ + vector > GetJacobianMatrix4GlobalShapeNorm() const { return this->m_JacobianMatrix4GlobalShapeNorm;} + + /** Get gradients for this warping point */ + vector< vector > GetGradients() const {return this->m_Gradients;} + + /** Get steepest descent image for this warping point */ + vector< vector > GetSteepestDescentImages4ShapeModel() const {return this->m_SteepestDescentImages4ShapeModel;} + + /** Get steepest descent image for global shape normalization for this warping point */ + vector< vector > GetSteepestDescentImages4GlobalShapeNorm() const { return this->m_SteepestDescentImages4GlobalShapeNorm;} + + + /** Set this warping point coordinates */ + void SetPosition(Point2f iPoint) {this->m_CVPosition = iPoint;} + void SetPosition(float x, float y) {this->m_CVPosition = Point2f(x, y);} + + /** Set this warping point triangle index in AAM model triangle vector */ + void SetTriangleIndex(unsigned int iTriangleIndex) {this->m_iTriangleIndex = iTriangleIndex;} + + /** Set this warping point index in AAM model point vector */ + void SetPointIndex(unsigned int iPointIndex) {this->m_iPointIndex = iPointIndex;} + + /** Set 3 vertexes of the triangle that this point belong to, with input in format of a vector of points */ + void SetTriangle2DStructure(const vector& inVertexes, const vector& iIndexes) + { + assert(inVertexes.size() == 3); + this->m_VOTriangle2DStructure = VO_Triangle2DStructure(inVertexes, iIndexes); + } + + + /** Set the triangle that this point belong to */ + void SetTriangle2DStructure(const VO_Triangle2DStructure& inAAMTriangle2d) + { + this->m_VOTriangle2DStructure = inAAMTriangle2d; + } + + /** Set Jacobian one 1*3 vector - alpha, beta, gamma */ + void SetJacobianOne(const vector& iJO) { this->m_Jacobian_One = iJO;} + + /** Set Jacobian matrix for this warping point */ + void SetJacobianMatrix(const vector > & iJM) { this->m_JacobianMatrix4ShapeModel = iJM;} + + /** Set Jacobian matrix for global shape normalization for this warping point */ + void SetJacobianMatrix4GlobalShapeNorm(const vector< vector >& iJM4GlobalShapeNorm) { this->m_JacobianMatrix4GlobalShapeNorm = iJM4GlobalShapeNorm;} + + /** Set gradients for this warping point */ + void SetGradients(const vector< vector >& iGradients) { this->m_Gradients = iGradients;} + + /** Set steepest descent image for this warping point */ + void SetSteepestDescentImages(const vector< vector >& iSDI) { this->m_SteepestDescentImages4ShapeModel = iSDI;} + + /** Set steepest descent image for global shape normalization for this warping point */ + void SetSteepestDescentImages4GlobalShapeNorm(const vector< vector >& iSDI4GlobalShapeNorm) { this->m_SteepestDescentImages4GlobalShapeNorm = iSDI4GlobalShapeNorm;} + +}; + +#endif // __VO_WARPINGPOINT_H__ + diff --git a/modules/smbuilding/smbuilding.vcxproj b/modules/smbuilding/smbuilding.vcxproj new file mode 100644 index 0000000..e979f1e --- /dev/null +++ b/modules/smbuilding/smbuilding.vcxproj @@ -0,0 +1,124 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {6B3E57CC-3CA2-4BD6-88D5-ED99990D8DDB} + smbuilding + + + + StaticLibrary + true + MultiByte + v110 + + + Application + false + true + MultiByte + v110 + + + + + + + + + + + + + + + Level3 + Disabled + C:\OpenCV2.4.5\opencv\build\include;./include;../include;../common/include;../cvcommon/include;../featureextraction/include;../integraltransform/include;C:\boost\boost_1_53_0;%(AdditionalIncludeDirectories) + NOMINMAX;_MBCS;%(PreprocessorDefinitions) + + + true + + + true + + + + + Level3 + MaxSpeed + true + true + _MBCS;BOOST_AVAILABLE;%(PreprocessorDefinitions) + C:\boost\boost_1_53_0;%(AdditionalIncludeDirectories) + + + true + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/modules/smbuilding/smbuilding.vcxproj.filters b/modules/smbuilding/smbuilding.vcxproj.filters new file mode 100644 index 0000000..563a867 --- /dev/null +++ b/modules/smbuilding/smbuilding.vcxproj.filters @@ -0,0 +1,159 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + \ No newline at end of file diff --git a/modules/smbuilding/src/VO_AAMBasic.cpp b/modules/smbuilding/src/VO_AAMBasic.cpp new file mode 100644 index 0000000..55fb965 --- /dev/null +++ b/modules/smbuilding/src/VO_AAMBasic.cpp @@ -0,0 +1,1525 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include +#include +#include +#include +#include +#include + + +#include "VO_AAMBasic.h" +#include "VO_CVCommon.h" + + + +/** + +@ note +Explained by JIA Pei, 2007-05-17 +Refer to +1) Cootes' "Comparing Variations on the Active Appearance Model Algorithm" +2) Cootes' "Statistical Models of Appearance for Computer Vision" page 56 +3) Stegmann's" AAM-API, Basic Active Appearance Models" + +This is for calculating the multivariate linear regression matrix +Explained by JIA Pei, 2007-05-17 +According to Cootes' "Comparing Variations on the Active Appearance Model Algorithm", +there are three types of parameters, p = [c|t|u]. +Here, we only consider the c and t parameters, why? +u is for texture transformation, so that the texture can be commensurate at the same scale + +Actually, the experiments designed in Stegmann's summary in AAM-API is just for estimating +the regression matrix R, in case of 1) the robustness; 2) make the training dataset more linear!! + +It's explained in Cootes' Comparing Variations on the Active Appearance Model Algorithm that +"Delta(r)/Delta(p) is estimated by numeric differentiation, systematically displacing each parameter +from the known optimal value on typical images and computing an average over the training set." + +Here, we could ignore the additional experimental samples, that's to say, the training data are just the 240 +faces from the dataset. we set a variable to control this whether we need this additional man-made samples or not. +*/ +void VO_AAMBasic::VO_CalcRegressionMatrices() +{ + // do model parameter experiments + { + // in VO_DoCParamExperiments, a matrix size of 80259*21120 is too big to be allocated + this->VO_DoCParamExperiments(); + + // estimate Rc using principal component regression + //this->VO_DoRegression( C, X, m_pModel->m_R_c ); + } + // do pose experiments + { + this->VO_DoPoseExperiments(); + + // estimate Rt using principal component regression + //this->VO_DoRegression( C, X, m_pModel->m_R_t ); + } + + // store negative versions to avoid any sign change at each iteration + // m_pModel->m_R_c *= -1; + // m_pModel->m_R_t *= -1; + + // gsl_multifit_linear_svd(); +} + + +// Totally, there are 17280 samples for the experiments, every is of size 80259. +// 80259*17280, too big to allocate. So, here, we use another methods for multivariate regression -- +// by saving all those data in files. +void VO_AAMBasic::VO_DoCParamExperiments() +{ + bool recordIntermediateImgs = false; + + int nExperiment = 0; + int totalExp4OneSample = this->m_vvCDisps.size() * this->m_vvCDisps[0].cols; + Mat_ X = Mat_::zeros(this->m_iNbOfTextures, totalExp4OneSample); // 80259*72 + Mat_ C = Mat_::zeros(this->m_iNbOfAppearanceEigens, totalExp4OneSample); // 12*72 + Mat_ currentConcatenatedParameters; + Mat_ currentShape, currentTexture; + VO_Shape currentShapeInstance; // built from the parameters + VO_Texture currentTextureInstance; // built from the parameters, but not sampled by using the shape parameters + VO_Texture delta_g; + Mat img; + Mat tempImage1, tempImage2, resImage1, resImage2; + + // for each training example in the training set + for(unsigned int i = 0; i < this->m_iNbOfSamples; i++) + { + if(this->m_iNbOfChannels == 1) + img = imread ( this->m_vStringTrainingImageNames[i].c_str (), 0 ); + else if (this->m_iNbOfChannels == 3) + img = imread ( this->m_vStringTrainingImageNames[i].c_str (), 1 ); + else + cerr << "We can't deal with image channels not equal to 1 or 3!" << endl; + + if(recordIntermediateImgs) + { + img.copyTo(tempImage1); + img.copyTo(tempImage2); + } + + for(unsigned int j = 0; j < this->m_vvCDisps.size(); j++) // 4 + { + for (int k = 0; k < this->m_vvCDisps[0].cols; k++) // 12 + { + // do displacement measures + currentConcatenatedParameters = this->m_MatAppearanceProject2Truncated.row(i); + + // adjust(shift) currentConcatenatedParameters to implement the experiments + currentConcatenatedParameters(0, k) += this->m_vvCDisps[j](0,k); + + // According to Cootes' "Comparing Variations on the Active Appearance Model Algorithm" - Equation (3) + + // Build the shape instance from the combined model +// cv::gemm(currentConcatenatedParameters, this->m_MatQs, 1, this->m_PCAAlignedShape.mean, 1, currentShape, GEMM_2_T ); + currentShape = currentConcatenatedParameters * this->m_MatQs.t() + this->m_PCAAlignedShape.mean; + + // Build the texture instance from the combined model +// cv::gemm(currentConcatenatedParameters, this->m_MatQg, 1, this->m_PCANormalizedTexture.mean, 1, currentTexture, GEMM_2_T ); + currentTexture = currentConcatenatedParameters * this->m_MatQg.t() + this->m_PCANormalizedTexture.mean; + + // Align from the displacement alignedshape to the shape of original size + currentShapeInstance.SetTheShape(currentShape, 2); + currentShapeInstance.AlignTo(this->m_vShapes[i]); + + // Obtain the original texture information from shape instance + if(!VO_TextureModel::VO_LoadOneTextureFromShape(currentShapeInstance, img, this->m_vTemplateTriangle2D, this->m_vTemplatePointWarpInfo, delta_g)) + continue; + + currentTextureInstance.SetTheTexture(currentTexture, delta_g.GetNbOfTextureRepresentation()); + + ////////////////////////////////////////////////////////////////////////// + // The following codes are just for intermediate display + if(recordIntermediateImgs) + { + // extracted from the real image + VO_TextureModel::VO_PutOneTextureToTemplateShape(delta_g, this->m_vTemplateTriangle2D, tempImage1); + + // build from the model + VO_TextureModel::VO_NormalizedTexture2ReferenceScale(currentTextureInstance, this->m_fAverageTextureStandardDeviation, currentTextureInstance); + VO_TextureModel::VO_PutOneTextureToTemplateShape(currentTextureInstance, this->m_vTemplateTriangle2D, tempImage2); + + VO_TextureModel::VO_WarpFromOneShapeToAnother(this->m_VOReferenceShape, currentShapeInstance, this->m_vTemplateTriangle2D, tempImage1, resImage1); + VO_TextureModel::VO_WarpFromOneShapeToAnother(this->m_VOReferenceShape, currentShapeInstance, this->m_vTemplateTriangle2D, tempImage2, resImage2); + + stringstream ssi, ssj, ssk; + string stri, strj, strk; + ssi << i; + ssj << j; + ssk << k; + ssi >> stri; + ssj >> strj; + ssk >> strk; + + string temp1Str = "CDisplaceLoadedImage" + stri + "_" + strj + "_" + strk + ".jpg"; + string temp2Str = "CDisplaceTextureImage" + stri + "_" + strj + "_" + strk + ".jpg"; + string res1Str = "CDisplaceLoadedWarpedImage" + stri + "_" + strj + "_" + strk + ".jpg"; + string res2Str = "CDisplaceTextureWarpedImage" + stri + "_" + strj + "_" + strk + ".jpg"; + + imwrite(temp1Str.c_str(), tempImage1 ); + imwrite(temp2Str.c_str(), tempImage2 ); + imwrite(res1Str.c_str(), resImage1 ); + imwrite(res2Str.c_str(), resImage2 ); + + ssi.clear(); + ssj.clear(); + ssk.clear(); + } + + ////////////////////////////////////////////////////////////////////////// + + delta_g.Normalize(); + delta_g -= currentTextureInstance; + + // Explained by JIA Pei. Here, the X matrix is too big to allocate a memory for calculation, 2007-05-30 + // insert the results into X and C + for (unsigned int n = 0; n < this->m_iNbOfTextures; n++) + { + X(n, nExperiment) = delta_g.GetATexture(n); + } + C(k, nExperiment) = this->m_vvCDisps[j](0,k); + + nExperiment++; + } + //cout << "Experiment" << nExperiment << "of" << X->cols << "done (c)..." << endl; + } + + ////////////////////////////////////////////////////////////////////////// + // just in order to save the data + // X, C + /* + string filestr = this->m_vimgFiles[i].substr (14,5) + ".txt"; + string folderstr1 = "./cexperiment"; + string folderstrX = folderstr1 + "/" + "X"; + string folderstrC = folderstr1 + "/" + "C"; + string folderfilestrX = folderstrX + "/" + filestr; + string folderfilestrC = folderstrC + "/" + filestr; + + MakeDirectory( folderstr1 ); + MakeDirectory( folderstrX ); + MakeDirectory( folderstrC ); + + fstream fp; + fp.open(folderfilestrX.c_str (), ios::out); + fp << this->m_vimgFiles[i].substr (14,5) << "-X" << endl; + for (unsigned int m = 0; m < X->rows; m++) + { + for (unsigned int n = 0; n < X->cols; n++) + { + fp << X(m, n) << " "; + } + fp << endl; + } + fp.close();fp.clear(); + + fp.open(folderfilestrC.c_str (), ios::out); + fp << this->m_vimgFiles[i].substr (14,5) << "-C" << endl; + for (unsigned int m = 0; m < C->rows; m++) + { + for (unsigned int n = 0; n < C->cols; n++) + { + fp << C(m, n) << " "; + } + fp << endl; + } + fp.close();fp.clear(); + */ + ////////////////////////////////////////////////////////////////////////// + } +} + +/** + * @author JIA Pei + * @version 2010-02-05 + * @brief Carry out pose displacement experiments + * @return void +*/ +void VO_AAMBasic::VO_DoPoseExperiments() +{ + bool recordIntermediateImgs = false; + + int nExperiment = 0; + int totalExp4OneSample = this->m_vvPoseDisps.size() * this->m_vvPoseDisps[0].cols; + Mat_ X = Mat_::zeros(this->m_iNbOfTextures, totalExp4OneSample); // 80259*72 + Mat_ P = Mat_::zeros(this->m_iNbOfAppearanceEigens, totalExp4OneSample); // 12*72 + Mat_ currentConcatenatedParameters; + Mat_ currentShape, currentTexture; + VO_Shape currentShapeInstance; // built from the parameters + VO_Texture currentTextureInstance; // built from the parameters, but not sampled by using the shape parameters + VO_Texture delta_g; + //Mat_ disp = Mat_::zeros(1, 4); + Mat_ posedisp = Mat_::zeros(1, 4); + + // just for displacement + Mat_ translation, disptranslation; + float scale = 1.0f, dispscale = 1.0f; + vector theta(1), dispangles(1); + Mat img; + Mat tempImage, resImage; + + + // for each training example in the training set + for(unsigned int i = 0; i < this->m_iNbOfSamples; i++) + { + nExperiment = 0; + + if(this->m_iNbOfChannels == 1) + img = imread ( this->m_vStringTrainingImageNames[i].c_str (), 0 ); + else if (this->m_iNbOfChannels == 3) + img = imread ( this->m_vStringTrainingImageNames[i].c_str (), 1 ); + else + cerr << "We can't deal with image channels not equal to 1 or 3!" << endl; + + if(recordIntermediateImgs) + { + img.copyTo(tempImage); + } + + for(unsigned int j = 0; j < this->m_vvPoseDisps.size(); j++) // 4 + { + for (int k = 0; k < this->m_vvPoseDisps[0].cols; k++) // 4 + { + posedisp = Mat_::zeros(posedisp.size()); + posedisp(0,k) = this->m_vvPoseDisps[j](0,k); + VO_Shape::GlobalShapeNormalization2SimilarityTrans(posedisp, dispscale, dispangles, disptranslation ); + + // do displacement measures + currentConcatenatedParameters = this->m_MatAppearanceProject2Truncated.row(i); + + // According to Cootes' "Comparing Variations on the Active Appearance Model Algorithm" - Equation (3) + + // Build the shape instance from the combined model +// cv::gemm(currentConcatenatedParameters, this->m_MatQs, 1, this->m_PCAAlignedShape.mean, 1, currentShape, GEMM_2_T ); + currentShape = currentConcatenatedParameters * this->m_MatQs.t() + this->m_PCAAlignedShape.mean; + + // Build the texture instance from the combined model +// cv::gemm(currentConcatenatedParameters, this->m_MatQg, 1, this->m_PCANormalizedTexture.mean, 1,currentTexture, GEMM_2_T); + currentTexture = currentConcatenatedParameters * this->m_MatQg.t() + this->m_PCANormalizedTexture.mean; + + // Calculate the align transformation + currentShapeInstance.SetTheShape(currentShape, 2); + currentShapeInstance.AlignTransformation(this->m_vShapes[i], scale, theta, translation); + + currentShapeInstance.Scale(dispscale * scale); + + vector tempTheta; + tempTheta.resize(1); + tempTheta[0] = dispangles[0] + theta[0]; + currentShapeInstance.Rotate( tempTheta ); + + Mat_ tempTranslate = Mat_::zeros(translation.size()); + tempTranslate(0,0) = disptranslation(0,0) + translation(0,0); + tempTranslate(1,0) = disptranslation(1,0) + translation(1,0); + currentShapeInstance.Translate(tempTranslate); + + if(!VO_TextureModel::VO_LoadOneTextureFromShape(currentShapeInstance, img, this->m_vTemplateTriangle2D, this->m_vTemplatePointWarpInfo, delta_g)) + continue; + + currentTextureInstance.SetTheTexture(currentTexture, delta_g.GetNbOfTextureRepresentation()); + + //// The following codes are just for intermediate display + if(recordIntermediateImgs) + { + VO_TextureModel::VO_PutOneTextureToTemplateShape(delta_g, this->m_vTemplateTriangle2D, tempImage); + VO_TextureModel::VO_WarpFromOneShapeToAnother(this->m_VOReferenceShape, currentShapeInstance, this->m_vTemplateTriangle2D, tempImage, resImage); + + stringstream ssi, ssj, ssk; + string stri, strj, strk; + ssi << i; + ssj << j; + ssk << k; + ssi >> stri; + ssj >> strj; + ssk >> strk; + + string temp1Str = "poseDisplaceLoadedImage" + stri + "_" + strj + "_" + strk + ".jpg"; + string res1Str = "poseDisplaceLoadedWarpedImage" + stri + "_" + strj + "_" + strk + ".jpg"; + + imwrite(temp1Str.c_str(), tempImage ); + imwrite(res1Str.c_str(), resImage ); + + ssi.clear(); + ssj.clear(); + ssk.clear(); + } + + + + delta_g.Normalize(); + delta_g -= currentTextureInstance; + + // Explained by JIA Pei. Here, the X matrix is too big to allocate a memory for calculation, 2007-05-30 + // insert the results into X and C + for (unsigned int n = 0; n < this->m_iNbOfTextures; n++) + { + X(n, nExperiment) = delta_g.GetATexture(n); + } + P(k, nExperiment) = this->m_vvPoseDisps[j](0,k); + + ++nExperiment; + } + //cout << "Experiment" << nExperiment << "of" << X->cols << "done (pose)..." << endl; + } + + ////////////////////////////////////////////////////////////////////////// + // just in order to save the data + // X, P + /* + string filestr = this->m_vimgFiles[i].substr (14,5) + ".txt"; + string folderstr1 = "./cexperiment"; + string folderstrX = folderstr1 + "/" + "X"; + string folderstrC = folderstr1 + "/" + "P"; + string folderfilestrX = folderstrX + "/" + filestr; + string folderfilestrC = folderstrC + "/" + filestr; + + MakeDirectory( folderstr1 ); + MakeDirectory( folderstrX ); + MakeDirectory( folderstrC ); + + fstream fp; + fp.open(folderfilestrX.c_str (), ios::out); + fp << this->m_vimgFiles[i].substr (14,5) << "-X" << endl; + for (unsigned int m = 0; m < X->rows; m++) + { + for (unsigned int n = 0; n < X->cols; n++) + { + fp << X(m, n) << " "; + } + fp << endl; + } + fp.close();fp.clear(); + + fp.open(folderfilestrC.c_str (), ios::out); + fp << this->m_vimgFiles[i].substr (14,5) << "-C" << endl; + for (unsigned int m = 0; m < C->rows; m++) + { + for (unsigned int n = 0; n < C->cols; n++) + { + fp << C(m, n) << " "; + } + fp << endl; + } + fp.close();fp.clear(); + */ + ////////////////////////////////////////////////////////////////////////// + } +} + + +/** + * @author JIA Pei + * @version 2010-02-05 + * @brief Carry out multi variate linear regression experiments + * @return void +*/ +void VO_AAMBasic::VO_DoRegression() +{ + +} + + +/** + * @author JIA Pei + * @version 2010-02-05 + * @brief Build gradient matrices + * @return void +*/ +void VO_AAMBasic::VO_CalcGradientMatrices() +{ + // do model parameter experiments + { + this->VO_EstCParamGradientMatrix(this->m_MatCParamGradientMatrix); + + // estimate Rc + this->m_MatRc = (this->m_MatCParamGradientMatrix.t()*this->m_MatCParamGradientMatrix).inv() + *(this->m_MatCParamGradientMatrix.t() ); + } + // do pose experiments, this is for global shape normalization + { + this->VO_EstPoseGradientMatrix(this->m_MatPoseGradientMatrix); + + // estimate Rt + this->m_MatRt = (this->m_MatPoseGradientMatrix.t()*this->m_MatPoseGradientMatrix).inv() + *(this->m_MatPoseGradientMatrix.t() ); + } +} + + +/** + * @author JIA Pei + * @version 2010-02-05 + * @brief Build gradient matrices in terms of C parameters + * @param oCParamGM - Output the built gradient matrix in terms of C parameters + * @return void +*/ +void VO_AAMBasic::VO_EstCParamGradientMatrix(Mat_& oCParamGM) +{ + bool recordIntermediateImgs = false; + + oCParamGM = Mat_::zeros(this->m_iNbOfTextures, this->m_vvCDisps[0].cols ); // 80259*12 + int nExperiment = 0; + Mat_ currentConcatenatedParameters, currentConcatenatedParametersPositiveDisp, currentConcatenatedParametersNegativeDisp; + Mat_ currentShapePositive, currentShapeNegative, currentTexturePositive, currentTextureNegative; + VO_Shape currentShapeInstancePositive, currentShapeInstanceNegative; + VO_Texture currentTextureInstancePositive, currentTextureInstanceNegative; + VO_Texture delta_g1, delta_g2, cDiff; + Mat img; + Mat tempImage1, tempImage2, tempImage3, tempImage4, resImage1, resImage2, resImage3, resImage4; + + // for each training example in the training set + for(unsigned int i = 0; i < this->m_iNbOfSamples; i++) + { + if(this->m_iNbOfChannels == 1) + img = imread ( this->m_vStringTrainingImageNames[i].c_str (), 0 ); + else if (this->m_iNbOfChannels == 3) + img = imread ( this->m_vStringTrainingImageNames[i].c_str (), 1 ); + else + cerr << "We can't deal with image channels not equal to 1 or 3!" << endl; + + if(recordIntermediateImgs) + { + img.copyTo(tempImage1); + img.copyTo(tempImage2); + img.copyTo(tempImage3); + img.copyTo(tempImage4); + } + + for(unsigned int j = 0; j < this->m_vvCDisps.size(); j = j+2) // 4 -- number of displacements for each shape parameter + { + for (int k = 0; k < this->m_vvCDisps[0].cols; k++) // 12 -- number of shape parameters + { + // do displacement measures + currentConcatenatedParameters = this->m_MatAppearanceProject2Truncated.row(i); + currentConcatenatedParameters.copyTo(currentConcatenatedParametersNegativeDisp); + currentConcatenatedParameters.copyTo(currentConcatenatedParametersPositiveDisp); + + // adjust(shift) currentConcatenatedParameters ... for calculating the Jacobian Matrix + currentConcatenatedParametersNegativeDisp(0, k) = currentConcatenatedParameters(0, k) + this->m_vvCDisps[j](0,k); + currentConcatenatedParametersPositiveDisp(0, k) = currentConcatenatedParameters(0, k) + this->m_vvCDisps[j+1](0,k); + + // According to Cootes' "Comparing Variations on the Active Appearance Model Algorithm" - Equation (3) + + // Build the shape instance from the combined model +// cv::gemm(currentConcatenatedParametersNegativeDisp, this->m_MatQs, 1, this->m_PCAAlignedShape.mean, 1, currentShapeNegative, GEMM_2_T ); +// cv::gemm(currentConcatenatedParametersPositiveDisp, this->m_MatQs, 1, this->m_PCAAlignedShape.mean, 1, currentShapePositive, GEMM_2_T ); + currentShapeNegative = currentConcatenatedParametersNegativeDisp * this->m_MatQs.t() + this->m_PCAAlignedShape.mean; + currentShapePositive = currentConcatenatedParametersPositiveDisp * this->m_MatQs.t() + this->m_PCAAlignedShape.mean; + + // Build the texture instance from the combined model +// cv::gemm(currentConcatenatedParametersNegativeDisp, this->m_MatQg, 1, this->m_PCANormalizedTexture.mean, 1, currentTextureNegative, GEMM_2_T ); +// cv::gemm(currentConcatenatedParametersPositiveDisp, this->m_MatQg, 1, this->m_PCANormalizedTexture.mean, 1, currentTexturePositive, GEMM_2_T ); + currentTextureNegative = currentConcatenatedParametersNegativeDisp * this->m_MatQg.t() + this->m_PCANormalizedTexture.mean; + currentTexturePositive = currentConcatenatedParametersPositiveDisp * this->m_MatQg.t() + this->m_PCANormalizedTexture.mean; + + // Align from the displacement alignedshape to the shape of original size + currentShapeInstanceNegative.SetTheShape(currentShapeNegative, 2); + currentShapeInstancePositive.SetTheShape(currentShapePositive, 2); + currentShapeInstanceNegative.AlignTo(this->m_vShapes[i]); + currentShapeInstancePositive.AlignTo(this->m_vShapes[i]); + + // Obtain the original texture information from shape instance + if(!VO_TextureModel::VO_LoadOneTextureFromShape(currentShapeInstanceNegative, img, this->m_vTemplateTriangle2D, this->m_vTemplatePointWarpInfo, delta_g1)) + continue; + if(!VO_TextureModel::VO_LoadOneTextureFromShape(currentShapeInstancePositive, img, this->m_vTemplateTriangle2D, this->m_vTemplatePointWarpInfo, delta_g2)) + continue; + + currentTextureInstanceNegative.SetTheTexture(currentTextureNegative, delta_g1.GetNbOfTextureRepresentation()); + currentTextureInstancePositive.SetTheTexture(currentTexturePositive, delta_g1.GetNbOfTextureRepresentation()); + + // The following codes are just for intermediate display + if(recordIntermediateImgs) + { + // extracted from the real image + VO_TextureModel::VO_PutOneTextureToTemplateShape(delta_g1, this->m_vTemplateTriangle2D, tempImage1); + VO_TextureModel::VO_PutOneTextureToTemplateShape(delta_g2, this->m_vTemplateTriangle2D, tempImage2); + + // build from the model + VO_TextureModel::VO_NormalizedTexture2ReferenceScale(currentTextureInstanceNegative, this->m_fAverageTextureStandardDeviation, currentTextureInstanceNegative); + VO_TextureModel::VO_NormalizedTexture2ReferenceScale(currentTextureInstancePositive, this->m_fAverageTextureStandardDeviation, currentTextureInstancePositive); + VO_TextureModel::VO_PutOneTextureToTemplateShape(currentTextureInstanceNegative, this->m_vTemplateTriangle2D, tempImage3); + VO_TextureModel::VO_PutOneTextureToTemplateShape(currentTextureInstancePositive, this->m_vTemplateTriangle2D, tempImage4); + +// VO_Texture tempTexture; +// Mat oImg(tempImage1); +// cout << this->m_VOReferenceShape << endl; +// imwrite("template.jpg", this->m_ImageTemplateFace); +// VO_TextureModel::VO_LoadOneTextureFromShape(this->m_VOReferenceShape, this->m_ImageTemplateFace, this->m_vTemplateTriangle2D, this->m_vTemplatePointWarpInfo, tempTexture); +// VO_TextureModel::VO_PutOneTextureToTemplateShape(tempTexture, this->m_vTemplateTriangle2D, oImg); +// imwrite("temp.jpg", oImg); + + + VO_TextureModel::VO_WarpFromOneShapeToAnother(this->m_VOReferenceShape, currentShapeInstanceNegative, this->m_vTemplateTriangle2D, tempImage1, resImage1); + VO_TextureModel::VO_WarpFromOneShapeToAnother(this->m_VOReferenceShape, currentShapeInstancePositive, this->m_vTemplateTriangle2D, tempImage2, resImage2); + VO_TextureModel::VO_WarpFromOneShapeToAnother(this->m_VOReferenceShape, currentShapeInstanceNegative, this->m_vTemplateTriangle2D, tempImage3, resImage3); + VO_TextureModel::VO_WarpFromOneShapeToAnother(this->m_VOReferenceShape, currentShapeInstancePositive, this->m_vTemplateTriangle2D, tempImage4, resImage4); + + stringstream ssi, ssj, ssj1, ssk; + string stri, strj, strj1, strk; + ssi << i; + ssj << j; + ssj1 << (j + 1); + ssk << k; + ssi >> stri; + ssj >> strj; + ssj1 >> strj1; + ssk >> strk; + + string temp1Str = "CDisplaceLoadedImage" + stri + "_" + strj + "_" + strk + ".jpg"; + string temp2Str = "CDisplaceLoadedImage" + stri + "_" + strj1 + "_" + strk + ".jpg"; + string temp3Str = "CDisplaceTextureImage" + stri + "_" + strj + "_" + strk + ".jpg"; + string temp4Str = "CDisplaceTextureImage" + stri + "_" + strj1 + "_" + strk + ".jpg"; + string res1Str = "CDisplaceLoadedWarpedImage" + stri + "_" + strj + "_" + strk + ".jpg"; + string res2Str = "CDisplaceLoadedWarpedImage" + stri + "_" + strj1 + "_" + strk + ".jpg"; + string res3Str = "CDisplaceTextureWarpedImage" + stri + "_" + strj + "_" + strk + ".jpg"; + string res4Str = "CDisplaceTextureWarpedImage" + stri + "_" + strj1 + "_" + strk + ".jpg"; + + imwrite(temp1Str.c_str(), tempImage1 ); + imwrite(temp2Str.c_str(), tempImage2 ); + imwrite(temp3Str.c_str(), tempImage3 ); + imwrite(temp4Str.c_str(), tempImage4 ); + imwrite(res1Str.c_str(), resImage1 ); + imwrite(res2Str.c_str(), resImage2 ); + imwrite(res3Str.c_str(), resImage3 ); + imwrite(res4Str.c_str(), resImage4 ); + + ssi.clear(); + ssj.clear(); + ssj1.clear(); + ssk.clear(); + } + + + + // Normalize the extracted(loaded) textures + delta_g1.Normalize(); + delta_g2.Normalize(); + delta_g1 -= currentTextureInstanceNegative; + delta_g2 -= currentTextureInstancePositive; + + // form central difference + cDiff = (delta_g2-delta_g1)/(this->m_vvCDisps[j+1](0,k) - this->m_vvCDisps[j](0,k)); + for (unsigned int n = 0; n < this->m_iNbOfTextures; n++) + { + oCParamGM(n, k) += cDiff.GetATexture(n); + } + nExperiment++; + } + } + } + + // normalize + // this->m_MatCParamGradientMatrix is a summation for 240 pictures (size of this->m_iNbOfSamples) + // 4 -- refer to 4 possible shifts for just one parameter (12 parameters altogether) + // 2 -- every pair of shifts, we got only one cDiff, which is actually stored in this->m_MatCParamGradientMatrix +// oCParamGM /= (this->m_iNbOfSamples * 4 / 2); + // nExperiment should be equal to this->m_iNbOfSamples * 4 / 2, if every texture can be successfully loaded. + oCParamGM /= nExperiment; +} + + +/** + * @author JIA Pei + * @version 2010-02-05 + * @brief Build gradient matrices in terms of pose + * @param oPoseGM - Output the built gradient matrix in terms of poses + * @return void +*/ +void VO_AAMBasic::VO_EstPoseGradientMatrix(Mat_& oPoseGM) +{ + bool recordIntermediateImgs = false; + + int nExperiment = 0; + oPoseGM = Mat_::zeros(this->m_iNbOfTextures, this->m_vvPoseDisps[0].cols); + Mat_ currentConcatenatedParameters; + Mat_ currentShape, currentTexture; + VO_Shape currentShapeInstance, currentShapeInstance1, currentShapeInstance2; + VO_Texture currentTextureInstance; + VO_Texture delta_g1, delta_g2, cDiff; + //Mat_ disp1 = Mat_::zeros(1, 4); + //Mat_ disp2 = Mat_::zeros(1, 4); + Mat_ posedisp1 = Mat_::zeros(1, 4); + Mat_ posedisp2 = Mat_::zeros(1, 4); + + // just for displacement + Mat_ translation, disptranslation1, disptranslation2; + float scale = 1.0f, dispscale1 = 1.0f, dispscale2 = 1.0f; + vector theta(1), dispangles1(1), dispangles2(1); + Mat img; + Mat tempImage1, tempImage2, resImage1, resImage2; + + + // for each training example in the training set + for(unsigned int i = 0; i < this->m_iNbOfSamples; i++) + { + if(this->m_iNbOfChannels == 1) + img = imread ( this->m_vStringTrainingImageNames[i].c_str (), 0 ); + else if (this->m_iNbOfChannels == 3) + img = imread ( this->m_vStringTrainingImageNames[i].c_str (), 1 ); + else + cerr << "We can't deal with image channels not equal to 1 or 3!" << endl; + + if(recordIntermediateImgs) + { + img.copyTo(tempImage1); + img.copyTo(tempImage2); + } + + for(unsigned int j = 0; j < this->m_vvPoseDisps.size(); j = j+2) // 4 -- number of displacements for each of the 4 pose parameters + { + for (int k = 0; k < this->m_vvPoseDisps[0].cols; k++) // 4 -- number of pose parameters + { + posedisp1 = Mat_::zeros(posedisp1.size()); + posedisp2 = Mat_::zeros(posedisp2.size()); + posedisp1(0,k) = this->m_vvPoseDisps[j](0,k); + posedisp2(0,k) = this->m_vvPoseDisps[j+1](0,k); + VO_Shape::GlobalShapeNormalization2SimilarityTrans(posedisp1, dispscale1, dispangles1, disptranslation1 ); + VO_Shape::GlobalShapeNormalization2SimilarityTrans(posedisp2, dispscale2, dispangles2, disptranslation2 ); + + // do displacement measures + currentConcatenatedParameters = this->m_MatAppearanceProject2Truncated.row(i); + + // According to Cootes' "Comparing Variations on the Active Appearance Model Algorithm" - Equation (3) + + // Build the shape instance from the combined model +// cv::gemm(currentConcatenatedParameters, this->m_MatQs, 1, this->m_PCAAlignedShape.mean, 1, currentShape, GEMM_2_T ); + currentShape = currentConcatenatedParameters * this->m_MatQs.t() + this->m_PCAAlignedShape.mean; + + // Build the texture instance from the combined model +// cv::gemm(currentConcatenatedParameters, this->m_MatQg, 1, this->m_PCANormalizedTexture.mean, 1,currentTexture, GEMM_2_T); + currentTexture = currentConcatenatedParameters * this->m_MatQg.t() + this->m_PCANormalizedTexture.mean; + + // Calculate the align transformation + currentShapeInstance.SetTheShape(currentShape, 2); + currentShapeInstance.AlignTransformation(this->m_vShapes[i], scale, theta, translation); + + currentShapeInstance1 = currentShapeInstance; + currentShapeInstance2 = currentShapeInstance; + + vector tempTheta(1, 0.0f); + Mat_ tempTranslate = Mat_::zeros(translation.size()); + tempTheta[0] = dispangles1[0] + theta[0]; + tempTranslate(0,0) = disptranslation1(0,0) + translation(0,0); + tempTranslate(1,0) = disptranslation1(1,0) + translation(1,0); + currentShapeInstance1.GlobalShapeNormalization2D(dispscale1 * scale, tempTheta, tempTranslate); + tempTheta[0] = dispangles2[0] + theta[0]; + tempTranslate(0,0) = disptranslation2(0,0) + translation(0,0); + tempTranslate(1,0) = disptranslation2(1,0) + translation(1,0); + currentShapeInstance2.GlobalShapeNormalization2D(dispscale2 * scale, tempTheta, tempTranslate); + + if(!VO_TextureModel::VO_LoadOneTextureFromShape(currentShapeInstance1, img, this->m_vTemplateTriangle2D, this->m_vTemplatePointWarpInfo, delta_g1)) + continue; + if(!VO_TextureModel::VO_LoadOneTextureFromShape(currentShapeInstance2, img, this->m_vTemplateTriangle2D, this->m_vTemplatePointWarpInfo, delta_g2)) + continue; + + currentTextureInstance.SetTheTexture(currentTexture, delta_g1.GetNbOfTextureRepresentation()); + + //// The following codes are just for intermediate display + if(recordIntermediateImgs) + { + VO_TextureModel::VO_PutOneTextureToTemplateShape(delta_g1, this->m_vTemplateTriangle2D, tempImage1); + VO_TextureModel::VO_PutOneTextureToTemplateShape(delta_g2, this->m_vTemplateTriangle2D, tempImage2); + VO_TextureModel::VO_WarpFromOneShapeToAnother(this->m_VOReferenceShape, currentShapeInstance1, this->m_vTemplateTriangle2D, tempImage1, resImage1); + VO_TextureModel::VO_WarpFromOneShapeToAnother(this->m_VOReferenceShape, currentShapeInstance2, this->m_vTemplateTriangle2D, tempImage2, resImage2); + + stringstream ssi, ssj, ssj1, ssk; + string stri, strj, strj1, strk; + ssi << i; + ssj << j; + ssj1 << (j + 1); + ssk << k; + ssi >> stri; + ssj >> strj; + ssj1 >> strj1; + ssk >> strk; + + string temp1Str = "poseDisplaceLoadedImage" + stri + "_" + strj + "_" + strk + ".jpg"; + string temp2Str = "poseDisplaceLoadedImage" + stri + "_" + strj1 + "_" + strk + ".jpg"; + string res1Str = "poseDisplaceLoadedWarpedImage" + stri + "_" + strj + "_" + strk + ".jpg"; + string res2Str = "poseDisplaceLoadedWarpedImage" + stri + "_" + strj1 + "_" + strk + ".jpg"; + + imwrite(temp1Str.c_str(), tempImage1 ); + imwrite(temp2Str.c_str(), tempImage2 ); + imwrite(res1Str.c_str(), resImage1 ); + imwrite(res2Str.c_str(), resImage2 ); + + ssi.clear(); + ssj.clear(); + ssj1.clear(); + ssk.clear(); + } + + + // Normalize the extracted(loaded) textures + delta_g1.Normalize(); + delta_g2.Normalize(); + delta_g2 -= currentTextureInstance; + delta_g1 -= currentTextureInstance; + + // form central difference + cDiff = (delta_g2-delta_g1)/(this->m_vvPoseDisps[j+1](0,k) - this->m_vvPoseDisps[j](0,k)); + for (unsigned int n = 0; n < this->m_iNbOfTextures; n++) + { + oPoseGM(n, k) += cDiff.GetATexture(n); + } + + nExperiment++; + } + } + } + + // normalize + // this->m_MatCParamGradientMatrix is a summation for 240 pictures (size of this->m_iNbOfSamples) + // 4 -- refer to 4 possible shifts for just one parameter (16 parameters altogether) + // 2 -- every pair of shifts, we got only one cDiff, which is actually stored in this->m_MatCParamGradientMatrix +// oPoseGM /= (this->m_iNbOfSamples * 4 / 2); + // nExperiment should be equal to this->m_iNbOfSamples * 4 / 2, if every texture can be successfully loaded. + oPoseGM /= nExperiment; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief appearance parameters constrain + * @param ioC Input and Output - appearance parameters + * @param nSigma Input - number of sigmas + * @return void +*/ +void VO_AAMBasic::VO_AppearanceParameterConstraint(Mat_& ioC, float nSigma) +{ + for (int i = 0; i < ioC.cols; ++i) + { + float ct = nSigma * sqrt(this->m_PCAAppearance.eigenvalues.at(i,0) ); + if ( ioC(0, i) > ct ) + { + ioC(0, i) = ct; + } + else if ( ioC(0, i) < -ct ) + { + ioC(0, i) = -ct; + } + } +} + + +/** + * @author JIA Pei + * @version 2010-02-05 + * @brief Shape and texture project to shape parameters and texture parameters, and then concatenated + * @param iShape Input - input shape + * @param iTexture Input - input texture + * @param b Output - projected concatenated parameters + * @return void +*/ +void VO_AAMBasic::VO_ShapeTexture2Appearance( VO_Shape iShape, VO_Texture iTexture, Mat_& app ) const +{ + Mat_ b_s = this->m_PCAAlignedShape.project( iShape.GetTheShapeInARow() ) + * this->m_MatWeightsScaleShape2Texture; // 1*116->1*15 +// cv::gemm(b_s_temp, this->m_MatWeightsScaleShape2Texture, 1.0, Mat(), 0.0, b_s); // 1*21->1*21, just for rescaling + Mat_ b_t = this->m_PCANormalizedTexture.project(iTexture.GetTheTextureInARow()); // 1*80259->1*36 + app = Mat_::zeros(1, b_s.cols + b_t.cols); + + Mat_ roib_s = app(Range(0, 1), Range(0, b_s.cols) ); + Mat_ roib_t = app(Range(0, 1), Range(b_s.cols, b_s.cols + b_t.cols) ); + b_s.copyTo(roib_s); + b_t.copyTo(roib_t); +} + + +/** +* @author JIA Pei +* @version 2010-04-05 +* @brief appearance project to appearance parameters +* @param app Input - input appearance, a row vector +* @param outC Output - projected appearance parameters, a row vector +* @return void +*/ +void VO_AAMBasic::VO_AppearanceProjectToCParam( const Mat_& app, + Mat_& outC) const +{ + this->m_PCAAppearance.project(app, outC); +} + + +/** +* @author JIA Pei +* @version 2010-04-05 +* @brief Shape parameters and texture parameters projected to concatenated parameters +* @param inS Input - shape model parameters, a row vector +* @param inT Input - texture model parameters, a row vector +* @param outC Output - output concatenated parameters, a row vector +* @return void +*/ +void VO_AAMBasic::VO_SParamTParamProjectToCParam( const Mat_& inS, + const Mat_& inT, + Mat_& outC) const +{ + Mat_ tempConcatenated = Mat_::zeros(1, this->m_iNbOfShapeEigens + this->m_iNbOfTextureEigens); + + Mat_ tempShape = tempConcatenated( Range(0,1), Range(0, this->m_iNbOfShapeEigens) ); + Mat_ tempTexture = tempConcatenated( Range(0,1), Range(this->m_iNbOfShapeEigens, this->m_iNbOfAppearance) ); + inS.copyTo(tempShape); + tempShape *= this->m_MatWeightsScaleShape2Texture; + inT.copyTo(tempTexture); + outC = this->m_PCAAppearance.project(tempConcatenated); +} + + +/** +* @author JIA Pei +* @version 2010-04-05 +* @brief Appearance parameters back project to appearance +* @param inC Input - input appearance parameters +* @param app Output - the appearance +* @return void +*/ +void VO_AAMBasic::VO_CParamBackProjectToAppearance(const Mat_& inC, Mat_& app) const +{ + this->m_PCAAppearance.backProject(inC, app); +} + + +/** +* @author JIA Pei +* @version 2010-02-05 +* @brief Concatenated parameters back project to shape and texture parameters +* @param inC Input - input concatenated parameters +* @param outS Output - back projected shape model parameters +* @param outT Output - back projected texture model parameters +* @return void +*/ +void VO_AAMBasic::VO_CParamBackProjectToSParamTParam(const Mat_& inC, Mat_& outS, Mat_& outT) const +{ + Mat_ tempConcatenated = this->m_PCAAppearance.backProject(inC); + + outS = tempConcatenated( Range(0,1), Range(0, this->m_iNbOfShapeEigens) ); + outT = tempConcatenated( Range(0,1), Range(this->m_iNbOfShapeEigens, this->m_iNbOfAppearance) ); + outS *= this->m_MatWeightsScaleShape2Texture.inv(); +} + + +/** +* @author JIA Pei +* @version 2010-02-05 +* @brief Concatenated parameters back project to aligned shape +* @param inC Input - input concatenated parameters, a row vector +* @param oShape Output - the back projected shape +* @return void +*/ +void VO_AAMBasic::VO_CParamBackProjectToAlignedShape(const Mat_& inC, VO_Shape& oShape, int dim) const +{ + // c back to shape +// cv::gemm(inC, this->m_MatQs, 1, this->m_PCAAlignedShape.mean, 1, oShape, GEMM_1_T + GEMM_2_T); + oShape.SetTheShape(inC * this->m_MatQs.t() + this->m_PCAAlignedShape.mean, dim); +} + + +/** +* @author JIA Pei +* @version 2010-02-05 +* @brief Concatenated parameters back project to aligned shape +* @param inC Input - input concatenated parameters +* @param oTexture Output - the back projected shape +* @return void +*/ +void VO_AAMBasic::VO_CParamBackProjectToNormalizedTexture(const Mat_& inC, VO_Texture& oTexture, int tr) const +{ + // c back to texture +// cv::gemm(inC, this->m_MatQg, 1, this->m_PCANormalizedTexture.mean, 1, oTexture, GEMM_1_T + GEMM_2_T); + oTexture.SetTheTexture(inC * this->m_MatQg.t() + this->m_PCANormalizedTexture.mean, tr); +} + + +/** +* @author JIA Pei +* @version 2010-02-05 +* @brief generate displacement vectors for later numerical analysis +* @note Explained by Stegmann: sets must *always* be in anti-symmetric pairs with the largest displacements first, + e.g. [ -.5, .5, -.25, .25 ] + According to (7.7) from Stegmann's AAM-API, there will be n=m(4k+24)=240*(4*12+24)=21120 displacements. + However, here, for the pose displacements, the amount is not 24 but 16 instead. n=m(4k+16)=240*(4*12+16)=21120 + * @ref 1) Cootes' "Comparing Variations on the Active Appearance Model Algorithm" + 2) Cootes' "Statistical Models of Appearance for Computer Vision" page 56 + 3) Stegmann's" AAM-API, Basic Active Appearance Models". +*/ +void VO_AAMBasic::VO_CreateDisplacementSets() +{ + // generate c and pose displacement sets + vector vStdDisp; + vector vXYDisp; + vector vScaleDisp; + vector vRotDisp; + + // Mikkel B. Stegmann's master's thesis displacement set + vStdDisp.push_back(-.25f); + vStdDisp.push_back( .25f); + vStdDisp.push_back(-.5f); + vStdDisp.push_back( .5f); + + // relative displacement set + vScaleDisp.push_back( .95f); + vScaleDisp.push_back(1.05f); + vScaleDisp.push_back( .90f); + vScaleDisp.push_back(1.10f); + + // in radian measure + vRotDisp.push_back( (float)(-5.0 / 180.0 * CV_PI) ); + vRotDisp.push_back( (float)( 5.0 / 180.0 * CV_PI) ); + vRotDisp.push_back( (float)(-10.0 / 180.0 * CV_PI) ); + vRotDisp.push_back( (float)( 10.0 / 180.0 * CV_PI) ); + + vXYDisp.push_back(-.05f); + vXYDisp.push_back( .05f); + vXYDisp.push_back(-.10f); + vXYDisp.push_back( .10f); + + // Pre-process for X, Y pose parameters + vector vXDisp, vYDisp; + vXDisp.resize (vXYDisp.size()); + vYDisp.resize (vXYDisp.size()); + for (unsigned int i = 0; i < vXYDisp.size(); i++) + { + vXDisp[i] = vXYDisp[i] * this->m_ImageTemplateFace.cols; + vYDisp[i] = vXYDisp[i] * this->m_ImageTemplateFace.rows; + } + + this->m_vvCDisps = VO_AAMBasic::VO_CalcCParamDisplacementVectors( vStdDisp, this->m_PCAAppearance.eigenvalues ); + this->m_vvPoseDisps = VO_AAMBasic::VO_CalcPoseDisplacementVectors( vScaleDisp, vRotDisp, vXDisp, vYDisp ); +} + + +/** + * @author Stegmann, JIA Pei + * @version 2010-02-05 + * @brief Generates a set of combined model parameter displacement vectors + where each parameter is displaced at a time according to the values in vStdDisp. + * @param vStdDisp Input - A row vector of parameter displacements in standard deviations of the corresponding parameter. + * @param ConcatenatedSD Input - Concatenated standard deviation, a column vector + * @return A vector of displacement vectors. +*/ +vector< Mat_ > VO_AAMBasic::VO_CalcCParamDisplacementVectors(const vector& vStdDisp, const Mat_& ConcatenatedSD) +{ + vector< Mat_ > cDisplacements; + cDisplacements.resize( vStdDisp.size() ); + for (unsigned int i = 0; i < cDisplacements.size(); i++) + { + cDisplacements[i] = Mat_::zeros(1, ConcatenatedSD.rows ); + } + + // calculate displacements, for each parameter + for (unsigned int i = 0; i < cDisplacements.size(); i++) + { + for(int j = 0; j < cDisplacements[0].cols; j++) + { + // for each displacement + cDisplacements[i](0,j) = vStdDisp[i] * ConcatenatedSD(j, 0); + } + } + + return cDisplacements; +} + + +/** +* @author JIA Pei +* @version 2010-02-05 +* @brief Build displacement sets for Pose parameters +* @param vScaleDisp Input - all input shapes +* @param vRotDisp Input - all input images +* @param vXDisp Input - 1 or 3 +* @param vYDisp Input - texture build method +* @param method Input - truncated percentage for shape model +* @return vector > - all pose displacement vectors +*/ +vector< Mat_ > VO_AAMBasic::VO_CalcPoseDisplacementVectors(const vector& vScaleDisp, + const vector& vRotDisp, + const vector& vXDisp, + const vector& vYDisp) +{ + vector< Mat_ > poseDisplacements; + poseDisplacements.resize( vXDisp.size() ); + //for (unsigned int i = 0; i < poseDisplacements.size(); i++) + //{ + // poseDisplacements[i] = Mat_::zeros(1, 4); // 4 refers to X, Y, Scale, and Rotate + //} + + //Mat_ singlePose = Mat_::zeros(1, 4); + float scale = 1.0f; + vector angles(1); + Mat_ translation = Mat_::zeros(2, 1); + + for(unsigned int i = 0; i < poseDisplacements.size(); i++) + { + //singlePose(0,0) = vScaleDisp[i]; + //singlePose(0,1) = vRotDisp[i]; + //singlePose(0,2) = vXDisp[i]; + //singlePose(0,3) = vYDisp[i]; + scale = vScaleDisp[i]; + angles[0] = vRotDisp[i]; + translation(0,0) = vXDisp[i]; + translation(1,0) = vYDisp[i]; + + VO_Shape::SimilarityTrans2GlobalShapeNormalization(scale, angles, translation, poseDisplacements[i]); + } + + return poseDisplacements; +} + + +/** + * @author JIA Pei + * @version 2010-04-03 + * @brief Build Appearance Model, Basic AAM + * @param allLandmarkFiles4Training Input - all training landmark files + * @param allImgFiles4Training Input - all training image files + * @param shapeinfoFileName Input - shape info file + * @param database Input - which database is it? + * @param channels Input - How many channels are to be used? + * @param levels Input - multiscale levels + * @param trm Input - texture representation method + * @param TPShape Input - truncated percentage for shape model + * @param TPTexture Input - truncated percentage for texture model + * @param TPConcatenated Input - truncated percentage for appearance model + * @param useKnownTriangles Input - use known triangle structures?? + * @return void + * @note Using "float* oProf" is much much faster than using "VO_Profile& oProf" or "vector" + */ +void VO_AAMBasic::VO_BuildAppearanceModel( const vector& allLandmarkFiles4Training, + const vector& allImgFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + unsigned int channels, + unsigned int levels, + int trm, + float TPShape, + float TPTexture, + float TPConcatenated, + bool useKnownTriangles) +{ + this->VO_BuildTextureModel( allLandmarkFiles4Training, + allImgFiles4Training, + shapeinfoFileName, + database, + channels, + trm, + TPShape, + TPTexture, + useKnownTriangles); + this->m_iNbOfPyramidLevels = levels; + this->m_iNbOfAppearance = this->m_iNbOfShapeEigens + this->m_iNbOfTextureEigens; + this->m_iNbOfEigenAppearanceAtMost = MIN(this->m_iNbOfSamples, this->m_iNbOfAppearance); + this->m_fTruncatedPercent_Appearance = TPConcatenated; + + this->m_MatWeightsScaleShape2Texture = Mat_::zeros( this->m_iNbOfShapeEigens, this->m_iNbOfShapeEigens); + float SumOfEigenValues_Shape = safeDoubleToFloat(cv::sum( this->m_PCAAlignedShape.eigenvalues ).val[0]); + float SumOfEigenValues_Texture = safeDoubleToFloat(cv::sum( this->m_PCANormalizedTexture.eigenvalues ).val[0]); + float val = (float)(SumOfEigenValues_Texture / SumOfEigenValues_Shape); + cv::setIdentity(this->m_MatWeightsScaleShape2Texture, val); + + //////////////////////////////////////////////////////////////////////////// + // Assign concatenated + //////////////////////////////////////////////////////////////////////////// + Mat_ matAlignedShapes = Mat_::zeros(this->m_iNbOfSamples, this->m_iNbOfShapes); + Mat_ matNormalizedTextures = Mat_::zeros(this->m_iNbOfSamples, this->m_iNbOfTextures); + for(unsigned int i = 0; i < this->m_iNbOfSamples; ++i) + { + Mat tmpRowShape = matAlignedShapes.row(i); + this->m_vAlignedShapes[i].GetTheShapeInARow().copyTo(tmpRowShape); + Mat tmpRowTexture = matNormalizedTextures.row(i); + this->m_vNormalizedTextures[i].GetTheTextureInARow().copyTo(tmpRowTexture); + } + Mat_ AlignedShapesProject2Truncated = this->m_PCAAlignedShape.project(matAlignedShapes); + Mat_ NormalizedTexturesProject2Truncated = this->m_PCANormalizedTexture.project(matNormalizedTextures); + Mat_ WeightedAlignedShapesProject2Truncated = AlignedShapesProject2Truncated * this->m_MatWeightsScaleShape2Texture; +// cv::gemm(AlignedShapesProject2Truncated, this->m_MatWeightsScaleShape2Texture, 1.0, Mat(), 0.0, WeightedAlignedShapesProject2Truncated); + + Mat_ matConcatenated = Mat_::zeros( this->m_iNbOfSamples, this->m_iNbOfAppearance); + Mat_ matMeanConcatenated = Mat_::zeros(1, this->m_iNbOfAppearance); // Obviously, matMeanConcatenated should be all zeros + + Mat roiShape = matConcatenated(Range(0, this->m_iNbOfSamples), Range(0, this->m_iNbOfShapeEigens) ); + Mat roiTexture = matConcatenated(Range(0, this->m_iNbOfSamples), Range(this->m_iNbOfShapeEigens, this->m_iNbOfAppearance) ); + WeightedAlignedShapesProject2Truncated.copyTo(roiShape); + NormalizedTexturesProject2Truncated.copyTo(roiTexture); + + //////////////////////////////////////////////////////////////////////////// + + this->m_PCAAppearance(matConcatenated, matMeanConcatenated, CV_PCA_DATA_AS_ROW, this->m_iNbOfEigenAppearanceAtMost ); + // to decide how many components to be selected + this->m_iNbOfAppearanceEigens = 0; + + double SumOfEigenValues = cv::sum( this->m_PCAAppearance.eigenvalues ).val[0]; + double ps = 0.0f; + + for(unsigned int i = 0; i < this->m_iNbOfEigenAppearanceAtMost; i++) + { + ps += this->m_PCAAppearance.eigenvalues.at(i, 0 ); + ++this->m_iNbOfAppearanceEigens; + if( ps/SumOfEigenValues >= this->m_fTruncatedPercent_Appearance) break; + } + // m_iNbOfAppearanceEigens decided. For simplicity, we carry out PCA once again. + this->m_PCAAppearance(matConcatenated, matMeanConcatenated, CV_PCA_DATA_AS_ROW, this->m_iNbOfAppearanceEigens ); + this->m_MatAppearanceProject2Truncated = this->m_PCAAppearance.project(matConcatenated); + + // extract the shape part of the combined eigen vectors + this->m_MatPcs = this->m_PCAAppearance.eigenvectors (Range(0, this->m_iNbOfAppearanceEigens), Range(0, this->m_iNbOfShapeEigens)); + + // extract the texture part of the combined eigen vectors + this->m_MatPcg = this->m_PCAAppearance.eigenvectors (Range(0, this->m_iNbOfAppearanceEigens), Range(this->m_iNbOfShapeEigens, this->m_iNbOfAppearance)); + + // calculate m_MatQs = m_PCAAlignedShape.eigenvectors * m_MatWeightsScaleShape2Texture^{-1} * m_MatPcs + this->m_MatQs = this->m_PCAAlignedShape.eigenvectors.t() * this->m_MatWeightsScaleShape2Texture.inv() * this->m_MatPcs.t(); + + // calculate m_MatQg = m_PCANormalizedTexture.eigenvectors * m_MatPcg + this->m_MatQg = this->m_PCANormalizedTexture.eigenvectors.t() * this->m_MatPcg.t(); + + this->VO_CreateDisplacementSets( ); + +// this->VO_CalcRegressionMatrices(); + this->VO_CalcGradientMatrices(); +} + + +/** + * @author JIA Pei + * @version 2010-02-11 + * @brief Save Appearance Model to a specified folder + * @param fn Input - the folder that AAMBasic to be saved to + * @return void +*/ +void VO_AAMBasic::VO_Save(const string& fd) +{ + VO_AXM::VO_Save(fd); + + string fn = fd+"/AppearanceModel"; + MakeDirectory(fn); + + fstream fp; + string tempfn; + + // AppearanceModel + tempfn = fn + "/AppearanceModel" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + + fp << "m_iNbOfAppearance" << endl << this->m_iNbOfAppearance << endl; // m_iNbOfAppearance + fp << "m_iNbOfEigenAppearanceAtMost" << endl << this->m_iNbOfEigenAppearanceAtMost << endl; // m_iNbOfEigenAppearanceAtMost + fp << "m_iNbOfAppearanceEigens" << endl << this->m_iNbOfAppearanceEigens << endl; // m_iNbOfAppearanceEigens + fp << "m_fTruncatedPercent_Appearance" << endl << this->m_fTruncatedPercent_Appearance << endl; // m_fTruncatedPercent_Appearance + + fp.close();fp.clear(); + + // m_MatWeightsScaleShape2Texture + tempfn = fn + "/m_MatWeightsScaleShape2Texture" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_MatWeightsScaleShape2Texture" << endl; + fp << this->m_MatWeightsScaleShape2Texture; + fp.close();fp.clear(); + + // m_PCAAppearanceMean + tempfn = fn + "/m_PCAAppearanceMean" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_PCAAppearanceMean" << endl; + fp << Mat_(this->m_PCAAppearance.mean); + fp.close();fp.clear(); + + // m_PCAAppearanceEigenValues + tempfn = fn + "/m_PCAAppearanceEigenValues" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_PCAAppearanceEigenValues" << endl; + fp << Mat_(this->m_PCAAppearance.eigenvalues); + fp.close();fp.clear(); + + // m_PCAAppearanceEigenVectors + tempfn = fn + "/m_PCAAppearanceEigenVectors" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_PCAAppearanceEigenVectors" << endl; + fp << Mat_(this->m_PCAAppearance.eigenvectors); + fp.close();fp.clear(); + + // m_MatPcs + tempfn = fn + "/m_MatPcs" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_MatPcs" << endl; + fp << this->m_MatPcs; + fp.close();fp.clear(); + + // m_MatPcg + tempfn = fn + "/m_MatPcg" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_MatPcg" << endl; + fp << this->m_MatPcg; + fp.close();fp.clear(); + + // m_MatQs + tempfn = fn + "/m_MatQs" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_MatQs" << endl; + fp << this->m_MatQs; + fp.close();fp.clear(); + + // m_MatQg + tempfn = fn + "/m_MatQg" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_MatQg" << endl; + fp << this->m_MatQg; + fp.close();fp.clear(); + + // m_MatRc + tempfn = fn + "/m_MatRc" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_MatRc" << endl; + fp << this->m_MatRc; + fp.close();fp.clear(); + + // m_MatRt + tempfn = fn + "/m_MatRt" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_MatRt" << endl; + fp << this->m_MatRt; + fp.close();fp.clear(); + + // m_MatCParamGradientMatrix + tempfn = fn + "/m_MatCParamGradientMatrix" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_MatCParamGradientMatrix" << endl; + fp << this->m_MatCParamGradientMatrix; + fp.close();fp.clear(); + + // m_MatPoseGradientMatrix, we may ignore this first + tempfn = fn + "/m_MatPoseGradientMatrix" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_MatPoseGradientMatrix" << endl; + fp << this->m_MatPoseGradientMatrix; + fp.close();fp.clear(); + +} + + +/** + * @author JIA Pei + * @version 2010-02-11 + * @brief Load all Appearance Model data from a specified folder + * @param fd Input - the folder that AppearanceModel to be loaded from + * @return void +*/ +void VO_AAMBasic ::VO_Load(const string& fd) +{ + VO_AXM::VO_Load(fd); + + string fn = fd+"/AppearanceModel"; + if (!MakeDirectory(fn) ) + { + cout << "AppearanceModel subfolder is not existing. " << endl; + exit(EXIT_FAILURE); + } + + // ifstream fp; + // string tempfn; + // string temp; + // + // // m_MatAppearanceProject2Truncated + // tempfn = fn + "/m_MatAppearanceProject2Truncated" + ".txt"; + // SafeInputFileOpen(fp, tempfn); + // fp >> temp; + // fp >> this->m_MatAppearanceProject2Truncated; + // fp.close();fp.clear(); + + // m_vvCDisps + + // m_vvPoseDisps + +} + + +/** + * @author JIA Pei + * @version 2010-02-11 + * @brief Load all AAM data from a specified folder for later fitting + * @param fd Input - the folder that AAM to be loaded from + * @return void +*/ +void VO_AAMBasic::VO_LoadParameters4Fitting(const string& fd) +{ + VO_AXM::VO_LoadParameters4Fitting(fd); + + string fn = fd+"/AppearanceModel"; + if (!MakeDirectory(fn) ) + { + cout << "AppearanceModel subfolder is not existing. " << endl; + exit(EXIT_FAILURE); + } + + ifstream fp; + string tempfn; + string temp; + + // AppearanceModel + tempfn = fn + "/AppearanceModel" + ".txt"; + SafeInputFileOpen(fp, tempfn); + + fp >> temp >> this->m_iNbOfAppearance; + fp >> temp >> this->m_iNbOfEigenAppearanceAtMost; + fp >> temp >> this->m_iNbOfAppearanceEigens; + fp >> temp >> this->m_fTruncatedPercent_Appearance; + fp.close();fp.clear(); + + this->m_PCAAppearance = cv::PCA(); + + // m_PCAAppearanceMean + this->m_PCAAppearance.mean = Mat_::zeros(1, this->m_iNbOfAppearance); + tempfn = fn + "/m_PCAAppearanceMean" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_PCAAppearance.mean; + fp.close();fp.clear(); + + // m_PCAAppearanceEigenValues + this->m_PCAAppearance.eigenvalues = Mat_::zeros(this->m_iNbOfAppearanceEigens, 1); + tempfn = fn + "/m_PCAAppearanceEigenValues" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_PCAAppearance.eigenvalues; + fp.close();fp.clear(); + + // m_PCAAppearanceEigenVectors + this->m_PCAAppearance.eigenvectors = Mat_::zeros(this->m_iNbOfAppearanceEigens, this->m_iNbOfAppearance); + tempfn = fn + "/m_PCAAppearanceEigenVectors" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_PCAAppearance.eigenvectors; + fp.close();fp.clear(); + + // m_MatWeightsScaleShape2Texture + this->m_MatWeightsScaleShape2Texture = Mat_::zeros(this->m_iNbOfShapeEigens, this->m_iNbOfShapeEigens); + tempfn = fn + "/m_MatWeightsScaleShape2Texture" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_MatWeightsScaleShape2Texture; + fp.close();fp.clear(); + + // m_MatPcs + this->m_MatPcs = Mat_::zeros(this->m_iNbOfAppearanceEigens, this->m_iNbOfShapeEigens); + tempfn = fn + "/m_MatPcs" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_MatPcs; + fp.close();fp.clear(); + + // m_MatPcg + this->m_MatPcg = Mat_::zeros(this->m_iNbOfAppearanceEigens, this->m_iNbOfTextureEigens); + tempfn = fn + "/m_MatPcg" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_MatPcg; + fp.close();fp.clear(); + + // m_MatQs + this->m_MatQs = Mat_::zeros(this->m_iNbOfShapes, this->m_iNbOfAppearanceEigens); + tempfn = fn + "/m_MatQs" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_MatQs; + fp.close();fp.clear(); + + // m_MatQg + this->m_MatQg = Mat_::zeros(this->m_iNbOfTextures, this->m_iNbOfAppearanceEigens); + tempfn = fn + "/m_MatQg" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_MatQg; + fp.close();fp.clear(); + + // m_MatRc + this->m_MatRc = Mat_::zeros(this->m_iNbOfAppearanceEigens, this->m_iNbOfTextures); + tempfn = fn + "/m_MatRc" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_MatRc; + fp.close();fp.clear(); + + // m_MatRt + this->m_MatRt = Mat_::zeros(4, this->m_iNbOfTextures); + tempfn = fn + "/m_MatRt" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_MatRt; + fp.close();fp.clear(); + + // m_MatCParamGradientMatrix + this->m_MatCParamGradientMatrix = Mat_::zeros(this->m_iNbOfTextures, this->m_iNbOfAppearanceEigens); + tempfn = fn + "/m_MatCParamGradientMatrix" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_MatCParamGradientMatrix; + fp.close();fp.clear(); + + // m_MatPoseGradientMatrix, we may ignore this first + this->m_MatPoseGradientMatrix = Mat_::zeros(this->m_iNbOfTextures, 4); + tempfn = fn + "/m_MatPoseGradientMatrix" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_MatPoseGradientMatrix; + fp.close();fp.clear(); + +} + diff --git a/modules/smbuilding/src/VO_AAMForwardIA.cpp b/modules/smbuilding/src/VO_AAMForwardIA.cpp new file mode 100644 index 0000000..32d6424 --- /dev/null +++ b/modules/smbuilding/src/VO_AAMForwardIA.cpp @@ -0,0 +1,171 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include +#include +#include +#include + +#include "VO_AAMForwardIA.h" + + + +/** + * @author JIA Pei + * @version 2010-04-03 + * @brief FCIA AAM + * @param allLandmarkFiles4Training Input - all training landmark files + * @param allImgFiles4Training Input - all training image files + * @param shapeinfoFileName Input - shape info file + * @param database Input - which database is it? + * @param levels Input - multiscale levels + * @param channels Input - How many channels are to be used? + * @param trm Input - texture representation method + * @param TPShape Input - truncated percentage for shape model + * @param TPTexture Input - truncated percentage for texture model + * @param useKnownTriangles Input - use known triangle structures?? + * @return void + * @note Using "float* oProf" is much much faster than using "VO_Profile& oProf" or "vector" + */ +void VO_AAMForwardIA::VO_BuildAAMFCIA( const vector& allLandmarkFiles4Training, + const vector& allImgFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + unsigned int levels, + unsigned int channels, + int trm, + float TPShape, + float TPTexture, + bool useKnownTriangles) +{ + this->VO_BuildTextureModel( allLandmarkFiles4Training, + allImgFiles4Training, + shapeinfoFileName, + database, + channels, + trm, + TPShape, + TPTexture, + useKnownTriangles); + +} + + +/** + * @author JIA Pei + * @version 2010-04-03 + * @brief Save AAMFCIA to a specified folder + * @param fn Input - the folder that AAMFCIA to be saved to +*/ +void VO_AAMForwardIA::VO_Save(const string& fd) +{ + VO_AXM::VO_Save(fd); + + string fn = fd+"/AAMFCIA"; + MakeDirectory(fn); + + fstream fp; + string tempfn; + +} + + +/** + * @author JIA Pei + * @version 2010-04-03 + * @brief Load all AAMFCIA data from a specified folder + * @param fd Input - the folder that AAMFCIA to be loaded from +*/ +void VO_AAMForwardIA::VO_Load(const string& fd) +{ + VO_AXM::VO_Load(fd); + + string fn = fd+"/AAMFCIA"; + if (!MakeDirectory(fn) ) + { + cout << "AAMFCIA subfolder is not existing. " << endl; + exit(EXIT_FAILURE); + } +} + + +/** + * @author JIA Pei + * @version 2010-04-03 + * @brief Load all AAMICIA data from a specified folder + * @param fd Input - the folder that AAMICIA to be loaded from +*/ +void VO_AAMForwardIA::VO_LoadParameters4Fitting(const string& fd) +{ + VO_AXM::VO_LoadParameters4Fitting(fd); + + string fn = fd+"/AAMFCIA"; + if (!MakeDirectory(fn) ) + { + cout << "AAMFCIA subfolder is not existing. " << endl; + exit(EXIT_FAILURE); + } + + fstream fp; + string tempfn; + string temp; + +} + diff --git a/modules/smbuilding/src/VO_AAMInverseIA.cpp b/modules/smbuilding/src/VO_AAMInverseIA.cpp new file mode 100644 index 0000000..b567d6f --- /dev/null +++ b/modules/smbuilding/src/VO_AAMInverseIA.cpp @@ -0,0 +1,700 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include +#include +#include +#include + +#include "VO_AAMInverseIA.h" +#include "VO_CVCommon.h" + + +/** + * @author JIA Pei + * @version 2010-04-03 + * @brief Calculate Gradients for template face, both in X and Y directions + * @return void +*/ +void VO_AAMInverseIA::VO_CalcTemplateFaceGradients () +{ + vector< vector > tblr; + tblr.resize(this->m_iNbOfPixels); + + for (unsigned int i = 0; i < this->m_iNbOfPixels; i++) + { + tblr[i].resize(4); + for (unsigned int j = 0; j < 4; j++) + { + tblr[i][j] = -1; + } + } + + + float xPos, yPos; + for (unsigned int i = 0; i < this->m_iNbOfPixels; i++) // 30132 + { + // x refer to column; while y refer to row. + xPos = this->m_vTemplatePointWarpInfo[i].GetPosition().x; + yPos = this->m_vTemplatePointWarpInfo[i].GetPosition().y; + + // Start searching for the top bottom left right point index in this->m_vNormalizedPointWarpInfo + + for (unsigned int j = 0; j < this->m_iNbOfPixels; j++) // 30132 + { + // left + if ( fabs( this->m_vTemplatePointWarpInfo[j].GetPosition().x - (xPos - 1.) ) < FLT_EPSILON && + fabs( this->m_vTemplatePointWarpInfo[j].GetPosition().y - yPos ) < FLT_EPSILON ) + { + tblr[i][0] = j; + } + + // right + if ( fabs(this->m_vTemplatePointWarpInfo[j].GetPosition().x - (xPos + 1.) ) < FLT_EPSILON && + fabs(this->m_vTemplatePointWarpInfo[j].GetPosition().y - yPos ) < FLT_EPSILON ) + { + tblr[i][1] = j; + } + + // top + if ( fabs(this->m_vTemplatePointWarpInfo[j].GetPosition().x - xPos ) < FLT_EPSILON && + fabs(this->m_vTemplatePointWarpInfo[j].GetPosition().y - (yPos - 1.) ) < FLT_EPSILON ) + { + tblr[i][2] = j; + } + + // bottom + if ( fabs(this->m_vTemplatePointWarpInfo[j].GetPosition().x - xPos ) < FLT_EPSILON && + fabs(this->m_vTemplatePointWarpInfo[j].GetPosition().y - (yPos + 1.) ) < FLT_EPSILON ) + { + tblr[i][3] = j; + } + } + } + + vector< vector > tempGradients, tempNormalizedGradients; + tempGradients.resize(this->m_iNbOfChannels); + tempNormalizedGradients.resize(this->m_iNbOfChannels); + for (unsigned int m = 0; m < this->m_iNbOfChannels; m++) + { + tempGradients[m].resize(2); + tempNormalizedGradients[m].resize(2); + } + + for (unsigned int i = 0; i < this->m_iNbOfPixels; i++) // 30132 + { + for (unsigned int m = 0; m < this->m_iNbOfChannels; m++) + { + tempGradients[m][0] = 0.0f; + tempGradients[m][1] = 0.0f; + tempNormalizedGradients[m][0] = 0.0f; + tempNormalizedGradients[m][1] = 0.0f; + } + ////////////////////////////////////////////////////////////////////////// + + // noted by JIA Pei, it is obvious that for the current point this->m_vNormalizedPointWarpInfo[i] + // it can't be both at the top and at the bottom; also, it can't be both at the left and the right + // So, tblr[0] and tblr[1] shouldn't be the same point; tblr[2] and tblr[3] shouldn't be the same point + // let's simply add an assert here + //assert(tblr[i][0] ^ tblr[i][1]); + //assert(tblr[i][2] ^ tblr[i][3]); + ////////////////////////////////////////////////////////////////////////// + // Noted by JIA Pei, we always assume the face is lighter than the background + // Calculate the gradient + // Have to allocate the space first, this should be changed in future + + // Explained by JIA Pei... At the very beginning, set all the gradients that can be calculated first. + // 2.0 / this->m_fAverageShapeSize very important here. Can't use 2.0 directly!!! + // neither the leftmost nor the rightmost + if ( (tblr[i][0] != -1) && (tblr[i][1] != -1) ) + { + for (unsigned int k = 0; k < this->m_iNbOfChannels; k++) + { + tempGradients[k][0] = (this->m_VOReferenceTexture.GetATexture(k, tblr[i][1]) - + this->m_VOReferenceTexture.GetATexture(k, tblr[i][0]) )/2.0f; + tempNormalizedGradients[k][0] = + ( this->m_VONormalizedMeanTexture.GetATexture(k, tblr[i][1]) - + this->m_VONormalizedMeanTexture.GetATexture(k, tblr[i][0]) ) / + (this->m_vNormalizedPointWarpInfo[tblr[i][1]].GetPosition().x - + this->m_vNormalizedPointWarpInfo[tblr[i][0]].GetPosition().x ); + } + } + + if ( (tblr[i][2] != -1) && (tblr[i][3] != -1) ) + { + for (unsigned int k = 0; k < this->m_iNbOfChannels; k++) + { + tempGradients[k][1] = (this->m_VOReferenceTexture.GetATexture(k, tblr[i][3]) - + this->m_VOReferenceTexture.GetATexture(k, tblr[i][2]) )/2.0f; + tempNormalizedGradients[k][1] = + ( this->m_VONormalizedMeanTexture.GetATexture(k, tblr[i][3]) - + this->m_VONormalizedMeanTexture.GetATexture(k, tblr[i][2]) ) / + (this->m_vNormalizedPointWarpInfo[tblr[i][3]].GetPosition().y - + this->m_vNormalizedPointWarpInfo[tblr[i][2]].GetPosition().y ); + } + } + + this->m_vTemplatePointWarpInfo[i].SetGradients (tempGradients); + this->m_vNormalizedPointWarpInfo[i].SetGradients (tempNormalizedGradients); + } + + for (unsigned int i = 0; i < this->m_iNbOfPixels; i++) // 30132 + { + for (unsigned int m = 0; m < this->m_iNbOfChannels; m++) + { + tempGradients[m][0] = 0.0f; + tempGradients[m][1] = 0.0f; + tempNormalizedGradients[m][0] = 0.0f; + tempNormalizedGradients[m][1] = 0.0f; + } + + // leftmost, not rightmost + if (tblr[i][0] == -1 && tblr[i][1] != -1) + { + for (unsigned int k = 0; k < this->m_iNbOfChannels; k++) + { + tempGradients[k][0] = this->m_vTemplatePointWarpInfo[ tblr[i][1] ].GetGradients()[k][0]; + tempNormalizedGradients[k][0] = this->m_vNormalizedPointWarpInfo[ tblr[i][1] ].GetGradients()[k][0]; + } + this->m_vTemplatePointWarpInfo[i].SetGradients (tempGradients); + this->m_vNormalizedPointWarpInfo[i].SetGradients (tempNormalizedGradients); + tblr[i][0] = -2; + } + // rightmost, not leftmost + else if (tblr[i][1] == -1 && tblr[i][0] != -1) + { + for (unsigned int k = 0; k < this->m_iNbOfChannels; k++) + { + tempGradients[k][0] = this->m_vTemplatePointWarpInfo[ tblr[i][0] ].GetGradients()[k][0]; + tempNormalizedGradients[k][0] = this->m_vNormalizedPointWarpInfo[ tblr[i][0] ].GetGradients()[k][0]; + } + this->m_vTemplatePointWarpInfo[i].SetGradients (tempGradients); + this->m_vNormalizedPointWarpInfo[i].SetGradients (tempNormalizedGradients); + tblr[i][1] = -2; + } + else if (tblr[i][1] == -1 && tblr[i][0] == -1) // leftmost and rightmost at the same time + { + for (unsigned int k = 0; k < this->m_iNbOfChannels; k++) + { + tempGradients[k][0] = 0; + tempNormalizedGradients[k][0] = 0; + } + this->m_vTemplatePointWarpInfo[i].SetGradients (tempGradients); + this->m_vNormalizedPointWarpInfo[i].SetGradients (tempNormalizedGradients); + tblr[i][0] = -2; + tblr[i][1] = -2; + } + + // topmost, not bottommost + if (tblr[i][2] == -1 && tblr[i][3] != -1) + { + for (unsigned int k = 0; k < this->m_iNbOfChannels; k++) + { + tempGradients[k][1] = this->m_vTemplatePointWarpInfo[ tblr[i][3] ].GetGradients()[k][1]; + tempNormalizedGradients[k][1] = this->m_vNormalizedPointWarpInfo[ tblr[i][3] ].GetGradients()[k][1]; + } + this->m_vTemplatePointWarpInfo[i].SetGradients (tempGradients); + this->m_vNormalizedPointWarpInfo[i].SetGradients (tempNormalizedGradients); + tblr[i][2] = -2; + } + // bottommost, not topmost + else if (tblr[i][3] == -1 && tblr[i][2] != -1) + { + for (unsigned int k = 0; k < this->m_iNbOfChannels; k++) + { + tempGradients[k][1] = this->m_vTemplatePointWarpInfo[ tblr[i][2] ].GetGradients()[k][1]; + tempNormalizedGradients[k][1] = this->m_vNormalizedPointWarpInfo[ tblr[i][2] ].GetGradients()[k][1]; + } + this->m_vTemplatePointWarpInfo[i].SetGradients (tempGradients); + this->m_vNormalizedPointWarpInfo[i].SetGradients (tempNormalizedGradients); + tblr[i][3] = -2; + } + else if (tblr[i][3] == -1 && tblr[i][2] == -1) // topmost and bottommost at the same time + { + for (unsigned int k = 0; k < this->m_iNbOfChannels; k++) + { + tempGradients[k][1] = 0; + tempNormalizedGradients[k][1] = 0; + } + this->m_vTemplatePointWarpInfo[i].SetGradients (tempGradients); + this->m_vNormalizedPointWarpInfo[i].SetGradients (tempNormalizedGradients); + tblr[i][2] = -2; + tblr[i][3] = -2; + } + } + + + Mat_ templateGradientX = Mat_::zeros( this->m_iNbOfChannels, this->m_iNbOfPixels ); + Mat_ templateGradientY = Mat_::zeros( this->m_iNbOfChannels, this->m_iNbOfPixels ); + VO_Texture templateTextureInstanceX, templateTextureInstanceY, templateTextureInstance; + + for (unsigned int i = 0; i < this->m_iNbOfPixels; i++) + { + for (unsigned int j = 0; j < this->m_iNbOfChannels; j++) + { + templateGradientX(j, i) = this->m_vTemplatePointWarpInfo[i].GetGradients()[j][0] + AVERAGEFACETEXTURE; + templateGradientY(j, i) = this->m_vTemplatePointWarpInfo[i].GetGradients()[j][1] + AVERAGEFACETEXTURE; + } + } + templateTextureInstanceX.SetTheTexture(templateGradientX); + templateTextureInstanceY.SetTheTexture(templateGradientY); + templateTextureInstanceX.Clamp(0.0f, 255.0f); + templateTextureInstanceY.Clamp(0.0f, 255.0f); + templateTextureInstance = m_VOReferenceTexture; + + VO_TextureModel::VO_PutOneTextureToTemplateShape(templateTextureInstanceX, this->m_vTemplateTriangle2D, this->m_IplImageTempFaceX); + VO_TextureModel::VO_PutOneTextureToTemplateShape(templateTextureInstanceY, this->m_vTemplateTriangle2D, this->m_IplImageTempFaceY); + VO_TextureModel::VO_PutOneTextureToTemplateShape(templateTextureInstance, this->m_vTemplateTriangle2D, this->m_IplImageTempFace); + +} + + +/** + * @author JIA Pei + * @version 2010-04-03 + * @brief Calculate steepest descent image for template face + * @return void +*/ +void VO_AAMInverseIA::VO_CalcSDI() +{ + // AAM Revisited equation (42) + // calculate the m_MatSimilarityTransform (for Global Shape Normalizing Transform) + this->m_MatSimilarityTransform = Mat_::zeros(4, this->m_iNbOfShapes); + for (unsigned int i = 0; i < this->m_iNbOfPoints; i++) + { + this->m_MatSimilarityTransform(0, i) = this->m_PCAAlignedShape.mean.at(0, i); + this->m_MatSimilarityTransform(0, i+this->m_iNbOfPoints) = this->m_PCAAlignedShape.mean.at(0, i+this->m_iNbOfPoints); + this->m_MatSimilarityTransform(1, i) = -this->m_PCAAlignedShape.mean.at(0, i+this->m_iNbOfPoints); + this->m_MatSimilarityTransform(1, i+this->m_iNbOfPoints) = this->m_PCAAlignedShape.mean.at(0, i); + this->m_MatSimilarityTransform(2, i) = 1.0f; + this->m_MatSimilarityTransform(2, i+this->m_iNbOfPoints) = 0.0f; + this->m_MatSimilarityTransform(3, i) = 0.0f; + this->m_MatSimilarityTransform(3, i+this->m_iNbOfPoints) = 1.0f; +// this->m_MatSimilarityTransform(2, i) = 1.0f/sqrt((float)this->m_iNbOfPoints); +// this->m_MatSimilarityTransform(3, i+this->m_iNbOfPoints) = 1.0f/sqrt((float)this->m_iNbOfPoints); + } + + // AAM Revisited, before (50) + // "evaluating the Jacobian at p = 0, q = 0." + // Explained by JIA Pei. The above citation means, when calculating the Jacobian + // partial(N)/partial(q) and partial(W)/partial(q), p=q=0 requires + // m_vNormalizedPointWarpInfo, rather than m_vTemplatePointWarpInfo + for (unsigned int i = 0; i < this->m_iNbOfPixels; i++) + { + this->m_vNormalizedPointWarpInfo[i].CalcJacobianOne(); + this->m_vNormalizedPointWarpInfo[i].CalcJacobianMatrix4ShapeModel(this->m_PCAAlignedShape.eigenvectors); + this->m_vNormalizedPointWarpInfo[i].CalcJacobianMatrix4GlobalShapeNorm(this->m_MatSimilarityTransform); + this->m_vNormalizedPointWarpInfo[i].CalcSteepestDescentImages4ShapeModel (this->m_iNbOfChannels); + this->m_vNormalizedPointWarpInfo[i].CalcSteepestDescentImages4GlobalShapeNorm (this->m_iNbOfChannels); + } + + this->m_MatSteepestDescentImages4ShapeModel = Mat_::zeros(this->m_iNbOfTextures, this->m_iNbOfShapeEigens); + this->m_MatSteepestDescentImages4GlobalShapeNormalization = Mat_::zeros(this->m_iNbOfTextures, 4); + + for (unsigned int i = 0; i < this->m_iNbOfPixels; i++) + { + for (unsigned int j = 0; j < this->m_iNbOfShapeEigens; j++) + { + for (unsigned int k = 0; k < this->m_iNbOfChannels; k++) + { + this->m_MatSteepestDescentImages4ShapeModel(this->m_iNbOfChannels * i + k, j) + = this->m_vNormalizedPointWarpInfo[i].GetSteepestDescentImages4ShapeModel()[k][j]; + } + } + } + + for (unsigned int i = 0; i < this->m_iNbOfPixels; i++) + { + for (unsigned int j = 0; j < 4; j++) + { + for (unsigned int k = 0; k < this->m_iNbOfChannels; k++) + { + this->m_MatSteepestDescentImages4GlobalShapeNormalization(this->m_iNbOfChannels * i + k, j) + = this->m_vNormalizedPointWarpInfo[i].GetSteepestDescentImages4GlobalShapeNorm()[k][j]; + } + } + } +} + + +/** + * @author JIA Pei + * @version 2010-04-03 + * @brief Calculate modified steepest descent image for template face - project out appearance variation + * @return void +*/ +void VO_AAMInverseIA::VO_CalcModifiedSDI() +{ + //project out appearance variation i.e. modify the steepest descent image + this->m_MatSteepestDescentImages = Mat_::zeros(this->m_iNbOfTextures, this->m_iNbOfShapeEigens+4); + this->m_MatModifiedSteepestDescentImages = Mat_::zeros(this->m_iNbOfTextures, this->m_iNbOfShapeEigens+4); + + for (unsigned int i = 0; i < this->m_iNbOfTextures; i++) + { + // AAM Revisited (63) + for (unsigned int j = 0; j < 4; j++) + { + this->m_MatSteepestDescentImages(i, j) = this->m_MatSteepestDescentImages4GlobalShapeNormalization(i, j); + } + // AAM Revisited (64) + for (unsigned int j = 0; j < this->m_iNbOfShapeEigens; j++) + { + this->m_MatSteepestDescentImages(i, 4+j) = this->m_MatSteepestDescentImages4ShapeModel(i, j); + } + } + + Mat_ oneCol = Mat_::zeros(this->m_iNbOfTextures, 1); + Mat_ spanedsum = Mat_::zeros(this->m_iNbOfTextures, 1); + Mat_ modifiedoneCol = Mat_::zeros(this->m_iNbOfTextures, 1); + Mat_ oneSpanRowTranspose = Mat_::zeros(this->m_iNbOfTextures, 1); + + for (int i = 0; i < this->m_MatSteepestDescentImages.cols; i++) + { + spanedsum = Mat_::zeros(this->m_iNbOfTextures, 1); + oneCol = this->m_MatSteepestDescentImages.col(i); + for (unsigned int j = 0; j < this->m_iNbOfTextureEigens; j++) + { + oneSpanRowTranspose = this->m_PCANormalizedTexture.eigenvectors.row(j).t(); + double weight = oneSpanRowTranspose.dot(oneCol); + + // dst(I)=src1(I)*alpha+src2(I)*beta+gamma + cv::addWeighted( spanedsum, 1.0, oneSpanRowTranspose, weight, 0.0, spanedsum ); + } + + cv::subtract(oneCol, spanedsum, modifiedoneCol); + Mat_ tmpCol = this->m_MatModifiedSteepestDescentImages.col(i); + modifiedoneCol.copyTo(tmpCol); + } +} + + +/** + * @author JIA Pei + * @version 2010-04-03 + * @brief Calculate inverse Hessian matrix for template face + * @return void +*/ +void VO_AAMInverseIA::VO_CalcInverseHessian() +{ + // HessianMatrix to zeros + Mat_ HessianMatrix = Mat_::zeros ( this->m_iNbOfShapeEigens+4, this->m_iNbOfShapeEigens+4); + + cv::gemm(this->m_MatModifiedSteepestDescentImages, this->m_MatModifiedSteepestDescentImages, 1, Mat(), 0, HessianMatrix, GEMM_1_T); + + cv::invert( HessianMatrix, this->m_MatHessianMatrixInverse, CV_SVD ); +} + + +/** + * @author JIA Pei + * @version 2010-04-03 + * @brief Calculate Hessian matrix * MSDI^T + * @return void +*/ +void VO_AAMInverseIA::VO_CalcICIAPreMatrix() +{ + cv::gemm(this->m_MatHessianMatrixInverse, this->m_MatModifiedSteepestDescentImages, 1, Mat(), 0, this->m_MatICIAPreMatrix, GEMM_2_T); +} + + +/** + * @author JIA Pei + * @version 2010-04-03 + * @brief ICIA AAM + * @param allLandmarkFiles4Training Input - all training landmark files + * @param allImgFiles4Training Input - all training image files + * @param shapeinfoFileName Input - shape info file + * @param database Input - which database is it? + * @param levels Input - multiscale levels + * @param channels Input - How many channels are to be used? + * @param trm Input - texture representation method + * @param TPShape Input - truncated percentage for shape model + * @param TPTexture Input - truncated percentage for texture model + * @param useKnownTriangles Input - use known triangle structures?? + * @return void + * @note Using "float* oProf" is much much faster than using "VO_Profile& oProf" or "vector" + */ + void VO_AAMInverseIA::VO_BuildAAMICIA( const vector& allLandmarkFiles4Training, + const vector& allImgFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + unsigned int channels, + unsigned int levels, + int trm, + float TPShape, + float TPTexture, + bool useKnownTriangles) +{ + this->VO_BuildTextureModel( allLandmarkFiles4Training, + allImgFiles4Training, + shapeinfoFileName, + database, + channels, + trm, + TPShape, + TPTexture, + useKnownTriangles); + this->m_iNbOfPyramidLevels = levels; + this->VO_CalcTemplateFaceGradients(); + this->VO_CalcSDI(); + this->VO_CalcModifiedSDI(); + this->VO_CalcInverseHessian(); + this->VO_CalcICIAPreMatrix(); +} + + +/** + * @author JIA Pei + * @version 2010-04-03 + * @brief Save AAMICIA to a specified folder + * @param fn Input - the folder that AAMICIA to be saved to +*/ +void VO_AAMInverseIA::VO_Save(const string& fd) +{ + VO_AXM::VO_Save(fd); + + string fn = fd+"/AAMICIA"; + MakeDirectory(fn); + + fstream fp; + string tempfn; + + // m_IplImageTempFaceX + tempfn = fn + "/m_IplImageTempFaceX.jpg"; + imwrite(tempfn.c_str(), this->m_IplImageTempFaceX); + + // m_IplImageTempFaceY + tempfn = fn + "/m_IplImageTempFaceY.jpg"; + imwrite(tempfn.c_str(), this->m_IplImageTempFaceY); + + // m_IplImageTempFace + tempfn = fn + "/m_IplImageTempFace.jpg"; + imwrite(tempfn.c_str(), this->m_IplImageTempFace); + + // m_MatSimilarityTransform + tempfn = fn + "/m_MatSimilarityTransform" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_MatSimilarityTransform" << endl; + fp << this->m_MatSimilarityTransform; + fp.close();fp.clear(); + + // m_MatSteepestDescentImages4ShapeModel + tempfn = fn + "/m_MatSteepestDescentImages4ShapeModel" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_MatSteepestDescentImages4ShapeModel" << endl; + fp << this->m_MatSteepestDescentImages4ShapeModel; + fp.close();fp.clear(); + + // m_MatSteepestDescentImages4GlobalShapeNormalization + tempfn = fn + "/m_MatSteepestDescentImages4GlobalShapeNormalization" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_MatSteepestDescentImages4GlobalShapeNormalization" << endl; + fp << this->m_MatSteepestDescentImages4GlobalShapeNormalization; + fp.close();fp.clear(); + + // m_MatSteepestDescentImages + tempfn = fn + "/m_MatSteepestDescentImages" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_MatSteepestDescentImages" << endl; + fp << this->m_MatSteepestDescentImages; + fp.close();fp.clear(); + + // m_MatModifiedSteepestDescentImages + tempfn = fn + "/m_MatModifiedSteepestDescentImages" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_MatModifiedSteepestDescentImages" << endl; + fp << this->m_MatModifiedSteepestDescentImages; + fp.close();fp.clear(); + + // m_MatHessianMatrixInverse + tempfn = fn + "/m_MatHessianMatrixInverse" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_MatHessianMatrixInverse" << endl; + fp << this->m_MatHessianMatrixInverse; + fp.close();fp.clear(); + + // m_MatICIAPreMatrix + tempfn = fn + "/m_MatICIAPreMatrix" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_MatICIAPreMatrix" << endl; + fp << this->m_MatICIAPreMatrix; + fp.close();fp.clear(); + +} + + +/** + * @author JIA Pei + * @version 2010-04-03 + * @brief Load all AAMICIA data from a specified folder + * @param fd Input - the folder that AAMICIA to be loaded from +*/ +void VO_AAMInverseIA ::VO_Load(const string& fd) +{ + VO_AXM::VO_Load(fd); + + string fn = fd+"/AAMICIA"; + if (!MakeDirectory(fn) ) + { + cout << "AAMICIA subfolder is not existing. " << endl; + exit(EXIT_FAILURE); + } + + fstream fp; + string tempfn; + string temp; + + //// AAMICIA + //tempfn = fn + "/AAMICIA" + ".txt"; + //SafeInputFileOpen(fp, tempfn); + //fp.close();fp.clear(); + + // m_MatSimilarityTransform + tempfn = fn + "/m_MatSimilarityTransform" + ".txt"; + SafeOutputFileOpen(fp, tempfn); + fp >> temp; // m_MatSimilarityTransform + fp >> this->m_MatSimilarityTransform; + fp.close();fp.clear(); + + // m_MatSteepestDescentImages4ShapeModel + tempfn = fn + "/m_MatSteepestDescentImages4ShapeModel" + ".txt"; + SafeOutputFileOpen(fp, tempfn); + fp >> temp; // m_MatSteepestDescentImages4ShapeModel + fp >> this->m_MatSteepestDescentImages4ShapeModel; + fp.close();fp.clear(); + + // m_MatSteepestDescentImages4GlobalShapeNormalization + tempfn = fn + "/m_MatSteepestDescentImages4GlobalShapeNormalization" + ".txt"; + SafeOutputFileOpen(fp, tempfn); + fp >> temp; // m_MatSteepestDescentImages4GlobalShapeNormalization + fp >> this->m_MatSteepestDescentImages4GlobalShapeNormalization; + fp.close();fp.clear(); + + // m_MatSteepestDescentImages + tempfn = fn + "/m_MatSteepestDescentImages" + ".txt"; + SafeOutputFileOpen(fp, tempfn); + fp >> temp; // m_MatSteepestDescentImages + fp >> this->m_MatSteepestDescentImages; + fp.close();fp.clear(); + + // m_MatModifiedSteepestDescentImages + tempfn = fn + "/m_MatModifiedSteepestDescentImages" + ".txt"; + SafeOutputFileOpen(fp, tempfn); + fp >> temp; // m_MatModifiedSteepestDescentImages + fp >> this->m_MatModifiedSteepestDescentImages; + fp.close();fp.clear(); + + // m_MatHessianMatrixInverse + tempfn = fn + "/m_MatHessianMatrixInverse" + ".txt"; + SafeOutputFileOpen(fp, tempfn); + fp >> temp; // m_MatHessianMatrixInverse + fp >> this->m_MatHessianMatrixInverse; + fp.close();fp.clear(); + + // m_MatICIAPreMatrix + tempfn = fn + "/m_MatICIAPreMatrix" + ".txt"; + SafeOutputFileOpen(fp, tempfn); + fp >> temp; // m_MatICIAPreMatrix + fp >> this->m_MatICIAPreMatrix; + fp.close();fp.clear(); + +} + + +/** + * @author JIA Pei + * @version 2010-04-03 + * @brief Load all AAMICIA data from a specified folder + * @param fd Input - the folder that AAMICIA to be loaded from +*/ +void VO_AAMInverseIA ::VO_LoadParameters4Fitting(const string& fd) +{ + VO_AXM::VO_LoadParameters4Fitting(fd); + + string fn = fd+"/AAMICIA"; + if (!MakeDirectory(fn) ) + { + cout << "AAMICIA subfolder is not existing. " << endl; + exit(EXIT_FAILURE); + } + + fstream fp; + string tempfn; + string temp; + + // m_MatICIAPreMatrix + this->m_MatICIAPreMatrix = Mat_::zeros(this->m_iNbOfShapeEigens+4, this->m_iNbOfTextures); + tempfn = fn + "/m_MatICIAPreMatrix" + ".txt"; + SafeOutputFileOpen(fp, tempfn); + fp >> temp; // m_MatICIAPreMatrix + fp >> this->m_MatICIAPreMatrix; + fp.close();fp.clear(); + + // m_MatSimilarityTransform + this->m_MatSimilarityTransform = Mat_::zeros(4, this->m_iNbOfShapes); + tempfn = fn + "/m_MatSimilarityTransform" + ".txt"; + SafeOutputFileOpen(fp, tempfn); + fp >> temp; // m_MatSimilarityTransform + fp >> this->m_MatSimilarityTransform; + fp.close();fp.clear(); + +} + diff --git a/modules/smbuilding/src/VO_AFM.cpp b/modules/smbuilding/src/VO_AFM.cpp new file mode 100644 index 0000000..d3b1c7e --- /dev/null +++ b/modules/smbuilding/src/VO_AFM.cpp @@ -0,0 +1,324 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include +#include +#include + +#include "VO_AFM.h" +#include "VO_CVCommon.h" + + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief feature parameters constrain + * @param ioC Input and Output - appearance parameters + * @param nSigma Input - number of sigmas + * @return void +*/ +void VO_AFM::VO_FeatureParameterConstraint(Mat_& ioF, float nSigma) +{ + for (int i = 0; i < ioF.cols; ++i) + { + float ct = nSigma * sqrt(this->m_PCANormalizedFeatures.eigenvalues.at(i,0) ); + if ( ioF(0, i) > ct ) + { + ioF(0, i) = ct; + } + else if ( ioF(0, i) < -ct ) + { + ioF(0, i) = -ct; + } + } +} + + +/** + * @author JIA Pei + * @version 2010-02-22 + * @brief Build Active Feature Model - quite similar to Edwards method, but proposed by Cristinace and modified by me. + * @param allLandmarkFiles4Training Input - all training landmark files + * @param allImgFiles4Training Input - all training image files + * @param shapeinfoFileName Input - shape info file + * @param database Input - which database is it? + * @param channels Input - How many channels are to be used? + * @param levels Input - multiscale levels + * @param trm Input - texture representation method + * @param TPShape Input - truncated percentage for shape model + * @param useKnownTriangles Input - use known triangle structures?? + * @param ltcMtd Input - local texture constrain methods + * @return void + * @note Using "float* oProf" is much much faster than using "VO_Profile& oProf" or "vector" + */ +void VO_AFM::VO_BuildFeatureModel ( const vector& allLandmarkFiles4Training, + const vector& allImgFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + unsigned int channels, + unsigned int levels, + int trm, + float TPShape, + bool useKnownTriangles, + unsigned int ltcMtd, + Size imgSize) +{ + if (allLandmarkFiles4Training.size() != allImgFiles4Training.size() ) + cerr << "allLandmarkFiles4Training should have the same number of allImgFiles4Training! " << endl; + + this->VO_BuildShapeModel(allLandmarkFiles4Training, shapeinfoFileName, database, TPShape, useKnownTriangles); + this->m_iNbOfPyramidLevels = levels; + this->m_iNbOfChannels = channels; + this->m_iTextureRepresentationMethod = trm; + this->m_iLocalFeatureExtractionMethod = ltcMtd; + this->m_localImageSize = imgSize; + this->m_vStringTrainingImageNames = allImgFiles4Training; + + this->VO_LoadFeatureTrainingData( this->m_iLocalFeatureExtractionMethod); +} + + +/** + * @author JIA Pei + * @version 2010-02-22 + * @brief Calculate Image Patch Rectangle + * @param iImg Input -- the concerned image + * @param pt Input -- the point + * @param imgSize Input -- image size + * @return Rect of size imgSize*imgSize + */ +Rect VO_AFM::VO_CalcImagePatchRect( const Mat& iImg, + const Point2f& pt, + Size imgSize) +{ + // ensure the small image patch is within the image + Rect rect; + if(pt.x - imgSize.width/2 >= 0) + { + if(pt.x + imgSize.width/2 < iImg.cols) + rect.x = cvRound( pt.x - imgSize.width/2); + else + rect.x = cvFloor(iImg.cols - imgSize.width); + } + else rect.x = 0; + + if(pt.y - imgSize.height/2 >= 0) + { + if(pt.y + imgSize.height/2 < iImg.rows) + rect.y = cvRound(pt.y - imgSize.height/2); + else + rect.y = cvFloor(iImg.rows - imgSize.height); + } + else rect.y = 0; + + rect.width = imgSize.width; + rect.height = imgSize.height; + + return rect; +} + + +/** + * @author JIA Pei + * @version 2010-02-05 + * @brief Load Training data for texture model + * @param mtd Input -- ltc method + * @return void +*/ +void VO_AFM::VO_LoadFeatureTrainingData(unsigned int mtd) +{ + this->m_vLTCs.resize(this->m_iNbOfSamples); + this->m_vNormalizedLTCs.resize(this->m_iNbOfSamples); + Mat img; + Mat_ oneLTC, tmpRow; + + for(unsigned int i = 0; i < this->m_iNbOfSamples; ++i) + { + unsigned int count = 0; + this->m_vLTCs[i].Resize(this->m_iNbOfPoints, this->m_localImageSize.width*this->m_localImageSize.height); + img = imread ( this->m_vStringTrainingImageNames[i].c_str (), 0 ); + + double start = (double)cvGetTickCount(); + // Explained by JIA Pei -- wavelet feature extraction + for(unsigned int j = 0; j < this->m_iNbOfPoints; ++j) + { + oneLTC = this->VO_LoadLTC4OneAnnotatedPoint(img, + this->m_vShapes[i], + j, + this->m_localImageSize, + mtd); + tmpRow = this->m_vLTCs[i].m_MatTexture.row(j); + oneLTC.copyTo(tmpRow); + } + + double end = (double)cvGetTickCount(); + double elapsed = (end - start) / (cvGetTickFrequency()*1000.0); + } +} + + +/** + * @author JIA Pei + * @version 2010-02-22 + * @brief Build wavelet for key points + * @param iImg Input -- the concerned image + * @param theShape Input -- the concerned shape + * @param ptIdx Input -- which point? + * @param imgSize Input -- the image size + * @param mtd Input -- LTC method + * @return Mat_ Output -- the extracted LTC + */ +Mat_ VO_AFM::VO_LoadLTC4OneAnnotatedPoint( const Mat& iImg, + const VO_Shape& theShape, + unsigned int ptIdx, + Size imgSize, + unsigned int mtd) +{ + Mat_ resLTC; + Point2f pt = theShape.GetA2DPoint(ptIdx); + Rect rect = this->VO_CalcImagePatchRect(iImg, pt, imgSize); + Mat imgPatch = iImg(rect); + + switch(mtd) + { + case VO_Features::LBP: + { + // initialize the image before wavelet transform + for(int i = 0; i < rect.height; ++i) + { + for(int j = 0; j < rect.width; ++j) + { + + } + } + + bool showWaveletImage = true; + if(showWaveletImage) + { + imwrite("originalImage.jpg", imgPatch); + // this->VO_HardSaveWaveletSingleChannelImage("waveletImage.jpg", waveParamsGray, imgSize); + // this->VO_HardSaveWaveletSingleChannelImage("inverseWaveletImage.jpg", waveParamsGray, imgSize); + } + } + default: + break; + } + + return resLTC; +} + + +void VO_AFM::VO_Save ( const string& fd ) +{ + VO_AXM::VO_Save(fd); + + // create AFM subfolder for just AFM model data + string fn = fd+"/AFM"; + MakeDirectory(fn); + + ofstream fp; + string tempfn; + + // AFM + tempfn = fn + "/AFM" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + + fp << "m_iLocalFeatureExtractionMethod" << endl << this->m_iLocalFeatureExtractionMethod << endl; + fp << "m_localImageSize" << endl << this->m_localImageSize.height << " " << this->m_localImageSize.width << endl; + + fp.close();fp.clear(); + + // m_PCANormalizedLTCMean + tempfn = fn + "/m_PCANormalizedLTCMean" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_PCANormalizedLTCMean" << endl; + fp << Mat_(this->m_PCANormalizedFeatures.mean); + fp.close();fp.clear(); + + // m_PCANormalizedLTCEigenvectors + tempfn = fn + "/m_PCANormalizedLTCEigenvectors" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_PCANormalizedLTCEigenvectors" << endl; + fp << Mat_(this->m_PCANormalizedFeatures.eigenvectors); + fp.close();fp.clear(); + + // m_PCANormalizedLTCEigenvalues + tempfn = fn + "/m_PCANormalizedLTCEigenvalues" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_PCANormalizedLTCEigenvalues" << endl; + fp << Mat_(this->m_PCANormalizedFeatures.eigenvalues); + fp.close();fp.clear(); + +} + + +void VO_AFM::VO_Load ( const string& fd ) +{ + VO_AXM::VO_Load(fd); +} + + +void VO_AFM::VO_LoadParameters4Fitting ( const string& fd ) +{ + VO_AXM::VO_LoadParameters4Fitting(fd); // Note, for ASMProfile fitting, no problem +} + diff --git a/modules/smbuilding/src/VO_ASMLTCs.cpp b/modules/smbuilding/src/VO_ASMLTCs.cpp new file mode 100644 index 0000000..2d0b217 --- /dev/null +++ b/modules/smbuilding/src/VO_ASMLTCs.cpp @@ -0,0 +1,543 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include +#include +#include + +#include "VO_ASMLTCs.h" +#include "VO_CVCommon.h" + +#include "VO_DirectFeatures.h" +#include "VO_LBPFeatures.h" +#include "VO_HaarFeatures.h" +#include "VO_GaborFeatures.h" +#include "VO_DaubechiesFeatures.h" + + +/** + * @author JIA Pei + * @version 2010-02-22 + * @brief Build Local Texture Constraint ASM + * @param allLandmarkFiles4Training Input - all training landmark files + * @param allImgFiles4Training Input - all training image files + * @param shapeinfoFileName Input - shape info file + * @param database Input - which database is it? + * @param channels Input - How many channels are to be used? + * @param levels Input - multiscale levels + * @param trm Input - texture representation method + * @param TPShape Input - truncated percentage for shape model + * @param useKnownTriangles Input - use known triangle structures?? + * @param ltcMtd Input - local texture constrain methods + * @return void + * @note Using "float* oProf" is much much faster than using "VO_Profile& oProf" or "vector" + */ +void VO_ASMLTCs::VO_BuildASMLTCs ( const vector& allLandmarkFiles4Training, + const vector& allImgFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + unsigned int channels, + unsigned int levels, + int trm, + float TPShape, + bool useKnownTriangles, + unsigned int ltcMtd, + Size imgSize) +{ + if (allLandmarkFiles4Training.size() != allImgFiles4Training.size() ) + cerr << "allLandmarkFiles4Training should have the same number of allImgFiles4Training! " << endl; + + this->VO_BuildShapeModel(allLandmarkFiles4Training, shapeinfoFileName, database, TPShape, useKnownTriangles); + this->m_iNbOfPyramidLevels = levels; + this->m_iNbOfChannels = channels; + this->m_iTextureRepresentationMethod = trm; + this->m_iLTCMethod = ltcMtd; + this->m_localImageSize = imgSize; + this->m_vStringTrainingImageNames = allImgFiles4Training; + + // Initialize all member variables + switch(this->m_iLTCMethod) + { + case VO_Features::LBP: + this->m_pVOfeatures = new VO_LBPFeatures(); + break; + case VO_Features::HAAR: + this->m_pVOfeatures = new VO_HaarFeatures(); + break; + case VO_Features::GABOR: + this->m_pVOfeatures = new VO_GaborFeatures(); + break; + case VO_Features::DAUBECHIES: + this->m_pVOfeatures = new VO_DaubechiesFeatures(); + break; + case VO_Features::DIRECT: + default: + this->m_pVOfeatures = new VO_DirectFeatures(); + break; + } + this->m_pVOfeatures->VO_GenerateAllFeatureInfo(this->m_localImageSize, 2); + this->m_iNbOfLTC4PerPoint = this->m_pVOfeatures->GetNbOfFeatures(); + + this->m_vvCVMInverseOfLTCCov.resize(this->m_iNbOfPyramidLevels); + this->m_vvLTCMeans.resize(this->m_iNbOfPyramidLevels); + for(unsigned int i = 0; i < this->m_iNbOfPyramidLevels; i++) + { + this->m_vvCVMInverseOfLTCCov[i].resize(this->m_iNbOfPoints); + this->m_vvLTCMeans[i].resize(this->m_iNbOfPoints); + } + + this->m_vvLTCs.resize(this->m_iNbOfSamples); + this->m_vvNormalizedLTCs.resize(this->m_iNbOfSamples); + for(unsigned int i = 0; i < this->m_iNbOfSamples; ++i) + { + this->m_vvLTCs[i].resize(this->m_iNbOfPyramidLevels); + this->m_vvNormalizedLTCs[i].resize(this->m_iNbOfPyramidLevels); + for(unsigned int j = 0; j < this->m_iNbOfPyramidLevels; ++j) + { + this->m_vvLTCs[i][j] = Mat_::zeros(this->m_iNbOfPoints, this->m_iNbOfLTC4PerPoint ); + this->m_vvNormalizedLTCs[i][j] = Mat_::zeros(this->m_iNbOfPoints, this->m_iNbOfLTC4PerPoint ); + } + } + + + this->VO_LoadLTCTrainingData(); + this->VO_CalcStatistics4AllLTCs(); +} + + +/** + * @author JIA Pei + * @version 2010-02-22 + * @brief Calculate Image Patch Rectangle + * @param iImg Input -- the concerned image + * @param pt Input -- the point in the center of the image patch + * @param imgSize Input -- image patch size + * @return Rect of size imgSize*imgSize + */ +Rect VO_ASMLTCs::VO_CalcImagePatchRect( const Mat& iImg, + const Point2f& pt, + Size imgSize) +{ + // ensure the small image patch is within the image + Rect rect; + if(pt.x - imgSize.width/2 >= 0) + { + if(pt.x + imgSize.width/2 < iImg.cols) + rect.x = cvRound( pt.x - imgSize.width/2); + else + rect.x = cvFloor(iImg.cols - imgSize.width); + } + else rect.x = 0; + + if(pt.y - imgSize.height/2 >= 0) + { + if(pt.y + imgSize.height/2 < iImg.rows) + rect.y = cvRound(pt.y - imgSize.height/2); + else + rect.y = cvFloor(iImg.rows - imgSize.height); + } + else rect.y = 0; + + rect.width = imgSize.width; + rect.height = imgSize.height; + + return rect; +} + + +/** + * @author JIA Pei + * @version 2010-02-05 + * @brief Load Training data for texture model + * @param mtd Input -- ltc method + * @return void +*/ +void VO_ASMLTCs::VO_LoadLTCTrainingData() +{ + Mat img, resizedImg; + VO_Shape resizedShape; + float PyrScale, scale, sampleScale; + float refScale = this->m_VOReferenceShape.GetCentralizedShapeSize(); + + for(unsigned int i = 0; i < this->m_iNbOfSamples; ++i) + { + img = imread ( this->m_vStringTrainingImageNames[i].c_str (), 0 ); + + sampleScale = this->m_vShapes[i].GetCentralizedShapeSize(); + + for( unsigned int j = 0; j < this->m_iNbOfPyramidLevels; ++j) + { + PyrScale = pow(2.0f, (float)j); + scale = refScale/sampleScale/PyrScale; + + resizedShape = this->m_vShapes[i]*scale; + cv::resize(img, resizedImg, Size( (int)(img.cols*scale), (int)(img.rows*scale) ) ); + +// static stringstream ss; +// static string ssi; +// ss << i; +// ss >> ssi; +// static string fn = ssi+".jpg"; +// imwrite(fn.c_str(), resizedImg); + + // Explained by JIA Pei -- wavelet feature extraction + for( unsigned int k = 0; k < this->m_iNbOfPoints; ++k ) + { + VO_ASMLTCs::VO_LoadLTC4OneAnnotatedPoint( resizedImg, + resizedShape, + k, + this->m_localImageSize, + this->m_pVOfeatures); + Mat oneLTC = this->m_pVOfeatures->m_MatFeatures; + Mat oneLTCNormalized = this->m_pVOfeatures->m_MatFeatures; + Mat tmpRow = this->m_vvLTCs[i][j].row(k); + oneLTC.copyTo( tmpRow ); + cv::normalize(oneLTC, oneLTCNormalized); + Mat tmpRow1 = this->m_vvNormalizedLTCs[i][j].row(k); + oneLTCNormalized.copyTo(tmpRow1); + } + } + } +} + + +/** + * @author JIA Pei + * @version 2010-02-22 + * @brief Build wavelet for key points + * @param iImg Input -- the concerned image + * @param theShape Input -- the concerned shape + * @param ptIdx Input -- which point? + * @param imgSize Input -- the image size + * @param mtd Input -- LTC method + * @param shiftX Input -- shift in X direction + * @param shiftY Input -- shift in Y direction + * @return Mat_ Output -- the extracted LTC + */ +void VO_ASMLTCs::VO_LoadLTC4OneAnnotatedPoint( const Mat& iImg, + const VO_Shape& theShape, + unsigned int ptIdx, + Size imgSize, + VO_Features* vofeatures, + int shiftX, + int shiftY) +{ + Point2f pt = theShape.GetA2DPoint(ptIdx); + pt.x += shiftX; + pt.y += shiftY; + Rect rect = VO_ASMLTCs::VO_CalcImagePatchRect(iImg, pt, imgSize); + Mat imgPatch = iImg(rect); + vofeatures->VO_GenerateAllFeatures(imgPatch); +} + + +void VO_ASMLTCs::VO_CalcStatistics4AllLTCs() +{ + Mat_ normalizedLTCs4SinglePoint = Mat_::zeros(this->m_iNbOfSamples, this->m_iNbOfLTC4PerPoint); + Mat_ Covar = Mat_::zeros(this->m_iNbOfLTC4PerPoint, this->m_iNbOfLTC4PerPoint ); + + for(unsigned int i = 0; i < this->m_iNbOfPyramidLevels; i++) + { + for(unsigned int j = 0; j < this->m_iNbOfPoints; j++) + { + for(unsigned int k = 0; k < this->m_iNbOfSamples; k++) + { + Mat_ tmpRow = normalizedLTCs4SinglePoint.row(k); + this->m_vvNormalizedLTCs[k][i].row(j).copyTo(tmpRow); + } + // OK! Now We Calculate the Covariance Matrix of prof for Landmark iPoint + cv::calcCovarMatrix(normalizedLTCs4SinglePoint, + Covar, + this->m_vvLTCMeans[i][j], + CV_COVAR_NORMAL+CV_COVAR_ROWS+CV_COVAR_SCALE, + CV_32F); + this->m_vvCVMInverseOfLTCCov[i][j] = Covar.inv(DECOMP_SVD); + } + } +} + + +/** + * @author JIA Pei + * @version 2010-02-22 + * @brief Save wavelet image , 1 single channel + * @param fn Input -- file name + * @param imgSize Input -- the transformed image + * @param displayMtd Input -- two choices, 1) show directly with clamp 2) normalize to 0-255 without clamp + * @return void + */ +void VO_ASMLTCs::VO_HardSaveWaveletSingleChannelImage( const string& fn, + Size imgSize, + unsigned int displayMtd) +{ + Mat img = Mat::ones(imgSize, CV_8UC1);(imgSize, CV_8UC1); + float onePixel = 0.0; + + //this looks unfinished, don't let it run. + assert(true); + double minPixel=0; + double maxPixel=0; + double pixelStretch = maxPixel - minPixel; + if(fabs(pixelStretch) < FLT_EPSILON) + { + img = img*(unsigned char) pixelStretch; + } + else + { + switch(displayMtd) + { + case CLAMP: + { + for(int i = 0; i < imgSize.height; i++) + { + for(int j = 0; j < imgSize.width; j++) + { + if(onePixel <= 0.0) + img.at(i, j) = 0; + else if(onePixel >= 255.0) + img.at(i, j) = 255; + else + img.at(i, j) = (unsigned char) onePixel; + } + } + } + break; + case STRETCH: + { + for(int i = 0; i < imgSize.height; i++) + { + for(int j = 0; j < imgSize.width; j++) + { +// onePixel = ( gsl_matrix_get (waveParams, i, j) - minPixel) / (maxPixel - minPixel) * 255.0; + img.at(i, j) = (unsigned char) onePixel; + } + } + } + break; + } + } + imwrite(fn, img); +} + + +/** + * @author JIA Pei + * @version 2010-06-06 + * @brief Save ASMLTC to a specified folder + * @param fd Input - the folder that ASM to be saved to +*/ +void VO_ASMLTCs::VO_Save ( const string& fd ) +{ + VO_AXM::VO_Save(fd); + + // create ASMLTCs subfolder for justASMLTCs model data + string fn = fd+"/ASMLTCs"; + MakeDirectory(fn); + + ofstream fp; + string tempfn; + + // ASMLTCs + tempfn = fn + "/ASMLTCs" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + + fp << "m_iLTCMethod" << endl << this->m_iLTCMethod << endl; + fp << "m_iNbOfLTC4PerPoint" << endl << this->m_iNbOfLTC4PerPoint << endl; + fp << "m_localImageSize" << endl << this->m_localImageSize.height << " " << this->m_localImageSize.width << endl; + fp.close();fp.clear(); + + // m_vvLTCMeans + tempfn = fn + "/m_vvLTCMeans" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_vvLTCMeans" << endl; + for (unsigned int i = 0; i < this->m_iNbOfPyramidLevels; i++) + { + for (unsigned int j = 0; j < this->m_iNbOfPoints; j++) + { + fp << "level " << i << " node " << j << endl; + fp << this->m_vvLTCMeans[i][j] << endl; + } + } + fp.close();fp.clear(); + + // m_vvCVMInverseOfLTCCov + tempfn = fn + "/m_vvCVMInverseOfLTCCov" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_vvCVMInverseOfLTCCov" << endl; + for (unsigned int i = 0; i < this->m_iNbOfPyramidLevels; i++) + { + for (unsigned int j = 0; j < this->m_iNbOfPoints; j++) + { + fp << "level " << i << " node " << j << endl; + fp << this->m_vvCVMInverseOfLTCCov[i][j] << endl; + } + } + fp << this->m_vvCVMInverseOfLTCCov << endl; + fp.close();fp.clear(); +} + + +/** +* @author JIA Pei +* @version 2010-06-06 +* @brief Load all ASMLTC data from a specified folder +* @param fd Input - the folder that ASMLTC to be loaded from +*/ +void VO_ASMLTCs::VO_Load ( const string& fd ) +{ + VO_AXM::VO_Load(fd); +} + + +/** +* @author JIA Pei +* @version 2010-06-06 +* @brief Load all ASMLTC data from a specified folder for later fitting +* @param fd Input - the folder that ASMLTC to be loaded from +*/ +void VO_ASMLTCs::VO_LoadParameters4Fitting ( const string& fd ) +{ + VO_AXM::VO_LoadParameters4Fitting(fd); // Note, for ASMProfile fitting, no problem + + string fn = fd+"/ASMLTCs"; + if (!MakeDirectory(fn) ) + { + cout << "VO_ASMLTCs subfolder is not existing. " << endl; + exit(EXIT_FAILURE); + } + + ifstream fp; + string tempfn; + string temp; + + // ASMLTCs + tempfn = fn + "/ASMLTCs" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp >> this->m_iLTCMethod; + fp >> temp >> this->m_iNbOfLTC4PerPoint; + fp >> temp >> this->m_localImageSize.height >> this->m_localImageSize.width; + fp.close();fp.clear(); + // Initialize all member variables + switch(this->m_iLTCMethod) + { + case VO_Features::LBP: + this->m_pVOfeatures = new VO_LBPFeatures(); + break; + case VO_Features::HAAR: + this->m_pVOfeatures = new VO_HaarFeatures(); + break; + case VO_Features::GABOR: + this->m_pVOfeatures = new VO_GaborFeatures(); + break; + case VO_Features::DAUBECHIES: + this->m_pVOfeatures = new VO_DaubechiesFeatures(); + break; + case VO_Features::DIRECT: + default: + this->m_pVOfeatures = new VO_DirectFeatures(); + break; + } + this->m_pVOfeatures->VO_GenerateAllFeatureInfo(this->m_localImageSize, 2); + + // m_vvLTCMeans + tempfn = fn + "/m_vvLTCMeans" + ".txt"; + SafeInputFileOpen(fp, tempfn); + getline(fp, temp); + this->m_vvLTCMeans.resize(this->m_iNbOfPyramidLevels); + for (unsigned int i = 0; i < this->m_iNbOfPyramidLevels; i++) + { + this->m_vvLTCMeans[i].resize(this->m_iNbOfPoints); + for (unsigned int j = 0; j < this->m_iNbOfPoints; j++) + { + fp >> temp; + while ( !fp.eof() && temp!="level") + { + fp >> temp; + } + getline(fp, temp); + this->m_vvLTCMeans[i][j] = Mat_::zeros(1, this->m_iNbOfLTC4PerPoint ); + fp >> this->m_vvLTCMeans[i][j]; + } + } + fp.close();fp.clear(); + + // m_vvCVMInverseOfLTCCov + tempfn = fn + "/m_vvCVMInverseOfLTCCov" + ".txt"; + SafeInputFileOpen(fp, tempfn); + this->m_vvCVMInverseOfLTCCov.resize(this->m_iNbOfPyramidLevels); + for (unsigned int i = 0; i < this->m_iNbOfPyramidLevels; i++) + { + this->m_vvCVMInverseOfLTCCov[i].resize(this->m_iNbOfPoints); + for (unsigned int j = 0; j < this->m_iNbOfPoints; j++) + { + fp >> temp; + while ( !fp.eof() && temp!= "level") + { + fp >> temp; + } + getline(fp, temp); + this->m_vvCVMInverseOfLTCCov[i][j] = Mat_::zeros( this->m_iNbOfLTC4PerPoint, this->m_iNbOfLTC4PerPoint); + fp >> this->m_vvCVMInverseOfLTCCov[i][j]; + } + } + fp.close();fp.clear(); +} + diff --git a/modules/smbuilding/src/VO_ASMNDProfiles.cpp b/modules/smbuilding/src/VO_ASMNDProfiles.cpp new file mode 100644 index 0000000..5d6cbb3 --- /dev/null +++ b/modules/smbuilding/src/VO_ASMNDProfiles.cpp @@ -0,0 +1,523 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include +#include +#include + +#include "VO_ASMNDProfiles.h" +#include "VO_CVCommon.h" + + +/** + * @author JIA Pei + * @version 2010-02-05 + * @brief Calculate how many profiles for each level + * @param nbOfProfilesPerPixelAtLevels Output - number of profiles per pixel at different levels + * @param NbOfLevels Input - profile k pixels refer to Cootes "Statistical Models of Appearance for Computer Vision" page 38 + * @param NbOfProfileInLevel0 Input - an odd natural number, number of samples for a single landmark + * @param ProduceMethod Input - method + * @return void +*/ +void VO_ASMNDProfiles::VO_ProduceLevelProfileNumbers( vector& nbOfProfilesPerPixelAtLevels, + unsigned int NbOfLevels, + unsigned int NbOfProfileInLevel0, + unsigned int ProduceMethod) +{ + ///////////////////////////////////////////////////////////////////////////////// + if(NbOfProfileInLevel0%2 == 0) + { + cerr << "Number of Profiles in level 0 must be an odd " << endl; + exit(1); + } + ///////////////////////////////////////////////////////////////////////////////// + + nbOfProfilesPerPixelAtLevels.resize(NbOfLevels); + nbOfProfilesPerPixelAtLevels[0] = NbOfProfileInLevel0; + switch(ProduceMethod) + { + case DESCENDING: + { + if( (NbOfProfileInLevel0 - (NbOfLevels-1)*2) <= 0 ) + { + cerr << "Too many multi scale levels. DESCENDING. VO_ASMNDProfiles. " << endl; + exit(1); + } + for(unsigned int i = 1; i < NbOfLevels; i++) + { + nbOfProfilesPerPixelAtLevels[i] = nbOfProfilesPerPixelAtLevels[0] - i*2; + } + } + break; + case PYRAMID: + { + if( ( (float)(NbOfProfileInLevel0 - 1) / ( pow( 2.0f, (float)(NbOfLevels) ) ) ) < 2.0 ) + { + cerr << "Too many multi scale levels. PYRAMID. VO_ASMNDProfiles. " << endl; + exit(1); + } + if( ( (NbOfProfileInLevel0 - 1) / (unsigned int) ( pow( 2.0f, (float)(NbOfLevels) ) ) ) % 2 != 0 ) + { + cerr << "Multi scale levels are not suitable for PYRAMID. " << endl; + exit(1); + } + for(unsigned int i = 1; i < NbOfLevels; i++) + { + unsigned int PyrScale = (unsigned int) ( pow(2.0f, (float)(i) ) ); + nbOfProfilesPerPixelAtLevels[i] = nbOfProfilesPerPixelAtLevels[0] / PyrScale + 1; + } + } + break; + case SAME: + default: + { + for(unsigned int i = 1; i < NbOfLevels; i++) + { + nbOfProfilesPerPixelAtLevels[i] = nbOfProfilesPerPixelAtLevels[0]; + } + } + break; + } +} + + +/** + * @author JIA Pei + * @version 2010-02-05 + * @brief Calculate all profiles + * @return void +*/ +void VO_ASMNDProfiles::VO_LoadProfileTrainingData (int loadFlag) +{ + Mat img, resizedImg; + VO_Shape resizedShape; + VO_Profile tempProfile; + float PyrScale, scale, sampleScale; + float refScale = this->m_VOReferenceShape.GetCentralizedShapeSize(); + + for( unsigned int i = 0; i < this->m_iNbOfSamples; ++i ) + { + img = imread ( this->m_vStringTrainingImageNames[i].c_str (), loadFlag ); + + sampleScale = this->m_vShapes[i].GetCentralizedShapeSize(); + + for( unsigned int j = 0; j < this->m_iNbOfPyramidLevels; ++j) + { + PyrScale = pow(2.0f, (float)j); + scale = refScale/sampleScale/PyrScale; + + resizedShape = this->m_vShapes[i]*scale; + cv::resize(img, resizedImg, Size( (int)(img.cols*scale), (int)(img.rows*scale) ) ); + +// static stringstream ss; +// static string ssi; +// ss << i; +// ss >> ssi; +// static string fn = ssi+".jpg"; +// imwrite(fn.c_str(), resizedImg); + + // Explained by JIA Pei -- wavelet feature extraction + for( unsigned int k = 0; k < this->m_iNbOfPoints; ++k ) + { + VO_Profile::VO_GetNDProfiles4OneLandmark ( resizedImg, + resizedShape, + this->m_vShape2DInfo, + k, + tempProfile, + this->m_iNbOfProfileDim, + this->m_iNbOfProfilesPerPixelAtLevels[j] ); + this->m_vvvNormalizedProfiles[i][j][k].SetProfile(tempProfile ); + this->m_vvvNormalizedProfiles[i][j][k].Normalize(); + } + } + } +} + + +/** + * @author JIA Pei + * @version 2010-02-05 + * @brief Calculate statistics for all profiles; Computer all landmarks' mean prof and covariance matrix + * @return void +*/ +void VO_ASMNDProfiles::VO_CalcStatistics4AllProfiles() +{ + // Calcuate Inverse of Sg + for(unsigned int i = 0; i < this->m_iNbOfPyramidLevels; i++) + { + Mat_ allProfiles = Mat_::zeros( this->m_iNbOfSamples, this->m_vvvNormalizedProfiles[0][i][0].GetProfileLength() ); + Mat_ Covar = Mat_::zeros(this->m_vvvNormalizedProfiles[0][i][0].GetProfileLength(), + this->m_vvvNormalizedProfiles[0][i][0].GetProfileLength() ); + Mat_ Mean = Mat_::zeros(1, this->m_vvvNormalizedProfiles[0][i][0].GetProfileLength() ); + for(unsigned int j = 0; j < this->m_iNbOfPoints; j++) + { + for(unsigned int k = 0; k < this->m_iNbOfProfileDim; k++) + { + for(unsigned int l = 0; l < this->m_iNbOfSamples; l++) + { + Mat_ tmpRow = allProfiles.row(l); + Mat_ tmp = this->m_vvvNormalizedProfiles[l][i][j].GetTheProfile().col(k).t(); + tmp.copyTo(tmpRow); + } + + // OK! Now We Calculate the Covariance Matrix of prof for Landmark iPoint + cv::calcCovarMatrix( allProfiles, Covar, Mean, CV_COVAR_NORMAL+CV_COVAR_ROWS+CV_COVAR_SCALE, CV_32F); +// cv::calcCovarMatrix( allProfiles, Covar, Mean, CV_COVAR_SCRAMBLED+CV_COVAR_ROWS, CV_32F); + this->m_vvMeanNormalizedProfile[i][j].Set1DimProfile(Mean.t(), k); + + // Explained by YAO Wei, 2008-1-29. + // Actually Covariance Matrix is semi-positive definite. But I am not sure + // whether it is invertible or not!!! + // In my opinion it is un-invert, since C = X.t() * X!!! + cv::invert(Covar, this->m_vvvCVMInverseOfSg[i][j][k], DECOMP_SVD); + } + } + } +} + + +/** + * @author JIA Pei + * @version 2010-02-22 + * @brief Build ND Profile ASM + * @param allLandmarkFiles4Training Input - all training landmark files + * @param allImgFiles4Training Input - all training image files + * @param shapeinfoFileName Input - shape info file + * @param database Input - which database is it? + * @param channels Input - How many channels are to be used? + * @param levels Input - multiscale levels + * @param profdim Input - ND profile, how many dimensions? + * @param kk Input - kk, refer to Cootes' paper "Statistical Model for Computer Vision" + * @param trm Input - texture representation method + * @param TPShape Input - truncated percentage for shape model + * @param useKnownTriangles Input - use known triangle structures?? + * @return void + */ +void VO_ASMNDProfiles::VO_BuildASMNDProfiles ( const vector& allLandmarkFiles4Training, + const vector& allImgFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + unsigned int channels, + unsigned int levels, + unsigned int profdim, + unsigned int kk, + int trm, + float TPShape, + bool useKnownTriangles ) +{ + if (allLandmarkFiles4Training.size() != allImgFiles4Training.size() ) + cerr << "allLandmarkFiles4Training should have the same number of allImgFiles4Training! " << endl; + + //Not checking actual supported channels image + if( (channels == 1 && (profdim == 1 || profdim == 2) ) || (profdim == 2*channels)) + cerr << profdim <<" m_iNbOfProfileDim should only be 1, or 2 times the number of channels!" << endl; + + this->VO_BuildShapeModel(allLandmarkFiles4Training, shapeinfoFileName, database, TPShape, useKnownTriangles); + this->m_iNbOfChannels = channels; + this->m_iNbOfPyramidLevels = levels; + //2 profiles per channel, unless 1D profiles + if(profdim == 1) + { + this->m_iNbOfProfileDim = profdim; + }else + { + this->m_iNbOfProfileDim = 2*channels; + } + + this->m_iTextureRepresentationMethod = trm; + this->m_vStringTrainingImageNames = allImgFiles4Training; + + // Initialize all member variables + VO_ASMNDProfiles::VO_ProduceLevelProfileNumbers(this->m_iNbOfProfilesPerPixelAtLevels, this->m_iNbOfPyramidLevels, 2*kk+1); // m_iNbOfProfilesPerPixelAtLevels + + this->m_vvvNormalizedProfiles.resize ( this->m_iNbOfSamples ); + for(unsigned int i = 0; i < this->m_iNbOfSamples; i++) + { + this->m_vvvNormalizedProfiles[i].resize ( this->m_iNbOfPyramidLevels ); + for(unsigned int j = 0; j< this->m_iNbOfPyramidLevels; j++) + { + this->m_vvvNormalizedProfiles[i][j].resize(this->m_iNbOfPoints); + } + } + + this->m_vvMeanNormalizedProfile.resize(this->m_iNbOfPyramidLevels); + this->m_vvvCVMInverseOfSg.resize(this->m_iNbOfPyramidLevels); + + + for(unsigned int i = 0; i < this->m_iNbOfPyramidLevels; i++) + { + this->m_vvMeanNormalizedProfile[i].resize(this->m_iNbOfPoints); + this->m_vvvCVMInverseOfSg[i].resize(this->m_iNbOfPoints); + for(unsigned int j = 0; j < this->m_iNbOfPoints; j++) + { + this->m_vvvCVMInverseOfSg[i][j].resize(this->m_iNbOfProfileDim); + this->m_vvMeanNormalizedProfile[i][j].Resize(this->m_iNbOfProfilesPerPixelAtLevels[i], this->m_iNbOfProfileDim); + for(unsigned int k = 0; k < this->m_iNbOfProfileDim; k++) + { + this->m_vvvCVMInverseOfSg[i][j][k] = Mat_::zeros(this->m_iNbOfProfilesPerPixelAtLevels[i], this->m_iNbOfProfilesPerPixelAtLevels[i]); + } + } + } + //can only force 1 channel + if(channels == 1) + { + //returns a grayscale image + this->VO_LoadProfileTrainingData (0); + }else + { + //should load the alpha channel as well. + this->VO_LoadProfileTrainingData(-1); + } + this->VO_CalcStatistics4AllProfiles(); +} + + +void VO_ASMNDProfiles::VO_Save ( const string& fd ) +{ + VO_AXM::VO_Save(fd); + + // create ASM subfolder for just ASM model data + string fn = fd+"/ASMNDProfiles"; + MakeDirectory(fn); + + ofstream fp; + string tempfn; + + // ASMNDProfiles + tempfn = fn + "/ASMNDProfiles" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_iNbOfProfileDim" << endl << this->m_iNbOfProfileDim << endl; + fp << "m_iNbOfProfilesPerPixelAtLevels[0]" << endl << this->m_iNbOfProfilesPerPixelAtLevels[0] << endl; + fp.close();fp.clear(); + + // m_vvMeanNormalizedProfile + tempfn = fn + "/m_vvMeanNormalizedProfile" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_vvMeanNormalizedProfile" << endl; + // fp << this->m_vvMeanNormalizedProfile; + // You can output everything by the above line, but you won't have level and node description in the output file. + for (unsigned int i = 0; i < this->m_iNbOfPyramidLevels; i++) + { + for (unsigned int j = 0; j < this->m_iNbOfPoints; j++) + { + fp << "level " << i << " node " << j << endl; + fp << this->m_vvMeanNormalizedProfile[i][j] << endl; + } + } + fp.close();fp.clear(); + + // m_vvvCVMInverseOfSg + tempfn = fn + "/m_vvvCVMInverseOfSg" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_vvvCVMInverseOfSg" << endl; + //fp << this->m_vvvCVMInverseOfSg; + // You can output everything by the above line, but you won't have level and node description in the output file. + for (unsigned int i = 0; i < this->m_iNbOfPyramidLevels; i++) + { + for (unsigned int j = 0; j < this->m_iNbOfPoints; j++) + { + for(unsigned int k = 0; k < this->m_iNbOfProfileDim; k++) + { + fp << "level " << i << " node " << j << " dim " << k << endl; + fp << this->m_vvvCVMInverseOfSg[i][j][k] << endl; + } + } + } + fp.close();fp.clear(); +} + + +void VO_ASMNDProfiles::VO_Load ( const string& fd ) +{ + VO_AXM::VO_Load(fd); + + string fn = fd+"/ASMNDProfiles"; + if (!MakeDirectory(fn) ) + { + cout << "ASMNDProfiles subfolder is not existing. " << endl; + exit(EXIT_FAILURE); + } + + ifstream fp; + string tempfn; + string temp; + + // ASMNDProfiles + tempfn = fn + "/ASMNDProfiles" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp >> this->m_iNbOfProfileDim; // m_iNbOfProfileDim + this->m_iNbOfProfilesPerPixelAtLevels.resize(this->m_iNbOfPyramidLevels); + fp >> temp >> this->m_iNbOfProfilesPerPixelAtLevels[0]; // m_iNbOfProfilesPerPixelAtLevels[0] + VO_ASMNDProfiles::VO_ProduceLevelProfileNumbers(this->m_iNbOfProfilesPerPixelAtLevels, this->m_iNbOfPyramidLevels, this->m_iNbOfProfilesPerPixelAtLevels[0]); + fp.close();fp.clear(); + + // m_vvMeanNormalizedProfile + tempfn = fn + "/m_vvMeanNormalizedProfile" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + this->m_vvMeanNormalizedProfile.resize(this->m_iNbOfPyramidLevels); + for (unsigned int i = 0; i < this->m_iNbOfPyramidLevels; i++) + { + this->m_vvMeanNormalizedProfile[i].resize(this->m_iNbOfPoints); + for (unsigned int j = 0; j < this->m_iNbOfPoints; j++) + { + fp >> this->m_vvMeanNormalizedProfile[i][j]; + } + } + fp.close();fp.clear(); + + // m_vvvCVMInverseOfSg + tempfn = fn + "/m_vvvCVMInverseOfSg" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + this->m_vvvCVMInverseOfSg.resize(this->m_iNbOfPyramidLevels); + for (unsigned int i = 0; i < this->m_iNbOfPyramidLevels; i++) + { + this->m_vvvCVMInverseOfSg[i].resize(this->m_iNbOfPoints); + for (unsigned int j = 0; j < this->m_iNbOfPoints; j++) + { + this->m_vvvCVMInverseOfSg[i][j].resize(this->m_iNbOfProfileDim); + for(unsigned int k = 0; k < this->m_iNbOfProfileDim; k++) + { + fp >> this->m_vvvCVMInverseOfSg[i][j][k]; + } + } + } + fp.close();fp.clear(); +} + + +void VO_ASMNDProfiles::VO_LoadParameters4Fitting ( const string& fd ) +{ + VO_AXM::VO_LoadParameters4Fitting(fd); + + string fn = fd+"/ASMNDProfiles"; + if (!MakeDirectory(fn) ) + { + cout << "ASMNDProfiles subfolder is not existing. " << endl; + exit(EXIT_FAILURE); + } + + ifstream fp; + string tempfn; + string temp; + + // ASMNDProfiles + tempfn = fn + "/ASMNDProfiles" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp >> this->m_iNbOfProfileDim; // m_iNbOfProfileDim + this->m_iNbOfProfilesPerPixelAtLevels.resize(this->m_iNbOfPyramidLevels); + fp >> temp >> this->m_iNbOfProfilesPerPixelAtLevels[0]; // m_iNbOfProfilesPerPixelAtLevels[0] + VO_ASMNDProfiles::VO_ProduceLevelProfileNumbers(this->m_iNbOfProfilesPerPixelAtLevels, this->m_iNbOfPyramidLevels, this->m_iNbOfProfilesPerPixelAtLevels[0]); + fp.close();fp.clear(); + + // m_vvMeanNormalizedProfile + tempfn = fn + "/m_vvMeanNormalizedProfile" + ".txt"; + SafeInputFileOpen(fp, tempfn); + getline(fp, temp); + this->m_vvMeanNormalizedProfile.resize(this->m_iNbOfPyramidLevels); + for (unsigned int i = 0; i < this->m_iNbOfPyramidLevels; i++) + { + this->m_vvMeanNormalizedProfile[i].resize(this->m_iNbOfPoints); + for (unsigned int j = 0; j < this->m_iNbOfPoints; j++) + { + fp >> temp; + while ( !fp.eof() && temp!="level") + { + fp >> temp; + } + getline(fp, temp); + this->m_vvMeanNormalizedProfile[i][j].Resize(this->m_iNbOfProfilesPerPixelAtLevels[i], this->m_iNbOfProfileDim); + fp >> this->m_vvMeanNormalizedProfile[i][j]; + } + } + + fp.close();fp.clear(); + + + // m_vvvCVMInverseOfSg + tempfn = fn + "/m_vvvCVMInverseOfSg" + ".txt"; + SafeInputFileOpen(fp, tempfn); + getline(fp, temp); + this->m_vvvCVMInverseOfSg.resize(this->m_iNbOfPyramidLevels); + for (unsigned int i = 0; i < this->m_iNbOfPyramidLevels; i++) + { + this->m_vvvCVMInverseOfSg[i].resize(this->m_iNbOfPoints); + for (unsigned int j = 0; j < this->m_iNbOfPoints; j++) + { + this->m_vvvCVMInverseOfSg[i][j].resize(this->m_iNbOfProfileDim); + for(unsigned int k = 0; k < this->m_iNbOfProfileDim; k++) + { + fp >> temp; + while ( !fp.eof() && temp!= "level") + { + fp >> temp; + } + getline(fp, temp); + this->m_vvvCVMInverseOfSg[i][j][k] = + Mat_::zeros( this->m_iNbOfProfilesPerPixelAtLevels[i], + this->m_iNbOfProfilesPerPixelAtLevels[i]); + fp >> this->m_vvvCVMInverseOfSg[i][j][k]; + } + } + } + fp.close();fp.clear(); +} + diff --git a/modules/smbuilding/src/VO_AXM.cpp b/modules/smbuilding/src/VO_AXM.cpp new file mode 100644 index 0000000..2da5efa --- /dev/null +++ b/modules/smbuilding/src/VO_AXM.cpp @@ -0,0 +1,201 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include +#include +#include + +#include "VO_AXM.h" + + +/** + * @author JIA Pei + * @version 2010-02-13 + * @brief Save ASM to a specified folder + * @param fd Input - the folder that ASM to be saved to + * @return void +*/ +void VO_AXM::VO_Save ( const string& fd ) +{ + switch(this->m_iMethod) + { + case AAM_BASIC: + case AAM_DIRECT: + case AAM_FAIA: + case AAM_CMUICIA: + case AAM_IAIA: + VO_TextureModel::VO_Save(fd); + break; + case ASM_PROFILEND: + case ASM_LTC: + VO_ShapeModel::VO_Save(fd); + break; + case CLM: + case AFM: + break; + } + + // create AXM subfolder for just AXM model data + string fn = fd+"/AXM"; + MakeDirectory(fn); + + ofstream fp; + string tempfn; + + // AXM + tempfn = fn + "/AXM" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_iNbOfPyramidLevels" << endl << this->m_iNbOfPyramidLevels << endl; + fp.close();fp.clear(); +} + + +/** + * @author JIA Pei + * @version 2010-02-13 + * @brief Load all trained data + * @param fd Input - the folder that ASM to be saved to + * @return void +*/ +void VO_AXM::VO_Load ( const string& fd ) +{ + switch(this->m_iMethod) + { + case AAM_BASIC: + case AAM_DIRECT: + case AAM_FAIA: + case AAM_CMUICIA: + case AAM_IAIA: + VO_TextureModel::VO_Load(fd); + break; + case ASM_PROFILEND: + case ASM_LTC: + VO_ShapeModel::VO_Load(fd); + break; + case CLM: + case AFM: + break; + } + + string fn = fd+"/AXM"; + if (!MakeDirectory(fn) ) + { + cout << "AXM subfolder is not existing. " << endl; + exit(EXIT_FAILURE); + } + + ifstream fp; + string tempfn; + string temp; + + // AXM + tempfn = fn + "/AXM" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp >> this->m_iNbOfPyramidLevels; // m_iNbOfPyramidLevels + fp.close();fp.clear(); +} + + +/** + * @author JIA Pei + * @version 2010-02-13 + * @brief Load all trained data for fitting + * @param fd Input - the folder that ASM to be saved to + * @return void +*/ +void VO_AXM::VO_LoadParameters4Fitting ( const string& fd ) +{ + switch(this->m_iMethod) + { + case AAM_BASIC: + case AAM_DIRECT: + case AAM_FAIA: + case AAM_CMUICIA: + case AAM_IAIA: + VO_TextureModel::VO_LoadParameters4Fitting(fd); + break; + case ASM_PROFILEND: + case ASM_LTC: + VO_ShapeModel::VO_LoadParameters4Fitting(fd); + break; + case CLM: + case AFM: + break; + } + + string fn = fd+"/AXM"; + if (!MakeDirectory(fn) ) + { + cout << "AXM subfolder is not existing. " << endl; + exit(EXIT_FAILURE); + } + + ifstream fp; + string tempfn; + string temp; + + // AXM + tempfn = fn + "/AXM" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp >> this->m_iNbOfPyramidLevels; // m_iNbOfPyramidLevels + fp.close();fp.clear(); +} + diff --git a/modules/smbuilding/src/VO_Edge.cpp b/modules/smbuilding/src/VO_Edge.cpp new file mode 100644 index 0000000..aff31e1 --- /dev/null +++ b/modules/smbuilding/src/VO_Edge.cpp @@ -0,0 +1,83 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_Edge.h" + + +ostream& operator<<(ostream& os, const VO_Edge& aamedge) +{ + os << aamedge.index1 << " " << aamedge.index2; + + return os; +} + + +istream& operator>>(istream& is, VO_Edge& aamedge) +{ + unsigned int idx; + is >> idx; + aamedge.index1 = idx; + is >> idx; + aamedge.index2 = idx; + + return is; +} + diff --git a/modules/smbuilding/src/VO_Ellipse.cpp b/modules/smbuilding/src/VO_Ellipse.cpp new file mode 100644 index 0000000..ee7fd73 --- /dev/null +++ b/modules/smbuilding/src/VO_Ellipse.cpp @@ -0,0 +1,215 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_Ellipse.h" + + +ostream& operator<<(ostream& os, const VO_Ellipse& ellipse) +{ + os << ellipse.m_fPhimin << " " + << ellipse.m_fPhimax << " " + << ellipse.m_fAxisXHalfLen << " " + << ellipse.m_fAxisYHalfLen << " " + << ellipse.m_fTheta << " " + << ellipse.m_Center.x << " " + << ellipse.m_Center.y; + + return os; +} + + +istream& operator>>(istream& is, VO_Ellipse& ellipse) +{ + is >> ellipse.m_fPhimin + >> ellipse.m_fPhimax + >> ellipse.m_fAxisXHalfLen + >> ellipse.m_fAxisYHalfLen + >> ellipse.m_fTheta + >> ellipse.m_Center.x + >> ellipse.m_Center.y; + + return is; +} + + +VO_Ellipse& VO_Ellipse::operator=(const VO_Ellipse& ellipse) +{ + this->CopyData(ellipse); + return (*this); +} + + +VO_Ellipse VO_Ellipse::operator*(float value) +{ + VO_Ellipse res(*this); + res.m_fAxisXHalfLen *= value; + res.m_fAxisYHalfLen *= value; + + return res; +} + + +VO_Ellipse& VO_Ellipse::operator*=(float value) +{ + this->m_fAxisXHalfLen *= value; + this->m_fAxisYHalfLen *= value; + + return *this; +} + + +void VO_Ellipse::Translate( const Mat_& translation ) +{ + this->m_Center.x += translation(0, 0); + this->m_Center.y += translation(1, 0); +} + + +void VO_Ellipse::ScaleCenter(float value) +{ + this->m_Center.x *= value; + this->m_Center.y *= value; +} + + +/** + * @brief Find the nearest point on the boundary to the input point + * @note to be done, tough. needs optimization */ +Point2f VO_Ellipse::FindNearestPointOnBoundary(const Point2f& pt) +{ + Point2f res; + + return res; +} + +/** + * @brief Find the intersect point on the boundary connecting from the COG and the input point + * @ref http://my.opera.com/Vorlath/blog/show.dml/476448 + * x = a*b*xx/sqrt((b*xx)2 + (a*yy)2) + * y = a*b*yy/sqrt((b*xx)2 + (a*yy)2) +*/ +Point2f VO_Ellipse::FindIntersectPointOnBoundary(const Point2f& pt) +{ + if(pt == this->m_Center) + { + cerr << "The input point shouldn't be the same as Ellipse COG!" << endl; + exit(1); + } + Point2f res; + float xx = pt.x*cos(this->m_fTheta) - pt.y*sin(this->m_fTheta) - this->m_Center.x; + float yy = pt.x*sin(this->m_fTheta) + pt.y*cos(this->m_fTheta) - this->m_Center.y; + float denominator = sqrt( pow(this->m_fAxisYHalfLen*xx, 2.0f) + + pow(this->m_fAxisXHalfLen*yy, 2.0f) ); + float x = this->m_fAxisXHalfLen*this->m_fAxisYHalfLen*xx/denominator + this->m_Center.x; + float y = this->m_fAxisXHalfLen*this->m_fAxisYHalfLen*yy/denominator + this->m_Center.y; + res.x = x*cos(this->m_fTheta) + y*sin(this->m_fTheta); + res.y = -x*sin(this->m_fTheta) + y*cos(this->m_fTheta); + + return res; +} + + +/** + * @brief Obtain point list within the ellipse according to its range + * @param ellipse - Input ellipse + */ +vector VO_Ellipse::VO_PointsInEllipse( const VO_Ellipse& ellipse) +{ + vector res; + + Rect brect = this->CalcBoundingRect(); + int startx = cvFloor(brect.x); + int starty = cvFloor(brect.y); + int finishx = cvCeil(brect.x+brect.width); + int finishy = cvCeil(brect.y+brect.height); + for(int i = startx; i <= finishx; i++ ) + { + for(int j = starty; j <= finishy; j++) + { + if( this->IsPointWithinEllipse( Point2f(static_cast(i), static_cast(j)) ) ) + res.push_back( Point(i,j) ); + } + } + + return res; +} + + +/** + * @brief Find bounding rectangle for multiple ellipses, this rectangle should cover all ellipses + * @param ellipses - Input ellipses + * @return Rect + */ +Rect VO_Ellipse::VO_CalcBoundingRect4MultipleEllipses(const vector& ellipses) +{ + unsigned int NbOfEllipses = ellipses.size(); + vector allVertexes; + + for(unsigned int i = 0; i < NbOfEllipses; i++) + { + Rect rect = ellipses[i].CalcBoundingRect(); + allVertexes.push_back( Point(rect.x, rect.y) ); + allVertexes.push_back( Point(rect.x+rect.width, rect.y+rect.height) ); + } + Mat pointsMat(allVertexes); + return cv::boundingRect( pointsMat ); +} + diff --git a/modules/smbuilding/src/VO_FacePart.cpp b/modules/smbuilding/src/VO_FacePart.cpp new file mode 100644 index 0000000..6d08917 --- /dev/null +++ b/modules/smbuilding/src/VO_FacePart.cpp @@ -0,0 +1,272 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_FacePart.h" + +ostream& operator<<(ostream& os, const VO_FacePart& facepart) +{ + switch( facepart.GetType() ) + { + case VO_FacePart::CHIN: + os << "m_vChin" << " "; + break; + case VO_FacePart::LEFTEYEBROW: + os << "m_vLeftEyebrow" << " "; + break; + case VO_FacePart::RIGHTEYEBROW: + os << "m_vRightEyebrow" << " "; + break; + case VO_FacePart::LEFTEYE: + os << "m_vLeftEye" << " "; + break; + case VO_FacePart::RIGHTEYE: + os << "m_vRightEye" << " "; + break; + case VO_FacePart::NOSE: + os << "m_vNose" << " "; + break; + case VO_FacePart::NOSTRIL: + os << "m_vNostril" << " "; + break; + case VO_FacePart::NOSETIP: + os << "m_vNoseTip" << " "; + break; + case VO_FacePart::LIPOUTERLINE: + os << "m_vLipOuterLine" << " "; + break; + case VO_FacePart::LIPINNERLINE: + os << "m_vLipInnerLine" << " "; + break; + case VO_FacePart::LIPUPPEROUTERLINE: + os << "m_vLipUpperOuterLine" << " "; + break; + case VO_FacePart::LIPUPPERINNERLINE: + os << "m_vLipUpperInnerLine" << " "; + break; + case VO_FacePart::LIPDOWNEROUTERLINE: + os << "m_vLipDownerOuterLine" << " "; + break; + case VO_FacePart::LIPDOWNERINNERLINE: + os << "m_vLipDownerInnerLine" << " "; + break; + case VO_FacePart::LEFTSIDEPOINTS: + os << "m_vLeftSidePoints" << " "; + break; + case VO_FacePart::RIGHTSIDEPOINTS: + os << "m_vRightSidePoints" << " "; + break; + case VO_FacePart::MIDLINEPOINTS: + os << "m_vMidlinePoints" << " "; + break; + case VO_FacePart::EYECORNERPOINTS: + os << "m_vEyeCornerPoints" << " "; + break; + case VO_FacePart::MOUTHCORNERPOINTS: + os << "m_vMouthCornerPoints" << " "; + break; + case VO_FacePart::PITCHAXISLINEPOINTS: + os << "m_vPitchAxisLinePoints" << " "; + break; + case VO_FacePart::LEFTIRIS: + os << "m_vLeftIris" << " "; + break; + case VO_FacePart::RIGHTIRIS: + os << "m_vRightIris" << " "; + break; + case VO_FacePart::LEFTEAR: + os << "m_vLeftEar" << " "; + break; + case VO_FacePart::RIGHTEAR: + os << "m_vRightEar" << " "; + break; + } + + if( facepart.IsClosedOrNot() ) + os << "closed" << " "; + else + os << "open" << " "; + + for(unsigned int i = 0; i < facepart.GetIndexes().size(); i++) + { + os << facepart.GetIndexes()[i] << " "; + } + + return os; +} + + +istream& operator>>(istream& is, VO_FacePart& facepart) +{ + string temp; + unsigned int tempUI; + + is >> temp; + if(temp == "m_vChin") + { + facepart.m_iType = VO_FacePart::CHIN; + } + else if(temp == "m_vLeftEyebrow") + { + facepart.m_iType = VO_FacePart::LEFTEYEBROW; + } + else if(temp == "m_vRightEyebrow") + { + facepart.m_iType = VO_FacePart::RIGHTEYEBROW; + } + else if(temp == "m_vLeftEye") + { + facepart.m_iType = VO_FacePart::LEFTEYE; + } + else if(temp == "m_vRightEye") + { + facepart.m_iType = VO_FacePart::RIGHTEYE; + } + else if(temp == "m_vNose") + { + facepart.m_iType = VO_FacePart::NOSE; + } + else if(temp == "m_vNostril") + { + facepart.m_iType = VO_FacePart::NOSTRIL; + } + else if(temp == "m_vNoseTip") + { + facepart.m_iType = VO_FacePart::NOSETIP; + } + else if(temp == "m_vLipOuterLine") + { + facepart.m_iType = VO_FacePart::LIPOUTERLINE; + } + else if(temp == "m_vLipInnerLine") + { + facepart.m_iType = VO_FacePart::LIPINNERLINE; + } + else if(temp == "m_vLipUpperOuterLine") + { + facepart.m_iType = VO_FacePart::LIPUPPEROUTERLINE; + } + else if(temp == "m_vLipUpperInnerLine") + { + facepart.m_iType = VO_FacePart::LIPUPPERINNERLINE; + } + else if(temp == "m_vLipDownerOuterLine") + { + facepart.m_iType = VO_FacePart::LIPDOWNEROUTERLINE; + } + else if(temp == "m_vLipDownerInnerLine") + { + facepart.m_iType = VO_FacePart::LIPDOWNERINNERLINE; + } + else if(temp == "m_vLeftSidePoints") + { + facepart.m_iType = VO_FacePart::LEFTSIDEPOINTS; + } + else if(temp == "m_vRightSidePoints") + { + facepart.m_iType = VO_FacePart::RIGHTSIDEPOINTS; + } + else if(temp == "m_vMidlinePoints") + { + facepart.m_iType = VO_FacePart::MIDLINEPOINTS; + } + else if(temp == "m_vEyeCornerPoints") + { + facepart.m_iType = VO_FacePart::EYECORNERPOINTS; + } + else if(temp == "m_vMouthCornerPoints") + { + facepart.m_iType = VO_FacePart::MOUTHCORNERPOINTS; + } + else if(temp == "m_vPitchAxisLinePoints") + { + facepart.m_iType = VO_FacePart::PITCHAXISLINEPOINTS; + } + else if(temp == "m_vLeftIris") + { + facepart.m_iType = VO_FacePart::LEFTIRIS; + } + else if(temp == "m_vRightIris") + { + facepart.m_iType = VO_FacePart::RIGHTIRIS; + } + else if(temp == "m_vLeftEar") + { + facepart.m_iType = VO_FacePart::LEFTEAR; + } + else if(temp == "m_vRightEar") + { + facepart.m_iType = VO_FacePart::RIGHTEAR; + } + + is >> temp; + if(temp == "open") + facepart.m_bClosed = false; + else if(temp == "closed") + facepart.m_bClosed = true; + + while (is >> tempUI) + { + facepart.m_vIndexes.push_back(tempUI); + } + + return is; +} + diff --git a/modules/smbuilding/src/VO_FaceParts.cpp b/modules/smbuilding/src/VO_FaceParts.cpp new file mode 100644 index 0000000..962951d --- /dev/null +++ b/modules/smbuilding/src/VO_FaceParts.cpp @@ -0,0 +1,295 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include +#include +#include +#include + +#include + + +#include "VO_FaceParts.h" + + +ostream& operator<<(ostream& os, const VO_FaceParts& faceparts) +{ + os << "#FaceParts" << endl; + os << faceparts.m_vChin << endl; + os << faceparts.m_vLeftEyebrow << endl; + os << faceparts.m_vRightEyebrow << endl; + os << faceparts.m_vLeftEye << endl; + os << faceparts.m_vRightEye << endl; + os << faceparts.m_vNose << endl; + os << faceparts.m_vNostril << endl; + os << faceparts.m_vNoseTip << endl; + os << faceparts.m_vLipOuterLine << endl; + os << faceparts.m_vLipInnerLine << endl; + os << faceparts.m_vLipUpperOuterLine << endl; + os << faceparts.m_vLipUpperInnerLine << endl; + os << faceparts.m_vLipLowerOuterLine << endl; + os << faceparts.m_vLipLowerInnerLine << endl; + os << faceparts.m_vLeftSidePoints << endl; + os << faceparts.m_vRightSidePoints << endl; + os << faceparts.m_vMidlinePoints << endl; + os << faceparts.m_vEyeCornerPoints << endl; + os << faceparts.m_vMouthCornerPoints << endl; + os << faceparts.m_vPitchAxisLinePoints << endl; + os << faceparts.m_vLeftIris << endl; + os << faceparts.m_vRightIris << endl; + os << faceparts.m_vLeftEar << endl; + os << faceparts.m_vRightEar << endl; + + return os; +} + + +istream& operator>>(istream& is, VO_FaceParts& faceparts) +{ + VO_FacePart onePart; + string line; + getline(is, line); + + bool isContainTriangleIndex = false; + + while ( getline(is, line) ) + { + + boost::trim(line); + + if(line == "#FaceParts") + break; + } + while ( getline(is, line) ) + { + cout << line << endl; + if(line == "TriangleIndexes:") + { + isContainTriangleIndex = true; + break; + } + istringstream iss(line); + iss >> onePart; + + faceparts.VO_SetOneFacePart(onePart); + onePart.clear(); + } + if(isContainTriangleIndex) + { + int size = 0; + is >> line; + istringstream iss(line); + iss >> size; + } + + return is; +} + + +/** + */ +VO_FacePart VO_FaceParts::VO_GetOneFacePart(unsigned int partIdx) const +{ + switch(partIdx ) + { + case VO_FacePart::WHOLEFACE: + break; + case VO_FacePart::CHIN: + return this->m_vChin; + case VO_FacePart::LEFTEYEBROW: + return this->m_vLeftEyebrow; + case VO_FacePart::RIGHTEYEBROW: + return this->m_vRightEyebrow; + case VO_FacePart::LEFTEYE: + return this->m_vLeftEye; + case VO_FacePart::RIGHTEYE: + return this->m_vRightEye; + case VO_FacePart::NOSE: + return this->m_vNose; + case VO_FacePart::NOSTRIL: + return this->m_vNostril; + case VO_FacePart::NOSETIP: + return this->m_vNoseTip; + case VO_FacePart::LIPOUTERLINE: + return this->m_vLipOuterLine; + case VO_FacePart::LIPINNERLINE: + return this->m_vLipInnerLine; + case VO_FacePart::LIPUPPEROUTERLINE: + return this->m_vLipUpperOuterLine; + case VO_FacePart::LIPUPPERINNERLINE: + return this->m_vLipUpperInnerLine; + case VO_FacePart::LIPDOWNEROUTERLINE: + return this->m_vLipLowerOuterLine; + case VO_FacePart::LIPDOWNERINNERLINE: + return this->m_vLipLowerInnerLine; + case VO_FacePart::LEFTSIDEPOINTS: + return this->m_vLeftSidePoints; + case VO_FacePart::RIGHTSIDEPOINTS: + return this->m_vRightSidePoints; + case VO_FacePart::MIDLINEPOINTS: + return this->m_vMidlinePoints; + case VO_FacePart::EYECORNERPOINTS: + return this->m_vEyeCornerPoints; + case VO_FacePart::MOUTHCORNERPOINTS: + return this->m_vMouthCornerPoints; + case VO_FacePart::PITCHAXISLINEPOINTS: + return this->m_vPitchAxisLinePoints; + case VO_FacePart::LEFTIRIS: + return this->m_vLeftIris; + case VO_FacePart::RIGHTIRIS: + return this->m_vRightIris; + case VO_FacePart::LEFTEAR: + return this->m_vLeftEar; + case VO_FacePart::RIGHTEAR: + return this->m_vRightEar; + case VO_FacePart::WHOLEMOUTH: + break; + case VO_FacePart::NOSECENTRALAREA: + break; + default: + throw; + } + throw;// unreachable code. + return VO_FacePart(); +} + + +/** + */ +void VO_FaceParts::VO_SetOneFacePart(const VO_FacePart& iFacePart) +{ + switch(iFacePart.GetType() ) + { + case VO_FacePart::CHIN: + this->m_vChin = iFacePart; + break; + case VO_FacePart::LEFTEYEBROW: + this->m_vLeftEyebrow = iFacePart; + break; + case VO_FacePart::RIGHTEYEBROW: + this->m_vRightEyebrow = iFacePart; + break; + case VO_FacePart::LEFTEYE: + this->m_vLeftEye = iFacePart; + break; + case VO_FacePart::RIGHTEYE: + this->m_vRightEye = iFacePart; + break; + case VO_FacePart::NOSE: + this->m_vNose = iFacePart; + break; + case VO_FacePart::NOSTRIL: + this->m_vNostril = iFacePart; + break; + case VO_FacePart::NOSETIP: + this->m_vNoseTip = iFacePart; + break; + case VO_FacePart::LIPOUTERLINE: + this->m_vLipOuterLine = iFacePart; + break; + case VO_FacePart::LIPINNERLINE: + this->m_vLipInnerLine = iFacePart; + break; + case VO_FacePart::LIPUPPEROUTERLINE: + this->m_vLipUpperOuterLine = iFacePart; + break; + case VO_FacePart::LIPUPPERINNERLINE: + this->m_vLipUpperInnerLine = iFacePart; + break; + case VO_FacePart::LIPDOWNEROUTERLINE: + this->m_vLipLowerOuterLine = iFacePart; + break; + case VO_FacePart::LIPDOWNERINNERLINE: + this->m_vLipLowerInnerLine = iFacePart; + break; + case VO_FacePart::LEFTSIDEPOINTS: + this->m_vLeftSidePoints = iFacePart; + break; + case VO_FacePart::RIGHTSIDEPOINTS: + this->m_vRightSidePoints = iFacePart; + break; + case VO_FacePart::MIDLINEPOINTS: + this->m_vMidlinePoints = iFacePart; + break; + case VO_FacePart::EYECORNERPOINTS: + this->m_vEyeCornerPoints = iFacePart; + break; + case VO_FacePart::MOUTHCORNERPOINTS: + this->m_vMouthCornerPoints = iFacePart; + break; + case VO_FacePart::PITCHAXISLINEPOINTS: + this->m_vPitchAxisLinePoints = iFacePart; + break; + case VO_FacePart::LEFTIRIS: + this->m_vLeftIris = iFacePart; + break; + case VO_FacePart::RIGHTIRIS: + this->m_vRightIris = iFacePart; + break; + case VO_FacePart::LEFTEAR: + this->m_vLeftEar = iFacePart; + break; + case VO_FacePart::RIGHTEAR: + this->m_vRightEar = iFacePart; + break; + } +} + diff --git a/modules/smbuilding/src/VO_FeatureModel.cpp b/modules/smbuilding/src/VO_FeatureModel.cpp new file mode 100644 index 0000000..2bc80c9 --- /dev/null +++ b/modules/smbuilding/src/VO_FeatureModel.cpp @@ -0,0 +1,205 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include +#include +#include +#include +#include +#include + +#include "VO_FeatureModel.h" +#include "VO_AnnotationDBIO.h" + + +VO_FeatureModel::VO_FeatureModel() +{ + this->init(); +} + + +/** Initialization */ +void VO_FeatureModel::init() +{ + this->m_iNbOfTotalFeatures = 0; + this->m_iNbOfEigenFeatureAtMost = 0; + this->m_iNbOfFeatureEigens = 0; + this->m_fTruncatedPercent_Feature = 0.95f; + this->m_vFeatures.clear(); + this->m_vNormalizedFeatures.clear(); +} + + +VO_FeatureModel::~VO_FeatureModel() +{ + this->m_vFeatures.clear(); + this->m_vNormalizedFeatures.clear(); +} + + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief Calculate point warping information + * @param iShape Input - the shape + * @param img Input - image + * @param templateTriangles Input - the composed face template triangles + * @param warpInfo Input - warping information for all pixels in template face + * @param oFeature Output - the extracted features + * @param trm Input - texture representation method + * @return bool loading succeed or not? +*/ +bool VO_FeatureModel::VO_LoadFeatures4OneFace( const VO_Shape& iShape, + const Mat& img, + const vector& templateTriangles, + const vector& warpInfo, + Mat_& oFeature, + int trm) +{ + + return true; +} + + +/** + * @author JIA Pei + * @version 2010-02-05 + * @brief Load Training data for texture model + * @param allLandmarkFiles4Training Input - all landmark file names + * @param shapeinfoFileName Input - all + * @param allImgFiles4Training Input - all input image file names + * @param channels Input - how to load each image + * @return void +*/ +bool VO_FeatureModel::VO_LoadFeatureTrainingData( const vector& allLandmarkFiles4Training, + const vector& allImgFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + unsigned int channels) +{ + // load auxiliary shape information + VO_Shape2DInfo::ReadShape2DInfo(shapeinfoFileName, this->m_vShape2DInfo, this->m_FaceParts); + CAnnotationDBIO::VO_LoadShapeTrainingData( allLandmarkFiles4Training, database, this->m_vShapes); + + this->m_vStringTrainingImageNames = allImgFiles4Training; + this->m_vFeatures.resize(this->m_iNbOfSamples); + this->m_vNormalizedFeatures.resize(this->m_iNbOfSamples); + Mat img; + + for(unsigned int i = 0; i < this->m_iNbOfSamples; ++i) + { + if(channels == 1) + img = imread ( allImgFiles4Training[i].c_str (), 0 ); + else if (channels == 3) + img = imread ( allImgFiles4Training[i].c_str (), 1 ); + else + cerr << "We can't deal with image channels not equal to 1 or 3!" << endl; + + double start = (double)cvGetTickCount(); + // Explained by JIA Pei -- Feature extraction over the whole image, so many methods. + if ( !VO_FeatureModel::VO_LoadFeatures4OneFace( this->m_vShapes[i], + img, + this->m_vTemplateTriangle2D, + this->m_vTemplatePointWarpInfo, + this->m_vFeatures[i], + this->m_iTextureRepresentationMethod) ) + + { + cout << "Texture Fail to Load at image " << i << endl; + return false; + } + + double end = (double)cvGetTickCount(); + double elapsed = (end - start) / (cvGetTickFrequency()*1000.0); + } + + return true; +} + + +/** + * @author JIA Pei + * @version 2010-02-05 + * @brief build Texture Model + * @param allLandmarkFiles4Training Input - all training landmark files + * @param allImgFiles4Training Input - all training image files + * @param shapeinfoFileName Input - shape info file + * @param database Input - which database is it? + * @param channels Input - How many channels are to be used? + * @param trm Input - texture representation method + * @param TPShape Input - truncated percentage for shape model + * @param TPTexture Input - truncated percentage for texture model + * @param useKnownTriangles Input - use known triangle structures?? + * @note Refer to "Statistical Models of Appearance for Computer Vision" page 31, Cootes + * @return void +*/ +void VO_FeatureModel::VO_BuildFeatureModel( const vector& allLandmarkFiles4Training, + const vector& allImgFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + unsigned int channels, + int trm, + float TPShape, + float TPTexture, + bool useKnownTriangles) +{ + +} diff --git a/modules/smbuilding/src/VO_Point2DDistributionModel.cpp b/modules/smbuilding/src/VO_Point2DDistributionModel.cpp new file mode 100644 index 0000000..4e85feb --- /dev/null +++ b/modules/smbuilding/src/VO_Point2DDistributionModel.cpp @@ -0,0 +1,251 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include +#include +#include +#include + +#include "VO_Point2DDistributionModel.h" +#include "VO_CVCommon.h" + + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief build point model + * @param allAlignedShapes Input - input aligned points. + * @return void +*/ +void VO_Point2DDistributionModel::VO_BuildPointDistributionModel( const vector& allAlignedShapes ) +{ + unsigned int NbOfSamples = allAlignedShapes.size(); + unsigned int NbOfPoints = allAlignedShapes[0].GetNbOfPoints(); + this->m_VONormalizedEllipses.resize(NbOfPoints); + Mat_ matAlignedPoints = Mat_::zeros(NbOfSamples, 2); + Mat_ matAlignedMeanPoint = Mat_::zeros(1, 2); + + for(unsigned int i = 0; i < NbOfPoints; ++i) + { + for(unsigned int j = 0; j < NbOfSamples; ++j) + { + matAlignedPoints(j, 0) = allAlignedShapes[j].GetA2DPoint(i).x; + matAlignedPoints(j, 1) = allAlignedShapes[j].GetA2DPoint(i).y; + } + PCA pca = PCA(matAlignedPoints, matAlignedMeanPoint, CV_PCA_DATA_AS_ROW, 2); + float xx = pca.eigenvectors.at(0,0); + float yy = pca.eigenvectors.at(0,1); + float theta = 0.0; + if( fabs(xx) < FLT_MIN ) theta = (float)(CV_PI/2.0); + else theta = atan(yy/xx)/safeDoubleToFloat(CV_PI*180.0); + Point2f pt = Point2f( pca.mean.at(0,0), + pca.mean.at(0,1)); + this->m_VONormalizedEllipses[i] = VO_Ellipse( pt, + 3.0f*sqrt(pca.eigenvalues.at(0,0)), + 3.0f*sqrt(pca.eigenvalues.at(1,0)), + 0, + 360, + theta ); + } +} + + +/** + * @author JIA Pei + * @version 2010-06-07 + * @brief Constrain single point + * @param pt Input and Output - the input and output point +*/ +void VO_Point2DDistributionModel::VO_ConstrainSinglePoint(Point2f& pt, const VO_Ellipse& ellipse) +{ + if( ! const_cast(ellipse).IsPointWithinEllipse(pt) ) + { + Point2f boundpt = const_cast(ellipse).FindIntersectPointOnBoundary(pt); + pt = boundpt; + } +} + + +/** + * @author JIA Pei + * @version 2010-06-07 + * @brief Scale all input ellipses + * @param iEllipses - input + * @param oEllipses - output +*/ +void VO_Point2DDistributionModel::VO_ScalePDMEllipses(const vector& iEllipses, float scale, vector& oEllipses) +{ + unsigned int NbOfEllipses = iEllipses.size(); + for(unsigned int i = 0; i < NbOfEllipses; i++) + { + oEllipses[i] = const_cast< vector& >(iEllipses)[i]*scale; + } +} + + +/** + * @author JIA Pei + * @version 2010-06-07 + * @brief Constrain all points respetively + * @param ioShape Input and Output - the input and output shape +*/ +void VO_Point2DDistributionModel::VO_ConstrainAllPoints(VO_Shape& ioShape) +{ + unsigned int NbOfPoints = ioShape.GetNbOfPoints(); + Point2f pt; + + for(unsigned int i = 0; i < NbOfPoints; i++) + { + pt = ioShape.GetA2DPoint(i); + VO_Point2DDistributionModel::VO_ConstrainSinglePoint( pt, this->m_VONormalizedEllipses[i] ); + ioShape.SetA2DPoint(pt, i); + } +} + + +/** + * @author JIA Pei + * @version 2010-02-22 + * @brief Save ASM to a specified folder + * @param fd Input - the folder that ASM to be saved to +*/ +void VO_Point2DDistributionModel::VO_Save(const string& fd) +{ + // create Point2DDistributionModel subfolder for just Point2DDistributionModel data + string fn = fd+"/Point2DDistributionModel"; + MakeDirectory(fn); + + fstream fp; + string tempfn; + + // Point2DDistributionModel + tempfn = fn + "/Point2DDistributionModel" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "NbOfPoints" << endl << this->m_VONormalizedEllipses.size() << endl; + fp.close();fp.clear(); + + // m_VONormalizedEllipses + tempfn = fn + "/m_VONormalizedEllipses" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_VONormalizedEllipses" << endl; + for(unsigned int i = 0; i < this->m_VONormalizedEllipses.size(); i++) + { + fp << this->m_VONormalizedEllipses[i] << endl; + } + fp.close();fp.clear(); +} + + +/** +* @author JIA Pei +* @version 2010-02-22 +* @brief Load all ASM data from a specified folder +* @param fd Input - the folder that ASM to be loaded from +*/ +void VO_Point2DDistributionModel::VO_Load(const string& fd) +{ + this->VO_LoadParameters4Fitting(fd); +} + + +/** +* @author JIA Pei +* @version 2010-02-22 +* @brief Load all ASM data from a specified folder for later fitting +* @param fd Input - the folder that ASM to be loaded from +*/ +void VO_Point2DDistributionModel::VO_LoadParameters4Fitting(const string& fd) +{ + string fn = fd+"/Point2DDistributionModel"; + if (!MakeDirectory(fn) ) + { + cout << "Point2DDistributionModel subfolder is not existing. " << endl; + exit(EXIT_FAILURE); + } + + ifstream fp; + string tempfn; + string temp; + unsigned int NbOfPoints; + + // Point2DDistributionModel + tempfn = fn + "/Point2DDistributionModel" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp >> NbOfPoints; + fp.close();fp.clear(); + + this->m_VONormalizedEllipses.resize(NbOfPoints); + + // m_VONormalizedEllipses + tempfn = fn + "/m_VONormalizedEllipses" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + this->m_VONormalizedEllipses.resize(NbOfPoints); + for(unsigned int i = 0; i < NbOfPoints; i++) + { + fp >> this->m_VONormalizedEllipses[i]; + } + fp.close();fp.clear(); +} + + + + diff --git a/modules/smbuilding/src/VO_Profile.cpp b/modules/smbuilding/src/VO_Profile.cpp new file mode 100644 index 0000000..a927e10 --- /dev/null +++ b/modules/smbuilding/src/VO_Profile.cpp @@ -0,0 +1,1027 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_Profile.h" +#include "VO_CVCommon.h" + + +ostream& operator<<(ostream& os, const VO_Profile& profile) +{ + os << profile.m_MatProf; + return os; +} + + +istream& operator>>(istream &is, VO_Profile& profile) +{ + is >> profile.m_MatProf; + return is; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator= overloading, similar to copy constructor + * @param iProfile The input profile + * @return VO_Profile& +*/ +VO_Profile& VO_Profile::operator=(const VO_Profile& iProfile) +{ + this->CopyData (iProfile); + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator= overloading, similar to copy constructor + * @param iProfile The input profile + * @return VO_Profile& +*/ +VO_Profile& VO_Profile::operator=(const Mat_& iProfile) +{ + iProfile.copyTo(m_MatProf); + return *this; +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief operator= overloading, similar to copy constructor +* @param value assign all values in VO_Profile to value +* @return VO_Profile& +*/ +VO_Profile& VO_Profile::operator=(float value) +{ + m_MatProf = value; + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator+ overloading, shift one AAM shape by value + * @param value + * @return VO_Profile +*/ +VO_Profile VO_Profile::operator+(float value) +{ + VO_Profile res(*this); + res.m_MatProf += value; + + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator+= overloading, add value to this profile + * @param value + * @return VO_Profile& +*/ +VO_Profile& VO_Profile::operator+=(float value) +{ + m_MatProf += value; + + return *this; +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief operator+ overloading, add two profiles to one +* @param iProfile the added profile +* @return VO_Profile +*/ +VO_Profile VO_Profile::operator+(const VO_Profile& iProfile) +{ + VO_Profile res(*this); + res.m_MatProf += iProfile.m_MatProf; + + return res; +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief operator+= overloading, add the input profile to this profile +* @param iProfile the added profile +* @return VO_Profile& +*/ +VO_Profile& VO_Profile::operator+=(const VO_Profile& iProfile) +{ + this->m_MatProf += iProfile.m_MatProf; + + return *this; +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief operator- overloading, shift one profile by -value +* @param value +* @return VO_Profile +*/ +VO_Profile VO_Profile::operator-(float value) +{ + VO_Profile res(*this); + res.m_MatProf -= value; + + return res; +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief operator-= overloading, subtract value from this profile +* @param value +* @return VO_Profile& +*/ +VO_Profile& VO_Profile::operator-=(float value) +{ + this->m_MatProf -= value; + + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator- overloading, subtract one profile from another + * @param iProfile the subtracted profile + * @return VO_Profile +*/ +VO_Profile VO_Profile::operator-(const VO_Profile& iProfile) +{ + VO_Profile res(*this); + res.m_MatProf -= iProfile.m_MatProf; + + return res; +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief operator-= overloading, subtract the input profile from this profile +* @param iProfile the subtracted profile +* @return VO_Profile& +*/ +VO_Profile& VO_Profile::operator-=(const VO_Profile& iProfile) +{ + this->m_MatProf -= iProfile.m_MatProf; + + return *this; +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief operator* overloading, scale a profile with input float value +* @param value scale size +* @return VO_Profile +*/ +VO_Profile VO_Profile::operator*(float value) +{ + VO_Profile res(*this); + res.m_MatProf *= value; + + return res; +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief operator*= overloading, scale this profile with input float value +* @param value +* @return VO_Profile& +*/ +VO_Profile& VO_Profile::operator*=(float value) +{ + this->m_MatProf *= value; + + return *this; +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief operator* overloading, element-wise product of two profiles +* @param iProfile profile to be dot producted +* @return float dot product +*/ +VO_Profile VO_Profile::operator*(const VO_Profile& iProfile) +{ + VO_Profile res(*this); + res.m_MatProf = res.m_MatProf.mul(iProfile.m_MatProf); + + return res; +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief operator* overloading, element-wise product of two profiles +* @param iProfile profile to be dot producted +* @return float dot product +*/ +VO_Profile& VO_Profile::operator*=(const VO_Profile& iProfile) +{ + this->m_MatProf = this->m_MatProf.mul(iProfile.m_MatProf); + + return *this; +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief operator/ overloading, scale a profile +* @param value 1.0/value = scale size +* @return VO_Profile +*/ +VO_Profile VO_Profile::operator/(float value) +{ + if( fabs(value) <= FLT_MIN ) + { + cerr << "Divided by zero!" << endl; + exit(EXIT_FAILURE); + } + + VO_Profile res(*this); + res.m_MatProf /= value; + + return res; +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief operator/= overloading, scale this profile with input float value +* @param value 1.0/value = the scaled value +* @return VO_Profile& +*/ +VO_Profile& VO_Profile::operator/=(float value) +{ + if( fabs(value) <= FLT_MIN ) + { + cerr << "Divided by zero!" << endl; + exit(EXIT_FAILURE); + } + + this->m_MatProf /= value; + + return *this; +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief operator/ overloading, scale a profile +* @param iProfile for element-wise division +* @return VO_Profile +*/ +VO_Profile VO_Profile::operator/(const VO_Profile& iProfile) +{ + for(int i = 0; i < iProfile.m_MatProf.rows; i++) + { + for(int j = 0; j < iProfile.m_MatProf.cols; j++) + { + if( fabs(iProfile.m_MatProf(i,j)) <= FLT_MIN ) + { + cerr << "Divided by zero!" << endl; + exit(EXIT_FAILURE); + } + } + } + + VO_Profile res(*this); + res.m_MatProf /= iProfile.m_MatProf; + + return res; +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief operator/= overloading, scale this profile with input float value +* @param iProfile for element-wise division +* @return VO_Profile& +*/ +VO_Profile& VO_Profile::operator/=(const VO_Profile& iProfile) +{ + for(int i = 0; i < iProfile.m_MatProf.rows; i++) + { + for(int j = 0; j < iProfile.m_MatProf.cols; j++) + { + if( fabs(iProfile.m_MatProf(i,j)) <= FLT_MIN ) + { + cerr << "Divided by zero!" << endl; + exit(EXIT_FAILURE); + } + } + } + + this->m_MatProf /= iProfile.m_MatProf; + + return *this; +} + + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief operator() overloading, obtain the profile value at row and col +* @param row input - row index +* @param col input - col index +* @return float& +*/ +float& VO_Profile::operator() (unsigned row, unsigned col) +{ + if ( row >= m_MatProf.rows || col >= m_MatProf.cols ) + cerr << "Matrix subscript out of bounds" << endl; + return m_MatProf.at(row, col); +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief operator() overloading, obtain the profile value at row and col +* @param row input - row index +* @param col input - col index +* @return float& +*/ +float VO_Profile::operator() (unsigned row, unsigned col) const +{ + if ( row >= m_MatProf.rows || col >= m_MatProf.cols ) + cerr << "Matrix subscript out of bounds" << endl; + return m_MatProf.at(row, col); +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief operator* overloading, dot product of two profiles +* @param iProfile profile to be dot producted +* @return float dot product +*/ +float VO_Profile::dot(const VO_Profile& iProfile) +{ + return safeDoubleToFloat(this->m_MatProf.dot(iProfile.m_MatProf)); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief resize the matrix + * @param length Input, profile length + * @param dim Input, profile dim + * @return void +*/ +void VO_Profile::Resize(unsigned int length, unsigned int dim) +{ + this->m_MatProf.release(); + this->m_MatProf = Mat_::zeros(length, dim); +} + +/** + * @author Colin B + * @version 2012-05-28 + * @brief Range check for before texture calculation + * @param inPt Input/Output point to bound + * @param rows Input height of image + * @param cols Input width of image + * @return void + */ +void VO_Profile::BoundPoint(Point2f& toBound,const int height,const int width){ + + if(static_cast(toBound.x) < 0) + toBound.x = 0.0f; + else if (static_cast(toBound.x+1) >= width-1) + toBound.x -= 2*(toBound.x - (width-1)) +2; + + assert(toBound.x+1 < width); + + if(static_cast(toBound.y) < 0) + toBound.y = 0.0f; + else if(static_cast(toBound.y+1) >= height-1) + toBound.y -= 2*(toBound.y - (height-1)) +2; + + assert(toBound.y+1 < height); +} + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Get a subprofile in 1 dimension, length*1 + * @param start Input start from which index + * @param length Input how long will the subprofile be + * @return Mat_ the subprofile + */ +Mat_ VO_Profile::GetSubProfile(int start, unsigned int length, unsigned int dimIdx) const +{ + if(start+length > this->m_MatProf.rows) + cerr << "VO_Profile start+length is bigger than m_MatProf.rows" << endl; + + Mat_ oProfile = Mat_::zeros(length, 1); + for(unsigned int i = 0; i < length; i++) + { + oProfile(i, 0) = this->m_MatProf(i+start, dimIdx); + } + + return oProfile; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Set 1 dimension Profile + * @param iOneDimProf Input 1D column vector, length*1 + * @return void + */ +void VO_Profile::Set1DimProfile(const Mat_& iOneDimProf, unsigned int idx) +{ + unsigned int NbOfDim = this->m_MatProf.cols; + unsigned int NbOfLength = this->m_MatProf.rows; + if(idx >= NbOfDim) + { + cerr << "idx shouldn't be bigger than the dim of m_MatProf!" << endl; + exit(1); + } + if(iOneDimProf.rows != NbOfLength) + { + cerr << "input profile length should be equal to the length of m_MatProf!" << endl; + exit(1); + } + for(unsigned int i = 0; i < NbOfLength; i++) + { + this->m_MatProf(i, idx) = iOneDimProf(i, 0); + } +} + +void VO_Profile::Set1DimProfile(const VO_Profile& iOneDimProf, unsigned int idx) +{ + this->Set1DimProfile(iOneDimProf.m_MatProf, idx); +} + +/** + * @author Colin B + * @author JIA Pei + * @version 2012-05-17 + * @brief Set 2dimension Profile + * @param iTwoDimProf Input 2D column vector, length*1 + * @return void + */ +void VO_Profile::Set2DimProfile(const Mat_& iTwoDimProf, unsigned int idx) +{ + unsigned int NbOfDim = this->m_MatProf.cols; + unsigned int NbOfLength = this->m_MatProf.rows; + if(idx >= NbOfDim) + { + cerr << "idx shouldn't be bigger than the dim of m_MatProf!" << endl; + exit(1); + } + if(iTwoDimProf.rows != NbOfLength) + { + cerr << "input profile length should be equal to the length of m_MatProf!" << endl; + exit(1); + } + for(unsigned int i = 0; i < NbOfLength; i++) + { + this->m_MatProf(i, idx) = iTwoDimProf(i, 0); + this->m_MatProf(i, idx+1) = iTwoDimProf(i, 1); + } +} + +void VO_Profile::Set2DimProfile(const VO_Profile& iTwoDimProf, unsigned int idx) +{ + this->Set2DimProfile(iTwoDimProf.m_MatProf, idx); +} + + +/** + * @author JIA Pei + * @version 2010-02-22 + * @brief Get ND profiles for a single landmark + * @param iImg Input -- the input image + * @param ThisPoint Input -- the concerned point + * @param oProf Output -- output profile in column format + * @param deltaX Input -- deltaX in some direction + * @param deltaY Input -- deltaY in some direction + * @param ProfileLength Input -- how many elements for a single profile + * @return void + * @note Using "float* oProf" is much much faster than using "VO_Profile& oProf" or "vector" + */ +void VO_Profile::VO_Get1DProfileInMat4OneLandmark ( const Mat& iImg, + const Point2f& ThisPoint, + VO_Profile& oProf, + const float deltaX, + const float deltaY, + const unsigned int ProfileLength) +{ +//static Mat oImg(iImg); +//Point pt1, pt2; +//int rgb = 255; +//Scalar profileColor = Scalar(rgb,rgb,rgb); +//unsigned int TheColor = 0; + unsigned int width = iImg.cols; + unsigned int height = iImg.rows; + unsigned int channels = iImg.channels(); + + //each image channel get a seperate profile + oProf = Mat_::zeros(ProfileLength, channels); + + // Emphasized by JIA Pei. k shouldn't be unsigned int in this function + int k = (ProfileLength-1)/2; + Point2f normalPoint; + + normalPoint.x = ThisPoint.x + ( -k-1 ) * deltaX; + normalPoint.y = ThisPoint.y + ( -k-1 ) * deltaY; + +//pt1 = normalPoint; +//cv::line( tempImg, pt1, pt1, colors[TheColor%8], 5, 0, 0 ); +//if(ptIdx == 84 || ptIdx == 85 || ptIdx == 86 ) +//{ +//cv::line( oImg, pt1, pt1, colors[TheColor%8], 5, 0, 0 ); +//} + + // make sure the point is within the image, otherwise, you can't extract the pixel RGB texture + BoundPoint(normalPoint,height,width); + + if(channels == 1) + { + float gray_prev = 0.0f; + float gray_curr = 0.0f; + + VO_TextureModel::VO_CalcSubPixelTexture ( normalPoint.x, normalPoint.y, iImg, &gray_prev ); + + for (int i = -k; i <= k; ++i) + { +//pt1 = normalPoint; + normalPoint.x = ThisPoint.x + i * deltaX; + normalPoint.y = ThisPoint.y + i * deltaY; +//pt2 = normalPoint; + +//{ +//rgb = int ( 255.0/(float)ProfileLength*(float)TheColor ); +//profileColor = Scalar(rgb,rgb,rgb); +//cv::line( oImg, pt1, pt2, profileColor, 2, 0, 0 ); +//++TheColor; +//} + + // make sure the point is within the image, otherwise, you can't extract the pixel RGB texture + BoundPoint(normalPoint,height,width); + + VO_TextureModel::VO_CalcSubPixelTexture ( normalPoint.x, normalPoint.y, iImg, &gray_curr ); + + oProf.m_MatProf(i+k, 0) = gray_curr - gray_prev; + gray_prev = gray_curr; + } + } + else if(channels == 2){ + + float gray_prev = 0.0f; + float gray_curr = 0.0f; + float depth_prev = 0.0f; + float depth_curr = 0.0f; + + //seperate pixel channels calculated + VO_TextureModel::VO_CalcSubPixelTexture ( normalPoint.x, normalPoint.y, iImg, &gray_prev, &depth_prev ); + + //silly loop for dealing with high profile dimensions + for (int i = -k; i <= k; ++i) + { + normalPoint.x = ThisPoint.x + i * deltaX; + normalPoint.y = ThisPoint.y + i * deltaY; + + // range check + BoundPoint(normalPoint,height,width); + + VO_TextureModel::VO_CalcSubPixelTexture ( normalPoint.x, normalPoint.y, iImg, &gray_curr, &depth_curr ); + + oProf.m_MatProf(i+k, 0) = gray_curr - gray_prev; + oProf.m_MatProf(i+k, 1) = depth_curr - depth_prev; + + gray_prev = gray_curr; + depth_prev = depth_curr; + } + } + else if (channels == 3) + { + Mat grayImg; + cv::cvtColor(iImg, grayImg, CV_BGR2GRAY); + float gray_prev = 0.0f; + float gray_curr = 0.0f; + + VO_TextureModel::VO_CalcSubPixelTexture ( normalPoint.x, normalPoint.y, grayImg, &gray_prev ); + + for (int i = -k; i <= k; ++i) + { +//pt1 = normalPoint; + normalPoint.x = ThisPoint.x + i * deltaX; + normalPoint.y = ThisPoint.y + i * deltaY; +//pt2 = normalPoint; + +//{ +//rgb = int ( 255.0/(float)ProfileLength*(float)TheColor ); +//profileColor = Scalar(rgb,rgb,rgb); +//cv::line( oImg, pt1, pt2, profileColor, 2, 0, 0 ); +//++TheColor; +//} + + // make sure the point is within the image, otherwise, you can't extract the pixel RGB texture + BoundPoint(normalPoint,height,width); + + VO_TextureModel::VO_CalcSubPixelTexture ( normalPoint.x, normalPoint.y, grayImg, &gray_curr ); + + oProf.m_MatProf(i+k, 0) = gray_curr - gray_prev; + gray_prev = gray_curr; + } +//////////////////////////////////////////////////////////////////////////////////////////// +// The following is dealing with 3 channels +// float b_prev = 0.0f, g_prev = 0.0f, r_prev = 0.0f; +// float b_curr = 0.0f, g_curr = 0.0f, r_curr = 0.0f; +// +// VO_TextureModel::VO_CalcSubPixelTexture ( normalPoint.x, normalPoint.y, iImg, &b_prev, & g_prev, &r_prev ); +// +// for (int i = -k; i <= k; ++i) +// { +////pt1 = normalPoint; +// normalPoint.x = ThisPoint.x + i * deltaX; +// normalPoint.y = ThisPoint.y + i * deltaY; +////pt2 = normalPoint; +// +////{ +////rgb = int ( 255.0/(float)ProfileLength*(float)TheColor ); +////profileColor = Scalar(rgb,rgb,rgb); +////cv::line( oImg, pt1, pt2, profileColor, 2, 0, 0 ); +////++TheColor; +////} +// + //// make sure the point is within the image, otherwise, you can't extract the pixel RGB texture + //BoundPoint(normalPoint,height,width); +// VO_TextureModel::VO_CalcSubPixelTexture ( normalPoint.x, normalPoint.y, iImg, &b_curr, &g_curr, &r_curr ); +// +// oProf(3*(i+k)+0, 0) = b_curr - b_prev; +// oProf(3*(i+k)+1, 0) = g_curr - g_prev; +// oProf(3*(i+k)+2, 0) = r_curr - r_prev; +// b_prev = b_curr; +// g_prev = g_curr; +// r_prev = r_curr; +// } +//////////////////////////////////////////////////////////////////////////////////////////// + } + else + { + cerr << "VO_Profile: image channels error!" << endl; + exit(1); + } + +//imwrite("test.jpg", oImg); +} + +/** + * @author JIA Pei + * @author Colin B + * @version 2010-02-22 + * @brief Get ND profiles for a single landmark + * @param iImg Input -- the input image + * @param ThisPoint Input -- the concerned point + * @param oProf Output -- output profile in column format + * @param deltaX Input -- deltaX in some direction + * @param deltaY Input -- deltaY in some direction + * @param ProfileLength Input -- how many elements for a single profile + * @return void + * @note Using "float* oProf" is much much faster than using "VO_Profile& oProf" or "vector" + */ +void VO_Profile::VO_Get2DProfileInMat4OneLandmark ( const Mat& iImg, + const Point2f& ThisPoint, + VO_Profile& oProf, + const float deltaX, + const float deltaY, + const unsigned int ProfileLength) +{ +//static Mat oImg(iImg); +//Point pt1, pt2; +//int rgb = 255; +//Scalar profileColor = Scalar(rgb,rgb,rgb); +//unsigned int TheColor = 0; + unsigned int width = iImg.cols; + unsigned int height = iImg.rows; + + //each image channel get a seperate profile + oProf = Mat_::zeros(ProfileLength, 2); + + // Emphasized by JIA Pei. k shouldn't be unsigned int in this function + int k = (ProfileLength-1)/2; + Point2f normalPoint; + + normalPoint.x = ThisPoint.x + ( -k-1 ) * deltaX; + normalPoint.y = ThisPoint.y + ( -k-1 ) * deltaY; + +//pt1 = normalPoint; +//cv::line( tempImg, pt1, pt1, colors[TheColor%8], 5, 0, 0 ); +//if(ptIdx == 84 || ptIdx == 85 || ptIdx == 86 ) +//{ +//cv::line( oImg, pt1, pt1, colors[TheColor%8], 5, 0, 0 ); +//} + + // make sure the point is within the image, otherwise, you can't extract the pixel RGB texture + BoundPoint(normalPoint,height,width); + + + float gray_prev = 0.0f; + float gray_curr = 0.0f; + float depth_prev = 0.0f; + float depth_curr = 0.0f; + + //seperate pixel channels calculated + VO_TextureModel::VO_CalcSubPixelTexture ( normalPoint.x, normalPoint.y, iImg, &gray_prev, &depth_prev ); + + //silly loop for dealing with high profile dimensions + for (int i = -k; i <= k; ++i) + { + normalPoint.x = ThisPoint.x + i * deltaX; + normalPoint.y = ThisPoint.y + i * deltaY; + + // range check and reflection padding + BoundPoint(normalPoint,height,width); + + VO_TextureModel::VO_CalcSubPixelTexture ( normalPoint.x, normalPoint.y, iImg, &gray_curr, &depth_curr ); + + oProf.m_MatProf(i+k, 0) = gray_curr - gray_prev; + oProf.m_MatProf(i+k, 1) = depth_curr - depth_prev; + + gray_prev = gray_curr; + depth_prev = depth_curr; + } + +//imwrite("test.jpg", oImg); +} + +/** + * @author JIA Pei + * @version 2010-02-22 + * @brief Get ND profiles for a single landmark + * @param iImg Input -- the input image + * @param iShape Input -- the training shape + * @param iShapeInfo Input -- shape info + * @param ptIdx Input -- the landmark index + * @param oProf Output -- output profile + * @param dim Input -- 1D, 2D, 4D? + * @param ProfileLength Input -- how many elements for a single profile, already multiply by 3 if iImg is of 3 channels + * @param pDeltaX Output -- deltaX in normal direction + * @param pDeltaY Output -- deltaY in normal direction + * @return void + * @note Using "float* oProf" is much much faster than using "VO_Profile& oProf" or "vector" + */ +void VO_Profile::VO_GetNDProfiles4OneLandmark ( const Mat& iImg, + const VO_Shape& iShape, + const vector& iShapeInfo, + unsigned int ptIdx, + VO_Profile& oProf, + unsigned int dim, + unsigned int channels, + unsigned int ProfileLength, + float* pDeltaX, + float* pDeltaY) +{ + //number of profile dimension might mean channels + oProf = Mat_::zeros( ProfileLength, dim ); + + /** Here, this is not compatible with 3D */ + Point2f PrevPoint = iShape.GetA2DPoint ( iShapeInfo[ptIdx].GetFrom() ); + Point2f ThisPoint = iShape.GetA2DPoint ( ptIdx ); + Point2f NextPoint = iShape.GetA2DPoint ( iShapeInfo[ptIdx].GetTo() ); + + float deltaX, deltaY; + float normX, normY; + float sqrtsum; + + // left side (connected from side) + deltaX = ThisPoint.x - PrevPoint.x; + deltaY = ThisPoint.y - PrevPoint.y; + sqrtsum = sqrt ( deltaX*deltaX + deltaY*deltaY ); + if ( sqrtsum < FLT_EPSILON ) sqrtsum = 1.0f; + deltaX /= sqrtsum; deltaY /= sqrtsum; // Normalize + // Firstly, normX normY record left side norm. + normX = -deltaY; + normY = deltaX; + + // right side (connected to side) + deltaX = NextPoint.x - ThisPoint.x; + deltaY = NextPoint.y - ThisPoint.y; + sqrtsum = sqrt ( deltaX*deltaX + deltaY*deltaY ); + if ( sqrtsum < FLT_EPSILON ) sqrtsum = 1.0f; + deltaX /= sqrtsum; deltaY /= sqrtsum; // Normalize + // Secondly, normX normY will average both left side and right side norm. + normX += -deltaY; + normY += deltaX; + + // Average left right side + sqrtsum = sqrt ( normX*normX + normY*normY ); + if ( sqrtsum < FLT_EPSILON ) sqrtsum = 1.0f; + normX /= sqrtsum; + normY /= sqrtsum; // Final Normalize + + ////////////////////////////////////////////////////////////////////////////// + // For the 1st dimension -- ASM_PROFILE1D + // terrific - speed up always. Explained by JIA Pei, coded by Yao Wei. + VO_Profile tmpCol; + switch(dim) + { + case 2: + { + float tangentX = -normY; + float tangentY = normX; + + VO_Profile::VO_Get1DProfileInMat4OneLandmark ( iImg, + ThisPoint, + tmpCol, + normX, + normY, + ProfileLength); + if(channels == 2){ + //sets dim 0,1 + oProf.Set2DimProfile(tmpCol, 0 ); + }else{ + oProf.Set1DimProfile(tmpCol, 0 ); + } + VO_Profile::VO_Get1DProfileInMat4OneLandmark ( iImg, + ThisPoint, + tmpCol, + tangentX, + tangentY, + ProfileLength); + if(channels == 2){ + //set dim 2,3 + oProf.Set2DimProfile(tmpCol, 2 ); + }else{ + oProf.Set1DimProfile(tmpCol, 1 ); + } + } + break; + case 4: + { + float tangentX = -normY; + float tangentY = normX; + if(channels == 2){ + //special case + VO_Profile::VO_Get2DProfileInMat4OneLandmark (iImg, ThisPoint, tmpCol, normX, normY, ProfileLength); + //sets dim 0,1 + oProf.Set2DimProfile(tmpCol, 0 ); + VO_Profile::VO_Get2DProfileInMat4OneLandmark (iImg, ThisPoint, tmpCol, tangentX, tangentY, ProfileLength); + //set dim 2,3 + oProf.Set2DimProfile(tmpCol, 2 ); + } + else{ + float tmp45X = 0.707106781f*normX-0.707106781f*normY; + float tmp45Y = 0.707106781f*normX+0.707106781f*normY; + float tmp135X = -0.707106781f*normX-0.707106781f*normY; + float tmp135Y = 0.707106781f*normX-0.707106781f*normY; + + VO_Profile::VO_Get1DProfileInMat4OneLandmark ( iImg, + ThisPoint, + tmpCol, + normX, + normY, + ProfileLength); + oProf.Set1DimProfile(tmpCol, 0 ); + VO_Profile::VO_Get1DProfileInMat4OneLandmark ( iImg, + ThisPoint, + tmpCol, + tangentX, + tangentY, + ProfileLength); + oProf.Set1DimProfile(tmpCol, 1 ); + VO_Profile::VO_Get1DProfileInMat4OneLandmark ( iImg, + ThisPoint, + tmpCol, + tmp45X, + tmp45Y, + ProfileLength); + oProf.Set1DimProfile(tmpCol, 2 ); + VO_Profile::VO_Get1DProfileInMat4OneLandmark ( iImg, + ThisPoint, + tmpCol, + tmp135X, + tmp135Y, + ProfileLength); + oProf.Set1DimProfile(tmpCol, 3 ); + } + } + break; + case 1: + default: + { + VO_Profile::VO_Get1DProfileInMat4OneLandmark ( iImg, + ThisPoint, + tmpCol, + normX, + normY, + ProfileLength); + oProf.Set1DimProfile(tmpCol, 0 ); + } + break; + } + ////////////////////////////////////////////////////////////////////////////// + + if(pDeltaX) *pDeltaX = normX; + if(pDeltaY) *pDeltaY = normY; +} + + +/** + * @brief Normalization for every dim + * 1D normalization - refer to Cootes "Statistical Models of Appearance for Computer Vision" page 38, (7.1) + * @note It's not a direct normalization over all elements in the matrix, it's basically column-wise normalization +*/ +void VO_Profile::Normalize() +{ + for(int i = 0; i < this->m_MatProf.cols; i++) + { + Mat oneCol = this->m_MatProf.col(i); + cv::normalize( oneCol, oneCol); + } +} + diff --git a/modules/smbuilding/src/VO_Shape.cpp b/modules/smbuilding/src/VO_Shape.cpp new file mode 100644 index 0000000..3805c08 --- /dev/null +++ b/modules/smbuilding/src/VO_Shape.cpp @@ -0,0 +1,1829 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include +#include +#include +#include + +#include "VO_Shape.h" +#include "VO_Triangle2DStructure.h" // a must, compare to line 25-26 in VO_Shape.h +#include "VO_CVCommon.h" +#include "VO_Common.h" + + +ostream& operator<<(ostream& os, const VO_Shape& shape) +{ + os << shape.m_MatShape; + return os; +} + + +istream& operator>>(istream &is, VO_Shape& shape) +{ + is >> shape.m_MatShape; + return is; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator= overloading, similar to copy constructor + * @param iShape The input shape + * @return VO_Shape& +*/ +VO_Shape& VO_Shape::operator=(const VO_Shape& iShape) +{ + this->CopyData (iShape); + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator= overloading, similar to copy constructor + * @param iShape The input shape + * @return VO_Shape& +*/ +VO_Shape& VO_Shape::operator=(const Mat_& iShape) +{ + iShape.copyTo(this->m_MatShape); + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator= overloading, similar to copy constructor + * @param value assign all values in VO_Shape to value + * @return VO_Shape& +*/ +VO_Shape& VO_Shape::operator=(float value) +{ + m_MatShape = value; + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator+ overloading, shift one AAM shape by value + * @param value + * @return VO_Shape +*/ +VO_Shape VO_Shape::operator+(float value) +{ + VO_Shape res(*this); + res.m_MatShape += value; + + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator+= overloading, add value to this AAM shape + * @param value + * @return VO_Shape& +*/ +VO_Shape& VO_Shape::operator+=(float value) +{ + m_MatShape += value; + + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator+ overloading, add two AAM shape to one + * @param iShape the added AAM shape + * @return VO_Shape +*/ +VO_Shape VO_Shape::operator+(const VO_Shape& iShape) +{ + VO_Shape res(*this); + res.m_MatShape += iShape.m_MatShape; + + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator+= overloading, add the input AAM shape to this AAM shape + * @param iShape the added AAM shape + * @return VO_Shape& +*/ +VO_Shape& VO_Shape::operator+=(const VO_Shape& iShape) +{ + this->m_MatShape += iShape.m_MatShape; + + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator- overloading, shift one AAM shape by -value + * @param value + * @return VO_Shape +*/ +VO_Shape VO_Shape::operator-(float value) +{ + VO_Shape res(*this); + res.m_MatShape -= value; + + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator-= overloading, subtract value from this AAM shape + * @param value + * @return VO_Shape& +*/ +VO_Shape& VO_Shape::operator-=(float value) +{ + this->m_MatShape -= value; + + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator- overloading, subtract one shape from another + * @param iShape the subtracted AAM shape + * @return VO_Shape +*/ +VO_Shape VO_Shape::operator-(const VO_Shape& iShape) +{ + VO_Shape res(*this); + res.m_MatShape -= iShape.m_MatShape; + + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator-= overloading, subtract the input AAM shape from this AAM shape + * @param iShape the subtracted AAM shape + * @return VO_Shape& +*/ +VO_Shape& VO_Shape::operator-=(const VO_Shape &iShape) +{ + this->m_MatShape -= iShape.m_MatShape; + + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator* overloading, scale a shape with input float value + * @param value scale size + * @return VO_Shape +*/ +VO_Shape VO_Shape::operator*(float value) +{ + VO_Shape res(*this); + res.m_MatShape *= value; + + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator*= overloading, scale this shape with input float value + * @param iShape the subtracted AAM shape + * @return VO_Shape& +*/ +VO_Shape& VO_Shape::operator*=(float value) +{ + this->m_MatShape *= value; + + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator* overloading, element-wise product of two AAM shapes + * @param iShape AAM shape to be dot producted + * @return float dot product +*/ +VO_Shape VO_Shape::operator*(const VO_Shape& iShape) +{ + VO_Shape res(*this); + res.m_MatShape = res.m_MatShape.mul(iShape.m_MatShape); + + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator* overloading, element-wise product of two AAM shapes + * @param iShape AAM shape to be dot producted + * @return float dot product +*/ +VO_Shape& VO_Shape::operator*=(const VO_Shape& iShape) +{ + this->m_MatShape = this->m_MatShape.mul(iShape.m_MatShape); + + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator/ overloading, scale a shape + * @param value 1.0/value = scale size + * @return VO_Shape +*/ +VO_Shape VO_Shape::operator/(float value) +{ + if( fabs(value) <= FLT_MIN ) + { + cerr << "Divided by zero!" << endl; + exit(EXIT_FAILURE); + } + + VO_Shape res(*this); + res.m_MatShape /= value; + + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator/= overloading, scale this shape with input float value + * @param value 1.0/value = the scaled value + * @return VO_Shape& +*/ +VO_Shape& VO_Shape::operator/=(float value) +{ + if( fabs(value) <= FLT_MIN ) + { + cerr << "Divided by zero!" << endl; + exit(EXIT_FAILURE); + } + + this->m_MatShape /= value; + + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator/ overloading, scale a shape + * @param iShape for element-wise division + * @return VO_Shape +*/ +VO_Shape VO_Shape::operator/(const VO_Shape& iShape) +{ + for(int i = 0; i < iShape.m_MatShape.rows; i++) + { + for(int j = 0; j < iShape.m_MatShape.cols; j++) + { + if( fabs(iShape.m_MatShape(i,j)) <= FLT_MIN ) + { + cerr << "Divided by zero!" << endl; + exit(EXIT_FAILURE); + } + } + } + + VO_Shape res(*this); + res.m_MatShape /= iShape.m_MatShape; + + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator/= overloading, scale this shape with input float value + * @param iShape for element-wise division + * @return VO_Shape& +*/ +VO_Shape& VO_Shape::operator/=(const VO_Shape& iShape) +{ + for(int i = 0; i < iShape.m_MatShape.rows; i++) + { + for(int j = 0; j < iShape.m_MatShape.cols; j++) + { + if( fabs(iShape.m_MatShape(i,j)) <= FLT_MIN ) + { + cerr << "Divided by zero!" << endl; + exit(EXIT_FAILURE); + } + } + } + + this->m_MatShape /= iShape.m_MatShape; + + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator() overloading, obtain the shape value at row and col + * @param row input - row index + * @param col input - col index + * @return float& +*/ +float& VO_Shape::operator() (unsigned row, unsigned col) +{ + if ( row >= m_MatShape.rows || col >= m_MatShape.cols ) + cerr << "Matrix subscript out of bounds" << endl; + return this->m_MatShape(row, col); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator() overloading, obtain the shape value at row and col + * @param row input - row index + * @param col input - col index + * @return float& +*/ +float VO_Shape::operator() (unsigned row, unsigned col) const +{ + if ( row >= m_MatShape.rows || col >= m_MatShape.cols ) + cerr << "Matrix subscript out of bounds" << endl; + return this->m_MatShape(row, col); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator* overloading, dot product of two AAM shapes + * @param iShape AAM shape to be dot producted + * @return float dot product +*/ +float VO_Shape::dot(const VO_Shape& iShape) +{ + return safeDoubleToFloat(this->m_MatShape.dot(iShape.m_MatShape)); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief resize the matrix + * @param rows + * @param cols + * @return void +*/ +void VO_Shape::Resize(unsigned int rows, unsigned int cols) +{ + this->m_MatShape.release(); + this->m_MatShape = Mat_::zeros(rows, cols); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculates the Center Of Gravity of the shape + * @return Nothing. +*/ +Mat_ VO_Shape::CenterOfGravity() const +{ + return this->Mean(); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Centralize this shape. + * @return Nothing. +*/ +void VO_Shape::Centralize( ) +{ + Mat_ center = this->CenterOfGravity(); + this->Translate(-center); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Translate this shape. + * @param x X-translation. + * @param y Y-translation. + * @return Nothing. +*/ +void VO_Shape::Translate( const Mat_& translation ) +{ + if (translation.rows != this->m_MatShape.rows ) + { + cerr << "translation Mat_ should have the same size as Point dimention!"; + exit(EXIT_FAILURE); + } + for(int i = 0; i < this->m_MatShape.rows; i++) + this->m_MatShape.row(i) += translation(i,0); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Scale this shape. + * @param s Scale factor + * @return void +*/ +void VO_Shape::Scale( float s) +{ + this->m_MatShape *= s; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Scale this shape + * @param svec -- input, scale factor + * @return void +*/ +void VO_Shape::Scale( const Mat_& svec) +{ + if (svec.rows != this->m_MatShape.rows ) + { + cerr << "scale Mat_ should have the same size as Point dimention!"; + exit(EXIT_FAILURE); + } + for(int i = 0; i < this->m_MatShape.rows; i++) + this->m_MatShape.row(i) *= svec(i,0); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Scale this shape in X direction + * @return void +*/ +void VO_Shape::ScaleX( float sX) +{ + this->m_MatShape.row(0) *= sX; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Scale this shape in Y direction + * @return void +*/ +void VO_Shape::ScaleY( float sY) +{ + this->m_MatShape.row(1) *= sY; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Scale this shape in Z direction + * @return void +*/ +void VO_Shape::ScaleZ( float sZ) +{ + this->m_MatShape.row(2) *= sZ; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Rotates the shape in 2-3D + * @param angles Rotation angle in radians + * @return Nothing. +*/ +void VO_Shape::Rotate( const vector& angles) +{ + unsigned int shapeDim = this->m_MatShape.rows; + unsigned int NbOfPoints = this->m_MatShape.cols; + unsigned int angleDim = shapeDim == 2 ? 1 : 3; // only 2D or 3D are considered + if ( angleDim != angles.size() ) + { + cerr << "Rotate angles should have the same dimension as point dimension" << endl; + exit(EXIT_FAILURE); + } + switch(shapeDim) + { + case 3: + { + float x, y, z; + + // Assuming the angles are in radians. + float cph = cos( angles[0] ); + float sph = sin( angles[0] ); + float cth = cos( angles[1] ); + float sth = sin( angles[1] ); + float cps = cos( angles[2] ); + float sps = sin( angles[2] ); + + // set up rotation matrix + // http://mathworld.wolfram.com/RotationMatrix.html + // this is equal to + /** + * 1 0 0 + * 0 cos(ph) sin(ph) + * 0 -sin(ph) cos(ph) + * + * cos(th) 0 -sin(th) + * 0 1 0 + * sin(th) 0 cos(th) + * + * cos(ps) sin(ps) 0 + * -sin(ps) cos(ps) 0 + * 0 0 1 + */ + float c00 = cth * cps; + float c01 = cth * sps; + float c02 = -sth; + float c10 = sph * sth * cps - cph * sps; + float c11 = sph * sth * sps + cph * cps; + float c12 = sph * cth; + float c20 = cph * sth * cps + sph * sps; + float c21 = cph * sth * sps - sph * cps; + float c22 = cph * cth; + + for(unsigned int i = 0; i < NbOfPoints; ++i) + { + x = this->m_MatShape(0, i); + y = this->m_MatShape(1, i); + z = this->m_MatShape(2, i); + this->m_MatShape(0, i) = c00*x+c01*y+c02*z; + this->m_MatShape(1, i) = c10*x+c11*y+c12*z; + this->m_MatShape(2, i) = c20*x+c21*y+c22*z; + } + } + break; + case 2: + default: + { + float x, y; + + // set up rotation matrix + float c00 = cos( angles[0] ); + float c01 = -sin( angles[0] ); + float c10 = sin( angles[0] ); + float c11 = cos( angles[0] ); + + for(unsigned int i = 0; i < NbOfPoints; i++) + { + x = this->m_MatShape(0, i); + y = this->m_MatShape(1, i); + this->m_MatShape(0, i) = c00*x+c01*y; + this->m_MatShape(1, i) = c10*x+c11*y; + } + } + break; + } +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Normalize the shape by translating to its Center Of Gravity + scaling by the reciprocal of the 2-norm + * @return nothing +*/ +void VO_Shape::Normalize() +{ + this->Centralize(); + + float norm = this->GetShapeNorm(); + + this->Scale( 1.0f/norm ); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Returns the 2-norm of this centralized shape + * @return The 2-norm of the shape after translating to origin +*/ +float VO_Shape::GetCentralizedShapeSize() const +{ + VO_Shape tmp(*this); + tmp.Centralize(); + + return tmp.GetShapeNorm(); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Get the norm of the shape + * @return float +*/ +float VO_Shape::GetShapeNorm() const +{ + return (float) cv::norm(this->m_MatShape); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Returns the rotation from this to ref (in radians). + Get the rotation between two shapes by minimizing the sum of squared point distances, + as described by Goodall (and Bookstein, Horn) using Singular Value Decomposition (SVD). + + Note that both shapes must be normalized with respect to scale and position beforehand. + This could be done by using VO_Shape::Normalize(). + * @param ref The reference shape. + * @return The estimated angle, theta, between the two shapes. +*/ +vector VO_Shape::GetRotation( const VO_Shape& ref ) const +{ + vector angles; + assert( ref.m_MatShape.size() == this->m_MatShape.size() ); + + Mat_ res = ref.m_MatShape * this->m_MatShape.t(); +// cv::gemm(ref.m_MatShape, this->m_MatShape, 1.0, Mat_(), 0.0, res, GEMM_2_T); + + double det = cv::determinant( res ); + + SVD svd(res); + + if(det < 0.0f) + { + Mat_ D = Mat_::eye(this->m_MatShape.rows, this->m_MatShape.rows); + D.at(D.rows - 1, D.cols - 1) = -1.0; + +// cv::gemm(svd.u, D, 1.0, Mat_(), 0.0, res); +// cv::gemm(res, svd.vt, 1.0, Mat_(), 0.0, res); + res = svd.u * D * svd.vt; + } + else + { +// cv::gemm(svd.u, svd.vt, 1.0, Mat_(), 0.0, res); + res = svd.u * svd.vt; + } + + switch(this->m_MatShape.rows) + { + case 3: + { + angles.resize(3); + // res now holds a normal 3x3 rotation matrix + // http://mathworld.wolfram.com/RotationMatrix.html + float sin_beta = -res.at(0, 2); + float tan_alpha = res.at(1, 2) / res.at(2, 2); + float tan_gamma = res.at(0, 1) / res.at(0, 0); + + angles[0] = (float)atan(tan_alpha); + angles[1] = (float)asin(sin_beta); + angles[2] = (float)atan(tan_gamma); + } + break; + case 2: + default: + { + angles.resize(1); + // res now holds a normal 2*2 rotation matrix + float cos_theta = res.at(0, 0); + float sin_theta = res.at(1, 0); + + // cos_theta should be less than 1.0; but here, cos_theta is calculated by matrix computation, + // in stead of by acos() function, so cos_theta might have some special values like cos_theta >= 1.0. + if ( ( fabs(1.0-cos_theta) < FLT_EPSILON ) || cos_theta >= 1.0 ) + { + // cos_theta = 1 => shapes are already aligned + angles[0] = 0.0f; + } + else if ( fabs(cos_theta) < FLT_EPSILON ) + { + // cos_theta = 0 => 90 degrees rotation + angles[0] = (float)(CV_PI/2.0); + } + // cos_theta should be bigger than -1.0; but here, cos_theta is calculated by matrix computation, + // in stead of by acos() function, so cos_theta might have some special values like cos_theta <= -1.0. + else if ( ( fabs(1.0+cos_theta) < FLT_EPSILON ) || cos_theta <= -1.0 ) + { + // cos_theta=-1 => 180 degrees rotation + angles[0] = (float)CV_PI; + } + else + { + // get the rotation in radians + float a_cos = (float)acos( cos_theta ); + float a_sin = (float)asin( sin_theta ); + + // http://en.wikipedia.org/wiki/Trigonometric_function + if (a_sin<0) + { + // lower half of the unit circle + angles[0] = -a_cos; + } + else + { + // upper half of the unit circle + angles[0] = a_cos; + } + } + } + break; + } + + return angles; +} + + +/** + * @author JIA Pei + * @version 2010-05-19 + * @brief Ensure the shape is within the size + * @param isize input parameter - the size + * @return void +*/ +void VO_Shape::ConstrainShapeInSize(const Size& isize) +{ + unsigned int NbOfPoints = this->GetNbOfPoints(); + for(unsigned int j = 0; j < NbOfPoints; j++) + { + if(this->m_MatShape(0, j) < FLT_MIN) + this->m_MatShape(0, j) = 0.0f; + else if(this->m_MatShape(0, j) > (isize.width -1.0f) ) + this->m_MatShape(0, j) = isize.width -1.0f; + + if(this->m_MatShape(1, j) < FLT_MIN) + this->m_MatShape(1, j) = 0.0f; + else if(this->m_MatShape(1, j) > (isize.height -1.0f) ) + this->m_MatShape(1, j) = isize.height -1.0f; + } +} + + +/** + * @author JIA Pei + * @version 2010-05-19 + * @brief Ensure the shape is within the image + * @param iImg input parameter - the image + * @return void +*/ +void VO_Shape::ConstrainShapeInImage(const Mat& iImg) +{ + unsigned int NbOfPoints = this->GetNbOfPoints(); + for(unsigned int j = 0; j < NbOfPoints; j++) + { + if(this->m_MatShape(0, j) < FLT_MIN) + this->m_MatShape(0, j) = 0.0f; + else if(this->m_MatShape(0, j) >= (iImg.cols -1.0f - FLT_MIN) ) + this->m_MatShape(0, j) = iImg.cols -1.0f - 0.1f; + + if(this->m_MatShape(1, j) < FLT_MIN) + this->m_MatShape(1, j) = 0.0f; + else if(this->m_MatShape(1, j) > (iImg.rows -1.0f - FLT_MIN) ) + this->m_MatShape(1, j) = iImg.rows -1.0f - 0.1f; + } +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Aligns this to 'ref' with respect to pose. + * @param ref input parameter - The reference shape + * @param scale output parameter - scale value, from this to ref + * @param angles output parameter - rotation angles, from this to ref + * @param translation output parameter - translation in both X, Y directions, from this to ref + * @return void +*/ +void VO_Shape::AlignTo( const VO_Shape& ref, float* scale, vector* angles, Mat_* translation ) +{ + // make a copy of 'ref' + VO_Shape refCpy( ref ); + Mat_ center1, center2; + + // translate, move this and refCpy to origin + center1 = this->CenterOfGravity( ); + this->Translate( -center1 ); + center2 = refCpy.CenterOfGravity( ); + refCpy.Translate( -center2 ); + if(translation) *translation = center2 - center1; + + // scale, using the 2-norm + float this_size = this->GetShapeNorm(); + float ref_size = refCpy.GetShapeNorm(); + if(scale) + { + *scale = ref_size/this_size; + this->Scale( *scale ); + } + else + this->Scale( ref_size/this_size ); + + // align rotation between this and refCpy + if(angles) + { + *angles = this->GetRotation( refCpy ); + this->Rotate( *angles ); + } + else + this->Rotate( this->GetRotation( refCpy ) ); + + // translate this to ref origin + this->Translate( center2 ); +} + + +/** + * @author JIA Pei + * @version 2010-05-24 + * @brief Affine transform of current shape + * @param affineMat input parameter - matrix for affine transform + * @return void +*/ +void VO_Shape::Affine2D(const Mat_& affineMat) +{ + unsigned int NbOfPoints = this->m_MatShape.cols; + Mat_ tmp = Mat_::ones(3, NbOfPoints); + Mat roi = tmp(Range(0, 2), Range(0, NbOfPoints) ); + this->m_MatShape.copyTo(roi); + this->m_MatShape = affineMat*tmp; +} + + +/** + * @author JIA Pei + * @version 2010-02-24 + * @brief Returns the transformation that aligns this to 'ref' + * @param ref input parameter - The reference shape + * @param scale output parameter - scale value, from this to ref + * @param angles output parameter - rotation angles, from this to ref + * @param translation output parameter - translation in both X, Y directions, from this to ref + * @return void +*/ +void VO_Shape::AlignTransformation( const VO_Shape& ref, float& scale, vector& angles, Mat_& translation ) const +{ + VO_Shape refCpy( ref ); + VO_Shape thisCpy( *this ); // a must, we don't want to change "this" right now + Mat_ center1, center2; + + // move thisCpy and refCpy to origin + center1 = thisCpy.CenterOfGravity( ); + thisCpy.Translate( -center1 ); + center2 = refCpy.CenterOfGravity( ); + refCpy.Translate( -center2 ); + translation = center2 - center1; + + // normalize scale, using the 2-norm + float this_size = thisCpy.GetShapeNorm(); + float ref_size = refCpy.GetShapeNorm(); + scale = ref_size/this_size; + thisCpy.Scale( scale ); + + // align rotation between thisCpy and refCpy + angles = thisCpy.GetRotation( refCpy ); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Proscrustes Analysis + * @param ref input parameter - The reference shape, a must!!!!!!!!!! + * @param scale output parameter - 2 norm of a shape + * @param angles output parameter - rotation angles, from this to ref + * @param translation output parameter - translation in both X, Y directions + * @return void + * @note http://en.wikipedia.org/wiki/Procrustes_analysis +*/ +void VO_Shape::ProcrustesAnalysis( const VO_Shape& ref, float &scale, vector& angles, Mat_& translation ) +{ + // move this to origin + translation = this->CenterOfGravity( ); + this->Translate( -translation ); + + // 2-norm = 1 + scale = this->GetShapeNorm(); + this->Scale( 1.0f/scale ); + + // align rotation between this and ref + VO_Shape refCpy( ref ); + Mat_ center = refCpy.CenterOfGravity( ); + refCpy.Translate( -center ); + float norm = refCpy.GetShapeNorm(); + refCpy.Scale( 1.0f/norm ); + + angles = this->GetRotation( refCpy ); + this->Rotate( angles ); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Inverse Proscrustes Analysis + * @param scale input parameter - scale value + * @param angles input parameter - rotation angles, from ref to this + * @param translation input parameter - translation in both X, Y directions + * @return void + * @note http://en.wikipedia.org/wiki/Procrustes_analysis +*/ +void VO_Shape::InverseProcrustesAnalysis( const float& scale, const vector& angles, const Mat_& translation ) +{ + // rotation back + this->Rotate(-angles); + + // 2-norm back + this->Scale(scale); + + // COG back + this->Translate(translation); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Global shape normalization -- refer to "AAM Revisited equation (42)" + * @param iShape Input -- the input shape + * @param oShape Output -- the output shape after global shape normalization + * @param t Input -- parameters required for global shape normalization, rigid variance, refer to "AAM Revisited equation (42)" + * @return void + * @note http://en.wikipedia.org/wiki/Procrustes_analysis +*/ +void VO_Shape::GlobalShapeNormalization2D(const VO_Shape& iShape, VO_Shape& oShape, const Mat_& q) +{ + unsigned int NbOfPoints = iShape.GetNbOfPoints(); + float x, y; + float a = q(0, 0) + 1; + float b = q(0,1); + float tx = q(0,2); + float ty = q(0,3); + + for(unsigned int i = 0; i < NbOfPoints; ++i) + { + x = iShape(0,i); + y = iShape(1,i); + oShape(0, i) = a*x-b*y+tx; + oShape(1, i) = b*x+a*y+ty; + } +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Global shape normalization -- refer to "AAM Revisited equation (42)" + * @param t Input -- parameters required for global shape normalization, rigid variance, refer to "AAM Revisited equation (42)" + * @return void + * @methods - 1) three points' pairs are used to calculate affine transform + * - 2) use my own align transform - COG -> rotation -> scaling... + * - 3) SVD: Y = AX + * x' a00 a01 a02 x + * (y') = (a10 a11 a12) * (y) , + * 1 a20 a21 a22 1 + * where a00 = cos(theta), a01 = -sin(theta) + * a10 = sin(theta), a11 = cons(theta) + * a02 = tx, a12 = ty + * a20 = a21 = 0; a22 = 1 + * However, the above values are not guaranteed during calculation + * @note http://en.wikipedia.org/wiki/Procrustes_analysis + * @ref AAM Revisited (42) (44) a=q0, b=q1, tx=q2, ty=q3 +*/ +void VO_Shape::GlobalShapeNormalization2D(const Mat_& q) +{ + VO_Shape tmp(*this); + unsigned int NbOfPoints = this->m_MatShape.cols; + float x, y; + float a = q(0, 0) + 1; + float b = q(0,1); + float tx = q(0,2); + float ty = q(0,3); + + for(unsigned int i = 0; i < NbOfPoints; ++i) + { + x = tmp(0,i); + y = tmp(1,i); + this->m_MatShape(0, i) = a*x-b*y+tx; + this->m_MatShape(1, i) = b*x+a*y+ty; + } +} + + +/** + * @author JIA Pei + * @version 2010-05-07 + * @brief Global shape normalization -- refer to "AAM Revisited equation (42)" + * @param scale Input -- for scaling + * @param angles Input -- for rotation + * @param translation Input -- for translation + * @return void +*/ +void VO_Shape::GlobalShapeNormalization2D(const VO_Shape& iShape, VO_Shape& oShape, float scale, const vector& angles, const Mat_& translation) +{ + oShape.clone(iShape); + oShape.GlobalShapeNormalization2D(scale, angles, translation); +} + + +/** + * @author JIA Pei + * @version 2010-05-07 + * @brief Global shape normalization -- refer to "AAM Revisited equation (42)" + * @param scale Input -- for scaling + * @param angles Input -- for rotation + * @param translation Input -- for translation + * @return void +*/ +void VO_Shape::GlobalShapeNormalization2D(float scale, const vector& angles, const Mat_& translation) +{ + this->Scale(scale); + this->Rotate( angles ); + this->Translate(translation); +} + + +/** + * @author JIA Pei + * @version 2010-05-07 + * @brief Similarity transform to Global shape normalization -- refer to "AAM Revisited equation (42) (44)" + Cootes' "Statistical Models of Appearance for Computer Vision" page 45 + (s, theta, tx, ty) ->( sx, sy, tx, ty) + * @param scale Input -- for scaling + * @param angles Input -- for rotation, a 1*1 vector + * @param translation Input -- for translation, a 2*1 column vector + * @param q Output -- global shape normalization vector, a 1*4 row vector + * @return void +*/ +void VO_Shape::SimilarityTrans2GlobalShapeNormalization(float scale, + const vector& angles, + const Mat_& translation, + Mat_& q) +{ + q = Mat_::zeros(1,4); + q(0,0) = scale*cos( angles[0] ) - 1.0f; + q(0,1) = scale*sin( angles[0] ); + q(0,2) = translation(0,0); + q(0,3) = translation(1,0); +} + + +/** + * @author JIA Pei + * @version 2010-05-07 + * @brief Global shape normalization 2 similarity transform -- refer to "AAM Revisited equation (42) (44)" + * @param scale Input -- for scaling + * @param angles Input -- for rotation + * @param translation Input -- for translation + * @param q Output -- global shape normalization vector, a 1*4 row vector + * @return void +*/ +void VO_Shape::GlobalShapeNormalization2SimilarityTrans( const Mat_& q, + float& scale, + vector& angles, + Mat_& translation ) +{ + scale = sqrt ( (1.0f + q(0,0)) * (1.0f + q(0,0)) + q(0,1) * q(0,1) ); // scale + angles.resize(1); + angles[0] = (float) atan2( (float)q(0,1), (float)(1.0 + q(0,0)) ); // rotation + translation = Mat_::zeros(2,1); + translation(0,0) = q(0,2); + translation(1,0) = q(0,3); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate min value in all directions + * @return min value in all directions +*/ +Mat_ VO_Shape::Min() const +{ + unsigned int NbOfDim = this->m_MatShape.rows; + Mat_ minVals = Mat_::zeros(NbOfDim, 1); + double minVal = 0; + for(unsigned int i = 0; i < NbOfDim; i++) + { + cv::minMaxLoc(this->m_MatShape.row(i), &minVal, 0, 0, 0); + minVals(i, 0) = safeDoubleToFloat(minVal); + } + + return minVals; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate min value in X direction + * @return min value in X direction +*/ +float VO_Shape::MinX() const +{ + double minVal = 0; + Mat_ oneRow = this->m_MatShape.row(0); + cv::minMaxLoc(oneRow, &minVal, 0, 0, 0); + + return safeDoubleToFloat(minVal); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate min value in Y direction + * @return min value in Y direction +*/ +float VO_Shape::MinY() const +{ + double minVal = 0; + Mat_ oneRow = this->m_MatShape.row(1); + cv::minMaxLoc(oneRow, &minVal, 0, 0, 0); + + return safeDoubleToFloat(minVal); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate min value in Z direction + * @return min value in Y direction +*/ +float VO_Shape::MinZ() const +{ + double minVal = 0; + Mat_ oneRow = this->m_MatShape.row(2); + cv::minMaxLoc(oneRow, &minVal, 0, 0, 0); + + return safeDoubleToFloat(minVal); +} + +Rect VO_Shape::GetShapeBoundRect() const +{ + double minX, maxX, minY, maxY; + cv::minMaxLoc(this->m_MatShape.row(0), &minX, &maxX, 0, 0); + cv::minMaxLoc(this->m_MatShape.row(1), &minY, &maxY, 0, 0); + Rect res; + res.x = static_cast(floor(minX)); + res.y = static_cast(floor(minY)); + res.width = static_cast(ceil(maxX)) - res.x ; + res.height = static_cast(ceil(maxY)) - res.y; + return res; +} + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate max value in all directions + * @return max value in all directions +*/ +Mat_ VO_Shape::Max() const +{ + unsigned int NbOfDim = this->m_MatShape.rows; + Mat_ maxVals = Mat_::zeros(NbOfDim, 1); + double maxVal = 0; + for(unsigned int i = 0; i < NbOfDim; i++) + { + cv::minMaxLoc(this->m_MatShape.row(i), 0, &maxVal, 0, 0); + maxVals(i, 0) = safeDoubleToFloat(maxVal); + } + + return maxVals; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate max value in X direction + * @return max value in X direction +*/ +float VO_Shape::MaxX() const +{ + double maxVal = 0; + Mat_ oneRow = this->m_MatShape.row(0); + cv::minMaxLoc(oneRow, 0, &maxVal, 0, 0); + + return safeDoubleToFloat(maxVal); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate max value in Y direction + * @return max value in Y direction +*/ +float VO_Shape::MaxY() const +{ + double maxVal = 0; + Mat_ oneRow = this->m_MatShape.row(1); + cv::minMaxLoc(oneRow, 0, &maxVal, 0, 0); + + return safeDoubleToFloat(maxVal); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate max value in Z direction + * @return max value in Y direction +*/ +float VO_Shape::MaxZ() const +{ + double maxVal = 0; + Mat_ oneRow = this->m_MatShape.row(2); + cv::minMaxLoc(oneRow, 0, &maxVal, 0, 0); + + return safeDoubleToFloat(maxVal); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate mean value in X direction + * @return mean value in X direction +*/ +Mat_ VO_Shape::Mean() const +{ + unsigned int NbOfDim = this->m_MatShape.rows; + Mat_ meanVals = Mat_::zeros(NbOfDim, 1); + for(unsigned int i = 0; i < NbOfDim; i++) + { + meanVals(i, 0) = safeDoubleToFloat(cv::mean(this->m_MatShape.row(i)).val[0]); + } + + return meanVals; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate mean value in X direction + * @return mean value in X direction +*/ +float VO_Shape::MeanX() const +{ + return safeDoubleToFloat(cv::mean(this->m_MatShape.row(0)).val[0]); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate mean value in Y direction + * @return mean value in Y direction +*/ +float VO_Shape::MeanY() const +{ + return safeDoubleToFloat(cv::mean(this->m_MatShape.row(1)).val[0]); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate mean value in Z direction + * @return mean value in Z direction +*/ +float VO_Shape::MeanZ() const +{ + return safeDoubleToFloat(cv::mean(this->m_MatShape.row(2)).val[0]); +} + + + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate max value in all directions + * @return max value in all directions +*/ +void VO_Shape::MinMaxX(double* minX, double* maxX) const +{ + double minVal = 0, maxVal = 0; + Mat_ oneRow = this->m_MatShape.row(0); + cv::minMaxLoc(oneRow, minX, maxX, 0, 0); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate max value in all directions + * @return max value in all directions +*/ +void VO_Shape::MinMaxY(double* minY, double* maxY) const +{ + double minVal = 0, maxVal = 0; + Mat_ oneRow = this->m_MatShape.row(1); + cv::minMaxLoc(oneRow, minY, maxY, 0, 0); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate max value in all directions + * @return max value in all directions +*/ +void VO_Shape::MinMaxZ(double* minZ, double* maxZ) const +{ + double minVal = 0, maxVal = 0; + Mat_ oneRow = this->m_MatShape.row(2); + cv::minMaxLoc(oneRow, minZ, maxZ, 0, 0); +} + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Find MinX and MinY of shape + * @return left top of shape as a point +*/ +Point VO_Shape::GetLeftTop() const +{ + Point res; + res.x = static_cast(MinX()); + res.y = static_cast(MinY()); + return res; +} + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Find MaxX and MaxY of shape + * @return right bottom of shape as a point +*/ +Point VO_Shape::GetRightBottom() const +{ + Point res; + res.x = static_cast(MaxX()); + res.y = static_cast(MaxY()); + return res; +} + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Find MinX and MaxY of shape + * @return left bottom of shape as a point +*/ +Point VO_Shape::GetLeftBottom() const +{ + Point res; + res.x = static_cast(MinX()); + res.y = static_cast(MaxY()); + return res; +} + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Find MaxX and MainY of shape + * @return right top of shape as a point +*/ +Point VO_Shape::GetRightTop() const +{ + Point res; + res.x = static_cast(MaxX()); + res.y = static_cast(MinY()); + return res; +} + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Get a subshape from shape + * @param iPtIdx -- indexes of point list + * @return the subshape +*/ +VO_Shape VO_Shape::GetSubShape(const vector& iPtIdx) const +{ + unsigned int NbOfSub = iPtIdx.size(); + Mat_ oShape = Mat_::zeros(this->m_MatShape.rows, NbOfSub); + Mat_ tmpCol; + unsigned int count = 0; + for(unsigned int i = 0; i < NbOfSub; i++) + { + tmpCol = oShape.col(count++); + this->m_MatShape.col(iPtIdx[i]).copyTo(tmpCol); + } + return oShape; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Combine two shapes + * @param shape1 -- shape 1 + * @param shape2 -- shape 2 + * @return the point +*/ +VO_Shape VO_Shape::Combine2Shapes(const VO_Shape& shape1, const VO_Shape& shape2) +{ + unsigned int NbOfPoints1 = shape1.GetNbOfPoints(); + unsigned int NbOfPoints2 = shape2.GetNbOfPoints(); + unsigned int NbOfDim1 = shape1.GetNbOfDim(); + unsigned int NbOfDim2 = shape2.GetNbOfDim(); + if( (NbOfPoints1==0 && NbOfPoints2 == 0) || (NbOfDim1==0 && NbOfDim2==0) ) + return Mat_(); + + if(NbOfPoints1!=0 && NbOfPoints2 != 0) + { + if(NbOfDim1 != NbOfDim2) + { + cerr <<"VO_Shape " << "The 2 shapes must have the same dim for combination" << endl; + exit(1); + } + } + + unsigned int NbOfDim = NbOfDim1==0? NbOfDim2:NbOfDim1; + unsigned int NbOfPoints = NbOfPoints1 + NbOfPoints2; + Mat_ res = Mat_::zeros(NbOfDim, NbOfPoints); + + for(unsigned int i = 0; i < NbOfDim; i++) + { + for(unsigned int j = 0; j < NbOfPoints1; j++) + { + res(i, j) = shape1(i,j); + } + + for(unsigned int j = 0; j < NbOfPoints2; j++) + { + res(i, j+shape1.GetNbOfPoints() ) = shape2(i,j); + } + } + + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Get a point in 2D + * @param idx -- indexes of point list + * @return the point +*/ +Point2f VO_Shape::GetA2DPoint(unsigned int idx) const +{ + Point2f res; + unsigned int count = 0; + switch(idx) + { + case LEFTMOST: + { + float xmin = FLT_MAX; + for(int i = 0; i < m_MatShape.cols; i++) + { + if(m_MatShape(0, i) < xmin) + { + xmin = m_MatShape(0, i); + count = i; + } + } + } + break; + case RIGHTMOST: + { + float xmax = -FLT_MAX; + for(int i = 0; i < m_MatShape.cols; i++) + { + if(m_MatShape(0, i) > xmax) + { + xmax = m_MatShape(0, i); + count = i; + } + } + } + break; + case TOPMOST: + { + float ymin = FLT_MAX; + for(int i = 0; i < m_MatShape.cols; i++) + { + if(m_MatShape(1, i) < ymin) + { + ymin = m_MatShape(1, i); + count = i; + } + } + } + break; + case BOTTOMMOST: + { + float ymax = -FLT_MAX; + for(int i = 0; i < m_MatShape.cols; i++) + { + if(m_MatShape(1, i) > ymax) + { + ymax = m_MatShape(1, i); + count = i; + } + } + } + break; + case CENTER: + { + Mat_ mean = this->Mean(); + res.x = mean(0,0); + res.y = mean(1,0); + return res; + } + break; + default: + { + count = idx; + } + break; + } + + res.x = this->m_MatShape.col(count)(0,0); + res.y = this->m_MatShape.col(count)(1,0); + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Get a point in 3D + * @param idx -- indexes of point list + * @return Point3f the point +*/ +Point3f VO_Shape::GetA3DPoint(unsigned int idx) const +{ + Point3f res; + unsigned int count = 0; + switch(idx) + { + case LEFTMOST: + { + float xmin = FLT_MAX; + for(int i = 0; i < m_MatShape.cols; i++) + { + if(m_MatShape(0, i) < xmin) + { + xmin = m_MatShape(0, i); + count = i; + } + } + } + break; + case RIGHTMOST: + { + float xmax = -FLT_MAX; + for(int i = 0; i < m_MatShape.cols; i++) + { + if(m_MatShape(0, i) > xmax) + { + xmax = m_MatShape(0, i); + count = i; + } + } + } + break; + case TOPMOST: + { + float ymin = FLT_MAX; + for(int i = 0; i < m_MatShape.cols; i++) + { + if(m_MatShape(1, i) < ymin) + { + ymin = m_MatShape(1, i); + count = i; + } + } + } + break; + case BOTTOMMOST: + { + float ymax = -FLT_MAX; + for(int i = 0; i < m_MatShape.cols; i++) + { + if(m_MatShape(1, i) > ymax) + { + ymax = m_MatShape(1, i); + count = i; + } + } + } + break; + case INNERMOST: + { + float zmin = FLT_MAX; + for(int i = 0; i < m_MatShape.cols; i++) + { + if(m_MatShape(2, i) < zmin) + { + zmin = m_MatShape(2, i); + count = i; + } + } + } + break; + case OUTERMOST: + { + float zmax = -FLT_MAX; + for(int i = 0; i < m_MatShape.cols; i++) + { + if(m_MatShape(2, i) > zmax) + { + zmax = m_MatShape(2, i); + count = i; + } + } + } + break; + case CENTER: + { + Mat_ mean = this->Mean(); + res.x = mean(0,0); + res.y = mean(1,0); + res.z = mean(2,0); + return res; + } + break; + default: + { + count = idx; + } + break; + } + + res.x = this->m_MatShape.col(count)(0,0); + res.y = this->m_MatShape.col(count)(1,0); + res.z = this->m_MatShape.col(count)(2,0); + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-17 + * @brief Judge whether one point is within a shape + * @param pt input the concerned point + * @param triangles input all triangles composing the shape + * @param int return the index of triangle containing pt +*/ +int VO_Shape::IsPointInShape(const Point2f& pt, const vector& triangles) const +{ + vector shapeTriangles = this->GetTriangle2DStructure(triangles); + return VO_Triangle2DStructure::IsPointInTriangles(pt, shapeTriangles); +} + + +/** + * @author JIA Pei + * @version 2010-02-17 + * @brief Get VO_Triangle2DStructure specific for this shape + * @param triangles Input all triangles composing the shape + * @return vector the index of triangle containing pt +*/ +vector VO_Shape::GetTriangle2DStructure(const vector triangles) const +{ + unsigned int NbOfTriangles = triangles.size(); + vector shapeTriangles = triangles; + vector vertexIdx; + + for(unsigned int i = 0; i < NbOfTriangles; ++i) + { + vertexIdx = triangles[i].GetVertexIndexes(); + shapeTriangles[i].m_MatShape(0,0) = this->m_MatShape(0, vertexIdx[0]); + shapeTriangles[i].m_MatShape(1,0) = this->m_MatShape(1, vertexIdx[0]); + shapeTriangles[i].m_MatShape(0,1) = this->m_MatShape(0, vertexIdx[1]); + shapeTriangles[i].m_MatShape(1,1) = this->m_MatShape(1, vertexIdx[1]); + shapeTriangles[i].m_MatShape(0,2) = this->m_MatShape(0, vertexIdx[2]); + shapeTriangles[i].m_MatShape(1,2) = this->m_MatShape(1, vertexIdx[2]); + // explained by JIA Pei, m_dD of shapeTriangles[i] is not calculated/required here in this function + } + return shapeTriangles; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Write shape into file "fn" + * @param fn Output - the file name + * @param shape Input - save annotation data from shape data structure to file +*/ +void VO_Shape::ExportShape(const string& fn, const VO_Shape& shape) +{ + fstream fp(fn.c_str(), ios::out); + fp << shape << endl; + fp.close();fp.clear(); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Write shape into file "fn" + * @param fn Output - the file name + * @param shapeMat Input - save annotation data from mat shape to file +*/ +void VO_Shape::ExportShape(const string& fn, const Mat_& shapeMat) +{ + fstream fp(fn.c_str(), ios::out); + fp << shapeMat << endl; + fp.close();fp.clear(); +} + diff --git a/modules/smbuilding/src/VO_Shape2DInfo.cpp b/modules/smbuilding/src/VO_Shape2DInfo.cpp new file mode 100644 index 0000000..6221b9b --- /dev/null +++ b/modules/smbuilding/src/VO_Shape2DInfo.cpp @@ -0,0 +1,153 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include +#include + +#include "VO_Shape2DInfo.h" + + +ostream& operator<<(ostream& os, const VO_Shape2DInfo& shapeinfo) +{ + os << shapeinfo.GetPath() << " " + << shapeinfo.GetType() << " " + << shapeinfo.GetIndex() << " " + << shapeinfo.GetFrom() << " " + << shapeinfo.GetTo(); + + return os; +} + + +istream& operator>>(istream &is, VO_Shape2DInfo& shapeinfo) +{ + unsigned int tempUI; + + // path + is >> tempUI; + shapeinfo.SetPath(tempUI); + // type + is >> tempUI; + shapeinfo.SetType(tempUI); + // current point index + is >> tempUI; + shapeinfo.SetIndex(tempUI); + // connects from + is >> tempUI; + shapeinfo.SetFrom(tempUI); + // connects to + is >> tempUI; + shapeinfo.SetTo(tempUI); + + return is; +} + + +/** +* @author JIA Pei +* @version 2010-02-05 +* @brief read a file to get the shape information +* @param filename input parameter, file to read from +* @param vector output parameter, a vector of all shape points' information +* @param VO_FaceParts output parameter, a vector of all parts information +*/ +void VO_Shape2DInfo::ReadShape2DInfo(const string& filename, vector& oShapeInfo, VO_FaceParts& faceparts) +{ + VO_Shape2DInfo tempShapeInfo; + VO_FacePart onePart; + + fstream fp; + string temp, line; + unsigned int tempUI; + unsigned int NbOfPoints; + + // ShapeInfo.txt + fp.open(filename.c_str (), ios::in); + fp >> tempUI; + getline(fp, temp); + NbOfPoints = tempUI; + oShapeInfo.resize(NbOfPoints); + getline(fp, temp); + for(unsigned int i = 0; i < NbOfPoints; i++) + { + // Path + fp >> tempUI; + tempShapeInfo.SetPath(tempUI); + // type + fp >> tempUI; + tempShapeInfo.SetType(tempUI); + // Index + fp >> tempUI; + tempShapeInfo.SetIndex(tempUI); + // connects from + fp >> tempUI; + tempShapeInfo.SetFrom(tempUI); + // connects to + fp >> tempUI; + tempShapeInfo.SetTo(tempUI); + + oShapeInfo[i] = tempShapeInfo; + } + fp >> faceparts; + + fp.close();fp.clear(); +} + diff --git a/modules/smbuilding/src/VO_ShapeFace.cpp b/modules/smbuilding/src/VO_ShapeFace.cpp new file mode 100644 index 0000000..6bab427 --- /dev/null +++ b/modules/smbuilding/src/VO_ShapeFace.cpp @@ -0,0 +1,102 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include +#include +#include +#include + +#include "VO_ShapeFace.h" + + +VO_FaceCompPos VO_ShapeFace::CalcFaceCompRects(const VO_FaceParts& faceparts) +{ + Rect face, lefteye, righteye, nose, mouth; + VO_Shape leftEyeShape, rightEyeShape, noseShape, nostrilShape, nosetipShape, wholenoseShape, mouthShape; + + // face + face = this->GetShapeBoundRect(); + + // left eye + leftEyeShape = this->GetSubShape(faceparts.VO_GetOneFacePart(VO_FacePart::LEFTEYE).GetIndexes()); + lefteye = leftEyeShape.GetShapeBoundRect(); + + // right eye + rightEyeShape = this->GetSubShape(faceparts.VO_GetOneFacePart(VO_FacePart::RIGHTEYE).GetIndexes()); + righteye = rightEyeShape.GetShapeBoundRect(); + + // nose + noseShape = this->GetSubShape(faceparts.VO_GetOneFacePart(VO_FacePart::NOSE).GetIndexes()); + nostrilShape = this->GetSubShape(faceparts.VO_GetOneFacePart(VO_FacePart::NOSTRIL).GetIndexes()); + nosetipShape = this->GetSubShape(faceparts.VO_GetOneFacePart(VO_FacePart::NOSETIP).GetIndexes()); + wholenoseShape = VO_Shape::Combine2Shapes(noseShape, nostrilShape); + wholenoseShape = VO_Shape::Combine2Shapes(wholenoseShape, nosetipShape); + nose = wholenoseShape.GetShapeBoundRect(); + + // mouth + mouthShape = this->GetSubShape(faceparts.VO_GetOneFacePart(VO_FacePart::LIPOUTERLINE).GetIndexes()); + mouth = mouthShape.GetShapeBoundRect(); + + m_VOFaceCompPos = VO_FaceCompPos("", &face, &lefteye, &righteye, &nose, &mouth); + + return m_VOFaceCompPos; +} + diff --git a/modules/smbuilding/src/VO_ShapeModel.cpp b/modules/smbuilding/src/VO_ShapeModel.cpp new file mode 100644 index 0000000..7cefecd --- /dev/null +++ b/modules/smbuilding/src/VO_ShapeModel.cpp @@ -0,0 +1,1149 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include +#include +#include +#include + +#include "VO_Triangle2DStructure.h" +#include "VO_ShapeModel.h" +#include "VO_CVCommon.h" +#include "VO_AnnotationDBIO.h" + + +/** Default Constructor */ +VO_ShapeModel::VO_ShapeModel() +{ + this->init(); +} + + +/** Initialization */ +void VO_ShapeModel::init() +{ + this->m_iNbOfSamples = 0; + this->m_iNbOfShapeDim = 0; + this->m_iNbOfPoints = 0; + this->m_iNbOfShapes = 0; + this->m_iNbOfShapeEigens = 0; + this->m_iNbOfEigenShapesAtMost = 0; + this->m_iNbOfEdges = 0; + this->m_iNbOfTriangles = 0; + this->m_fAverageShapeSize = 0.0f; + this->m_fTruncatedPercent_Shape = 0.95f; + this->m_vShapes.clear(); + this->m_vAlignedShapes.clear(); + this->m_vShape2DInfo.clear(); + this->m_vEdge.clear(); + this->m_vTemplateTriangle2D.clear(); + this->m_vNormalizedTriangle2D.clear(); +} + + +/** Destructor */ +VO_ShapeModel::~VO_ShapeModel() +{ + this->m_vShapes.clear(); + this->m_vAlignedShapes.clear(); + this->m_vShape2DInfo.clear(); + this->m_vEdge.clear(); + this->m_vTemplateTriangle2D.clear(); + this->m_vNormalizedTriangle2D.clear(); +} + + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Align all shapes before PCA + * @param vShapes Input - all shapes before alignment + * @param alignedShapes Output - all shapes after alignment + * @return float return average shape size of all shapes + * @note make sure all alignedShapes, meanshape as well, are of size "1" !!! +*/ +float VO_ShapeModel::VO_AlignAllShapes(const vector& vShapes, vector& alignedShapes) +{ + unsigned int NbOfSamples = vShapes.size(); + unsigned int NbOfAnglesDim = (vShapes[0].GetNbOfDim() == 2) ? 1:3; // 3 or 2, if 2, change to 1 + float averageShapeSize = 0.0f; + alignedShapes = vShapes; + + ////////////////////////////////////////////////////////////////////////////////////// + // First estimation is mean of all VO_Shape + VO_Shape meanAlignedShape; + for(unsigned int i = 0; i < NbOfSamples; ++i) + { + averageShapeSize += alignedShapes[i].GetCentralizedShapeSize(); + alignedShapes[i].Normalize(); + } + averageShapeSize /= (float)NbOfSamples; // Why this is the average shape size without minor errors? + // Because later do-while loop doesn't change the normalized shape size + VO_ShapeModel::VO_CalcMeanShape( alignedShapes, meanAlignedShape ); + ////////////////////////////////////////////////////////////////////////////////////// + + ////////////////////////////////////////////////////////////////////////////////////// + // iterative Procrustes Analysis + // do a number of alignment iterations until the mean shape estimate is stable + float diff, diff_max = FLT_EPSILON; + int max_iter = 10, iter = 1; + VO_Shape tempMeanShape; + vector theta, angles; + theta.resize( NbOfAnglesDim ); + angles.resize(NbOfAnglesDim); + Mat_ rots = Mat_::zeros(NbOfAnglesDim, NbOfSamples); + + do + { + // normalize and align all other shapes to the mean shape estimate + for(unsigned int i=0;i < NbOfSamples;i++) + { + // align the i-th shape to the estimate of the mean shape + alignedShapes[i].AlignTo( meanAlignedShape, NULL, &theta, NULL ); + for(unsigned int j = 0; j < NbOfAnglesDim; ++j) + rots(j,i) = theta[j]; // record the rotation + + // re-scale to unit size to avoid the so-called 'shrinking effect' + // i.e. the alignment error goes towards zero, when the shapes are downscaled + alignedShapes[i].Scale( 1.0f/alignedShapes[i].GetShapeNorm () ); + } + tempMeanShape = meanAlignedShape; + // estimate the new mean shape + VO_ShapeModel::VO_CalcMeanShape( alignedShapes, meanAlignedShape ); + + // if first iteration, adjust the orientation of the mean shape, so the rotation of the training set to the mean shape is + // -- on average -- zero or put more clearly: "make the meanshape have a mean orientation" + if (iter==1) + { + for(unsigned int k = 0; k < NbOfAnglesDim; ++k) + { + angles[k] = safeDoubleToFloat(cv::mean( rots.row(k) ).val[0]); + } + meanAlignedShape.Rotate ( angles ); + } + + diff = (tempMeanShape-meanAlignedShape).GetShapeNorm(); + + ++iter; + + }while( fabs(diff)/meanAlignedShape.GetShapeNorm() > diff_max && iter < max_iter ); + + ////////////////////////////////////////////////////////////////////////////////////// + return averageShapeSize; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Rescale the alignedshapes so that meanshape's 2-norm could be "1" + * @param meanAlignedShape Input - meanalignedshape that all alignedshapes should rescale to + * @param alignedShapes Input and Output - aligned shape and aligned rescaled shape +*/ +void VO_ShapeModel::VO_RescaleAllAlignedShapes(const VO_Shape& meanAlignedShape, + vector& alignedShapes) +{ + unsigned int NbOfSamples = alignedShapes.size(); + + // Explained by JIA Pei. 2010-02-07. scale back so that the mean shape size is equal to 1.0 + for(unsigned int i = 0; i < NbOfSamples; i++) + { + VO_ShapeModel::VO_RescaleOneAlignedShape(meanAlignedShape, alignedShapes[i]); + } +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Rescale the alignedShape to the already trained meanAligneShape + * @param meanAlignedShape Input - meanalignedshape that all alignedshapes should rescale to + * @param alignedShape Input and Output - aligned shape and aligned rescaled shape + * @return void +*/ +void VO_ShapeModel::VO_RescaleOneAlignedShape(const VO_Shape& meanAlignedShape, VO_Shape& alignedShape) +{ + float ts = sqrt ( fabs(alignedShape.dot(meanAlignedShape) ) ); + alignedShape.Scale( 1.0f/ts ); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate mean shape + * @param vShapes Input - all shapes + * @param meanShape Output - mean shape + * @return void +*/ +void VO_ShapeModel::VO_CalcMeanShape(const vector& vShapes, VO_Shape& meanShape) +{ + unsigned int NbOfSamples = vShapes.size(); + meanShape = vShapes[0]; + + for(unsigned int i = 1; i < NbOfSamples; i++) + { + meanShape += vShapes[i]; + } + + meanShape /= (float)NbOfSamples; +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief Judge is point "pt" inside the convex hull "ch" +* @param pt Input - the point +* @param ch Input - convex hull +* @param includingHull Input flag - whether including the boundary +* @return bool including or excluding +*/ +bool VO_ShapeModel::VO_IsPointInConvexHull(const Point2f pt, const Mat_& ch, bool includingHull) +{ + if(!includingHull) + { + if (cv::pointPolygonTest( ch, pt, true ) > 0.0 ) + return true; + } + else + { + if (cv::pointPolygonTest( ch, pt, true ) >= 0.0) + return true; + } + + return false; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Judge whether edge indexed by (ind1+ind2) is already in the vector of "edges" + * @param edges edge collection + * @param ind1 first index of the edge to be judged + * @param ind2 second index of the edge to be judged + * @return bool counted or not +*/ +bool VO_ShapeModel::VO_EdgeHasBeenCounted(const vector& edges, unsigned int ind1, unsigned int ind2) +{ + unsigned int NbOfEdges = edges.size (); + for (unsigned int i = 0; i < NbOfEdges; i++) + { + if ( ( (edges[i].GetIndex1() == ind1) && (edges[i].GetIndex2() == ind2) ) + || ( (edges[i].GetIndex1() == ind2) && (edges[i].GetIndex2() == ind1) ) ) + { + return true; + } + } + + return false; +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief Judge whether triangle t is already in the vector of "triangles". +* @param triangles triangle collection +* @param t the triangle to be judged + * @return bool whether the triangle "t" has been counted or not +*/ +bool VO_ShapeModel::VO_TriangleHasBeenCounted(const vector& triangles, const vector& t) +{ + set tTriangle; + set sTriangle; + unsigned int NbOfTriangles = triangles.size (); + + for (unsigned int i = 0; i < NbOfTriangles; i ++) + { + // These two clear() are very important, cannot be displaced by empty(). + tTriangle.clear (); + sTriangle.clear (); + for (unsigned int j = 0; j < 3; j++ ) + { + tTriangle.insert (triangles[i].GetVertexIndex(j) ); + sTriangle.insert (t[j]); + } + if (tTriangle == sTriangle) + { + return true; // has been counted already + } + } + + return false; // not counted yet +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Build Edges + * @param vertexes Input - all vertexes/points composing the shape + * @param Subdiv Input - sub division which is already computed beforehand + * @param outEdges Output - edges + * @return unsigned int Number of edges +*/ +unsigned int VO_ShapeModel::VO_BuildEdges(const VO_Shape& iShape, const CvSubdiv2D* Subdiv, vector& outEdges) +{ + unsigned int NbOfPoints = iShape.GetNbOfPoints(); + CvSeqReader reader; + + cvStartReadSeq( (CvSeq*)(Subdiv->edges), &reader, 0 ); + + for(int i = 0; i < Subdiv->edges->total; i++ ) + { + CvQuadEdge2D* edge = (CvQuadEdge2D*)(reader.ptr); + + if( CV_IS_SET_ELEM( edge )) + { + Point2f org; + Point2f dst; + + CvSubdiv2DPoint* org_pt = cvSubdiv2DEdgeOrg((CvSubdiv2DEdge)edge); + CvSubdiv2DPoint* dst_pt = cvSubdiv2DEdgeDst((CvSubdiv2DEdge)edge); + + if( org_pt && dst_pt ) + { + org = org_pt->pt; + dst = dst_pt->pt; + + for (unsigned int j = 0; j < NbOfPoints; j++) + { + // if the current edge convex (org points, not the dst point) is in our point list + if ( (fabs ( org.x - iShape.GetACol(j)(0, 0) ) < FLT_EPSILON ) + && ( fabs ( org.y - iShape.GetACol(j)(1, 0) ) < FLT_EPSILON ) ) + { + for (unsigned int k = 0; k < NbOfPoints; k++) + { + // With the above org point, we search around for the dst point(s), + // which make org-dst an edge during cvSubdivDelaunay2DInsert() + if ( ( fabs (dst.x - iShape.GetACol(k)(0, 0) ) < FLT_EPSILON ) + && ( fabs (dst.y - iShape.GetACol(k)(1, 0) ) < FLT_EPSILON ) ) + { + // Already tested, this->m_vEdge is definitely correct! + outEdges.push_back ( VO_Edge(j,k) ); + } + } + } + } + } + } + + CV_NEXT_SEQ_ELEM( Subdiv->edges->elem_size, reader ); + } + + return outEdges.size(); +} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief Build AAM Triangles +* @param iShape Input - the input shape +* @param edges Input - to built edges +* @param outTriangles Output - the output triangles +* @return unsigned int Number of triangles +*/ +unsigned int VO_ShapeModel::VO_BuildTriangles( const VO_Shape& iShape, + const vector& edges, + vector& outTriangles) +{ + outTriangles.clear(); + unsigned int NbOfEdges = edges.size (); + unsigned int NbOfPoints = iShape.GetNbOfPoints(); + + for (unsigned int i = 0; i < NbOfEdges; i++) + { + unsigned int ind1 = edges[i].GetIndex1(); + unsigned int ind2 = edges[i].GetIndex2(); + + for (unsigned int j = 0; j < NbOfPoints; j++) + { + // For each edge, there are at most 2 triangles that could be added + if( VO_ShapeModel::VO_EdgeHasBeenCounted(edges, ind1, j) && VO_ShapeModel::VO_EdgeHasBeenCounted(edges, ind2, j) ) + { + vector iVertexes; + iVertexes.resize (3); + iVertexes[0] = ind1; + iVertexes[1] = ind2; + iVertexes[2] = j; + // If the triangle has not been counted yet + if (!VO_ShapeModel::VO_TriangleHasBeenCounted(outTriangles, iVertexes) ) + { + Mat_ vVertexes = iShape.GetSubShape(iVertexes).GetTheShape(); + // Note: Explained by JIA Pei, 2009-08-09, triangle vertex sequence should be adjusted here + VO_Triangle2DStructure temp(vVertexes, iVertexes); + temp.AdjustVertexSequence(); + outTriangles.push_back (temp); + } + } + } + } + + return outTriangles.size(); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculation triangulation mesh + For IMM 58 points dataset, 19 edges on the convex hull, 133 inner edges. + Totally, 133+19=152 edges + (133*2+19)/3=95 triangles + * @param iShape Input - the input shape + * @param edges Input - the constructed edges + * @param triangles Input - the constructed triangles + * @return void +*/ +void VO_ShapeModel::VO_BuildTemplateMesh( const VO_Shape& iShape, + vector& edges, + vector& triangles ) +{ + unsigned int NbOfPoints = iShape.GetNbOfPoints(); + + ////////////////////////////////////////////////////////////////////////// + // Build Delaunay Triangulation Sub Divisions + // Later VO_BuildEdges need DTSubdiv information + CvSubdiv2D* tempCVSubdiv = NULL; + + Rect rect = iShape.GetShapeBoundRect(); + + CvMemStorage* DelaunayStorage = cvCreateMemStorage(0); + + // By JIA Pei, 2006-09-20. How to release this storage? + tempCVSubdiv = cvCreateSubdivDelaunay2D( rect, DelaunayStorage ); + + for( unsigned int i = 0; i < NbOfPoints; i++ ) + { + Point2f onePoint = iShape.GetA2DPoint(i); + cvSubdivDelaunay2DInsert( tempCVSubdiv, onePoint); + } + ////////////////////////////////////////////////////////////////////////// + + unsigned int NbOfEdges = VO_ShapeModel::VO_BuildEdges(iShape, tempCVSubdiv, edges); + unsigned int NbOfTriangles = VO_ShapeModel::VO_BuildTriangles (iShape, edges, triangles); + + // How to release CvSubdiv2D* m_CVSubdiv is still a problem. + if (tempCVSubdiv) cvClearSubdivVoronoi2D( tempCVSubdiv ); + cvReleaseMemStorage( &DelaunayStorage ); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculation triangulation mesh + For IMM 58 points dataset, 19 edges on the convex hull, 133 inner edges. + Totally, 133+19=152 edges + (133*2+19)/3=95 triangles + * @param iShape Input - the input shape + * @param iFaceParts Input - face parts + * @param edges Input - the constructed edges + * @param triangles Input - the constructed triangles +*/ +void VO_ShapeModel::VO_BuildTemplateMesh(const VO_Shape& iShape, + const VO_FaceParts& iFaceParts, + vector& edges, + vector& triangles ) +{ + Mat_ vVertexes; + unsigned int NbOfPoints = iShape.GetNbOfPoints(); + + vector< vector < unsigned int > > ti = iFaceParts.GetTriangleIndexes(); + unsigned int NbOfTriangles = ti.size(); + + triangles.resize(NbOfTriangles); + for(unsigned int i = 0; i < NbOfTriangles; ++i) + { + vVertexes = iShape.GetSubShape(ti[i]).GetTheShape(); + triangles[i] = VO_Triangle2DStructure(vVertexes, ti[i]); + + if( !VO_ShapeModel::VO_EdgeHasBeenCounted(edges, ti[i][0], ti[i][1]) ) + edges.push_back( VO_Edge(ti[i][0], ti[i][1]) ); + + if( !VO_ShapeModel::VO_EdgeHasBeenCounted(edges, ti[i][0], ti[i][2]) ) + edges.push_back( VO_Edge(ti[i][0], ti[i][2]) ); + + if( !VO_ShapeModel::VO_EdgeHasBeenCounted(edges, ti[i][1], ti[i][2]) ) + edges.push_back( VO_Edge(ti[i][1], ti[i][2]) ); + } +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Is iShape inside img + * @param iShape Input - the input shape + * @param img Input - the image + * @return bool +*/ +bool VO_ShapeModel::VO_IsShapeInsideImage(const VO_Shape& iShape, const Mat& img) +{ + Rect_ rect = iShape.GetShapeRect(); + + // The following line should be avoided. explained by JIA Pei. Try the second line! + //if ( (rect.x > 0) && (rect.y > 0) && ( (rect.x + rect.width) <= img.cols ) && ( (rect.y + rect.height) <= img.rows ) ) + if ( (rect.x >= 0.0) && (rect.y >= 0.0) && ( (rect.x + rect.width) < (float)img.cols - 1.0f - FLT_EPSILON ) && ( (rect.y + rect.height) < (float)img.rows - 1.0f - FLT_EPSILON ) ) + return true; + else + return false; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Reference scale texture back to aligned one + * @param inShape Input - reference shape + * @param shapeSD Input - shape standard deviation + * @param outShape Output - output aligned shape + * @return void +*/ +void VO_ShapeModel::VO_ReferenceShapeBack2Aligned(const VO_Shape& inShape, float shapeSD, VO_Shape& outShape) +{ + outShape = inShape; + Mat_ center = outShape.CenterOfGravity( ); // centralize first + outShape.Translate( -center ); + outShape.Scale(1.0f/shapeSD ); // scale second + // note: no rotation!!! +} + + +/** + * @author JIA Pei + * @version 2010-02-09 + * @brief calculate the boundary rectangle of a list of triangles + * @param triangles Input - A list of triangles + * @return Rect the boundary rectangle. +*/ +Rect VO_ShapeModel::VO_CalcBoundingRectFromTriangles(const vector & triangles) +{ + VO_Shape shape = VO_Triangle2DStructure::Triangle2D2Shape(triangles); + + return shape.GetShapeBoundRect(); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief shape parameters constrain + * @param ioP Input and Output - shape parameters + * @param nSigma Input - number of sigmas + * @return void +*/ +void VO_ShapeModel::VO_ShapeParameterConstraint(Mat_& ioP, float nSigma) +{ + for (int i = 0; i < ioP.cols; ++i) + { + float ct = nSigma * sqrt(this->m_PCAAlignedShape.eigenvalues.at(i,0) ); + if ( ioP(0, i) > ct ) + { + ioP(0, i) = ct; + } + else if ( ioP(0, i) < -ct ) + { + ioP(0, i) = -ct; + } + } +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief aligned shape project to shape parameters + * @param inP Input - input shape + * @param oShape Output - the projected shape parameters + * @return void +*/ +void VO_ShapeModel::VO_AlignedShapeProjectToSParam(const VO_Shape& iShape, Mat_& outP) const +{ + this->m_PCAAlignedShape.project(iShape.GetTheShapeInARow(), outP); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief shape parameters back project to aligned shape + * @param inP Input - input shape parameters + * @param oShape Output - the back projected shape + * @param dim dim - input + * @return void +*/ +void VO_ShapeModel::VO_SParamBackProjectToAlignedShape(const Mat_& inP, VO_Shape& oShape, int dim) const +{ + oShape.SetTheShape(this->m_PCAAlignedShape.backProject(inP), dim); +} + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief shape parameters back project to aligned shape +* @param inP Input - input shape parameters +* @param oShapeMat Output - the back projected shape in a row matrix +* @return void +*/ +void VO_ShapeModel::VO_SParamBackProjectToAlignedShape(const Mat_& inP, Mat_& oShapeMat) const +{ + oShapeMat = this->m_PCAAlignedShape.backProject(inP); +} + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate all parameters for an arbitrary shape + * @param iShape Input - the reference shape + * @param oShape Output - output shape + * @param outP Output - shape parameters + * @param norm Output - 2-norm, scaling + * @param angle Output - rotation in 2D + * @param translation Output - translation + * @return void +*/ +void VO_ShapeModel::VO_CalcAllParams4AnyShapeWithConstrain( const Mat_& iShape, + Mat_& oShape, + Mat_& outP, + float& norm, + vector& angles, + Mat_& translation ) +{ + VO_Shape oS(iShape, this->m_iNbOfShapeDim); + oS.ProcrustesAnalysis( this->m_VOAlignedMeanShape, norm, angles, translation ); + this->VO_AlignedShapeProjectToSParam(oS, outP); + this->VO_ShapeParameterConstraint(outP); + this->VO_SParamBackProjectToAlignedShape(outP, oS, this->m_iNbOfShapeDim); + oS.InverseProcrustesAnalysis( norm, angles, translation ); + oShape = oS.GetTheShapeInARow(); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate all parameters for an arbitrary shape + * @param iShape Input - the input shape + * @param oShape Output - output shape + * @param outP Output - shape parameters, a 1*m_iNbOfShapeDim row vector + * @param outQ Output - similarity transform parameters, a 1*4 row vector + * @return void +*/ +//void VO_ShapeModel::VO_CalcAllParams4AnyShapeWithConstrain( const Mat_& iShape, +// Mat_& oShape, +// Mat_& outP, +// Mat_& outQ ) +//{ +// float norm; +// vector angles; +// Mat_ translation; +// +// this->VO_CalcAllParams4AnyShapeWithConstrain( iShape, +// oShape, +// outP, +// norm, +// angles, +// translation ); +// VO_Shape::SimilarityTrans2GlobalShapeNormalization(norm, -angles, translation, outQ); +//} + + +/** +* @author JIA Pei +* @version 2010-02-07 +* @brief Calculate all parameters for an arbitrary shape +* @param ioShape Input and output - input/output shape +* @param outP Output - shape parameters +* @param norm Output - 2-norm, scaling +* @param angles Output - rotation +* @param translation Output - translation +* @return void +*/ +void VO_ShapeModel::VO_CalcAllParams4AnyShapeWithConstrain( VO_Shape& ioShape, + Mat_& outP, + float& norm, + vector& angles, + Mat_& translation ) +{ + ioShape.ProcrustesAnalysis( this->m_VOAlignedMeanShape, norm, angles, translation ); + this->VO_AlignedShapeProjectToSParam(ioShape, outP); + this->VO_ShapeParameterConstraint(outP); + this->VO_SParamBackProjectToAlignedShape(outP, ioShape, ioShape.GetNbOfDim()); + ioShape.InverseProcrustesAnalysis( norm, angles, translation ); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Calculate all parameters for an arbitrary shape + * @param ioShape Input and output - input/output shape + * @param outP Output -- shape parameters + * @param outQ Output -- similarity transform parameters + * @return void +*/ +//void VO_ShapeModel::VO_CalcAllParams4AnyShapeWithConstrain(VO_Shape& ioShape, Mat_& outP, Mat_& outQ ) +//{ +// float norm; +// vector angles; +// Mat_ translation; +// +// this->VO_CalcAllParams4AnyShapeWithConstrain( ioShape, +// outP, +// norm, +// angles, +// translation ); +// VO_Shape::SimilarityTrans2GlobalShapeNormalization(norm, -angles, translation, outQ); +//} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Build up a shape from all rigid and non-rigid parameters + * @param inP Input -- shape parameters, for nonrigid + * @param inQ Input -- similarity transform parameters, for rigid + * @param oShape output -- output shape + * @return void +*/ +//void VO_ShapeModel::VO_BuildUpShapeFromRigidNonRigidParams(const Mat_& inP, const Mat_& inQ, VO_Shape& oShape ) +//{ +// float norm; +// vector angles; +// Mat_ translation; +// +// this->VO_SParamBackProjectToAlignedShape(inP, oShape, 2); +//// VO_Shape::GlobalShapeNormalization2SimilarityTrans(inQ, norm, angles, translation); +//// oShape.InverseProcrustesAnalysis( norm, angles, translation ); +// oShape.GlobalShapeNormalization2D(inQ); +//} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief build shape model + * @param allLandmarkFiles4Training Input - all landmark files + * @param shapeinfoFileName Input - shape info file + * @param database Input - which database is it? + * @param TPShape Input - truncated percentage for shape model + * @param useKnownTriangles Input - use known triangle structures?? + * @return void +*/ +void VO_ShapeModel::VO_BuildShapeModel( const vector& allLandmarkFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + float TPShape, + bool useKnownTriangles) +{ + // load auxiliary shape information + VO_Shape2DInfo::ReadShape2DInfo(shapeinfoFileName, this->m_vShape2DInfo, this->m_FaceParts); + CAnnotationDBIO::VO_LoadShapeTrainingData( allLandmarkFiles4Training, database, this->m_vShapes); + + this->m_iNbOfSamples = this->m_vShapes.size(); + this->m_iNbOfShapeDim = this->m_vShapes[0].GetNbOfDim(); + this->m_iNbOfPoints = this->m_vShapes[0].GetNbOfPoints(); + this->m_iNbOfShapes = this->m_iNbOfShapeDim * this->m_iNbOfPoints; + this->m_iNbOfEigenShapesAtMost = MIN(this->m_iNbOfSamples, this->m_iNbOfShapes); + this->m_fTruncatedPercent_Shape = TPShape; + Mat_ matAlignedShapes = Mat_::zeros(this->m_iNbOfSamples, this->m_iNbOfShapes); + Mat_ matAlignedMeanShape = Mat_::zeros(1, this->m_iNbOfShapes); + + // Align all shapes + this->m_fAverageShapeSize = VO_ShapeModel::VO_AlignAllShapes(this->m_vShapes, this->m_vAlignedShapes); + VO_ShapeModel::VO_CalcMeanShape(this->m_vAlignedShapes, this->m_VOAlignedMeanShape); + + // Calculate reference shape + this->m_VOReferenceShape = this->m_VOAlignedMeanShape*this->m_fAverageShapeSize; + Mat_ refMin = this->m_VOReferenceShape.Min(); + this->m_VOReferenceShape.Translate( -refMin ); + + ////////////////////////////////////////////////////////////////////////// + // Build VO_Point2DDistributionModel ///////////////////////////////////// + this->m_VOPDM.VO_BuildPointDistributionModel(this->m_vAlignedShapes); + ////////////////////////////////////////////////////////////////////////// + + matAlignedMeanShape = this->m_VOAlignedMeanShape.GetTheShapeInARow(); + for(unsigned int i = 0; i < this->m_iNbOfSamples; ++i) + { + Mat tmpRow = matAlignedShapes.row(i); + this->m_vAlignedShapes[i].GetTheShapeInARow().copyTo(tmpRow); + } + this->m_PCAAlignedShape(matAlignedShapes, matAlignedMeanShape, CV_PCA_DATA_AS_ROW, this->m_iNbOfEigenShapesAtMost ); + // to decide how many components to be selected + this->m_iNbOfShapeEigens = 0; + + double SumOfEigenValues = cv::sum( this->m_PCAAlignedShape.eigenvalues ).val[0]; + double ps = 0.0f; + + for(unsigned int i = 0; i < this->m_iNbOfEigenShapesAtMost; i++) + { + ps += this->m_PCAAlignedShape.eigenvalues.at( i,0 ); + ++this->m_iNbOfShapeEigens; + if( ps/SumOfEigenValues >= this->m_fTruncatedPercent_Shape) break; + } + // m_iNbOfShapeEigens decided. For simplicity, we carry out PCA once again. + this->m_PCAAlignedShape(matAlignedShapes, matAlignedMeanShape, CV_PCA_DATA_AS_ROW, this->m_iNbOfShapeEigens ); + + ////////////////////////////////////////////////////////////////////////// + // Calculate template shape mesh + ////////////////////////////////////////////////////////////////////////// + if (this->m_FaceParts.GetTriangleIndexes().size() != 0) + useKnownTriangles = true; + else + useKnownTriangles = false; + if( !useKnownTriangles ) + VO_ShapeModel::VO_BuildTemplateMesh(this->m_VOReferenceShape, this->m_vEdge, this->m_vTemplateTriangle2D ); + else + VO_ShapeModel::VO_BuildTemplateMesh(this->m_VOReferenceShape, this->m_FaceParts, this->m_vEdge, this->m_vTemplateTriangle2D ); + + this->m_iNbOfEdges = this->m_vEdge.size(); + this->m_iNbOfTriangles = this->m_vTemplateTriangle2D.size(); + + // 2) Calculate m_vNormalizedTriangle2D - make sure the mesh has been built first, so that we have m_iNbOfTriangles + vector tempVertexes; + vector iVertexes; + tempVertexes.resize(3); + Mat_ threePoints; + + for (unsigned int i = 0; i < this->m_iNbOfTriangles; i++) + { + iVertexes = this->m_vTemplateTriangle2D[i].GetVertexIndexes(); + threePoints = this->m_VOAlignedMeanShape.GetSubShape(iVertexes).GetTheShape(); + for(unsigned int j = 0; j < 3; j++) + tempVertexes[j] = Point2f(threePoints.at(0, j), threePoints.at(1,j)); + + this->m_vNormalizedTriangle2D.push_back (VO_Triangle2DStructure(tempVertexes, iVertexes)); + } +} + + +/** + * @author JIA Pei + * @version 2010-02-22 + * @brief Save ASM to a specified folder + * @param fd Input - the folder that ASM to be saved to +*/ +void VO_ShapeModel::VO_Save(const string& fd) +{ + // create ShapeModel subfolder for just ShapeModel data + string fn = fd+"/ShapeModel"; + MakeDirectory(fn); + + fstream fp; + string tempfn; + + // ShapeModel + tempfn = fn + "/ShapeModel" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_iNbOfSamples" << endl << this->m_iNbOfSamples << endl; // m_iNbOfSamples + fp << "m_iNbOfShapeDim" << endl << this->m_iNbOfShapeDim << endl; // m_iNbOfShapeDim + fp << "m_iNbOfPoints" << endl << this->m_iNbOfPoints << endl; // m_iNbOfPoints + fp << "m_iNbOfShapes" << endl << this->m_iNbOfShapes << endl; // m_iNbOfShapes + fp << "m_iNbOfEigenShapesAtMost" << endl << this->m_iNbOfEigenShapesAtMost << endl; // m_iNbOfEigenShapesAtMost + fp << "m_iNbOfShapeEigens" << endl << this->m_iNbOfShapeEigens << endl; // m_PCAAlignedShape + fp << "m_iNbOfEdges" << endl << this->m_iNbOfEdges << endl; // m_iNbOfEdges + fp << "m_iNbOfTriangles" << endl << this->m_iNbOfTriangles << endl; // m_iNbOfTriangles + fp << "m_fAverageShapeSize" << endl << this->m_fAverageShapeSize << endl; // m_fAverageShapeSize + fp << "m_fTruncatedPercent_Shape" << endl << this->m_fTruncatedPercent_Shape << endl; // m_fTruncatedPercent_Shape + fp.close();fp.clear(); + + // m_PCAAlignedShapeMean=m_MatAlignedMeanShapes + tempfn = fn + "/m_PCAAlignedShapeMean" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_PCAAlignedShapeMean" << endl; + fp << Mat_(this->m_PCAAlignedShape.mean) << endl; + fp.close();fp.clear(); + + // m_PCAAlignedShapeEigenValues + tempfn = fn + "/m_PCAAlignedShapeEigenValues" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_PCAAlignedShapeEigenValues" << endl; + fp << Mat_(this->m_PCAAlignedShape.eigenvalues) << endl; + fp.close();fp.clear(); + + // m_PCAAlignedShapeEigenVectors + tempfn = fn + "/m_PCAAlignedShapeEigenVectors" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_PCAAlignedShapeEigenVectors" << endl; + fp << Mat_(this->m_PCAAlignedShape.eigenvectors) << endl; + fp.close();fp.clear(); + + // m_VOAlignedMeanShape + tempfn = fn + "/m_VOAlignedMeanShape" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_VOAlignedMeanShape" << endl; + fp << this->m_VOAlignedMeanShape << endl; + fp.close();fp.clear(); + + // m_VOReferenceShape + tempfn = fn + "/m_VOReferenceShape" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_VOReferenceShape" << endl; + fp << this->m_VOReferenceShape << endl; + fp.close();fp.clear(); + + // m_vShapes + tempfn = fn + "/m_vShapes" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_vShapes" << endl; + fp << m_vShapes << endl; + fp.close();fp.clear(); + + // m_vAlignedShapes + tempfn = fn + "/m_vAlignedShapes" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_vAlignedShapes" << endl; + fp << this->m_vAlignedShapes << endl; + fp.close();fp.clear(); + + // m_vShape2DInfo + tempfn = fn + "/m_vShape2DInfo" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << this->m_vShape2DInfo.size() << endl; + fp << "m_vShape2DInfo" << endl; + fp << this->m_vShape2DInfo << endl; + fp << this->m_FaceParts << endl; + fp.close();fp.clear(); + + // m_vEdge + tempfn = fn + "/m_vEdge" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_vEdge" << endl; + fp << this->m_vEdge << endl; + fp.close();fp.clear(); + + // m_vTemplateTriangle2D + tempfn = fn + "/m_vTemplateTriangle2D" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_vTemplateTriangle2D" << endl; + fp << this->m_vTemplateTriangle2D << endl; + fp.close();fp.clear(); + + // m_vNormalizedTriangle2D + tempfn = fn + "/m_vNormalizedTriangle2D" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_vNormalizedTriangle2D" << endl; + fp << this->m_vNormalizedTriangle2D << endl; + fp.close();fp.clear(); + + // m_VOPDM + this->m_VOPDM.VO_Save(fd); +} + + +/** + * @author JIA Pei + * @version 2010-02-22 + * @brief Load all ASM data from a specified folder + * @param fd Input - the folder that ASM to be loaded from +*/ +void VO_ShapeModel::VO_Load(const string& fd) +{ + this->VO_LoadParameters4Fitting(fd); + + string fn = fd+"/ShapeModel"; + if (!MakeDirectory(fn) ) + { + cout << "ShapeModel subfolder is not existing. " << endl; + exit(EXIT_FAILURE); + } + + ifstream fp; + string tempfn; + string temp; + + // m_vShapes + tempfn = fn + "/m_vShapes" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + this->m_vShapes.resize(this->m_iNbOfSamples); + fp >> this->m_vShapes; + fp.close();fp.clear(); + + // m_vAlignedShapes + tempfn = fn + "/m_vAlignedShapes" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_vAlignedShapes; + fp.close();fp.clear(); +} + + +/** + * @author JIA Pei + * @version 2010-02-22 + * @brief Load all ASM data from a specified folder for later fitting + * @param fd Input - the folder that ASM to be loaded from +*/ +void VO_ShapeModel::VO_LoadParameters4Fitting(const string& fd) +{ + string fn = fd+"/ShapeModel"; + if (!MakeDirectory(fn) ) + { + cout << "ShapeModel subfolder is not existing. " << endl; + exit(EXIT_FAILURE); + } + + ifstream fp; + string tempfn; + string temp; + + // ShapeModel + tempfn = fn + "/ShapeModel" + ".txt"; + SafeInputFileOpen(fp,tempfn); + fp >> temp >> this->m_iNbOfSamples; + fp >> temp >> this->m_iNbOfShapeDim; + fp >> temp >> this->m_iNbOfPoints; + fp >> temp >> this->m_iNbOfShapes; + fp >> temp >> this->m_iNbOfEigenShapesAtMost; + fp >> temp >> this->m_iNbOfShapeEigens; + fp >> temp >> this->m_iNbOfEdges; + fp >> temp >> this->m_iNbOfTriangles; + fp >> temp >> this->m_fAverageShapeSize; + fp >> temp >> this->m_fTruncatedPercent_Shape; + fp.close();fp.clear(); + + this->m_PCAAlignedShape = cv::PCA(); + + // m_PCAAlignedShapeMean + this->m_PCAAlignedShape.mean = Mat_::zeros(1, this->m_iNbOfShapes); + tempfn = fn + "/m_PCAAlignedShapeMean" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_PCAAlignedShape.mean; + fp.close();fp.clear(); + + // m_PCAAlignedShapeEigenValues + this->m_PCAAlignedShape.eigenvalues = Mat_::zeros(this->m_iNbOfShapeEigens, 1); + tempfn = fn + "/m_PCAAlignedShapeEigenValues" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_PCAAlignedShape.eigenvalues; + fp.close();fp.clear(); + + // m_PCAAlignedShapeEigenVectors + this->m_PCAAlignedShape.eigenvectors = Mat_::zeros(this->m_iNbOfShapeEigens, this->m_iNbOfShapes); + tempfn = fn + "/m_PCAAlignedShapeEigenVectors" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_PCAAlignedShape.eigenvectors; + fp.close();fp.clear(); + + // m_VOAlignedMeanShape + this->m_VOAlignedMeanShape.m_MatShape = Mat_::zeros(this->m_iNbOfShapeDim, this->m_iNbOfPoints); + tempfn = fn + "/m_VOAlignedMeanShape" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_VOAlignedMeanShape; + fp.close();fp.clear(); + + // m_VOReferenceShape + this->m_VOReferenceShape.m_MatShape = Mat_::zeros(this->m_iNbOfShapeDim, this->m_iNbOfPoints); + tempfn = fn + "/m_VOReferenceShape" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_VOReferenceShape; + fp.close();fp.clear(); + + // m_vShape2DInfo & m_FaceParts + this->m_vShape2DInfo.resize(this->m_iNbOfPoints); + tempfn = fn + "/m_vShape2DInfo" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; // 58 + fp >> temp; + fp >> this->m_vShape2DInfo; + fp >> this->m_FaceParts; + fp.close();fp.clear(); + + // m_vEdge + this->m_vEdge.resize(this->m_iNbOfEdges); + tempfn = fn + "/m_vEdge" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_vEdge; + fp.close();fp.clear(); + + // m_vTemplateTriangle2D + this->m_vTemplateTriangle2D.resize(this->m_iNbOfTriangles); + tempfn = fn + "/m_vTemplateTriangle2D" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_vTemplateTriangle2D; + fp.close();fp.clear(); + + // m_vNormalizedTriangle2D + this->m_vNormalizedTriangle2D.resize(this->m_iNbOfTriangles); + tempfn = fn + "/m_vNormalizedTriangle2D" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + this->m_vNormalizedTriangle2D.resize(this->m_iNbOfTriangles); + fp >> this->m_vNormalizedTriangle2D; + fp.close();fp.clear(); + + this->m_VOPDM.VO_LoadParameters4Fitting(fd); +} + diff --git a/modules/smbuilding/src/VO_Texture.cpp b/modules/smbuilding/src/VO_Texture.cpp new file mode 100644 index 0000000..ae9f727 --- /dev/null +++ b/modules/smbuilding/src/VO_Texture.cpp @@ -0,0 +1,576 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include +#include +#include +#include +#include +#include + +#include "VO_Texture.h" +#include "VO_CVCommon.h" + + +ostream& operator<<(ostream& os, const VO_Texture& texture) +{ + os << texture.m_MatTexture; + return os; +} + + +istream& operator>>(istream &is, VO_Texture& texture) +{ + is >> texture.m_MatTexture; + return is; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator= overloading, similar to copy constructor + * @param iTexture The input texture + * @return VO_Texture& + */ +VO_Texture& VO_Texture::operator=(const VO_Texture& iTexture) +{ + this->CopyData (iTexture); + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator= overloading, similar to copy constructor + * @param iTexture The input texture + * @return VO_Texture& + */ +VO_Texture& VO_Texture::operator=(const Mat_& iTexture) +{ + iTexture.copyTo(m_MatTexture); + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator= overloading, similar to copy constructor + * @param value assign all values in VO_Texture to value + * @return VO_Texture& + */ +VO_Texture& VO_Texture::operator=(float value) +{ + m_MatTexture = value; + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator+ overloading, shift one AAM texture by value + * @param value + * @return VO_Texture + */ +VO_Texture VO_Texture::operator+(float value) +{ + VO_Texture res(*this); + res.m_MatTexture += value; + + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator+= overloading, add value to this AAM texture + * @param value + * @return VO_Texture& +*/ +VO_Texture& VO_Texture::operator+=(float value) +{ + m_MatTexture += value; + + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator+ overloading, add two AAM texture to one + * @param iTexture the added AAM texture + * @return VO_Texture +*/ +VO_Texture VO_Texture::operator+(const VO_Texture &iTexture) +{ + VO_Texture res(*this); + res.m_MatTexture += iTexture.m_MatTexture; + + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator+= overloading, add the input AAM texture to this AAM texture + * @param iTexture the added AAM texture + * @return VO_Texture& +*/ +VO_Texture& VO_Texture::operator+=(const VO_Texture &iTexture) +{ + this->m_MatTexture += iTexture.m_MatTexture; + + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator- overloading, shift one AAM texture by -value + * @param value + * @return VO_Texture +*/ +VO_Texture VO_Texture::operator-(float value) +{ + VO_Texture res(*this); + res.m_MatTexture -= value; + + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator-= overloading, subtract value from this AAM texture + * @param value + * @return VO_Texture& +*/ +VO_Texture& VO_Texture::operator-=(float value) +{ + this->m_MatTexture -= value; + + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator- overloading, subtract one texture from another + * @param iTexture the subtracted AAM texture + * @return VO_Texture +*/ +VO_Texture VO_Texture::operator-(const VO_Texture &iTexture) +{ + VO_Texture res(*this); + res.m_MatTexture -= iTexture.m_MatTexture; + + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator-= overloading, subtract the input AAM texture from this AAM texture + * @param iTexture the subtracted AAM texture + * @return VO_Texture& +*/ +VO_Texture& VO_Texture::operator-=(const VO_Texture &iTexture) +{ + this->m_MatTexture -= iTexture.m_MatTexture; + + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator* overloading, scale one texture by value + * @param value scale size + * @return VO_Texture +*/ +VO_Texture VO_Texture::operator*(float value) +{ + VO_Texture res(*this); + res.m_MatTexture *= value; + + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator-= overloading, scale this AAM texture by value + * @param value scale size + * @return VO_Texture& +*/ +VO_Texture& VO_Texture::operator*=(float value) +{ + this->m_MatTexture *= value; + + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator* overloading, scale one texture by value + * @param value scale size + * @return VO_Texture +*/ +VO_Texture VO_Texture::operator*(const VO_Texture& iTexture) +{ + VO_Texture res(*this); + res.m_MatTexture = res.m_MatTexture.mul(iTexture.m_MatTexture); + + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator-= overloading, scale this AAM texture by value + * @param value scale size + * @return VO_Texture& +*/ +VO_Texture& VO_Texture::operator*=(const VO_Texture& iTexture) +{ + this->m_MatTexture = this->m_MatTexture.mul(iTexture.m_MatTexture); + + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator/ overloading, scale a texture + * @param value 1.0/value = scale size + * @return VO_Texture +*/ +VO_Texture VO_Texture::operator/(float value) +{ + if( fabs(value) <= FLT_MIN ) + { + cerr << "Divided by zero!" << endl; + exit(EXIT_FAILURE); + } + + VO_Texture res(*this); + res.m_MatTexture /= value; + + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator/= overloading, scale this texture with input float value + * @param value 1.0/value = the scaled value + * @return VO_Texture& +*/ +VO_Texture& VO_Texture::operator/=(float value) +{ + if( fabs(value) <= FLT_MIN ) + { + cerr << "Divided by zero!" << endl; + exit(EXIT_FAILURE); + } + + this->m_MatTexture /= value; + + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-06 + * @brief operator/ overloading, scale a shape + * @param iTexture for element-wise division + * @return VO_Texture +*/ +VO_Texture VO_Texture::operator/(const VO_Texture& iTexture) +{ + for(int i = 0; i < iTexture.m_MatTexture.rows; i++) + { + for(int j = 0; j < iTexture.m_MatTexture.cols; j++) + { + if( fabs(iTexture.m_MatTexture(i,j)) <= FLT_MIN ) + { + cerr << "Divided by zero!" << endl; + exit(EXIT_FAILURE); + } + } + } + + VO_Texture res(*this); + res.m_MatTexture /= iTexture.m_MatTexture; + + return res; +} + + +/** + * @author JIA Pei + * @version 2010-02-06 + * @brief operator/= overloading, scale this shape with input float value + * @param iTexture for element-wise division + * @return VO_Texture& +*/ +VO_Texture& VO_Texture::operator/=(const VO_Texture& iTexture) +{ + for(int i = 0; i < iTexture.m_MatTexture.rows; i++) + { + for(int j = 0; j < iTexture.m_MatTexture.cols; j++) + { + if( fabs(iTexture.m_MatTexture(i,j)) <= FLT_MIN ) + { + cerr << "Divided by zero!" << endl; + exit(EXIT_FAILURE); + } + } + } + + this->m_MatTexture /= iTexture.m_MatTexture; + + return *this; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator() overloading, obtain the texture value at row and col + * @param row input - row index + * @param col input - col index + * @return float& +*/ +float& VO_Texture::operator() (unsigned row, unsigned col) +{ + if ( row >= m_MatTexture.rows || col >= m_MatTexture.cols ) + cerr << "Matrix subscript out of bounds" << endl; + return this->m_MatTexture(row, col); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator() overloading, obtain the texture value at row and col + * @param row input - row index + * @param col input - col index + * @return float& +*/ +float VO_Texture::operator() (unsigned row, unsigned col) const +{ + if ( row >= m_MatTexture.rows || col >= m_MatTexture.cols ) + cerr << "Matrix subscript out of bounds" << endl; + return this->m_MatTexture(row, col); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief operator* overloading, dot product of two AAM textures + * @param iTexture AAM texture to be dot producted + * @return float dot product +*/ +float VO_Texture::dot(const VO_Texture& iTexture) +{ + return safeDoubleToFloat(this->m_MatTexture.dot(iTexture.m_MatTexture)); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief resize the matrix + * @param rows + * @param cols + * @return void +*/ +void VO_Texture::Resize(unsigned int rows, unsigned int cols) +{ + this->m_MatTexture.release(); + this->m_MatTexture = Mat_::zeros(rows, cols); +} + + +/** + * @author JIA Pei + * @version 2010-02-20 + * @brief centralize the texture, average to 0 + * @param none + * @return void +*/ +void VO_Texture::Centralize() +{ + float avg = safeDoubleToFloat(cv::mean(this->m_MatTexture).val[0]); + this->m_MatTexture -= avg; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief scale + * @param s scale this AAM texture by s + * @return VO_Texture + */ +void VO_Texture::Scale( float s) +{ + this->m_MatTexture *= s; +} + + +/** + * @author JIA Pei + * @version 2010-04-07 + * @brief shift + * @param s shift this AAM texture by s + * @return VO_Texture + */ +void VO_Texture::Shift( float s) +{ + this->m_MatTexture += s; +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Normalize this AAM texture + * @return void +*/ +void VO_Texture::Normalize() +{ + this->Centralize(); + + float norm = this->GetTextureNorm(); + + this->Scale( 1.0f/norm ); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief calculate 2-norm of AAM texture + * @return 2-norm of AAM texture +*/ +float VO_Texture::GetStandardizedTextureNorm() const +{ + VO_Texture tmp(*this); + tmp.Centralize(); + + return tmp.GetTextureNorm(); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief calculate 2-norm of AAM texture + * @return 2-norm of AAM texture +*/ +float VO_Texture::GetTextureNorm() const +{ + return safeDoubleToFloat(norm(this->m_MatTexture)); +} + + +/** + * @author JIA Pei + * @version 2010-02-07 + * @brief Clamp the texture, constrain between infimum and supremum + * @param inf Input -- infimum + * @param sup Input -- supremum + * @return void +*/ +void VO_Texture::Clamp(float inf, float sup) +{ + for(int i = 0; i < m_MatTexture.rows; ++i) + { + for(int j = 0; j < m_MatTexture.cols; ++j) + { + if( m_MatTexture(i, j) < inf) + m_MatTexture(i, j) = inf; + else if (m_MatTexture(i, j) > sup) + m_MatTexture(i, j) = sup; + } + } +} + diff --git a/modules/smbuilding/src/VO_TextureModel.cpp b/modules/smbuilding/src/VO_TextureModel.cpp new file mode 100644 index 0000000..20fd9f8 --- /dev/null +++ b/modules/smbuilding/src/VO_TextureModel.cpp @@ -0,0 +1,2062 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include +#include +#include +#include + + +// Explained by JIA Pei. 2009-07-31. #include is a must for and +//#ifdef _WIN32 +//#include +//#endif +//#include +//#include +//#include + +#include "VO_TextureModel.h" +#include "VO_CVCommon.h" + + +/** Default Constructor */ +VO_TextureModel::VO_TextureModel() +{ + this->init(); +} + + +/** Initialization */ +void VO_TextureModel::init() +{ + this->m_iTextureRepresentationMethod = VO_Features::DIRECT; + this->m_iNbOfTextureRepresentations = 0; + this->m_iNbOfChannels = 1; + this->m_iNbOfPixels = 0; + this->m_iNbOfTextures = 0; + this->m_iNbOfEigenTexturesAtMost = 0; + this->m_iNbOfTextureEigens = 0; + this->m_fAverageTextureStandardDeviation = 0.0f; + this->m_fTruncatedPercent_Texture = 0.95f; + this->m_vTextures.clear(); + this->m_vNormalizedTextures.clear(); + this->m_vTemplatePointWarpInfo.clear(); + this->m_vNormalizedPointWarpInfo.clear(); +} + + +VO_TextureModel::~VO_TextureModel() +{ + this->m_vTextures.clear(); + this->m_vNormalizedTextures.clear(); + this->m_vTemplatePointWarpInfo.clear(); + this->m_vNormalizedPointWarpInfo.clear(); +} + + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief Calculate point warping information + * @param triangles Input - a vector of all triangles + * @param ch Input - convex hull + * @param warpInfo Input - information of a vector of warping points + * @return unsigned int points (actually pixels) inside the template face + * @note Obviously, triangles and ch contain duplicated information +*/ +unsigned int VO_TextureModel::VO_CalcPointWarpingInfo(const vector & templateTriangles, vector& warpInfo) +{ + unsigned int NbOfPixels = 0; + + Rect rect = VO_TextureModel::VO_CalcBoundingRectFromTriangles(templateTriangles); + + for (int i = 0; i < rect.height; i++) + { + for (int j = 0; j < rect.width; j++) + { + // JIA Pei. 2006-11-25. You will see the following (int) is very important + // without (int), the result here is not correct at all!! + Point2f pt = Point2f((float)j, (float)i); + + int k = VO_Triangle2DStructure::IsPointInTriangles(pt,templateTriangles); + if(k>=0) + { + VO_WarpingPoint tempPixelTriangle; + + // necessary for all methods + tempPixelTriangle.SetPosition(pt); + tempPixelTriangle.SetTriangleIndex(k); + tempPixelTriangle.SetPointIndex(NbOfPixels); + tempPixelTriangle.SetTriangle2DStructure(templateTriangles[k] ); + + // Very important!! Note by JIA Pei, push_back cannot perform on 2-D vectors + warpInfo.push_back (tempPixelTriangle); + + NbOfPixels ++; + } + } + } + + return NbOfPixels; +} + + +/** +* @author JIA Pei, YAO Wei + * @version 2010-02-10 + * @brief Load one VO_Texture from an image based on VO_Shape + * @param iShape Input - the shape + * @param img Input - image + * @param templateTriangles Input - the composed face template triangles + * @param warpInfo Input - warping information for all pixels in template face + * @param oTexture Output - the extracted texture + * @param trm Input - texture representation method + * @return bool loading succeed or not? +*/ +bool VO_TextureModel::VO_LoadOneTextureFromShape(const VO_Shape& iShape, + const Mat& img, + const vector& templateTriangles, + const vector& warpInfo, + VO_Texture& oTexture, + int trm) +{ + ofstream fs; + fs.open("loadTextureTime.txt", ios::app); + +double time0 = (double)cvGetTickCount(); + + // make sure all shape points are inside the image + if ( !VO_ShapeModel::VO_IsShapeInsideImage(iShape, img) ) + { + cout << "The shape goes out of the image" << endl; + cout << "Shape =" << iShape << endl; + cout << "cols=" << img.cols << " rows=" << img.rows << endl; + + return false; + } + + + unsigned int NbOfShapeDim = iShape.GetNbOfDim(); + unsigned int NbOfPixels = warpInfo.size(); + unsigned int NbOfChannels = img.channels(); + unsigned int NbOfTextures = NbOfPixels*NbOfChannels; + unsigned int NbOfTriangles = templateTriangles.size(); + oTexture.m_MatTexture = Mat_::zeros(NbOfChannels, NbOfPixels); + + Point2f src[3], dst[3]; + vector< Mat_ > matWarping; + matWarping.resize(NbOfTriangles); + + + // calculate all the mapping (for speeding up) 95 mapping totally + for (unsigned int j = 0; j < NbOfTriangles; j++ ) + { + // Get the affine transformation for each triangle pair. + src[0] = templateTriangles[j].GetA2DPoint(0); + src[1] = templateTriangles[j].GetA2DPoint(1); + src[2] = templateTriangles[j].GetA2DPoint(2); + + dst[0] = iShape.GetA2DPoint(templateTriangles[j].m_vVertexIndexes[0]); + dst[1] = iShape.GetA2DPoint(templateTriangles[j].m_vVertexIndexes[1]); + dst[2] = iShape.GetA2DPoint(templateTriangles[j].m_vVertexIndexes[2]); + + matWarping[j] = cv::getAffineTransform( src, dst ); + } + + Rect rect = iShape.GetShapeBoundRect(); + // Why +1? A must. + // The bounded rectangle could be (0, 0, 2, 2) + // That means the width of the image is 2-0+1=3 (from the aspect of pixel) + rect.width +=1; rect.height +=1; + +double time1 = (double)cvGetTickCount(); +double elapsed = (time1 - time0 )/ (cvGetTickFrequency()*1000.); +fs << "Before Mapping -- Step 1 of warping time: " << elapsed << "millisec." << endl; + + Mat Img2BExtracted; + + switch(trm) + { + case VO_Features::LAPLACE: + { + Mat tempImg2BExtracted = Mat::zeros(img.size(), CV_32F); + img.convertTo( tempImg2BExtracted, tempImg2BExtracted.type() ); + + // Explained by JIA Pei, 2008-03-09. + // Why I don't do cvCopy(tempImg2BExtracted, tempExcavated); right here, + // but after cvSmooth, cvLaplace, cvAbs? + // Because we may deal with the boundary and corners much better in this way. + cv::GaussianBlur( tempImg2BExtracted, tempImg2BExtracted, Size(5, 5), 0.5, 0.5); + cv::Laplacian( tempImg2BExtracted, tempImg2BExtracted, tempImg2BExtracted.depth(), 3); + cv::abs(tempImg2BExtracted); + + Img2BExtracted = tempImg2BExtracted(rect); + } + break; + case VO_Features::HARRISCORNER: + { + Mat tempImg2BExtracted; + + // Explained by JIA Pei, 2008-03-09. + // Why I don't do cvCopy(tempImg2BExtracted, tempExcavated); right here, + // but after cvCornerHarris? + // Because we may deal with the boundary and corners much better in this way. + // cvCornerHarris is for one channel + + switch (NbOfChannels) + { + case GRAYCHANNELS: + { + cv::cornerHarris( img, tempImg2BExtracted, 3, 3, 0.02); + cv::abs(tempImg2BExtracted); + } + break; + case COLORCHANNELS: + default: + { + vector bgr, bgrHC; + bgrHC.resize(3); + cv::split(img, bgr); + + for(unsigned int i = 0; i < 3; i++) + { + cv::cornerHarris( bgr[i], bgrHC[i], 3, 3, 0.02); + } + + cv::merge(bgrHC, tempImg2BExtracted); + cv::abs(tempImg2BExtracted); + } + break; + } + + Img2BExtracted = tempImg2BExtracted(rect); + } + break; + case VO_Features::GABOR: // This might be very slow + { + } + break; + case VO_Features::SEGMENTATION: + { + Mat tempImg2BExtracted = Mat::zeros(img.size(), CV_32F); + img.convertTo( tempImg2BExtracted, tempImg2BExtracted.type() ); + + cv::GaussianBlur( tempImg2BExtracted, tempImg2BExtracted, Size(5, 5), 0.5, 0.5); + Img2BExtracted = tempImg2BExtracted(rect ); + + cv::threshold( Img2BExtracted, Img2BExtracted, 0, 255, THRESH_BINARY + THRESH_OTSU ); // only for single channel images + } + break; + case VO_Features::HISTOGRAMEQUALIZED: + { + Img2BExtracted = img(rect); + + switch (NbOfChannels) + { + case GRAYCHANNELS: + cv::equalizeHist( Img2BExtracted, Img2BExtracted ); + break; + case COLORCHANNELS: + default: + { + vector bgr; + cv::split(Img2BExtracted, bgr); + + for(unsigned int i = 0; i < 3; i++) + cv::equalizeHist( bgr[i], bgr[i] ); + + cv::merge(bgr, Img2BExtracted); + } + break; + } + } + break; + case VO_Features::DIRECT: + { + Img2BExtracted = img(rect); + } + default: + break; + } + +double time2 = (double)cvGetTickCount(); +elapsed = (time2 - time1)/ (cvGetTickFrequency()*1000.); +fs << "Mapping -- Step 2 of warping time: " << elapsed << "millisec." << endl; + + //Mat Img4Display = Mat::zeros(Img2BExtracted.size(), CV_8U); + //Img2BExtracted.convertTo(Img4Display, Img4Display.type()); + //imwrite("testtest.jpg", Img4Display); + + float x; + float y; + register int X; + register int Y ; + register int X1; + register int Y1; + register float s; + register float t; + register float s1; + register float t1; + float XX, YY; + + if(Img2BExtracted.type() == CV_32FC3) + { + for (unsigned int i = 0; i < NbOfPixels; i++ ) + { + // Explained by Yao Wei. This part causes the slow speed of the warping. +// // warp from m_vTemplateTriangle2D[j] (template pixel in the specific triangle) to each shape/image +// warpsrc(0, 0) = warpInfo[i].m_CVPosition.x; +// warpsrc(1, 0) = warpInfo[i].m_CVPosition.y; +// warpsrc(2, 0) = 1.0; +//// cv::gemm(matWarping[warpInfo[i].m_iTriangleIndex], warpsrc, 1.0, Mat(), 0.0, warpeddst); +// warpeddst = matWarping[warpInfo[i].m_iTriangleIndex] * warpsrc; + + // warp from m_vTemplateTriangle2D[j] (template pixel in the specific triangle) to each shape/image + const Mat_& sss = matWarping[warpInfo[i].m_iTriangleIndex]; + XX = sss(0,0)*warpInfo[i].m_CVPosition.x + sss(0,1)*warpInfo[i].m_CVPosition.y + +sss(0,2); + YY = sss(1,0)*warpInfo[i].m_CVPosition.x + sss(1,1)*warpInfo[i].m_CVPosition.y + +sss(1,2); + + // Revised by Yao Wei. Now, much much faster !!! + x = XX - rect.x; + y = YY - rect.y; + X = cvFloor(x); + Y = cvFloor(y); + X1 = cvCeil(x); + Y1 = cvCeil(y); + s=x-X; + t=y-Y; + s1 = 1.0f - s; + t1 = 1.0f - t; + + for (int j = 0; j < 3; j++) + { + oTexture.m_MatTexture(j, i) = + ( t1 * Img2BExtracted.at(Y, X)[j] + + t * Img2BExtracted.at(Y1, X)[j] ) * s1 + + ( t1 * Img2BExtracted.at(Y, X1)[j] + + t * Img2BExtracted.at(Y1, X1)[j] ) * s; + } + } + } + else if(Img2BExtracted.type() == CV_8UC3) + { + for (unsigned int i = 0; i < NbOfPixels; i++ ) + { + // Explained by Yao Wei. This part causes the slow speed of the warping. +// // warp from m_vTemplateTriangle2D[j] (template pixel in the specific triangle) to each shape/image +// warpsrc(0, 0) = warpInfo[i].m_CVPosition.x; +// warpsrc(1, 0) = warpInfo[i].m_CVPosition.y; +// warpsrc(2, 0) = 1.0; +//// cv::gemm(matWarping[warpInfo[i].m_iTriangleIndex], warpsrc, 1.0, Mat(), 0.0, warpeddst); +// warpeddst = matWarping[warpInfo[i].m_iTriangleIndex] * warpsrc; + + // warp from m_vTemplateTriangle2D[j] (template pixel in the specific triangle) to each shape/image + const Mat_& sss = matWarping[warpInfo[i].m_iTriangleIndex]; + XX = sss(0,0)*warpInfo[i].m_CVPosition.x + sss(0,1)*warpInfo[i].m_CVPosition.y + +sss(0,2); + YY = sss(1,0)*warpInfo[i].m_CVPosition.x + sss(1,1)*warpInfo[i].m_CVPosition.y + +sss(1,2); + + // Revised by Yao Wei. Now, much much faster !!! + x = XX - rect.x; + y = YY - rect.y; + X = cvFloor(x); + Y = cvFloor(y); + X1 = cvCeil(x); + Y1 = cvCeil(y); + s=x-X; + t=y-Y; + s1 = 1.0f - s; + t1 = 1.0f - t; + + for (int j = 0; j < 3; j++) + { + oTexture.m_MatTexture(j, i) = + ( t1 * Img2BExtracted.at(Y, X)[j] + + t * Img2BExtracted.at(Y1, X)[j] ) * s1 + + ( t1 * Img2BExtracted.at(Y, X1)[j] + + t * Img2BExtracted.at(Y1, X1)[j] ) * s; + } + } + } + else if(Img2BExtracted.type() == CV_32FC1) + { + for (unsigned int i = 0; i < NbOfPixels; i++ ) + { + // Explained by Yao Wei. This part causes the slow speed of the warping. +// // warp from m_vTemplateTriangle2D[j] (template pixel in the specific triangle) to each shape/image +// warpsrc(0, 0) = warpInfo[i].m_CVPosition.x; +// warpsrc(1, 0) = warpInfo[i].m_CVPosition.y; +// warpsrc(2, 0) = 1.0; +//// cv::gemm(matWarping[warpInfo[i].m_iTriangleIndex], warpsrc, 1.0, Mat(), 0.0, warpeddst); +// warpeddst = matWarping[warpInfo[i].m_iTriangleIndex] * warpsrc; + + // warp from m_vTemplateTriangle2D[j] (template pixel in the specific triangle) to each shape/image + const Mat_& sss = matWarping[warpInfo[i].m_iTriangleIndex]; + XX = sss(0,0)*warpInfo[i].m_CVPosition.x + sss(0,1)*warpInfo[i].m_CVPosition.y + +sss(0,2); + YY = sss(1,0)*warpInfo[i].m_CVPosition.x + sss(1,1)*warpInfo[i].m_CVPosition.y + +sss(1,2); + + // Revised by Yao Wei. Now, much much faster !!! + x = XX - rect.x; + y = YY - rect.y; + X = cvFloor(x); + Y = cvFloor(y); + X1 = cvCeil(x); + Y1 = cvCeil(y); + s=x-X; + t=y-Y; + s1 = 1.0f - s; + t1 = 1.0f - t; + + oTexture.m_MatTexture(0, i) = + ( t1 * Img2BExtracted.at(Y, X) + + t * Img2BExtracted.at(Y1, X) ) * s1 + + ( t1 * Img2BExtracted.at(Y, X1) + + t * Img2BExtracted.at(Y1, X1) ) * s; + } + } + else if(Img2BExtracted.type() == CV_8UC1) + { + for (unsigned int i = 0; i < NbOfPixels; i++ ) + { + // Explained by Yao Wei. This part causes the slow speed of the warping. +// // warp from m_vTemplateTriangle2D[j] (template pixel in the specific triangle) to each shape/image +// warpsrc(0, 0) = warpInfo[i].m_CVPosition.x; +// warpsrc(1, 0) = warpInfo[i].m_CVPosition.y; +// warpsrc(2, 0) = 1.0; +//// cv::gemm(matWarping[warpInfo[i].m_iTriangleIndex], warpsrc, 1.0, Mat(), 0.0, warpeddst); +// warpeddst = matWarping[warpInfo[i].m_iTriangleIndex] * warpsrc; + + + // warp from m_vTemplateTriangle2D[j] (template pixel in the specific triangle) to each shape/image + const Mat_& sss = matWarping[warpInfo[i].m_iTriangleIndex]; + XX = sss(0,0)*warpInfo[i].m_CVPosition.x + sss(0,1)*warpInfo[i].m_CVPosition.y + +sss(0,2); + YY = sss(1,0)*warpInfo[i].m_CVPosition.x + sss(1,1)*warpInfo[i].m_CVPosition.y + +sss(1,2); + + // Revised by Yao Wei. Now, much much faster !!! + x = XX - rect.x; + y = YY - rect.y; + X = cvFloor(x); + Y = cvFloor(y); + X1 = cvCeil(x); + Y1 = cvCeil(y); + s=x-X; + t=y-Y; + s1 = 1.0f - s; + t1 = 1.0f - t; + + oTexture.m_MatTexture(0, i) = + ( t1 * Img2BExtracted.at(Y, X) + + t * Img2BExtracted.at(Y1, X) ) * s1 + + ( t1 * Img2BExtracted.at(Y, X1) + + t * Img2BExtracted.at(Y1, X1) ) * s; + } + } +double time3 = (double)cvGetTickCount(); +elapsed = (time3 - time2)/ (cvGetTickFrequency()*1000.); +fs << "Mapping -- Step 3 of warping time: " << elapsed << "millisec." << endl; + +//Rect rect1 = VO_TextureModel::VO_CalcBoundingRectFromTriangles(templateTriangles); +//Mat otmpImg; +//if(NbOfChannels == 1) +// otmpImg = Mat::zeros(Size(rect1.width, rect1.height), CV_8UC1); +//else if(NbOfChannels == 3) +// otmpImg = Mat::zeros(Size(rect1.width, rect1.height), CV_8UC3); +//VO_TextureModel::VO_PutOneTextureToTemplateShape(oTexture, templateTriangles, otmpImg); +//imwrite("ttttt.jpg", otmpImg); +///** How many triangles totally */ +//string trianglestr; +//stringstream ssi; +//string stri; +//static int NbOfImages = 0; +//ssi << NbOfImages; +//ssi >> stri; +//if(stri.length() == 2) +// trianglestr = "0" + stri + ".jpg"; +//else if(stri.length() == 1) +// trianglestr = "00" + stri + ".jpg"; +//else +// trianglestr = stri + ".jpg"; +// +//imwrite(trianglestr.c_str(), otmpImg); +// +//ssi.clear(); +//NbOfImages++; + + fs.close(); + return true; + +} + + +/** +* @author JIA Pei +* @version 2010-02-10 +* @brief Normalize all textures - 2-norm all to "1" +* @param vTextures Input - all textures before normalization +* @param normalizedTextures Output - all textures after normalization +* @return return float average texture size of all textures +*/ +float VO_TextureModel::VO_NormalizeAllTextures(const vector& vTextures, vector& normalizedTextures) +{ + unsigned int NbOfSamples = vTextures.size(); + unsigned int NbOfPixels = vTextures[0].GetNbOfPixels(); + float averageTextureStandardDeviation = 0.0f; + + normalizedTextures = vTextures; + + for(unsigned int i = 0; i < NbOfSamples; i++) + { + averageTextureStandardDeviation += normalizedTextures.at(i).GetStandardizedTextureNorm(); + normalizedTextures[i].Normalize(); + } + averageTextureStandardDeviation /= (float)NbOfSamples; + + return averageTextureStandardDeviation; +} + + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief Rescale the normalizetextures so that meantexture's 2-norm could be "1" + * @param meanNormalizeTexture Input - meannormalizedtexture that all normalizedtextures should rescale to + * @param normalizedTextures Input and Output - normalized texture and normalized rescaled texture + * @return void +*/ +void VO_TextureModel::VO_RescaleAllNormalizedTextures( const VO_Texture& meanNormalizeTexture, + vector& normalizedTextures) +{ + unsigned int NbOfSamples = normalizedTextures.size(); + + // Explained by JIA Pei. 2010-02-10. scale back so that the mean shape size is equal to 1.0 + for(unsigned int i = 0; i < NbOfSamples; i++) + { + VO_TextureModel::VO_RescaleOneNormalizedTexture( meanNormalizeTexture, normalizedTextures[i] ); + } +} + + +/** + * @author JIA Pei + * @version 2010-02-20 + * @brief Rescale the normalizedTexture to the already trained meanNormalizeTexture + * @param meanNormalizeTexture Input - meanNormalizeTexture that all normalizedTextures should rescale to + * @param normalizedTexture Input and Output - normalized texture and normalized rescaled texture +*/ +void VO_TextureModel:: VO_RescaleOneNormalizedTexture(const VO_Texture& meanNormalizeTexture, + VO_Texture& normalizedTexture) +{ + float ts = sqrt ( fabs(normalizedTexture.dot(meanNormalizeTexture) ) ); + normalizedTexture.Scale( 1.0f/ts ); +} + + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief Calculate mean texture + * @param vTextures Input - all textures + * @param meanTexture Output - mean texture +*/ +void VO_TextureModel::VO_CalcMeanTexture(const vector& vTextures, VO_Texture& meanTexture) +{ + unsigned int NbOfSamples = vTextures.size(); + meanTexture = vTextures[0]; + + for(unsigned int i = 1; i < NbOfSamples; i++) + { + meanTexture += vTextures[i]; + } + + meanTexture /= (float)NbOfSamples; +} + + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief Put one texture into the template shape with a corresponding warp + * @param texture Input + * @param triangles Input - template triangle, contains warping information + * @param oImg Output + * @note All texture vectors are of the same size. So, you can't put it into the respective triangulation + for respective image. You've got to put a texture into a template shape first, then VO_WarpFromOneShapeToAnother + * @return void +*/ +void VO_TextureModel::VO_PutOneTextureToTemplateShape(const VO_Texture& texture, const vector & triangles, Mat& oImg) +{ + unsigned int NbOfChannels = texture.GetNbOfTextureRepresentation() >= 3 ? 3:1; + VO_Shape shape = VO_Triangle2DStructure::Triangle2D2Shape(triangles); + Rect rect = shape.GetShapeBoundRect(); + if(NbOfChannels == 1) + oImg = Mat::zeros(Size(rect.width, rect.height), CV_8UC1); + else if(NbOfChannels == 3) + oImg = Mat::zeros(Size(rect.width, rect.height), CV_8UC3); + + Point2f pt; + // Get the image m_ImageTemplateFace, for alignment! + int PixelsInFaceTemplate = 0; + + if(NbOfChannels == 1) + { + for (int Y = 0; Y < oImg.rows; Y++) + { + for (int X = 0; X < oImg.cols; X++) + { + pt.x = static_cast(X); + pt.y = static_cast(Y); + + int k = VO_Triangle2DStructure::IsPointInTriangles(pt, triangles); + if(k>=0) + { + oImg.at(Y, X) = (unsigned char)texture.m_MatTexture(0, PixelsInFaceTemplate); + PixelsInFaceTemplate++; + } + } + } + } + else if(NbOfChannels == 3) + { + for (int Y = 0; Y < oImg.rows; Y++) + { + for (int X = 0; X < oImg.cols; X++) + { + pt.x = static_cast(X); + pt.y = static_cast(Y); + + int k = VO_Triangle2DStructure::IsPointInTriangles(pt, triangles); + if(k>=0) + { + for (unsigned int chan = 0; chan < NbOfChannels; chan++) + { + oImg.at(Y, X)[chan] = (unsigned char)texture.m_MatTexture(chan, PixelsInFaceTemplate); + } + PixelsInFaceTemplate++; + } + } + } + } +} + + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief Put one texture into the template shape with a corresponding warp + * @param iShape Input - shape warp from + * @param oShape Input - shape warp to + * @param triangles Input - contains warp information + * @param iImg Input - input image, extracted texture from + * @param oImg Output - output image, put the texture on to this image + * @note All texture vectors are of the same size. So, you can't put it into the respective triangulation + for respective image. You've got to put a texture into a template shape first, then VO_WarpFromOneShapeToAnother +*/ +unsigned int VO_TextureModel::VO_WarpFromOneShapeToAnother( const VO_Shape& iShape, + const VO_Shape& oShape, + const vector & triangles, + const Mat& iImg, + Mat& oImg) +{ + unsigned int NbOfChannels = iImg.channels(); + unsigned int NbOfPoints = iShape.GetNbOfPoints(); + unsigned int NbOfTriangles = triangles.size(); + unsigned int NbOfPixels = 0; + + Point2f src[3], dst[3]; + vector< Mat_ > matWarping; + matWarping.resize(NbOfTriangles); + vector warpedTriangles = oShape.GetTriangle2DStructure(triangles); + Rect rect_in = iShape.GetShapeBoundRect(); + Rect rect_out = oShape.GetShapeBoundRect(); + oImg = Mat::zeros( Size(rect_out.width, rect_out.height), iImg.type() ); + unsigned int vIdx0, vIdx1, vIdx2; + + // calculate all the possible mapping (for speeding up) 95 mapping totally + for (unsigned int i = 0; i < NbOfTriangles; i++ ) + { + vIdx0 = triangles[i].m_vVertexIndexes[0]; + vIdx1 = triangles[i].m_vVertexIndexes[1]; + vIdx2 = triangles[i].m_vVertexIndexes[2]; + + // Get the affine transformation for each triangle pair. + src[0] = oShape.GetA2DPoint(vIdx0); + src[1] = oShape.GetA2DPoint(vIdx1); + src[2] = oShape.GetA2DPoint(vIdx2); + + dst[0] = iShape.GetA2DPoint(vIdx0); + dst[1] = iShape.GetA2DPoint(vIdx1); + dst[2] = iShape.GetA2DPoint(vIdx2); + + matWarping[i] = cv::getAffineTransform( src, dst ); + } + + float XX, YY; + Point2f pt; + + if(NbOfChannels == 1) + { + float gray = 0.0; + + for (int Y = 0; Y < oImg.rows; Y++) + { + for (int X = 0; X < oImg.cols; X++) + { + pt.x = static_cast(X + rect_out.x); + pt.y = static_cast(Y + rect_out.y); + + int k = VO_Triangle2DStructure::IsPointInTriangles(pt, warpedTriangles); + if(k>=0) + { + const Mat_& sss = matWarping[k]; + XX = sss(0,0)*pt.x + sss(0,1)*pt.y + sss(0,2); + YY = sss(1,0)*pt.x + sss(1,1)*pt.y + sss(1,2); + + // Since the above calculations are basically matrix calculation, + // they won't be able to always ensure XX or YY are always within the image. + // Therefore, a constrain in the following is a must!!! + if (XX < 0.0f) XX = 0.0f; + if (YY < 0.0f) YY = 0.0f; + if (XX > ((float)(rect_in.width - 1) - 0.001f)) XX = (float) (rect_in.width - 1) - 0.001f; + if (YY > ((float)(rect_in.height - 1) - 0.001f)) YY = (float) (rect_in.height - 1) - 0.001f; + +// // warp from m_vTemplateTriangle2D[j] (template pixel in the specific triangle) to each shape/image +// warpsrc(0, 0) = pt.x; +// warpsrc(1, 0) = pt.y; +// warpsrc(2, 0) = 1.0; +// +// // cv::gemm(matWarping[k], warpsrc, 1.0, Mat(), 0.0, warpeddst); +// warpeddst = matWarping[k] * warpsrc; + + // extract the B,G,R on each shape/image respectively into textures + // warpeddst.at(0, 0) is for x while + // warpeddst.at(1, 0) is for y. But, + // x determines cols, while y determines rows. +// VO_TextureModel::VO_CalcSubPixelTexture ( warpeddst(0, 0), warpeddst(1, 0), iImg, &gray); + VO_TextureModel::VO_CalcSubPixelTexture ( XX, YY, iImg, &gray); + oImg.at(Y, X) = (unsigned char) gray; + + NbOfPixels ++; + } + } + } + } + else if(NbOfChannels == 3) + { + float b = 0.0, g = 0.0, r = 0.0; + + for (int Y = 0; Y < oImg.rows; Y++) + { + for (int X = 0; X < oImg.cols; X++) + { + pt.x = static_cast(X + rect_out.x); + pt.y = static_cast(Y + rect_out.y); + + int k = VO_Triangle2DStructure::IsPointInTriangles(pt, warpedTriangles); + if(k>=0) + { + const Mat_& sss = matWarping[k]; + XX = sss(0,0)*pt.x + sss(0,1)*pt.y + sss(0,2); + YY = sss(1,0)*pt.x + sss(1,1)*pt.y + sss(1,2); + + // Since the above calculations are basically matrix calculation, + // they won't be able to always ensure XX or YY are always within the image. + // Therefore, a constrain in the following is a must!!! + if (XX < 0.0f) XX = 0.0f; + if (YY < 0.0f) YY = 0.0f; + if (XX > ((float)(rect_in.width - 1) - 0.001f)) XX = (float) (rect_in.width - 1) - 0.001f; + if (YY > ((float)(rect_in.height - 1) - 0.001f)) YY = (float) (rect_in.height - 1) - 0.001f; + +// // warp from m_vTemplateTriangle2D[j] (template pixel in the specific triangle) to each shape/image +// warpsrc(0, 0) = pt.x; +// warpsrc(1, 0) = pt.y; +// warpsrc(2, 0) = 1.0; +// +// // cv::gemm(matWarping[k], warpsrc, 1.0, Mat(), 0.0, warpeddst); +// warpeddst = matWarping[k] * warpsrc; + + // extract the B,G,R on each shape/image respectively into textures + // warpeddst.at(0, 0) is for x while + // warpeddst.at(1, 0) is for y. But, + // x determines cols, while y determines rows. +// VO_TextureModel::VO_CalcSubPixelTexture (warpeddst(0, 0), warpeddst(1, 0), iImg, &b, &g, &r); + VO_TextureModel::VO_CalcSubPixelTexture (XX, YY, iImg, &b, &g, &r); + oImg.at(Y, X)[0] = (unsigned char) b; + oImg.at(Y, X)[1] = (unsigned char) g; + oImg.at(Y, X)[2] = (unsigned char) r; + + NbOfPixels ++; + } + } + } + } + return NbOfPixels; +} + + +/** + * @brief image morphing + * @param iShape1 Input the first shape + * @param iShape2 Input the second shape + * @param oShapes Output intermediate shapes during the process of morphing + * @param triangles Input a list of triangles + * @param iImg1 Input the first image + * @param iImg2 Input the second image + * @param oImgs Output the output images + * @param step Input between 0 to 1, Normally, 0.1 or 0.2 are good selections + * @note Current version requires iShape1 and iShape2 have the same size + **/ +void VO_TextureModel::VO_Morphing(const VO_Shape& iShape1, const VO_Shape& iShape2, vector& oShapes, const vector& triangles, const Mat& iImg1, const Mat& iImg2, vector& oImgs, float step) +{ + assert (iImg1.cols == iImg2.cols && iImg1.rows == iImg2.rows); + unsigned int NbOfFrames = cvRound(1.0/step); + oShapes.resize(NbOfFrames+1); + oImgs.resize(NbOfFrames+1); + + Mat morph1, morph2; + + oShapes[0] = iShape1; + oShapes[NbOfFrames] = iShape2; + iImg1.copyTo(oImgs[0]); + iImg2.copyTo(oImgs[NbOfFrames]); + + for(unsigned int i = 1; i < NbOfFrames; ++i) + { + oShapes[i] = const_cast(iShape1)*(step*i) + const_cast(iShape2)*(1.0f-step*i); + VO_TextureModel::VO_WarpFromOneShapeToAnother(iShape1, oShapes[i], triangles, iImg1, morph1); + VO_TextureModel::VO_WarpFromOneShapeToAnother(iShape2, oShapes[i], triangles, iImg2, morph2); + cv::addWeighted( morph1, step*i, morph2, 1.0-step*i, 0.0, oImgs[i] ); + } +} + + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief Put one texture into the template shape with a corresponding warp + * @param iShape Input - shape warp from + * @param oShape Input - shape warp to + * @param triangles Input - contains warp information + * @param iImg Input - input image, extracted texture from + * @param oImg Output - output image, put the texture on to this image + * @note All texture vectors are of the same size. So, you can't put it into the respective triangulation + for respective image. You've got to put a texture into a template shape first, then VO_WarpFromOneShapeToAnother +*/ +void VO_TextureModel::VO_PutOneTextureToOneShape(const VO_Texture& texture, const VO_Shape& oShape, const vector & triangles, Mat& oImg) +{ + Mat intermediateImg (oImg); + VO_TextureModel::VO_PutOneTextureToTemplateShape(texture, triangles, intermediateImg); + VO_Shape intermediateShape = VO_Triangle2DStructure::Triangle2D2Shape(triangles); + VO_TextureModel::VO_WarpFromOneShapeToAnother(intermediateShape, oShape, triangles, intermediateImg, oImg); +} + + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief Calculate sub pixel texture of a point in a gray-level image + * @param gray - output the gray level + * @note In the pictures, x represents cols, y represents rows! + * In Mat& image, x represents cols, y represents rows as well!!! + * @return void +*/ +void VO_TextureModel::VO_CalcSubPixelTexture(float x, float y, const Mat& image, float* gray) +{ + assert(image.channels() == GRAYCHANNELS); +// double sec = (double)cvGetTickCount(); // Now, pretty fast already for gray-level images + + int Y0 = (int)y; + int X0 = (int)x; + int Y1 = (int)(y+1); + int X1 = (int)(x+1); + + float s=x-X0; + float t=y-Y0; + float t1=1.0f-t; + float s1=1.0f-s; + + float tempA = t1 * image.at(Y0, X0) + + t * image.at(Y1, X0); + float tempB = t1 * image.at(Y0, X1) + + t * image.at(Y1, X1); + *gray = tempA * s1 + tempB * s; + +// sec = ((double)cvGetTickCount() - sec )/ (cvGetTickFrequency()); +// cout << "Interpolation time cost: " << sec << " millisec" << endl; +} + + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief Calculate sub pixel texture of a point in a color image + * @param gray - output the gray level + * @note In the pictures, x represents cols, y represents rows! + In Mat& image, x represents cols, y represents rows as well!!! + * @return void +*/ +void VO_TextureModel::VO_CalcSubPixelTexture(float x, float y, const Mat& image, float* b, float* g, float* r) +{ + assert(image.channels() == COLORCHANNELS); + +// double sec = (double)cvGetTickCount(); + + int Y0 = (int)y; + int X0 = (int)x; + int Y1 = (int)(y+1); + int X1 = (int)(x+1); + + float s=x-X0; + float t=y-Y0; + float s1=1.0f-s; + float t1=1.0f-t; + + + float tempA_b = t1 * image.at(Y0, X0)[0] + + t * image.at(Y1, X0)[0]; + float tempB_b = t1 * image.at(Y0, X1)[0] + + t * image.at(Y1, X1)[0]; + float tempA_g = t1 * image.at(Y0, X0)[1] + + t * image.at(Y1, X0)[1]; + float tempB_g = t1 * image.at(Y0, X1)[1] + + t * image.at(Y1, X1)[1]; + float tempA_r = t1 * image.at(Y0, X0)[2] + + t * image.at(Y1, X0)[2]; + float tempB_r = t1 * image.at(Y0, X1)[2] + + t * image.at(Y1, X1)[2]; + *b = tempA_b * s1 + tempB_b * s; + *g = tempA_g * s1 + tempB_g * s; + *r = tempA_r * s1 + tempB_r * s; + +// sec = ((double)cvGetTickCount() - sec )/ (cvGetTickFrequency()); +// cout << "Interpolation time cost: " << sec << " millisec" << endl; +} + +/** + * @author JIA Pei + * @author Colin B + * @version 2012-04-19 + * @brief Calculate sub pixel texture of a point in a color image + * @param gray - output the gray level + * @note In the pictures, x represents cols, y represents rows! + In Mat& image, x represents cols, y represents rows as well!!! + * @return void +*/ +void VO_TextureModel::VO_CalcSubPixelTexture(float x, float y, const Mat& image, float* g, float* d) +{ + //Two channels for one of color and one of depth, extra channels are ignored (hopefully?) + assert(image.channels() >= 2); + +// double sec = (double)cvGetTickCount(); + + int Y0 = static_cast(y); + int X0 = static_cast(x); + int Y1 = static_cast(y+1); + int X1 = static_cast(x+1); + + assert(X1 < image.rows); + assert(Y1 < image.cols); + + float s=x-X0; + float t=y-Y0; + float s1=1.0f-s; + float t1=1.0f-t; + + + float tempA_g = t1 * image.at(Y0, X0)[0] + t * image.at(Y1, X0)[0]; + //debug stuff + //bool test1 = image.dims <= 2 && image.data; + //bool test2 = (unsigned)Y0 < (unsigned)image.size.p[0]; + //bool test3 = (unsigned)(X1*2) < (unsigned)(image.size.p[1]*image.channels()); + //bool test3a = (unsigned)(X1*3) < (unsigned)(image.size.p[1]*image.channels()); + //int test3b = (unsigned)(image.size.p[1]*image.channels()); + //int test3c = image.size.p[1]; + //int test3e = image.channels(); + //bool test4 = ((((sizeof(size_t)<<28)|0x8442211) >> CV_MAT_DEPTH(8)*4) & 15) == image.elemSize1(); + //int test5 = image.depth(); + + //float test_y0x1_0 = image.at(Y0, X1)[0];//expects 3 channel spacing? + //float test_y1x1_0 = image.at(Y1, X1)[0]; + float tempB_g = t1 * image.at(Y0, X1)[0] + t * image.at(Y1, X1)[0]; + //float test_y0x0_1 = image.at(Y0, X0)[1]; + //float test_y1x0_1 = image.at(Y1, X0)[1]; + float tempA_d = t1 * image.at(Y0, X0)[1] + t * image.at(Y1, X0)[1]; + //float test_y0x1_1 = image.at(Y0, X0)[1]; + //float test_y1x1_1 = image.at(Y1, X0)[1]; + float tempB_d = t1 * image.at(Y0, X1)[1] + t * image.at(Y1, X1)[1]; + *g = tempA_g * s1 + tempB_g * s; + *d = tempA_d * s1 + tempB_d * s; + +// sec = ((double)cvGetTickCount() - sec )/ (cvGetTickFrequency()); +// cout << "Interpolation time cost: " << sec << " millisec" << endl; +} + + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief Calculate sub pixel texture of a point in image + * @return vector - a vector of size 1 or 3 + * @note In the pictures, x represents cols, y represents rows! + In Mat& image, x represents cols, y represents rows as well!!! + * @return void +*/ +vector VO_TextureModel::VO_CalcSubPixelTexture(float x, float y, const Mat& image) +{ +// double sec = (double)cvGetTickCount(); + + vector result; + unsigned int channels = image.channels(); + if (!(channels == 1 || channels == 3)) + { + cerr << "Cannot deal with images with nChannels != 1 or 3." << endl; + exit(EXIT_FAILURE); + } + + int Y0 = (int)y; + int X0 = (int)x; + int Y1 = (int)(y+1); + int X1 = (int)(x+1); + + float s = x-X0; + float t = y-Y0; + float s1 = 1.0f-s; + float t1 = 1.0f-t; + + float LeftTopB, LeftTopG, LeftTopR; + float LeftBottomB, LeftBottomG, LeftBottomR; + float RightTopB, RightTopG, RightTopR; + float RightBottomB, RightBottomG, RightBottomR; + + VO_TextureModel::VO_CalcPixelRGB(X0, Y0, image, LeftTopB, LeftTopG, LeftTopR); + VO_TextureModel::VO_CalcPixelRGB(X0, Y1, image, LeftBottomB, LeftBottomG, LeftBottomR); + VO_TextureModel::VO_CalcPixelRGB(X1, Y0, image, RightTopB, RightTopG, RightTopR); + VO_TextureModel::VO_CalcPixelRGB(X1, Y1, image, RightBottomB, RightBottomG, RightBottomR); + + float tempAB, tempAG, tempAR; + float tempBB, tempBG, tempBR; + + if(channels == 1) + { + tempAB = t1 * LeftTopB + t * LeftBottomB; + tempBB = t1 * RightTopB + t * RightBottomB; + result.push_back (tempAB * s1 + tempBB * s); + } + else + { + tempAB = t1 * LeftTopB + t * LeftBottomB; + tempBB = t1 * RightTopB + t * RightBottomB; + result.push_back (tempAB * s1 + tempBB * s); + tempAG = t1 * LeftTopG + t * LeftBottomG; + tempBG = t1 * RightTopG + t * RightBottomG; + result.push_back (tempAG * s1 + tempBG * s); + tempAR = t1 * LeftTopR + t * LeftBottomR; + tempBR = t1 * RightTopR + t * RightBottomR; + result.push_back (tempAR * s1 + tempBR * s); + } + +// sec = ((double)cvGetTickCount() - sec )/ (cvGetTickFrequency());Vec3b +// cout << "Interpolation time cost: " << sec << " millisec" << endl; + + return result; +} + + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief Calculate RGB value for a point in image + * @param image + * @return no return value, in order to speed up !! +*/ +void VO_TextureModel::VO_CalcPixelRGB(int x, int y, const Mat& image, float& B, float& G, float& R) +{ + unsigned int channels = image.channels(); + if (!(channels == 1 || channels == 3)) + { + cerr << "Cannot deal with images with nChannels != 1 or 3." << endl; + exit(EXIT_FAILURE); + } + + switch(image.depth()) + { + case IPL_DEPTH_8U: + { + if(channels == 1) + B = (float)image.at(y, x ); + else + { + B = (float)image.at(y, x*3 ); + G = (float)image.at(y, x*3+1 ); + R = (float)image.at(y, x*3+2 ); + } + } + break; + case IPL_DEPTH_8S: + { + if(channels == 1) + B = (float)image.at(y, x ); + else + { + B = (float)image.at(y, x*3 ); + G = (float)image.at(y, x*3+1 ); + R = (float)image.at(y, x*3+2 ); + } + } + break; + case IPL_DEPTH_16S: + { + if(channels == 1) + B = (float)image.at(y, x ); + else + { + B = (float)image.at(y, x*3 ); + G = (float)image.at(y, x*3+1 ); + R = (float)image.at(y, x*3+2 ); + } + } + break; + case IPL_DEPTH_16U: + { + if(channels == 1) + B = (float)image.at(y, x ); + else + { + B = (float)image.at(y, x*3 ); + G = (float)image.at(y, x*3+1 ); + R = (float)image.at(y, x*3+2 ); + } + } + break; + case IPL_DEPTH_32S: + { + if(channels == 1) + B = (float)image.at(y, x ); + else + { + B = (float)image.at(y, x*3 ); + G = (float)image.at(y, x*3+1 ); + R = (float)image.at(y, x*3+2 ); + } + } + break; + case IPL_DEPTH_32F: + { + if(channels == 1) + B = image.at(y, x ); + else + { + B = image.at(y, x*3 ); + G = image.at(y, x*3+1 ); + R = image.at(y, x*3+2 ); + } + } + break; + case IPL_DEPTH_64F: + { + if(channels == 1) + B = (float)image.at(y, x ); + else + { + B = (float)image.at(y, x*3 ); + G = (float)image.at(y, x*3+1 ); + R = (float)image.at(y, x*3+2 ); + } + } + break; + } +} + + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief Normalized texture to reference scale texture + * @param inTexture Input - input texture + * @param textureSD Input - texture standard deviation + * @param outTexture Output - output texture in reference scale + * @return void +*/ +void VO_TextureModel::VO_NormalizedTexture2ReferenceScale(const VO_Texture& inTexture, float textureSD, VO_Texture& outTexture) +{ + outTexture = const_cast (inTexture) * textureSD + AVERAGEFACETEXTURE; + + outTexture.Clamp(0.0, 255.0); +} + + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief Reference scale texture back to normalized one + * @param inTexture Input - reference texture + * @param textureSD Input - texture standard deviation + * @param outTexture Output - output normalized texture + * @return void +*/ +void VO_TextureModel::VO_ReferenceTextureBack2Normalize(const VO_Texture& inTexture, float textureSD, VO_Texture& outTexture) +{ + outTexture = ( const_cast (inTexture) - AVERAGEFACETEXTURE ) / textureSD; +} + + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief Put edges on template face + * @param edges Input - all edges information + * @param templateShape Input - template shape + * @param iImg Input - template image + * @param oImg Output - output image with edges on oImg + * @return void +*/ +void VO_TextureModel::VO_PutEdgesOnTemplateFace(const vector& edges, + const VO_Shape& templateShape, + const Mat& iImg, + Mat& oImg) +{ + unsigned int NbOfEdges = edges.size(); + Point2f iorg,idst; + + iImg.copyTo(oImg); + + for (unsigned int i = 0; i < NbOfEdges; i++) + { + iorg = templateShape.GetA2DPoint(edges[i].GetIndex1() ); + idst = templateShape.GetA2DPoint(edges[i].GetIndex2() ); + cv::line( oImg, iorg, iorg, colors[8], 2, 0, 0 ); + cv::line( oImg, iorg, idst, colors[8], 1, 0, 0 ); + } +} + + +///** +// * @author JIA Pei +// * @version 2010-02-10 +// * @brief Put convex hull on template face +// * @param ch Input - convex hull +// * @param iImg Input - template image +// * @param oImg Output - output images with convex hull on oImg +//*/ +//void VO_TextureModel::VO_PutConvexHullOnTemplateFace(const Mat& iImg, Mat& oImg) +//{ +// unsigned int NbOfPointsOnConvexHull = ch.cols; +// Point iorg,idst; +// +// iImg.copyTo(oImg); +// +// for (unsigned int i = 0; i < NbOfPointsOnConvexHull; i++) +// { +// if(i != NbOfPointsOnConvexHull - 1 ) +// { +// iorg = cvPointFrom32f( ch.at(0, i ) ); +// idst = cvPointFrom32f( ch.at(0, i+1 ) ); +// cv::line( oImg, iorg, iorg, colors[8], 2, 0, 0 ); // emphasize the vertex +// cv::line( oImg, iorg, idst, colors[8], 1, 0, 0 ); +// } +// else +// { +// iorg = cvPointFrom32f( ch.at(0, i ) ); +// idst = cvPointFrom32f( ch.at(0, 0 ) ); +// cv::line( oImg, iorg, iorg, colors[8], 2, 0, 0 ); // emphasize the vertex +// cv::line( oImg, iorg, idst, colors[8], 1, 0, 0 ); +// } +// } +//} +// +// +///** +// * @author JIA Pei +// * @version 2010-02-10 +// * @brief Put concave pixels on template face - inside convex hull but out of template face +// * @param ch Input - convex hull +// * @param triangles Input - all triangles information +// * @param iImg Input - template image +// * @param oImg Output - output images with concave pixels on oImg +//*/ +//void VO_TextureModel::VO_PutConcavePixelsOnTemplateFace(const Mat& ch, const vector & triangles, +// const Mat& iImg, Mat& oImg) +//{ +// iImg.copyTo(oImg); +// +// Point2f pt; +// for (unsigned int i = 0; i < iImg.rows; i++) +// { +// for (unsigned int j = 0; j < iImg.cols; j++) +// { +// pt.x = (float)j; +// pt.y = (float)i; +// +// if( VO_TextureModel::VO_IsPointInConvexHull(pt, ch, true) && !VO_TextureModel::VO_IsPointInTemplateFace(triangles, pt) ) +// { +// Point iorg = cvPointFrom32f(pt); +// cv::line( oImg, iorg, iorg, colors[8], 1, 0, 0 ); // Here, must not emphasize the points +// } +// } +// } +//} + + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief Put every single triangle onto template face + * @param triangles Input - all triangles information + * @param iImg Input - template image + * @param oImgs Output - output images with single triangle on every single oImg + * @return void +*/ +void VO_TextureModel::VO_PutTrianglesOnTemplateFace(const vector & triangles, + const Mat& iImg, + vector& oImgs) +{ + unsigned int NbOfTriangles = triangles.size(); + oImgs.resize(NbOfTriangles); + + vector iorgFloat, idstFloat; + iorgFloat.resize(3); + idstFloat.resize(3); + + for(unsigned int i = 0; i < NbOfTriangles; i++) + { + iImg.copyTo(oImgs[i]); + + iorgFloat[0] = triangles[i].GetA2DPoint(0); + iorgFloat[1] = triangles[i].GetA2DPoint(1); + iorgFloat[2] = triangles[i].GetA2DPoint(2); + idstFloat[0] = triangles[i].GetA2DPoint(1); + idstFloat[1] = triangles[i].GetA2DPoint(2); + idstFloat[2] = triangles[i].GetA2DPoint(0); + + cv::line( oImgs[i], iorgFloat[0], idstFloat[0], colors[8], 1, 0, 0 ); + cv::line( oImgs[i], iorgFloat[1], idstFloat[1], colors[8], 1, 0, 0 ); + cv::line( oImgs[i], iorgFloat[2], idstFloat[2], colors[8], 1, 0, 0 ); + } +} + + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief draw all PDM ellipses onto the textured template face + * @param iShape Input - the input shape + * @param iImg Input - the input image + * @param oImg Output - the output image + * @return void + */ +void VO_TextureModel::VO_PutPDMEllipsesOnTemplateFace(const vector& ellipses, const Mat& iImg, Mat& oImg) +{ + unsigned int NbOfEllipses = ellipses.size(); + iImg.copyTo(oImg); + Point2f iPoint; + + for(unsigned int i = 0; i < NbOfEllipses; i++) + { + Point2f pt = ellipses.at(i).GetCOG(); + + cv::ellipse(oImg, Point( (int)pt.x, (int)pt.y ), + Size(static_cast(ellipses.at(i).GetAxisXHalfLen()), static_cast(ellipses.at(i).GetAxisYHalfLen())), + ellipses.at(i).GetAngle(), + ellipses.at(i).GetStartAngle(), + ellipses.at(i).GetEndAngle(), + colors[7], + 1, + 0, + 0); + } +} + + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief draw the shape's key points onto the textured template face + * @param iShape Input - the input shape + * @param iImg Input - the input image + * @param oImg Output - the output image + * @return void + */ +void VO_TextureModel::VO_PutShapeOnTemplateFace(const VO_Shape& iShape, const Mat& iImg, Mat& oImg) +{ + unsigned int NbOfPoints = iShape.GetNbOfPoints(); + iImg.copyTo(oImg); + Point2f iPoint; + + for(unsigned int i = 0; i < NbOfPoints; i++) + { + iPoint = iShape.GetA2DPoint(i); + + cv::line( oImg, iPoint, iPoint, colors[7], 2, 0, 0 ); + } +} + + +/** + * @author JIA Pei + * @version 2010-04-07 + * @brief texture parameters constrain + * @param ioT Input and Output - texture parameters + * @param nSigma Input - number of sigmas + * @return void +*/ +void VO_TextureModel::VO_TextureParameterConstraint(Mat_& ioT, float nSigma) +{ + for (int i = 0; i < ioT.cols; ++i) + { + float ct = nSigma * sqrt(this->m_PCANormalizedTexture.eigenvalues.at(i,0) ); + if ( ioT(0, i) > ct ) + { + ioT(0, i) = ct; + } + else if ( ioT(0, i) < -ct ) + { + ioT(0, i) = -ct; + } + } +} + + +/** +* @author JIA Pei +* @version 2010-04-10 +* @brief texture parameters back project to normalized texture +* @param iTexture Input - input normalized texture +* @param outT Output - the projected texture parameters +*/ +void VO_TextureModel::VO_NormalizedTextureProjectToTParam(const VO_Texture& iTexture, Mat_& outT) const +{ + this->m_PCANormalizedTexture.project(iTexture.GetTheTextureInARow(), outT); +} + + +/** +* @author JIA Pei +* @version 2010-02-10 +* @brief texture parameters back project to normalized texture +* @param inC Input - input texture parameters +* @param oTexture Output - the back projected texture in a row +* @param tr Input - Number of texture representations +*/ +void VO_TextureModel::VO_TParamBackProjectToNormalizedTexture(const Mat_& inT, VO_Texture& oTexture, int tr) const +{ + oTexture.SetTheTexture(this->m_PCANormalizedTexture.backProject(inT), tr); +} + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief texture parameters back project to normalized texture + * @param inC Input - input texture parameters + * @param oTexture Output - the back projected texture in a row +*/ +void VO_TextureModel::VO_TParamBackProjectToNormalizedTexture(const Mat_& inT, Mat_& oTextureMat) const +{ + oTextureMat = this->m_PCANormalizedTexture.backProject(inT); +} + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief Calculate all parameters for an arbitrary texture + * @param iTexture Input - input texture + * @param oTexture Output - output texture + * @param outT Output - texture parameters + * @return void +*/ +void VO_TextureModel::VO_CalcAllParams4AnyTexture(const Mat_& iTexture, Mat_& oTexture, Mat_& outT) +{ + // Here, for VO_Texture; there is no point to know how many texture representations for each VO_Texture + VO_Texture oT(iTexture); + oT.Normalize(); + oTexture = oT.GetTheTextureInARow(); + this->m_PCANormalizedTexture.project(oTexture, outT ); +} + + +/** + * @author JIA Pei + * @version 2010-02-10 + * @brief Calculate all parameters for an arbitrary texture + * @param ioTexture Input and Output - input texture, output normalized texture + * @param outT Output - output texture model parameters + * @return void +*/ +void VO_TextureModel::VO_CalcAllParams4AnyTexture(VO_Texture& ioTexture, Mat_& outT) +{ + ioTexture.Normalize(); + this->m_PCANormalizedTexture.project(ioTexture.GetTheTextureInARow(), outT ); +} + + +/** + * @author JIA Pei + * @version 2010-02-05 + * @brief Load Training data for texture model + * @return void +*/ +bool VO_TextureModel::VO_LoadTextureTrainingData( const vector& allImgFiles4Training, + unsigned int channels, + int trm ) +{ + this->m_vStringTrainingImageNames = allImgFiles4Training; + this->m_iNbOfChannels = channels; + this->m_iTextureRepresentationMethod = trm; + + this->m_vTextures.resize(this->m_iNbOfSamples); + this->m_vNormalizedTextures.resize(this->m_iNbOfSamples); + Mat img; + + for(unsigned int i = 0; i < this->m_iNbOfSamples; ++i) + { + if(this->m_iNbOfChannels == 1) + img = imread ( this->m_vStringTrainingImageNames[i].c_str (), 0 ); + else if (this->m_iNbOfChannels == 3) + img = imread ( this->m_vStringTrainingImageNames[i].c_str (), 1 ); + else + cerr << "We can't deal with image channels not equal to 1 or 3!" << endl; + + double start = (double)cvGetTickCount(); + // Explained by JIA Pei -- warping + if ( !VO_TextureModel::VO_LoadOneTextureFromShape( this->m_vShapes[i], + img, + this->m_vTemplateTriangle2D, + this->m_vTemplatePointWarpInfo, + this->m_vTextures[i], + this->m_iTextureRepresentationMethod) ) + { + cout << "Texture Fail to Load at image " << i << endl; + return false; + } + + double end = (double)cvGetTickCount(); + double elapsed = (end - start) / (cvGetTickFrequency()*1000.0); + } + + return true; +} + + +/** + * @author JIA Pei + * @version 2010-02-05 + * @brief build Texture Model + * @param allLandmarkFiles4Training Input - all training landmark files + * @param allImgFiles4Training Input - all training image files + * @param shapeinfoFileName Input - shape info file + * @param database Input - which database is it? + * @param channels Input - How many channels are to be used? + * @param trm Input - texture representation method + * @param TPShape Input - truncated percentage for shape model + * @param TPTexture Input - truncated percentage for texture model + * @param useKnownTriangles Input - use known triangle structures?? + * @note Refer to "Statistical Models of Appearance for Computer Vision" page 31, Cootes + * @return void +*/ +void VO_TextureModel::VO_BuildTextureModel( const vector& allLandmarkFiles4Training, + const vector& allImgFiles4Training, + const string& shapeinfoFileName, + unsigned int database, + unsigned int channels, + int trm, + float TPShape, + float TPTexture, + bool useKnownTriangles) +{ + if (allLandmarkFiles4Training.size() != allImgFiles4Training.size() ) + cerr << "allLandmarkFiles4Training should have the same number of allImgFiles4Training! " << endl; + + this->VO_BuildShapeModel(allLandmarkFiles4Training, shapeinfoFileName, database, TPShape, useKnownTriangles); + this->m_iNbOfPixels = VO_TextureModel::VO_CalcPointWarpingInfo(this->m_vTemplateTriangle2D, this->m_vTemplatePointWarpInfo); + this->VO_LoadTextureTrainingData( allImgFiles4Training, channels, trm); + + this->m_iNbOfTextureRepresentations = this->m_vTextures[0].GetNbOfTextureRepresentation(); + this->m_iNbOfTextures = this->m_iNbOfPixels*this->m_iNbOfTextureRepresentations; + this->m_iNbOfEigenTexturesAtMost = MIN(this->m_iNbOfSamples, this->m_iNbOfTextures); + this->m_fTruncatedPercent_Texture = TPTexture; + Mat_ matNormalizedTextures = Mat_::zeros(this->m_iNbOfSamples, this->m_iNbOfTextures); + Mat_ matNormalizedMeanTextures = Mat_::zeros(1, m_iNbOfTextures); + + // Normalize all textures + this->m_fAverageTextureStandardDeviation = VO_TextureModel::VO_NormalizeAllTextures(this->m_vTextures, this->m_vNormalizedTextures); + VO_TextureModel::VO_CalcMeanTexture(this->m_vNormalizedTextures, this->m_VONormalizedMeanTexture); + // Calculate reference texture + VO_TextureModel::VO_NormalizedTexture2ReferenceScale(this->m_VONormalizedMeanTexture, this->m_fAverageTextureStandardDeviation, this->m_VOReferenceTexture); +//VO_TextureModel::VO_PutOneTextureToTemplateShape(this->m_VOReferenceTexture, this->m_vTemplateTriangle2D, this->m_ImageTemplateFace); +//imwrite("template.jpg",this->m_ImageTemplateFace); + + matNormalizedMeanTextures = this->m_VONormalizedMeanTexture.GetTheTextureInARow(); + for(unsigned int i = 0; i < this->m_iNbOfSamples; ++i) + { + Mat_ tmpRow = matNormalizedTextures.row(i); + this->m_vNormalizedTextures[i].GetTheTextureInARow().copyTo(tmpRow); + } + this->m_PCANormalizedTexture(matNormalizedTextures, matNormalizedMeanTextures, CV_PCA_DATA_AS_ROW, this->m_iNbOfEigenTexturesAtMost ); + // to decide how many components to be selected + this->m_iNbOfTextureEigens = 0; + + double SumOfEigenValues = cv::sum( this->m_PCANormalizedTexture.eigenvalues ).val[0]; + double ps = 0.0f; + + for(unsigned int i = 0; i < this->m_iNbOfEigenTexturesAtMost; i++) + { + ps += this->m_PCANormalizedTexture.eigenvalues.at(i,0 ); + ++this->m_iNbOfTextureEigens; + if( ps/SumOfEigenValues >= this->m_fTruncatedPercent_Texture) break; + } + // m_iNbOfTextureEigens decided. For simplicity, we carry out PCA once again. + this->m_PCANormalizedTexture(matNormalizedTextures, matNormalizedMeanTextures, CV_PCA_DATA_AS_ROW, this->m_iNbOfTextureEigens ); + + ////////////////////////////////////////////////////////////////////////// + // Calculate m_vNormalizedPointWarpInfo + ////////////////////////////////////////////////////////////////////////// + Point2f src[3]; + Point2f dst[3]; + + // warp from reference triangles to normalized triangles, it's basically just a translation and scaling + vector< Mat_ > matWarping; + matWarping.resize(this->m_iNbOfTriangles); + + // calculate all the possible mapping (for speeding up) 95 mapping totally + // Here, actually, a translation and a scaling can do this as well! + for (unsigned int k = 0; k < this->m_iNbOfTriangles; k++ ) + { + // Get the affine transformation for each triangle pair. + src[0] = this->m_vTemplateTriangle2D[k].GetA2DPoint(0); + src[1] = this->m_vTemplateTriangle2D[k].GetA2DPoint(1); + src[2] = this->m_vTemplateTriangle2D[k].GetA2DPoint(2); + + dst[0] = this->m_VOAlignedMeanShape.GetA2DPoint( this->m_vTemplateTriangle2D[k].GetVertexIndex(0) ); + dst[1] = this->m_VOAlignedMeanShape.GetA2DPoint( this->m_vTemplateTriangle2D[k].GetVertexIndex(1) ); + dst[2] = this->m_VOAlignedMeanShape.GetA2DPoint( this->m_vTemplateTriangle2D[k].GetVertexIndex(2) ); + + matWarping[k] = cv::getAffineTransform( src, dst ); + } + + unsigned int triangleIndex; + Point2f pt, pt0; + Mat_ warpsrc = Mat_::ones(3, 1); + Mat_ warpeddst = Mat_::zeros(2, 1); + for (unsigned int i = 0; i < this->m_iNbOfPixels; i++) + { + // JIA Pei. 2006-11-25. You will see the following (int) is very important + // without (int), the result here is not correct at all!! + pt = this->m_vTemplatePointWarpInfo[i].GetPosition(); + triangleIndex = this->m_vTemplatePointWarpInfo[i].GetTriangleIndex(); + + warpsrc(0, 0) = pt.x; + warpsrc(1, 0) = pt.y; + warpsrc(2, 0) = 1.0; + + warpeddst = matWarping[triangleIndex] * warpsrc; + pt0.x = warpeddst.at(0, 0); + pt0.y = warpeddst.at(1, 0); + + VO_WarpingPoint tempNormalizedPixelTriangle; + + tempNormalizedPixelTriangle.SetPosition(pt0); + tempNormalizedPixelTriangle.SetTriangleIndex(triangleIndex); + tempNormalizedPixelTriangle.SetPointIndex(i); + tempNormalizedPixelTriangle.SetTriangle2DStructure(this->m_vNormalizedTriangle2D[triangleIndex] ); + + // Very important!! Note by JIA Pei, push_back cannot perform on vector< vector < > > + this->m_vNormalizedPointWarpInfo.push_back (tempNormalizedPixelTriangle); + } + ////////////////////////////////////////////////////////////////////////// + + // Calculate template images + VO_TextureModel::VO_PutOneTextureToTemplateShape(this->m_VOReferenceTexture, this->m_vTemplateTriangle2D, this->m_ImageTemplateFace); + VO_TextureModel::VO_PutEdgesOnTemplateFace(this->m_vEdge, this->m_VOReferenceShape, this->m_ImageTemplateFace, this->m_ImageEdges); + + ////////////////////////////////////////////////////////////////////////// + VO_Shape tmpRefShape = this->m_VOAlignedMeanShape*this->m_fAverageShapeSize; + vector refEllipses = this->m_VOPDM.GetPDMEllipses(); + refEllipses.resize(this->m_iNbOfPoints); + for(unsigned int i = 0; i < this->m_iNbOfPoints; i++) + { + refEllipses[i].ScaleCenter(this->m_fAverageShapeSize); + refEllipses[i] *= this->m_fAverageShapeSize; + //refEllipses[i] *= this->m_fAverageShapeSize*3.0; +// Rect rect = refEllipses[i].CalcBoundingRect(); +// Mat tmpImg= Mat::zeros( rect.height, rect.width, this->m_ImageTemplateFace.type()); +// Mat_ ellipseMin = Mat_::zeros(2, 1); +// ellipseMin(0,0) = rect.x; +// ellipseMin(1,0) = rect.y; +// refEllipses[i].Translate(-ellipseMin); +// +// Point2f pt = refEllipses[i].GetCOG(); +// cv::ellipse(tmpImg, Point( (int)pt.x, (int)pt.y ), +// Size(refEllipses[i].GetAxisXHalfLen(), refEllipses[i].GetAxisYHalfLen()), +// refEllipses[i].GetAngle(), +// refEllipses[i].GetStartAngle(), +// refEllipses[i].GetEndAngle(), +// colors[3], +// 1, +// 0, +// 0); +// imwrite("tmp.jpg", tmpImg); + } + Rect brect = VO_Ellipse::VO_CalcBoundingRect4MultipleEllipses(refEllipses); + Mat_ ellipseMin = Mat_::zeros(2, 1); + ellipseMin(0,0) = static_cast(brect.x); + ellipseMin(1,0) = static_cast(brect.y); + this->m_ImageEllipses = Mat::zeros(brect.height, brect.width, this->m_ImageTemplateFace.type()); + tmpRefShape.Translate( -ellipseMin ); + for(unsigned int i = 0; i < this->m_iNbOfPoints; i++) + refEllipses[i].Translate(-ellipseMin); + VO_TextureModel::VO_PutShapeOnTemplateFace(tmpRefShape, this->m_ImageEllipses, this->m_ImageEllipses); + // imwrite("edges.jpg", this->m_ImageEllipses); + VO_TextureModel::VO_PutPDMEllipsesOnTemplateFace(refEllipses, this->m_ImageEllipses, this->m_ImageEllipses); + + ////////////////////////////////////////////////////////////////////////// +} + + +/** + * @author JIA Pei + * @version 2010-02-13 + * @brief Save AAM to a specified folder + * @param fd Input - the folder that AAM to be saved to + * @return void +*/ +void VO_TextureModel ::VO_Save(const string& fd) +{ + VO_ShapeModel::VO_Save(fd); + + string fn = fd+"/TextureModel"; + MakeDirectory(fn); + + fstream fp; + string tempfn; + + // TextureModel + tempfn = fn + "/TextureModel" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + + fp << "m_iTextureRepresentationMethod" << endl << this->m_iTextureRepresentationMethod << endl; // m_iTextureRepresentationMethod + fp << "m_iNbOfTextureRepresentations" << endl << this->m_iNbOfTextureRepresentations << endl; // m_iNbOfTextureRepresentations + fp << "m_iNbOfChannels" << endl << this->m_iNbOfChannels << endl; // m_iNbOfChannels + fp << "m_iNbOfPixels" << endl << this->m_iNbOfPixels << endl; // m_iNbOfPixels + fp << "m_iNbOfTextures" << endl << this->m_iNbOfTextures << endl; // m_iNbOfTextures + fp << "m_iNbOfEigenTexturesAtMost" << endl << this->m_iNbOfEigenTexturesAtMost << endl; // m_iNbOfEigenTexturesAtMost + fp << "m_iNbOfTextureEigens" << endl << this->m_iNbOfTextureEigens << endl; // m_iNbOfTextureEigens + fp << "m_fAverageTextureStandardDeviation" << endl << this->m_fAverageTextureStandardDeviation << endl; // m_fAverageTextureStandardDeviation + fp << "m_fTruncatedPercent_Texture" << endl << this->m_fTruncatedPercent_Texture << endl; // m_fTruncatedPercent_Texture + fp.close();fp.clear(); + + // m_PCANormalizedTextureMean + tempfn = fn + "/m_PCANormalizedTextureMean" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_PCANormalizedTextureMean" << endl; + fp << Mat_(this->m_PCANormalizedTexture.mean); + fp.close();fp.clear(); + + // m_PCANormalizedTextureEigenValues + tempfn = fn + "/m_PCANormalizedTextureEigenValues" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_PCANormalizedTextureEigenValues" << endl; + fp << Mat_(this->m_PCANormalizedTexture.eigenvalues); + fp.close();fp.clear(); + + // m_PCANormalizedTextureEigenVectors + tempfn = fn + "/m_PCANormalizedTextureEigenVectors" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_PCANormalizedTextureEigenVectors" << endl; + fp << Mat_(this->m_PCANormalizedTexture.eigenvectors); + fp.close();fp.clear(); + + // m_VONormalizedMeanTexture + tempfn = fn + "/m_VONormalizedMeanTexture" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_VONormalizedMeanTexture" << endl; + fp << this->m_VONormalizedMeanTexture; + fp.close();fp.clear(); + + // m_VOReferenceTexture + tempfn = fn + "/m_VOReferenceTexture" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_VOReferenceTexture" << endl; + fp << this->m_VOReferenceTexture; + fp.close();fp.clear(); + + // m_vTextures + tempfn = fn + "/m_vTextures" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_vTextures" << endl; + fp << this->m_vTextures; + fp.close();fp.clear(); + + // m_vNormalizedTextures + tempfn = fn + "/m_vNormalizedTextures" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_vNormalizedTextures" << endl; + fp << this->m_vNormalizedTextures; + fp.close();fp.clear(); + + // m_vTemplatePointWarpInfo + tempfn = fn + "/m_vTemplatePointWarpInfo" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_vTemplatePointWarpInfo" << endl; + fp << this->m_vTemplatePointWarpInfo; + fp.close();fp.clear(); + + // m_vNormalizedPointWarpInfo + tempfn = fn + "/m_vNormalizedPointWarpInfo" + ".txt"; + fp.open(tempfn.c_str (), ios::out); + fp << "m_vNormalizedPointWarpInfo" << endl; + fp << this->m_vNormalizedPointWarpInfo; + fp.close();fp.clear(); + + /** Template face image */ + tempfn = fn + "/Reference.jpg"; + imwrite(tempfn.c_str(), this->m_ImageTemplateFace); + + /** Image of edges */ + tempfn = fn + "/edges.jpg"; + imwrite(tempfn.c_str(), this->m_ImageEdges); + + /** Image of ellipse */ + tempfn = fn + "/ellipses.jpg"; + imwrite(tempfn.c_str(), this->m_ImageEllipses); + + /** How many triangles totally */ + vector imageTriangles; + VO_TextureModel::VO_PutTrianglesOnTemplateFace(this->m_vTemplateTriangle2D, this->m_ImageTemplateFace, imageTriangles); + string trianglestr; + stringstream ssi; + string stri; + for (unsigned int i = 0; i < imageTriangles.size(); i++) + { + ssi << i; + ssi >> stri; + if(stri.length() == 2) + trianglestr = fn + "/triangle0" + stri + ".jpg"; + else if(stri.length() == 1) + trianglestr = fn + "/triangle00" + stri + ".jpg"; + else + trianglestr = fn + "/triangle" + stri + ".jpg"; + + imwrite(trianglestr.c_str(), imageTriangles[i] ); + + ssi.clear(); + } +} + + +/** + * @author JIA Pei + * @version 2010-02-13 + * @brief Load all Texture Modeldata from a specified folder + * @param fd Input - the folder that Texture Model to be loaded from + * @return void +*/ +void VO_TextureModel ::VO_Load(const string& fd) +{ + VO_ShapeModel::VO_Load(fd); + + this->VO_LoadParameters4Fitting(fd); + + string fn = fd+"/TextureModel"; + if (!MakeDirectory(fn) ) + { + cout << "TextureModel subfolder is not existing. " << endl; + exit(EXIT_FAILURE); + } + + ifstream fp; + string tempfn; + string temp; + + // m_vTextures + tempfn = fn + "/m_vTextures" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + this->m_vTextures.resize(this->m_iNbOfSamples); + fp >> this->m_vTextures; + fp.close();fp.clear(); + + // m_vNormalizedTextures + tempfn = fn + "/m_vNormalizedTextures" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + this->m_vNormalizedTextures.resize(this->m_iNbOfSamples); + fp >> this->m_vNormalizedTextures; + fp.close();fp.clear(); + + /** Image of edges */ + tempfn = fn + "/edges.jpg"; + this->m_ImageEdges = imread(tempfn.c_str(), CV_LOAD_IMAGE_ANYCOLOR ); + +} + + +/** + * @author JIA Pei + * @version 2010-02-13 + * @brief Load all AAM data from a specified folder for later fitting + * @param fd Input - the folder that AAM to be loaded from + * @return void +*/ +void VO_TextureModel::VO_LoadParameters4Fitting(const string& fd) +{ + VO_ShapeModel::VO_LoadParameters4Fitting(fd); + + string fn = fd+"/TextureModel"; + if (!MakeDirectory(fn) ) + { + cout << "TextureModel subfolder is not existing. " << endl; + exit(EXIT_FAILURE); + } + + ifstream fp; + string tempfn; + string temp; + + // TextureModel + tempfn = fn + "/TextureModel" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp >> this->m_iTextureRepresentationMethod; + fp >> temp >> this->m_iNbOfTextureRepresentations; + fp >> temp >> this->m_iNbOfChannels; + fp >> temp >> this->m_iNbOfPixels; + fp >> temp >> this->m_iNbOfTextures; + fp >> temp >> this->m_iNbOfEigenTexturesAtMost; + fp >> temp >> this->m_iNbOfTextureEigens; + fp >> temp >> this->m_fAverageTextureStandardDeviation; + fp >> temp >> this->m_fTruncatedPercent_Texture; + fp.close();fp.clear(); + + this->m_PCANormalizedTexture = cv::PCA(); + + // m_PCANormalizedTextureMean + this->m_PCANormalizedTexture.mean = Mat_::zeros(1, this->m_iNbOfTextures); + tempfn = fn + "/m_PCANormalizedTextureMean" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_PCANormalizedTexture.mean; + fp.close();fp.clear(); + + // m_PCANormalizedTextureEigenValues + this->m_PCANormalizedTexture.eigenvalues = Mat_::zeros(this->m_iNbOfTextureEigens, 1); + tempfn = fn + "/m_PCANormalizedTextureEigenValues" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_PCANormalizedTexture.eigenvalues; + fp.close();fp.clear(); + + // m_PCANormalizedTextureEigenVectors + this->m_PCANormalizedTexture.eigenvectors = Mat_::zeros(this->m_iNbOfTextureEigens, this->m_iNbOfTextures); + tempfn = fn + "/m_PCANormalizedTextureEigenVectors" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_PCANormalizedTexture.eigenvectors; + fp.close();fp.clear(); + + // m_VONormalizedMeanTexture + this->m_VONormalizedMeanTexture.m_MatTexture = Mat_::zeros(this->m_iNbOfTextureRepresentations, this->m_iNbOfPixels); + tempfn = fn + "/m_VONormalizedMeanTexture" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_VONormalizedMeanTexture; + fp.close();fp.clear(); + + // m_VOReferenceTexture + this->m_VOReferenceTexture.m_MatTexture = Mat_::zeros(this->m_iNbOfTextureRepresentations, this->m_iNbOfPixels); + tempfn = fn + "/m_VOReferenceTexture" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + fp >> this->m_VOReferenceTexture; + fp.close();fp.clear(); + + // m_vTemplatePointWarpInfo + tempfn = fn + "/m_vTemplatePointWarpInfo" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + this->m_vTemplatePointWarpInfo.resize(this->m_iNbOfPixels); + fp >> this->m_vTemplatePointWarpInfo; + fp.close();fp.clear(); + for (unsigned int i = 0; i < this->m_iNbOfPixels; i++) + { + this->m_vTemplatePointWarpInfo[i].SetTriangle2DStructure( this->m_vTemplateTriangle2D[this->m_vTemplatePointWarpInfo[i].GetTriangleIndex ()] ); + } + + // m_vNormalizedPointWarpInfo + tempfn = fn + "/m_vNormalizedPointWarpInfo" + ".txt"; + SafeInputFileOpen(fp, tempfn); + fp >> temp; + this->m_vNormalizedPointWarpInfo.resize(this->m_iNbOfPixels); + fp >> this->m_vNormalizedPointWarpInfo; + fp.close();fp.clear(); + for (unsigned int i = 0; i < this->m_iNbOfPixels; i++) + { + this->m_vNormalizedPointWarpInfo[i].SetTriangle2DStructure( this->m_vNormalizedTriangle2D[this->m_vNormalizedPointWarpInfo[i].GetTriangleIndex ()] ); + } + + /** Template face image */ + tempfn = fn + "/Reference.jpg"; + this->m_ImageTemplateFace = imread(tempfn.c_str(), CV_LOAD_IMAGE_ANYCOLOR ); +} + diff --git a/modules/smbuilding/src/VO_Triangle2DStructure.cpp b/modules/smbuilding/src/VO_Triangle2DStructure.cpp new file mode 100644 index 0000000..e61e1fe --- /dev/null +++ b/modules/smbuilding/src/VO_Triangle2DStructure.cpp @@ -0,0 +1,276 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include +#include "VO_Triangle2DStructure.h" + + +ostream& operator<<(ostream& os, const VO_Triangle2DStructure& aamtriangle2d) +{ + os << aamtriangle2d.GetVertexIndex(0) << " " + << aamtriangle2d.m_MatShape(0,0) << " " + << aamtriangle2d.m_MatShape(1,0)<< " " + << aamtriangle2d.GetVertexIndex(1) << " " + << aamtriangle2d.m_MatShape(0,1) << " " + << aamtriangle2d.m_MatShape(1,1) << " " + << aamtriangle2d.GetVertexIndex(2) << " " + << aamtriangle2d.m_MatShape(0,2) << " " + << aamtriangle2d.m_MatShape(1,2) << " " + << aamtriangle2d.GetdD(); + + return os; +} + +istream& operator>>(istream &is, VO_Triangle2DStructure& aamtriangle2d) +{ + vector vIndexes; + Mat_ vVertexes = Mat_::zeros(2,3); + Point2f tmpPoint; + int tempint; + float tempdD; + + for(unsigned int j = 0; j < 3; j++) + { + is >> tempint; + vIndexes.push_back(tempint); + is >> tmpPoint.x >> tmpPoint.y; + vVertexes(0,j) = tmpPoint.x; + vVertexes(1,j) = tmpPoint.y; + } + is >> tempdD; + aamtriangle2d.SetTheShape(vVertexes); + aamtriangle2d.SetVertexIndexes(vIndexes); + aamtriangle2d.SetdD( tempdD ); + + return is; +} + + +/** + * @brief static function, a list of triangles to a shape + * @param triangles - Input a list of triangles + * @param VO_Shape - Output shape in Mat_ + * @return void + */ +VO_Shape VO_Triangle2DStructure::Triangle2D2Shape(const vector & triangles) +{ + vector vi4oneTriangle; + vector countedIdx; + vector ptList; + + for(unsigned int i = 0; i < triangles.size(); ++i) + { + vi4oneTriangle = triangles[i].GetVertexIndexes(); + + for(unsigned int j = 0; j < 3; ++j) + { + if( !IsContaining(countedIdx, vi4oneTriangle[j]) ) + { + countedIdx.push_back(vi4oneTriangle[j]); + ptList.push_back( triangles[i].GetA2DPoint(j) ); + } + } + } + + Mat_ mat = Mat_::zeros(2, ptList.size()); + + for(unsigned int i = 0; i < ptList.size(); ++i) + { + mat(0, i) = ptList[i].x; + mat(1, i) = ptList[i].y; + } + return mat; +} + + + +/** + * @brief Adjust vertex sequence + * @return void + */ +void VO_Triangle2DStructure::AdjustVertexSequence() +{ + Point2f FirstVertex, SecondVertex, ThirdVertex; + unsigned int FirstIndex, SecondIndex, ThirdIndex; + + if ( this->m_MatShape(1,0) <= this->m_MatShape(1,1)) + { + if ( this->m_MatShape(1,0) <= this->m_MatShape(1,2)) + { + FirstVertex = this->GetA2DPoint(0); + FirstIndex = this->m_vVertexIndexes[0]; + if ( this->determinant( ) == COUNTER_CLOCKWISE ) + { + SecondVertex = this->GetA2DPoint(1); + ThirdVertex = this->GetA2DPoint(2); + SecondIndex = this->m_vVertexIndexes[1]; + ThirdIndex = this->m_vVertexIndexes[2]; + } + else + { + SecondVertex = this->GetA2DPoint(2); + ThirdVertex = this->GetA2DPoint(1); + SecondIndex = this->m_vVertexIndexes[2]; + ThirdIndex = this->m_vVertexIndexes[1]; + } + } + else + { + FirstVertex = this->GetA2DPoint(2); + FirstIndex = this->m_vVertexIndexes[2]; + if ( this->determinant( ) == COUNTER_CLOCKWISE ) + { + SecondVertex = this->GetA2DPoint(0); + ThirdVertex = this->GetA2DPoint(1); + SecondIndex = this->m_vVertexIndexes[0]; + ThirdIndex = this->m_vVertexIndexes[1]; + } + else + { + SecondVertex = this->GetA2DPoint(1); + ThirdVertex = this->GetA2DPoint(0); + SecondIndex = this->m_vVertexIndexes[1]; + ThirdIndex = this->m_vVertexIndexes[0]; + } + } + } + else + { + if ( this->m_MatShape(1,1) <= this->m_MatShape(1,2)) + { + FirstVertex = this->GetA2DPoint(1); + FirstIndex = this->m_vVertexIndexes[1]; + if ( this->determinant( ) == COUNTER_CLOCKWISE ) + { + SecondVertex = this->GetA2DPoint(2); + ThirdVertex = this->GetA2DPoint(0); + SecondIndex = this->m_vVertexIndexes[2]; + ThirdIndex = this->m_vVertexIndexes[0]; + } + else + { + SecondVertex = this->GetA2DPoint(0); + ThirdVertex = this->GetA2DPoint(2); + SecondIndex = this->m_vVertexIndexes[0]; + ThirdIndex = this->m_vVertexIndexes[2]; + } + } + else + { + FirstVertex = this->GetA2DPoint(2); + FirstIndex = this->m_vVertexIndexes[2]; + if ( this->determinant( ) == COUNTER_CLOCKWISE ) + { + SecondVertex = this->GetA2DPoint(0); + ThirdVertex = this->GetA2DPoint(1); + SecondIndex = this->m_vVertexIndexes[0]; + ThirdIndex = this->m_vVertexIndexes[1]; + } + else + { + SecondVertex = this->GetA2DPoint(1); + ThirdVertex = this->GetA2DPoint(0); + SecondIndex = this->m_vVertexIndexes[1]; + ThirdIndex = this->m_vVertexIndexes[0]; + } + } + } + this->m_MatShape(0,0) = FirstVertex.x; + this->m_MatShape(1,0) = FirstVertex.y; + this->m_MatShape(0,1) = SecondVertex.x; + this->m_MatShape(1,1) = SecondVertex.y; + this->m_MatShape(0,2) = ThirdVertex.x; + this->m_MatShape(1,2) = ThirdVertex.y; + this->m_vVertexIndexes[0] = FirstIndex; + this->m_vVertexIndexes[1] = SecondIndex; + this->m_vVertexIndexes[2] = ThirdIndex; + this->Calc_dD(); +} + + +/** + * @author JIA Pei + * @version 2010-02-17 + * @brief Judge whether one point is within a convex/concave hull of a shape + * @param pt input the concerned point + * @param triangles input all triangles composing the shape + * @param int return the index of triangle containing pt +*/ +int VO_Triangle2DStructure::IsPointInTriangles(const Point2f& pt, const vector& triangles) +{ + unsigned int NbOfTriangles = triangles.size(); + Mat contour; + + for (unsigned int k = 0; k < NbOfTriangles; k++) + { + contour = triangles[k].ToPointList(); + + // Here, we can actually give out all the local variables except the BGR, + // the BGR is not shape based and will be given by function VO_LoadTexturesFromShapes() + if (cv::pointPolygonTest( contour, pt, false ) > 0.0) + { + return k; + } + } + + return -1; +} + diff --git a/modules/smbuilding/src/VO_WarpingPoint.cpp b/modules/smbuilding/src/VO_WarpingPoint.cpp new file mode 100644 index 0000000..9c7df87 --- /dev/null +++ b/modules/smbuilding/src/VO_WarpingPoint.cpp @@ -0,0 +1,271 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2008-04-03 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_WarpingPoint.h" + + + + +ostream& operator<<(ostream& os, const VO_WarpingPoint& warpingpoint) +{ + os << warpingpoint.m_iPointIndex << " " + << warpingpoint.m_CVPosition.x << " " + << warpingpoint.m_CVPosition.y << " " + << warpingpoint.m_iTriangleIndex; + + return os; +} + +istream& operator>>(istream& is, VO_WarpingPoint& warpingpoint) +{ + int tempint; + CvPoint2D32f tempPoint; + + is >> tempint; + warpingpoint.SetPointIndex(tempint); + is >> tempPoint.x >> tempPoint.y; + warpingpoint.SetPosition(tempPoint); + is >> tempint; + warpingpoint.SetTriangleIndex(tempint); + + return is; +} + +/** + * @author JIA Pei + * @version 2010-02-08 + * @brief Calculate the first item of Jacobian + * @return void + * @note http://visionopen.com/cv/aam.php +*/ +void VO_WarpingPoint::CalcJacobianOne() +{ + float x, y, x1, x2, x3, y1, y2, y3; + this->m_Jacobian_One.resize(3); + + x = this->m_CVPosition.x; + y = this->m_CVPosition.y; + + x1 = this->m_VOTriangle2DStructure.GetA2DPoint(0).x; + x2 = this->m_VOTriangle2DStructure.GetA2DPoint(1).x; + x3 = this->m_VOTriangle2DStructure.GetA2DPoint(2).x; + y1 = this->m_VOTriangle2DStructure.GetA2DPoint(0).y; + y2 = this->m_VOTriangle2DStructure.GetA2DPoint(1).y; + y3 = this->m_VOTriangle2DStructure.GetA2DPoint(2).y; + + this->m_Jacobian_One[0] = y*x3-y3*x+x*y2-x2*y+x2*y3-x3*y2; + this->m_Jacobian_One[1] = -y*x3+x1*y+x3*y1+y3*x-x1*y3-x*y1; + this->m_Jacobian_One[2] = -x*y2+x*y1+x1*y2+x2*y-x2*y1-x1*y; + + // make sure m_Jacobian_One is bigger than 0 + for (unsigned int i = 0; i < this->m_Jacobian_One.size(); i++) + { + this->m_Jacobian_One[i] /= this->m_VOTriangle2DStructure.GetdD(); + if (this->m_Jacobian_One[i] < FLT_EPSILON) + this->m_Jacobian_One[i] = 0.0; + } +} + + +/** + * @author JIA Pei + * @version 2010-02-08 + * @brief Calculate the Jacobian matrix + * @param TruncatedAlignedShapesEigenVectors Input the input parameters + * @return void +*/ +void VO_WarpingPoint::CalcJacobianMatrix4ShapeModel(const Mat_& TruncatedAlignedShapesEigenVectors) +{ + unsigned int NbOfShapesParameters = TruncatedAlignedShapesEigenVectors.rows; + unsigned int NbOfShapePoints = TruncatedAlignedShapesEigenVectors.cols/2; + // for a certain temppixeltriangle.m_CVPosition, the Jacobian[i] are different + + this->m_JacobianMatrix4ShapeModel.resize (2); + this->m_JacobianMatrix4ShapeModel[0].resize(NbOfShapesParameters); + this->m_JacobianMatrix4ShapeModel[1].resize(NbOfShapesParameters); + + for (unsigned int np = 0; np < NbOfShapesParameters; np++) + { + this->m_JacobianMatrix4ShapeModel[0][np] = + this->m_Jacobian_One[0] * TruncatedAlignedShapesEigenVectors.at( np, this->m_VOTriangle2DStructure.GetVertexIndex(0) ) + + this->m_Jacobian_One[1] * TruncatedAlignedShapesEigenVectors.at( np, this->m_VOTriangle2DStructure.GetVertexIndex(1) ) + + this->m_Jacobian_One[2] * TruncatedAlignedShapesEigenVectors.at( np, this->m_VOTriangle2DStructure.GetVertexIndex(2) ); + + this->m_JacobianMatrix4ShapeModel[1][np] = + this->m_Jacobian_One[0] * TruncatedAlignedShapesEigenVectors.at( np, this->m_VOTriangle2DStructure.GetVertexIndex(0)+NbOfShapePoints ) + + this->m_Jacobian_One[1] * TruncatedAlignedShapesEigenVectors.at( np, this->m_VOTriangle2DStructure.GetVertexIndex(1)+NbOfShapePoints ) + + this->m_Jacobian_One[2] * TruncatedAlignedShapesEigenVectors.at( np, this->m_VOTriangle2DStructure.GetVertexIndex(2)+NbOfShapePoints ); + } +} + + +/** + * @author JIA Pei + * @version 2010-02-08 + * @brief Calculate the Jacobian matrix for global shape normalization + * @param SimilarityTransformMatrix + * @return void +*/ +void VO_WarpingPoint::CalcJacobianMatrix4GlobalShapeNorm(const Mat_& SimilarityTransformMatrix) +{ + unsigned int NbOfSimilarityTransformParameters = SimilarityTransformMatrix.rows; + unsigned int NbOfShapePoints = SimilarityTransformMatrix.cols/2; + // for a certain temppixeltriangle.m_CVPosition, the Jacobian[i] are different + + this->m_JacobianMatrix4GlobalShapeNorm.resize (2); + this->m_JacobianMatrix4GlobalShapeNorm[0].resize(NbOfSimilarityTransformParameters); + this->m_JacobianMatrix4GlobalShapeNorm[1].resize(NbOfSimilarityTransformParameters); + + for (unsigned int np = 0; np < NbOfSimilarityTransformParameters; np++) + { + this->m_JacobianMatrix4GlobalShapeNorm[0][np] = + this->m_Jacobian_One[0] * SimilarityTransformMatrix.at(np, this->m_VOTriangle2DStructure.GetVertexIndex(0) ) + + this->m_Jacobian_One[1] * SimilarityTransformMatrix.at(np, this->m_VOTriangle2DStructure.GetVertexIndex(1) ) + + this->m_Jacobian_One[2] * SimilarityTransformMatrix.at(np, this->m_VOTriangle2DStructure.GetVertexIndex(2) ); + + this->m_JacobianMatrix4GlobalShapeNorm[1][np] = + this->m_Jacobian_One[0] * SimilarityTransformMatrix.at(np, this->m_VOTriangle2DStructure.GetVertexIndex(0)+NbOfShapePoints ) + + this->m_Jacobian_One[1] * SimilarityTransformMatrix.at(np, this->m_VOTriangle2DStructure.GetVertexIndex(1)+NbOfShapePoints ) + + this->m_Jacobian_One[2] * SimilarityTransformMatrix.at(np, this->m_VOTriangle2DStructure.GetVertexIndex(2)+NbOfShapePoints ); + } +} + + +/** + * @author JIA Pei + * @version 2010-02-08 + * @brief Calculate the Steepest Descent Image (SDI) + * @return void +*/ +void VO_WarpingPoint::CalcSteepestDescentImages4ShapeModel(unsigned int inChannels) +{ + this->m_SteepestDescentImages4ShapeModel.resize (inChannels); + + for (unsigned int i = 0; i < this->m_SteepestDescentImages4ShapeModel.size (); i++) + { + this->m_SteepestDescentImages4ShapeModel[i].resize (this->m_JacobianMatrix4ShapeModel[0].size ()); + } + + for (unsigned int i = 0; i < this->m_SteepestDescentImages4ShapeModel[0].size (); i++) + { + for (unsigned int k = 0; k < inChannels; k++) + { + this->m_SteepestDescentImages4ShapeModel[k][i] = + this->m_Gradients[k][0] * this->m_JacobianMatrix4ShapeModel[0][i] + + this->m_Gradients[k][1] * this->m_JacobianMatrix4ShapeModel[1][i]; + } + } +} + + +/** + * @author JIA Pei + * @version 2010-02-08 + * @brief Calculate the Steepest Descent Image (SDI) for global shape normalization + * @return void +*/ +void VO_WarpingPoint::CalcSteepestDescentImages4GlobalShapeNorm(unsigned int inChannels) +{ + this->m_SteepestDescentImages4GlobalShapeNorm.resize (inChannels); + + for (unsigned int i = 0; i < this->m_SteepestDescentImages4GlobalShapeNorm.size (); i++) + { + this->m_SteepestDescentImages4GlobalShapeNorm[i].resize (this->m_JacobianMatrix4GlobalShapeNorm[0].size ()); + } + + for (unsigned int i = 0; i < this->m_SteepestDescentImages4GlobalShapeNorm[0].size (); i++) + { + for (unsigned int k = 0; k < inChannels; k++) + { + this->m_SteepestDescentImages4GlobalShapeNorm[k][i] = + this->m_Gradients[k][0] * this->m_JacobianMatrix4GlobalShapeNorm[0][i] + + this->m_Gradients[k][1] * this->m_JacobianMatrix4GlobalShapeNorm[1][i]; + } + } +} + + +/** + * @author JIA Pei + * @version 2010-05-22 + * @brief Static function to warp one single point + * @return void +*/ +void VO_WarpingPoint::WarpOnePoint( const Point2f& iPt, + const VO_Triangle2DStructure& its, + Point2f& oPt, + const Point2f& ov1, + const Point2f& ov2, + const Point2f& ov3) +{ + Point2f iv1 = its.GetA2DPoint(0); + Point2f iv2 = its.GetA2DPoint(1); + Point2f iv3 = its.GetA2DPoint(2); + double c = 1.0/its.GetdD(); + + float alpha = safeDoubleToFloat((iPt.y*iv3.x-iv3.y*iPt.x+iPt.x*iv2.y-iv2.x*iPt.y+iv2.x*iv3.y-iv3.x*iv2.y)*c); + float belta = safeDoubleToFloat((-iPt.y*iv3.x+iv1.x*iPt.y+iv3.x*iv1.y+iv3.y*iPt.x-iv1.x*iv3.y-iPt.x*iv1.y)*c); + float gamma = 1.0f - alpha - belta; + + oPt.x = alpha*ov1.x + belta*ov2.x + gamma*ov3.x; + oPt.y = alpha*ov1.y + belta*ov2.y + gamma*ov3.y; +} + diff --git a/modules/smfitting/CMakeLists.txt b/modules/smfitting/CMakeLists.txt new file mode 100644 index 0000000..4e7e073 --- /dev/null +++ b/modules/smfitting/CMakeLists.txt @@ -0,0 +1,6 @@ +include_directories("${CMAKE_CURRENT_SOURCE_DIR}/../common/include" "${CMAKE_CURRENT_SOURCE_DIR}/../cvcommon/include" +"${CMAKE_CURRENT_SOURCE_DIR}/../comalgs/include" "${CMAKE_CURRENT_SOURCE_DIR}/../ensembletraining/include" +"${CMAKE_CURRENT_SOURCE_DIR}/../integraltransform/include" "${CMAKE_CURRENT_SOURCE_DIR}/../featureextraction/include" +"${CMAKE_CURRENT_SOURCE_DIR}/../smbuilding/include") +set(deps opencv_core opencv_imgproc opencv_highgui) +define_vosm_module(smfitting vosm_common vosm_cvcommon vosm_featureextraction vosm_integraltransform vosm_smbuilding ${deps}) diff --git a/modules/smfitting/include/VO_Fitting2DSM.h b/modules/smfitting/include/VO_Fitting2DSM.h new file mode 100644 index 0000000..2989905 --- /dev/null +++ b/modules/smfitting/include/VO_Fitting2DSM.h @@ -0,0 +1,289 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_2DSMFitting__ +#define __VO_2DSMFitting__ + +#include "opencv/cv.h" +#include "opencv/highgui.h" + + +#include "VO_Shape.h" +#include "VO_Texture.h" +#include "VO_Shape2DInfo.h" +#include "VO_WarpingPoint.h" +#include "VO_AXM.h" + +using namespace std; +using namespace cv; + +/** + * @author JIA Pei + * @brief Generalized class for 2D statistical model fitting algorithms. + */ +class VO_Fitting2DSM +{ +public: + /** Maximum iteration times */ + static const int EPOCH = 10; + + /** pClose */ + static float pClose; + + //Inner helper class might be more appropriate somwhere else + class DrawMeshInfo{ + public: + float pyrScale; + float f2Scale; + VO_Shape drawPts; + VO_AXM* modelPtr; + DrawMeshInfo(float ps, float f2s, VO_Shape dps, VO_AXM* mdlptr): pyrScale(ps), f2Scale(f2s), drawPts(dps), modelPtr(mdlptr){}; + }; + +protected: + /** number of pyramid levels */ + unsigned int m_iNbOfPyramidLevels; + + /** which fitting method is now undertaken */ + VO_AXM::TYPE m_iFittingMethod; + + /** number of iterations */ + unsigned int m_iIteration; + + /** fitting time */ + float m_fFittingTime; + + // The following 3 parameters are just for first estimation from the aligned shape to the real size shape instance. + // If we take pose into our consideration, (C - only deal with non-rigid transform and pose - global shape normalizatioin) + // the following 3 parameters are of course will not change during the fitting iterations. + /** scaling in Procrustes Analysis */ + float m_fScale; + + /** rotation in Procrustes Analysis */ + vector m_vRotateAngles; +// float m_fRot; + + /** center of gravity for translation, in Procrustes Analysis */ + Mat_ m_MatCenterOfGravity; + + /** original input image */ + Mat m_ImageInput; + + /** image to be processed, the image under processing might not be the same channel as original input image */ + Mat m_ImageProcessing; + + /** output image, the final image of the fitting process*/ + Mat m_ImageOutput; + + /** Shape parameter */ + Mat_ m_MatModelAlignedShapeParam; + + /** Texture parameter */ + Mat_ m_MatModelNormalizedTextureParam; + + /** current fitting shape instance */ + VO_Shape m_VOTemplateAlignedShape; + VO_Shape m_VOModelAlignedShape; + VO_Shape m_VOFittingShape; + VO_Shape m_VOEstimatedShape; + + /** current fitting texture instance */ + VO_Texture m_VOTemplateNormalizedTexture; + VO_Texture m_VOModelNormalizedTexture; + VO_Texture m_VOFittingTexture; + VO_Texture m_VOTextureError; + VO_Texture m_VOEstimatedTextureError; + + /** current fitting normalized texture instance extracted from original image, but with shape parameters */ + Mat_ m_MatNormalizedTextureInstanceExtractedFromImage; + + /** triangle structure */ + vector m_vTriangle2D; + + /** shape information */ + vector m_vShape2DInfo; + + /** face parts information, seems to have the duplicate information as m_vShape2DInfo, but not! */ + VO_FaceParts m_FaceParts; + + /** point warp information of all pixels */ + vector m_vPointWarpInfo; + + /** Initialization */ + void init(); + +public: + enum {USEGLOBALSHAPENORMALIZATION = 1, USESIMILARITYTRANSFORM = 2}; + + //add more techniques here as necessary, keeping numtechs updated + enum MULTICHANNELTECH{ FULLHYBRID = 0, FIRSTCHANNELONLY = 1, SECONDCHANNELONLY = 2, THIRDCHANNELONLY = 3, HYBRIDPYRAMID = 4, HYBRIDSUBSET = 5, NUMTECHS = 6}; + + + /** constructor */ + VO_Fitting2DSM(); + + /** destructor */ + virtual ~VO_Fitting2DSM(); + + /** Calculate texture difference between model texture built from C and the texture built from image */ + float VO_CalcErrorImage( const Mat& iImg, + const VO_Shape& iShape, + const VO_Texture& iTexture, + VO_Texture& textureDiff); + + /** Fitting the object until convergence for the input image */ + float VO_StartFitting(const Mat& iImage, + vector& oMeshInfos, + const VO_AXM::TYPE fittingMethod, + const Point2f& ptLeftEyeCenter, + const Point2f& ptRightEyeCenter, + const Point2f& ptMouthCenter, + const unsigned int epoch, + const unsigned int pyramidlevel, + const bool record, + const vector& fittingTechs, + const bool doAffineWarp = true); + + static void VO_DrawMeshOnImage( const DrawMeshInfo dmi,const Mat& iImg, Mat& oImg); + + /** Draw mesh on the input image and save to the output image */ + static void VO_DrawMesh(const VO_Shape& iShape, + const VO_AXM* iAXMModel, + Mat& oImg); + + /** Draw a line on the fitted image and save to the output image */ + static void VO_DrawAline( const VO_Shape& iShape, + const VO_Shape& theSubshape, + const vector& iLine, + Mat& oImg, + unsigned int dir = HORIZONTAL, + bool ws = false, + unsigned int offset = 0, + unsigned int ci = 0); + + /** Draw a point on the fitted image and save to the output image */ + static void VO_DrawAPoint(const Point2f& pt, Mat& ioImg); + + /** Gets and Sets */ + Mat GetFittedImage() const {return this->m_ImageOutput; } + void SetInputImage(const Mat& iImg) + { + iImg.copyTo(this->m_ImageInput); + } + + void SetProcessingImage(const Mat& iImg, const VO_AXM* aammodel) + { + unsigned int NbOfChannels = aammodel->GetNbOfChannels(); + + if (iImg.channels() == NbOfChannels) + { + iImg.copyTo(this->m_ImageProcessing ); + } + else if(NbOfChannels == 2) + { + /* Keep as 3 channel image, or it segfaults later for ? reason.*/ + iImg.copyTo(this->m_ImageProcessing); + } + else if(iImg.channels() == GRAYCHANNELS && NbOfChannels == COLORCHANNELS ) + { + cv::cvtColor(iImg, this->m_ImageProcessing, CV_GRAY2BGR); + } + else if(iImg.channels() == COLORCHANNELS && NbOfChannels == GRAYCHANNELS ) + { + cv::cvtColor(iImg, this->m_ImageProcessing, CV_BGR2GRAY); + } + else + { + cerr << " Can't handle channel issue in Built Statistcal Model ! " << endl; + exit(EXIT_FAILURE); + } + } + + float GetFittingTime() const { return this->m_fFittingTime; } + unsigned int GetNbOfIterations() const { return this->m_iIteration; } + const VO_FaceParts& GetFaceParts() const { return this->m_FaceParts; } + VO_Shape VO_GetFittedShape() const { return this->m_VOFittingShape; } + VO_Texture VO_GetFittedTexture() + { + VO_TextureModel::VO_LoadOneTextureFromShape(this->m_VOFittingShape, + this->m_ImageInput, + this->m_vTriangle2D, + this->m_vPointWarpInfo, + this->m_VOFittingTexture, + VO_Features::DIRECT); + return this->m_VOFittingTexture; + } + + /** N Points mapping from aligned shape to real size shape */ + static Mat_ VO_FirstEstimationBySingleWarp(const VO_FaceParts& iFaceParts, + const VO_Shape& iShape, + const Point2f& ptLeftEyeCenter, + const Point2f& ptRightEyeCenter, + const Point2f& ptMouthCenter ); + + /** */ + static VO_Shape VO_FirstEstimationByScaling(const VO_Shape& iShape, + const Rect& rect); + +}; + +#endif // __VO_2DSMFitting__ + diff --git a/modules/smfitting/include/VO_FittingAAMBasic.h b/modules/smfitting/include/VO_FittingAAMBasic.h new file mode 100644 index 0000000..9659e02 --- /dev/null +++ b/modules/smfitting/include/VO_FittingAAMBasic.h @@ -0,0 +1,141 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_FittingAAMBasic__ +#define __VO_FittingAAMBasic__ + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_CVCommon.h" +#include "VO_AAMBasic.h" +#include "VO_Fitting2DSM.h" + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Basic AAM fitting algorithm. + */ +class VO_FittingAAMBasic : public VO_Fitting2DSM +{ +public: + /** dampen coefficients */ + static vector k_values; + +private: + /** For non-rigid transform */ + Mat_ m_MatDeltaC; + Mat_ m_MatEstimatedC; + Mat_ m_MatCurrentC; + + /** For rigid transform */ + Mat_ m_MatDeltaT; + Mat_ m_MatEstimatedT; + Mat_ m_MatCurrentT; + + /** errors */ + float m_E; + float m_E_previous; + + /** Initialization */ + void init(); + + /** calculate the real-size modeled shape, from the trained appearance model */ + void VO_CParamTParam2FittingShape( const Mat_& c, + const Mat_& t, + VO_Texture& modelNormalizedTexture, + VO_Shape& oShape, + float& scale, + vector& rotateAngles, + Mat_& matCOG, + unsigned int mtd = VO_Fitting2DSM::USESIMILARITYTRANSFORM); + +public: + VO_AAMBasic* m_VOAAMBasic; + + /** Constructor */ + VO_FittingAAMBasic(); + + /** Destructor */ + ~VO_FittingAAMBasic(); + + /** Load Basic AAM fitting training results */ + void VO_LoadParameters4Fitting(const string& fd); + + /** Start Basic AAM fitting, for static images, recording all iterations of every single image */ + float VO_BasicAAMFitting(const Mat& iImg, vector& oMeshInfo, unsigned int epoch = EPOCH, bool record = false); + + /** Start Basic AAM fitting, for dynamic image sequence */ + float VO_BasicAAMFitting(const Mat& iImg, VO_Shape& ioShape, Mat& oImg, unsigned int epoch = EPOCH); + + /** Start Direct AAM fitting, for static images, recording all iterations of every single image */ + float VO_DirectAAMFitting(const Mat& iImg, vector& oMeshInfo, unsigned int epoch = EPOCH, bool record = false); + + /** Start Direct AAM fitting, for dynamic image sequence */ + float VO_DirectAAMFitting(const Mat& iImg, VO_Shape& ioShape, Mat& oImg, unsigned int epoch = EPOCH); +}; + +#endif + diff --git a/modules/smfitting/include/VO_FittingAAMForwardIA.h b/modules/smfitting/include/VO_FittingAAMForwardIA.h new file mode 100644 index 0000000..cd31186 --- /dev/null +++ b/modules/smfitting/include/VO_FittingAAMForwardIA.h @@ -0,0 +1,108 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_FittingAAMForwardIA__ +#define __VO_FittingAAMForwardIA__ + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_CVCommon.h" +#include "VO_AAMForwardIA.h" +#include "VO_Fitting2DSM.h" + + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Forward image alignment (IA) AAM fitting algorithm. + * @note FIXME + */ +class VO_FittingAAMForwardIA : public VO_Fitting2DSM +{ +private: + void init(); + +public: + VO_AAMForwardIA* m_VOAAMForwardIA; + + /** Constructor */ + VO_FittingAAMForwardIA(); + + /** Destructor */ + ~VO_FittingAAMForwardIA(); + + /** Load FCIA AAM fitting training results */ + void VO_LoadParameters4Fitting(const string& fd); + + /** Start Forward Additive Image Alignment fitting, for static images, recording all iterations of every single image */ + float VO_FAIAAAMFitting(const Mat& iImg, vector& oMeshInfo, unsigned int epoch = EPOCH, bool record = false); + + /** Start Forward Additive Image Alignment fitting, for dynamic image sequence */ + float VO_FAIAAAMFitting(const Mat& iImg, VO_Shape& ioShape, Mat& oImg, unsigned int epoch = EPOCH); +}; + +#endif + diff --git a/modules/smfitting/include/VO_FittingAAMInverseIA.h b/modules/smfitting/include/VO_FittingAAMInverseIA.h new file mode 100644 index 0000000..20b351f --- /dev/null +++ b/modules/smfitting/include/VO_FittingAAMInverseIA.h @@ -0,0 +1,152 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_FittingAAMInverseIA__ +#define __VO_FittingAAMInverseIA__ + +#include + +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_CVCommon.h" +#include "VO_AAMInverseIA.h" +#include "VO_Fitting2DSM.h" + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief Inverse image alignment (IA) AAM fitting algorithms, + * including IAIA and CMU ICIA. + */ +class VO_FittingAAMInverseIA : public VO_Fitting2DSM +{ +private: + /** For non-rigid transform */ + Mat_ m_MatCurrentP; + Mat_ m_MatEstimatedP; + Mat_ m_MatDeltaP; // 1*20 + + /** For rigid transform */ + Mat_ m_MatCurrentQ; + Mat_ m_MatEstimatedQ; + Mat_ m_MatDeltaQ; // 1*4 + + /** For both rigid and non-rigid transform */ + Mat_ m_MatDeltaPQ; + + /** errors */ + float m_E; + float m_E_previous; + + /** Initialization */ + void init(); + + void VO_PParamQParam2ModelAlignedShape( const Mat_& p, + const Mat_& q, + VO_Shape& oAlignedShape); + + /** calculate the real-size modeled shape, from the trained appearance model */ + void VO_PParamQParam2FittingShape( const Mat_& p, + const Mat_& q, + VO_Shape& oShape, + float& scale, + vector& rotateAngles, + Mat_& matCOG, + unsigned int mtd = VO_Fitting2DSM::USEGLOBALSHAPENORMALIZATION); + + /** The process of CMU Inverse Compositional. Developed by Yao Wei! */ + void VO_CMUInverseCompositional( const Mat_& matDeltaP, + const Mat_& matDeltaQ, + const VO_Shape& s, + VO_Shape& NewS); + +public: + VO_AAMInverseIA* m_VOAAMInverseIA; + + /** Constructor */ + VO_FittingAAMInverseIA(); + + /** Destructor */ + ~VO_FittingAAMInverseIA(); + + /** Load ICIA AAM fitting training results */ + void VO_LoadParameters4Fitting(const string& fd); + + /** Start Inverse Additive Image Alignment fitting, for static images, recording all iterations of every single image */ + float VO_IAIAAAMFitting(const Mat& iImg, vector& oMeshInfo, unsigned int epoch = EPOCH, bool record = false); + + /** Start Inverse Additive Image Alignment fitting, for dynamic image sequence */ + float VO_IAIAAAMFitting(const Mat& iImg, VO_Shape& ioShape, Mat& oImg, unsigned int epoch = EPOCH); + + /** Start CMU Inverse Compositional Image Alignment fitting, for static images, recording all iterations of every single image */ + float VO_ICIAAAMFitting(const Mat& iImg, vector& oMeshInfo, unsigned int epoch = EPOCH, bool record = false); + + /** Start CMU Inverse Compositional Image Alignment fitting, for dynamic image sequence */ + float VO_ICIAAAMFitting(const Mat& iImg, VO_Shape& ioShape, Mat& oImg, unsigned int epoch = EPOCH); + +}; + +#endif + diff --git a/modules/smfitting/include/VO_FittingAFM.h b/modules/smfitting/include/VO_FittingAFM.h new file mode 100644 index 0000000..faaff84 --- /dev/null +++ b/modules/smfitting/include/VO_FittingAFM.h @@ -0,0 +1,109 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_FittingAFM__ +#define __VO_FittingAFM__ + +#include +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_CVCommon.h" +#include "VO_AFM.h" +#include "VO_Fitting2DSM.h" + + +using namespace std; +using namespace cv; + + +/** + * @author JIA Pei + * @brief AFM fitting algorithm. + * @note FIXME + */ +class VO_FittingAFM : public VO_Fitting2DSM +{ +private: + void init(); + +public: + VO_AFM* m_VOAFM; + + /** Constructor */ + VO_FittingAFM(); + + /** Destructor */ + ~VO_FittingAFM(); + + /** Load AFM fitting training results */ + void VO_LoadParameters4Fitting(const string& fd); + + /** Start AFM fitting, for static images, recording all iterations of every single image */ + float VO_AFMFitting(const Mat& iImg, vector& oMeshInfo, unsigned int afmType = VO_AXM::AFM, unsigned int epoch = EPOCH, bool record = false); + + /** Start AFM fitting, for dynamic image sequence */ + float VO_AFMFitting(const Mat& iImg, VO_Shape& ioShape, Mat& oImg, unsigned int afmType = VO_AXM::AFM, unsigned int epoch = EPOCH); + +}; + +#endif + diff --git a/modules/smfitting/include/VO_FittingASMLTCs.h b/modules/smfitting/include/VO_FittingASMLTCs.h new file mode 100644 index 0000000..3772304 --- /dev/null +++ b/modules/smfitting/include/VO_FittingASMLTCs.h @@ -0,0 +1,152 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_FittingASMLTCs__ +#define __VO_FittingASMLTCs__ + +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_CVCommon.h" +#include "VO_Profile.h" +#include "VO_ASMLTCs.h" +#include "VO_Fitting2DSM.h" + +using namespace std; +using namespace cv; + +/** + * @author JIA Pei + * @brief Local texture constrained ASM fitting algorithms. + */ +class VO_FittingASMLTCs : public VO_Fitting2DSM +{ +private: + /** scale between original input image and search image */ + float m_fScale2; + + /** Initialization */ + void init(); + +public: + VO_ASMLTCs* m_VOASMLTC; + + /** constructor */ + VO_FittingASMLTCs(); + + /** destructor */ + ~VO_FittingASMLTCs(); + + static float VO_FindBestMatchingLTC( const VO_ASMLTCs* asmmodel, + const Mat& iImg, + const VO_Shape& iShape, + const vector& iShapeInfo, + const Mat_& iMean, + const Mat_& iCovInverse, + unsigned int ptIdx, + unsigned int offSetTolerance, + Point2f& ioLocation, + unsigned int LTCType ); + + static int UpdateShape(const VO_ASMLTCs* asmmodel, + const Mat& iImg, + VO_Shape& ioShape, + const vector& iShapeInfo, + const vector< Mat_ >& iMeans, + const vector< Mat_ >& iCovInverses, + unsigned int offSetTolerance = 3); + + void PyramidFit( VO_Shape& iShape, + const Mat& iImg, + vector& oMeshInfo, + unsigned int iLev, + float PClose = 0.90f, + unsigned int epoch = VO_Fitting2DSM::EPOCH, + bool record = false); + + void PyramidFit( VO_Shape& ioShape, + const Mat& iImg, + unsigned int iLev, + float PClose = 0.90f, + unsigned int epoch = VO_Fitting2DSM::EPOCH); + + /** Load ASM LTC fitting trained data */ + void VO_LoadParameters4Fitting(const string& fd); + + /** Start ASM LTC fitting, for static images, recording all iterations of every single image */ + float VO_ASMLTCFitting( const Mat& iImg, + vector& oMeshInfo, + unsigned int trmethod = VO_Features::DIRECT, + unsigned int epoch = EPOCH, + unsigned int pyramidlevel = 3, + bool record = false); + + /** Start ASM LTC fitting, for dynamic image sequence */ + float VO_ASMLTCFitting( const Mat& iImg, + VO_Shape& ioShape, + Mat& oImg, + unsigned int trmethod = VO_Features::DIRECT, + unsigned int epoch = 4, + unsigned int pyramidlevel = 3); +}; + +#endif // __VO_FittingASMLTCs__ + diff --git a/modules/smfitting/include/VO_FittingASMNDProfiles.h b/modules/smfitting/include/VO_FittingASMNDProfiles.h new file mode 100644 index 0000000..253a176 --- /dev/null +++ b/modules/smfitting/include/VO_FittingASMNDProfiles.h @@ -0,0 +1,196 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#ifndef __VO_FittingASMNDProfiles__ +#define __VO_FittingASMNDProfiles__ + +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "VO_CVCommon.h" +#include "VO_ASMNDProfiles.h" +#include "VO_Fitting2DSM.h" + +using namespace std; +using namespace cv; + + + +/** + * @author JIA Pei + * @brief 1D and 2D Profile ASM fitting algorithms. + */ +class VO_FittingASMNDProfiles : public VO_Fitting2DSM +{ +private: + /** scale between original input image and search image */ + float m_fScale2; + + /** Initialization */ + void init(); + + int FindBestOffset( + const Mat& inputMat, + const Mat& firstChannelImg, + const Mat& secondChannelImg, + const Mat& thirdChannelImg, + const Point2f& ThisPoint, + const vector< VO_Profile >& iMean, + const vector< vector< Mat_ > >& iCovInverse, + const unsigned int offSetTolerance, + const VO_Fitting2DSM::MULTICHANNELTECH fitTech, + const int ptIndex, + const int ptDir, + const Point2f& dirDistPt ); +public: + + VO_ASMNDProfiles* m_VOASMNDProfile; + + /** constructor */ + VO_FittingASMNDProfiles(); + + /** destructor */ + ~VO_FittingASMNDProfiles(); + + static int VO_FindBestMatchingProfile1D( const Mat& iImg, + const Point2f& ThisPoint, + const Mat_& iMean, + const Mat_& iCovInverse, + const unsigned int ProfileLength, + const unsigned int offSetTolerance, + const float DeltaX, + const float DeltaY); + + static int VO_FindBestMatchingProfile2D(const Mat& iImg, + const Point2f& ThisPoint, + const VO_Profile iMean, + const vector< Mat_ > iCovInverse, + const unsigned int ProfileLength, + const unsigned int offSetTolerance, + const float DeltaX, + const float DeltaY, + const int dir); + + unsigned int UpdateShape( const VO_ASMNDProfiles* asmmodel, + const Mat& iImg, + VO_Shape& ioShape, + const vector& iShapeInfo, + const vector< VO_Profile >& iMean, + const vector< vector< Mat_ > >& iCovInverse, + const unsigned int offSetTolerance, + const vector< VO_Fitting2DSM::MULTICHANNELTECH > & fitTechs); + + void PyramidFit( VO_Shape& ioShape, + const Mat& iImg, + vector& oMeshInfo, + const unsigned int iLev, + const float PClose, + const unsigned int epoch, + const bool record, + const vector& fitTechs); + + void PyramidFit( VO_Shape& ioShape, + const Mat& iImg, + unsigned int iLev, + float PClose = 0.90f, + unsigned int epoch = VO_Fitting2DSM::EPOCH, + unsigned int profdim = 2); + + void Fit( const Mat& iImage, + const float PyrScale, + const int nQualifyingDisplacements, + const unsigned int iLev, + const unsigned int epoch, + const vector& fitTech, + const bool record, + VO_Shape& ioShape, + vector& oMeshInfo); + + void StagedPyramidFit( VO_Shape& ioShape, + const Mat& iImg, + vector& oMeshInfo, + unsigned int iLev, + float PClose = 0.90f, + unsigned int epoch = VO_Fitting2DSM::EPOCH, + bool record = false); + + /** Load ASM fitting training results */ + void VO_LoadParameters4Fitting(const string& fd); + + /** Start ASM ND Profile fitting, for static images, recording all iterations of every single image */ + float VO_ASMNDProfileFitting( const Mat& iImg, + vector& oMeshInfo, + const unsigned int epoch, + const unsigned int pyramidlevel, + const bool record, + const vector& fittingTechniques); + + /** Start ASM ND Profile fitting, for dynamic image sequence */ + float VO_ASMNDProfileFitting( const Mat& iImg, + VO_Shape& ioShape, + Mat& oImg, + unsigned int epoch = 4, + unsigned int pyramidlevel = 4, + unsigned int profdim = 2); +}; + +#endif // __VO_FittingASMNDProfiles__ + diff --git a/modules/smfitting/smfitting.vcxproj b/modules/smfitting/smfitting.vcxproj new file mode 100644 index 0000000..ad67be0 --- /dev/null +++ b/modules/smfitting/smfitting.vcxproj @@ -0,0 +1,90 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {57706470-A0D4-455D-AB60-626EAE943596} + smfitting + + + + StaticLibrary + true + MultiByte + v110 + + + Application + false + true + MultiByte + v110 + + + + + + + + + + + + + + + Level3 + Disabled + ./include;../include;../smbuilding/include;../common/include;../cvcommon/include;../featureextraction/include;../integraltransform/include;C:\OpenCV2.4.5\opencv\build\include;C:\boost\boost_1_53_0;%(AdditionalIncludeDirectories) + NOMINMAX;_MBCS;%(PreprocessorDefinitions) + + + true + + + true + + + + + Level3 + MaxSpeed + true + true + + + true + true + true + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/modules/smfitting/smfitting.vcxproj.filters b/modules/smfitting/smfitting.vcxproj.filters new file mode 100644 index 0000000..4e08b30 --- /dev/null +++ b/modules/smfitting/smfitting.vcxproj.filters @@ -0,0 +1,63 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + \ No newline at end of file diff --git a/modules/smfitting/src/VO_Fitting2DSM.cpp b/modules/smfitting/src/VO_Fitting2DSM.cpp new file mode 100644 index 0000000..88e1eb3 --- /dev/null +++ b/modules/smfitting/src/VO_Fitting2DSM.cpp @@ -0,0 +1,453 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include +#include + + +#include "VO_CVCommon.h" +#include "VO_AXM.h" +#include "VO_Fitting2DSM.h" +#include "VO_FittingAAMBasic.h" +#include "VO_FittingAAMForwardIA.h" +#include "VO_FittingAAMInverseIA.h" +#include "VO_FittingAFM.h" +#include "VO_FittingASMLTCs.h" +#include "VO_FittingASMNDProfiles.h" +#include "VO_FaceKeyPoint.h" + + +float VO_Fitting2DSM::pClose = 0.90f; + +/** Initialization */ +void VO_Fitting2DSM::init() +{ + this->m_iNbOfPyramidLevels = 3; + this->m_iFittingMethod = VO_AXM::AFM; + this->m_iIteration = 0; + this->m_fFittingTime = 0.0f; + this->m_fScale = 1.0f; + this->m_vRotateAngles.resize(1); + this->m_MatCenterOfGravity = Mat_::zeros(2, 1); + this->m_vTriangle2D.clear(); + this->m_vShape2DInfo.clear(); + this->m_FaceParts.clear(); + this->m_vPointWarpInfo.clear(); +} + + +/** Constructor */ +VO_Fitting2DSM::VO_Fitting2DSM() +{ + this->init(); +} + + +/** Destructor */ +VO_Fitting2DSM::~VO_Fitting2DSM() +{ + this->m_vTriangle2D.clear(); + this->m_vShape2DInfo.clear(); + this->m_FaceParts.clear(); + this->m_vPointWarpInfo.clear(); +} + + +/** + *@author JIA Pei + * + *@brief Obtain the first shape instance + * + *@methods - 1) three points' pairs are used to calculate affine transform + * - 2) use my own align transform - COG -> rotation -> scaling... + * - 3) SVD: Y = AX + * x' a00 a01 a02 x + * (y') = (a10 a11 a12) * (y) , + * 1 a20 a21 a22 1 + * where a00 = cos(theta), a01 = -sin(theta) + * a10 = sin(theta), a11 = cons(theta) + * a02 = tx, a12 = ty + * a20 = a21 = 0; a22 = 1 + * However, the above values are not guaranteed during calculation +**/ +Mat_ VO_Fitting2DSM::VO_FirstEstimationBySingleWarp(const VO_FaceParts& iFaceParts, + const VO_Shape& iShape, + const Point2f& ptLeftEyeCenter, + const Point2f& ptRightEyeCenter, + const Point2f& ptMouthCenter ) +{ + unsigned int NbOfKeyPoints = 3; + Mat fpts(1, NbOfKeyPoints, CV_32FC2); + Mat tpts(1, NbOfKeyPoints, CV_32FC2); + + Point2f pt; + VO_KeyPoint::CalcFaceKeyPoint(pt, iShape, iFaceParts, VO_KeyPoint::LEFTEYECENTER); + fpts.at(0,0) = pt; + VO_KeyPoint::CalcFaceKeyPoint(pt, iShape, iFaceParts, VO_KeyPoint::RIGHTEYECENTER); + fpts.at(0,1) = pt; + VO_KeyPoint::CalcFaceKeyPoint(pt, iShape, iFaceParts, VO_KeyPoint::MOUTHCENTER); + fpts.at(0,2) = pt; + + tpts.at(0,0) = ptLeftEyeCenter; + tpts.at(0,1) = ptRightEyeCenter; + tpts.at(0,2) = ptMouthCenter; + + // Explained by JIA Pei. For only 3 points, the affine transform can be computed by "getAffineTransform" +// Mat_ matWarping = cv::getAffineTransform( iShapeKeyPoints, detectKeyPoints ); + + // For more than 3 points, we need "estimateRigidTransform" + Mat_ matWarping = cv::estimateRigidTransform( fpts, tpts, true ); + + return matWarping; +} + + +VO_Shape VO_Fitting2DSM::VO_FirstEstimationByScaling( const VO_Shape& iShape, + const Rect& rect ) +{ + VO_Shape res = iShape; + Rect_ rect0 = iShape.GetShapeRect(); + float fScaleX = static_cast(rect.width)/rect0.width *0.80f; + float fScaleY = static_cast(rect.height)/rect0.height *0.80f; + res.ScaleX(fScaleX); + res.ScaleY(fScaleY); + rect0 = iShape.GetShapeBoundRect(); + Mat_ translation = Mat_::zeros(2, 1); + float centerX = (float)rect.x + (float)rect.width/2.0f; + float centerY = (float)rect.y + (float)rect.height/2.0f; + float center0X = (float)rect0.x + (float)rect0.width/2.0f; + float center0Y = (float)rect0.x + (float)rect0.height/2.0f; + translation(0,0) = centerX - center0X; + translation(1,0) = centerY - center0Y; + res.Translate( translation ); + return res; +} + + +/** + * @author JIA Pei + * @version 2010-05-15 + * @brief Calculate error image by + * @param +*/ +float VO_Fitting2DSM::VO_CalcErrorImage(const Mat& iImg, + const VO_Shape& iShape, + const VO_Texture& iTexture, + VO_Texture& textureDiff) +{ + float E = FLT_MAX; + + // extract the real texture on the image from the model shape calculated above + if ( VO_TextureModel::VO_LoadOneTextureFromShape(iShape, iImg, this->m_vTriangle2D, this->m_vPointWarpInfo, this->m_VOFittingTexture ) ) + { + this->m_VOFittingTexture.Normalize(); + textureDiff = this->m_VOFittingTexture - iTexture; + E = textureDiff.GetTextureNorm(); + } + + return E; +} + + +/** + * @author JIA Pei + * @version 2010-05-14 + * @brief Start object tracking + * @param iImage Input -- The current image frame for tracking + * @param oImages Output -- output images - for converging iterations + * @param fittingMethod Input -- fitting method + * @param ptLeftEyeCenter Input -- the detected left eye center + * @param ptRightEyeCenter Input -- the detected right eye center + * @param ptMouthCenter Input -- the detected mouth center + * @param epoch Input -- the iteration epoch + * @param pyramidlevel Input -- pyramid levels, normally 3 +*/ +float VO_Fitting2DSM::VO_StartFitting( const Mat& iImage, + vector& oMeshInfo, + const VO_AXM::TYPE fittingMethod, + const Point2f& ptLeftEyeCenter, + const Point2f& ptRightEyeCenter, + const Point2f& ptMouthCenter, + const unsigned int epoch, + const unsigned int pyramidlevel, + const bool record, + const vector& fittingTechs, + const bool doAffineWarp) +{ + this->m_fFittingTime = 0.0f; + this->SetInputImage(iImage); + if(doAffineWarp){ + // The following function is important!! + // Although it will be done only once during the whole fitting process, it will roughly give out the rough warping (global shape normalization )relationship between m_MatAlignedShapeInstance and m_MatShapeInstance + // Once this->m_fScale, this->m_vRotateAngles, this->m_MatCenterOfGravity are calculated, they will not change during the whole fitting process + this->m_VOFittingShape.clone(this->m_VOTemplateAlignedShape); + this->m_VOFittingShape.Affine2D( VO_Fitting2DSM::VO_FirstEstimationBySingleWarp( + this->m_FaceParts, + this->m_VOFittingShape, + ptLeftEyeCenter, + ptRightEyeCenter, + ptMouthCenter) + ); + //Rect rect = fd.GetDetectedFaceWindow2SM(); // Explained by JIA Pei, it seems GetDetectedFaceWindow2SM doesn't work fine here + //Rect rect = fd.GetDetectedFaceWindow(); + //this->m_VOFittingShape = VO_Fitting2DSM::VO_FirstEstimationByScaling(this->m_VOTemplateAlignedShape, fd); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageInput); + + //Mat tempImg = this->m_ImageInput.clone(); + //for(unsigned int i = 0; i < this->m_VOFittingShape.GetNbOfPoints(); i++) + //{ + // Point2f pt = this->m_VOFittingShape.GetA2DPoint(i); + // VO_Fitting2DSM::VO_DrawAPoint(pt, tempImg); + //} + //imwrite("temp.jpg", tempImg); + } + + this->m_iFittingMethod = fittingMethod; + this->m_iNbOfPyramidLevels = pyramidlevel; + switch(this->m_iFittingMethod) + { + case VO_AXM::AAM_BASIC: + { + this->m_fFittingTime = dynamic_cast(this)->VO_BasicAAMFitting(iImage, oMeshInfo, epoch, record); + } + break; + case VO_AXM::AAM_DIRECT: + { + this->m_fFittingTime = dynamic_cast(this)->VO_DirectAAMFitting(iImage, oMeshInfo, epoch, record); + } + break; + case VO_AXM::CLM: + { +// this->m_fFittingTime = + } + break; + case VO_AXM::AFM: + { + this->m_fFittingTime = dynamic_cast(this)->VO_AFMFitting(iImage, oMeshInfo, this->m_iFittingMethod, epoch, record); + } + break; + case VO_AXM::AAM_IAIA: + { + this->m_fFittingTime = dynamic_cast(this)->VO_IAIAAAMFitting(iImage, oMeshInfo, epoch, record); + } + break; + case VO_AXM::AAM_CMUICIA: + { + this->m_fFittingTime = dynamic_cast(this)->VO_ICIAAAMFitting(iImage, oMeshInfo, epoch, record); + } + break; + case VO_AXM::AAM_FAIA: + { + this->m_fFittingTime = dynamic_cast(this)->VO_FAIAAAMFitting(iImage, oMeshInfo, epoch, record); + } + break; + case VO_AXM::ASM_LTC: + { + this->m_fFittingTime = dynamic_cast(this)->VO_ASMLTCFitting(iImage, oMeshInfo, VO_Features::DIRECT, epoch, this->m_iNbOfPyramidLevels, record); + } + break; + case VO_AXM::ASM_PROFILEND: + { + this->m_fFittingTime = dynamic_cast(this)->VO_ASMNDProfileFitting(iImage, oMeshInfo, epoch, this->m_iNbOfPyramidLevels, record, fittingTechs); + } + break; + default: + { + this->m_fFittingTime = dynamic_cast(this)->VO_ASMNDProfileFitting(iImage, oMeshInfo, epoch, this->m_iNbOfPyramidLevels, record, fittingTechs); + } + break; + } + if (oMeshInfo.size() > 0 ) VO_DrawMeshOnImage(oMeshInfo.back(),this->m_ImageInput,this->m_ImageOutput); + else this->m_ImageOutput = this->m_ImageInput; + +// vector angles; +// VO_AAMFaceRecognition::CalcFittedFaceAngle2D(angles, this->m_VOShape, this->m_FaceParts); +// imwrite("test.jpg", this->m_ImageOutput); + +// return this->m_ImageOutput; + return this->m_fFittingTime; +} + +/** + * @author JIA Pei + * @version 2010-05-07 + * @brief draw a point on the image + * @param iShape Input -- the input shape + * @param iAAMModel Input -- the model + * @param ioImg Input and Output -- the image + * @return void + */ +void VO_Fitting2DSM::VO_DrawMesh(const VO_Shape& iShape, const VO_AXM* iModel, Mat& ioImg) +{ + Point iorg,idst; + vector edges = iModel->GetEdge(); + unsigned int NbOfEdges = iModel->GetNbOfEdges(); + + for (unsigned int i = 0; i < NbOfEdges; i++) + { + iorg = cvPointFrom32f( iShape.GetA2DPoint( edges[i].GetIndex1() ) ); + idst = cvPointFrom32f( iShape.GetA2DPoint( edges[i].GetIndex2() ) ); + // Edge + cv::line( ioImg, iorg, idst, colors[8], 1, 0, 0 ); + // Key points + cv::circle( ioImg, iorg, 2, colors[0], -1, 8, 0 ); + cv::circle( ioImg, idst, 2, colors[0], -1, 8, 0 ); + } +} + +void VO_Fitting2DSM::VO_DrawMeshOnImage(DrawMeshInfo meshStruct,const Mat& iImage, Mat& oImage){ + //i hope this dosen't leak memory. + oImage = Mat(iImage.size(), iImage.type(), iImage.channels()); + Mat oImageROI = oImage(Range (0, (int)(oImage.rows/meshStruct.pyrScale) ), Range (0, (int)(oImage.cols/meshStruct.pyrScale) ) ); + cv::resize(iImage, oImageROI, oImageROI.size() ); + VO_Fitting2DSM::VO_DrawMesh(meshStruct.drawPts / meshStruct.f2Scale, meshStruct.modelPtr, oImage); + +} + +/** + * @author JIA Pei + * @version 2010-05-07 + * @brief draw a point on the image + * @param iShape Input -- the input shape + * @param theSubshape Output -- and input, the image drawn with the point + * @param iLine Input -- the line + * @param oImg Output-- output image + * @param dir Input -- direction + * @param ws Input -- + * @param offset Input -- add some offset at both ends of the line segment itself + * @param ci Input -- color index + * @return void + */ +void VO_Fitting2DSM::VO_DrawAline( const VO_Shape& iShape, + const VO_Shape& theSubshape, + const vector& iLine, + Mat& oImg, + unsigned int dir, + bool ws, + unsigned int offset, + unsigned int ci) +{ + switch(dir) + { + case VERTICAL: + { + float A = iLine[0]; + float B = iLine[1]; + float C = iLine[2]; + Point2f ptf1, ptf2; + if(ws) + { + ptf1.y = iShape.MinY() - offset; + ptf2.y = iShape.MaxY() + offset; + } + else + { + ptf1.y = theSubshape.MinY() - offset; + ptf2.y = theSubshape.MaxY() + offset; + } + ptf1.x = -(C + B*ptf1.y)/A; + ptf2.x = -(C + B*ptf2.y)/A; + Point pt1 = cvPointFrom32f( ptf1 ); + Point pt2 = cvPointFrom32f( ptf2 ); + cv::line( oImg, pt1, pt2, colors[ci], 2, 0, 0); + } + break; + case HORIZONTAL: + default: + { + float A = iLine[0]; + float B = iLine[1]; + float C = iLine[2]; + Point2f ptf1, ptf2; + if(ws) + { + ptf1.x = iShape.MinX() - offset; + ptf2.x = iShape.MaxX() + offset; + } + else + { + ptf1.x = theSubshape.MinX() - offset; + ptf2.x = theSubshape.MaxX() + offset; + } + ptf1.y = -(C + A*ptf1.x)/B; + ptf2.y = -(C + A*ptf2.x)/B; + Point pt1 = cvPointFrom32f( ptf1 ); + Point pt2 = cvPointFrom32f( ptf2 ); + cv::line( oImg, pt1, pt2, colors[ci], 2, 0, 0); + } + break; + } +} + + +/** + * @author JIA Pei + * @version 2010-05-07 + * @brief draw a point on the image + * @param pt Input -- the point + * @param oImg Input and Output -- the image drawn with the point + * @return void + */ +void VO_Fitting2DSM::VO_DrawAPoint(const Point2f& pt, Mat& ioImg) +{ + Point tempPt = cvPointFrom32f(pt); + cv::circle( ioImg, tempPt, 2, colors[5], -1, 8, 0 ); +} + diff --git a/modules/smfitting/src/VO_FittingAAMBasic.cpp b/modules/smfitting/src/VO_FittingAAMBasic.cpp new file mode 100644 index 0000000..201591d --- /dev/null +++ b/modules/smfitting/src/VO_FittingAAMBasic.cpp @@ -0,0 +1,621 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include +#include + +#include "VO_FittingAAMBasic.h" + + +/** For damp, damp coefficients */ +vector VO_FittingAAMBasic::k_values; + + +/** Default Constructor */ +VO_FittingAAMBasic::VO_FittingAAMBasic() +{ + this->init(); +} + +/** Destructor */ +VO_FittingAAMBasic::~VO_FittingAAMBasic() +{ + if(this->m_VOAAMBasic) delete this->m_VOAAMBasic; this->m_VOAAMBasic = NULL; + } + +/** Initialization */ +void VO_FittingAAMBasic::init() +{ + VO_Fitting2DSM::init(); + this->m_VOAAMBasic = new VO_AAMBasic(); + this->m_E = 0.0f; + this->m_E_previous = 0.0f; + VO_FittingAAMBasic::k_values.push_back(1.0f); + VO_FittingAAMBasic::k_values.push_back(0.5f); + VO_FittingAAMBasic::k_values.push_back(0.25f); + VO_FittingAAMBasic::k_values.push_back(0.125f); + VO_FittingAAMBasic::k_values.push_back(0.0625f); + VO_FittingAAMBasic::k_values.push_back(0.0f); +} + + +/** + * @author JIA Pei + * @version 2010-05-14 + * @brief obtain the reconstructed real-size shape using c parameters and t parameters + * @param c Input -- PCA appearance parameters, including both shape non-rigid variance + texture variance + * @param t Input -- parameters required for global shape normalization, rigid variance, refer to "AAM Revisited equation (42)" + * @param modelAlignedShape Output -- the modeled aligned shape + * @param modelNormalizedTexture Output -- the modeled normalized texture + * @param oShape Output -- output shape after global shape normalization. Please make sure global shape normalization is not Procrustes Analysis. + * @param scale Input and Output -- scaling + * @param rotateAngles Input and Output -- rotation + * @param matCOG Input and Output -- translation + */ +void VO_FittingAAMBasic::VO_CParamTParam2FittingShape( const Mat_& c, + const Mat_& t, + VO_Texture& modelNormalizedTexture, + VO_Shape& oShape, + float& scale, + vector& rotateAngles, + Mat_& matCOG, + unsigned int mtd) +{ + // generate shape and texture from C parameters + this->m_VOAAMBasic->VO_CParamBackProjectToSParamTParam(c, this->m_MatModelAlignedShapeParam, this->m_MatModelNormalizedTextureParam); + this->m_VOAAMBasic->VO_SParamBackProjectToAlignedShape(this->m_MatModelAlignedShapeParam, this->m_VOModelAlignedShape); + this->m_VOAAMBasic->VO_TParamBackProjectToNormalizedTexture(this->m_MatModelNormalizedTextureParam, modelNormalizedTexture, this->m_VOAAMBasic->GetNbOfChannels() ); + //this->VO_CParamBackProjectToAlignedShape(c, this->m_MatModeledAlignedShapeInstance); + //this->VO_CParamBackProjectToNormalizedTexture(c, this->m_MatModeledNormalizedTextureInstance); + + switch(mtd) + { + case VO_Fitting2DSM::USEGLOBALSHAPENORMALIZATION: + { + + } + break; + case VO_Fitting2DSM::USESIMILARITYTRANSFORM: + default: + { + float updatescale = 1.0; + vector updateangles(1); + Mat_ updatetranslation = Mat_::zeros(2, 1); + VO_Shape::GlobalShapeNormalization2SimilarityTrans(t, updatescale, updateangles, updatetranslation ); + scale *= updatescale; + rotateAngles[0] = -rotateAngles[0]+updateangles[0]; + matCOG(0,0) += updatetranslation(0,0); + matCOG(1,0) += updatetranslation(1,0); + + // shape adjustment by t parameters - pose transform + VO_Shape::GlobalShapeNormalization2D(this->m_VOModelAlignedShape, oShape, scale, rotateAngles, matCOG); + } + break; + } +} + + +/** + * @author JIA Pei + * @version 2010-05-18 + * @brief Load all AAM data from a specified folder for later fitting, to member variable m_VOAAMBasic + * @param fd Input - the folder that AAM to be loaded from +*/ +void VO_FittingAAMBasic::VO_LoadParameters4Fitting(const string& fd) +{ + this->m_VOAAMBasic->VO_LoadParameters4Fitting(fd); + + // VO_Fitting2DSM + this->m_VOTemplateAlignedShape = this->m_VOAAMBasic->m_VOAlignedMeanShape; + this->m_VOTemplateNormalizedTexture = this->m_VOAAMBasic->m_VONormalizedMeanTexture; + this->m_vTriangle2D = this->m_VOAAMBasic->m_vNormalizedTriangle2D; + this->m_vShape2DInfo = this->m_VOAAMBasic->m_vShape2DInfo; + this->m_FaceParts = this->m_VOAAMBasic->m_FaceParts; + this->m_vPointWarpInfo = this->m_VOAAMBasic->m_vNormalizedPointWarpInfo; + + // VO_FittingAAMBasic + this->m_MatDeltaC = Mat_::zeros(1, this->m_VOAAMBasic->m_iNbOfAppearanceEigens); + this->m_MatEstimatedC = Mat_::zeros(1, this->m_VOAAMBasic->m_iNbOfAppearanceEigens); + this->m_MatCurrentC = Mat_::zeros(1, this->m_VOAAMBasic->m_iNbOfAppearanceEigens); + this->m_MatDeltaT = Mat_::zeros(1, 4); + this->m_MatEstimatedT = Mat_::zeros(1, 4); + this->m_MatCurrentT = Mat_::zeros(1, 4); +} + + +/** + * @author JIA Pei + * @version 2010-05-20 + * @brief Basic AAM Fitting, for static images, so that we record the whole fitting process + * @param iImg Input - image to be fitted + * @param oImages Output - the fitted shape + * @param epoch Input - the iteration epoch +*/ +float VO_FittingAAMBasic::VO_BasicAAMFitting( const Mat& iImg, + vector& oMeshInfo, + unsigned int epoch, + bool record) +{ + double t = (double)cvGetTickCount(); + + this->SetProcessingImage(iImg, this->m_VOAAMBasic); + this->m_iIteration = 0; + + if(record) + { + oMeshInfo.push_back(DrawMeshInfo(1,1,this->m_VOFittingShape,this->m_VOAAMBasic)); + } + + // Get m_MatModelAlignedShapeParam and m_fScale, m_vRotateAngles, m_MatCenterOfGravity + this->m_VOAAMBasic->VO_CalcAllParams4AnyShapeWithConstrain( this->m_VOFittingShape, + this->m_MatModelAlignedShapeParam, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); + if(record) + { + oMeshInfo.push_back(DrawMeshInfo(1,1,this->m_VOFittingShape,this->m_VOAAMBasic)); + } + + // Get m_MatModelNormalizedTextureParam + VO_TextureModel::VO_LoadOneTextureFromShape(this->m_VOFittingShape, + this->m_ImageProcessing, + this->m_vTriangle2D, + this->m_vPointWarpInfo, + this->m_VOFittingTexture ); + // estimate the texture model parameters + this->m_VOAAMBasic->VO_CalcAllParams4AnyTexture(this->m_VOFittingTexture, + this->m_MatModelNormalizedTextureParam); + + // Calculate m_MatCurrentC + this->m_VOAAMBasic->VO_SParamTParamProjectToCParam( this->m_MatModelAlignedShapeParam, + this->m_MatModelNormalizedTextureParam, + this->m_MatCurrentC ); + // Set m_MatCurrentT, m_MatDeltaT, m_MatEstimatedT, m_MatDeltaC, m_MatEstimatedC, etc. + this->m_MatCurrentT = Mat_::zeros(this->m_MatCurrentT.size()); + this->m_MatDeltaT = Mat_::zeros(this->m_MatDeltaT.size()); + this->m_MatEstimatedT = Mat_::zeros(this->m_MatEstimatedT.size()); + this->m_MatDeltaC = Mat_::zeros(this->m_MatDeltaC.size()); + this->m_MatEstimatedC = Mat_::zeros(this->m_MatEstimatedC.size()); + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // explained by JIA Pei. 2010-05-20 + // For the first round, this->m_VOFittingShape should not change after calling "VO_CParamTParam2FittingShape" + // But this is not the case. why? + // Before calling VO_CParamTParam2FittingShape, this->m_VOFittingShape is calculated by + // a) assigning m_VOTemplateAlignedShape + // b) align to the real-size face using detected eyes and mouth + // c) constrain the shape within the image + // d) constrain the shape parameters and calculate those rigid transform parameters + // cout << this->m_VOFittingShape << endl; + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // Estimate m_VOFittingShape and m_VOFittingTexture + this->VO_CParamTParam2FittingShape( this->m_MatCurrentC, + this->m_MatCurrentT, + this->m_VOModelNormalizedTexture, + this->m_VOFittingShape, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity ); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); // Remember to call ConstrainShapeInImage() whenever you update m_VOFittingShape + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // When calling VO_CParamTParam2FittingShape, this->m_VOFittingShape is calculated by + // a) c parameters to reconstruct shape parameters + // b) shape parameters to reconstruct shape + // c) align to the real-size face by global shape normalization + // cout << this->m_VOFittingShape << endl; + ////////////////////////////////////////////////////////////////////////////////////////////////////// + if(record) + { + oMeshInfo.push_back(DrawMeshInfo(1,1,this->m_VOFittingShape,this->m_VOAAMBasic)); + } + + this->m_E_previous = this->m_E = this->VO_CalcErrorImage(this->m_ImageProcessing, + this->m_VOFittingShape, + this->m_VOModelNormalizedTexture, + this->m_VOTextureError); + + do + { + ++this->m_iIteration; + float estScale = this->m_fScale; + vector estRotateAngles = this->m_vRotateAngles; + Mat_ estCOG = this->m_MatCenterOfGravity.clone(); + bool cBetter = false; + bool poseBetter = false; + + /**First shape parameters, c parameters. refer to equation (9.3) + * Cootes "Statistical Model of Appearance for Computer Vision" */ + cv::gemm(this->m_VOTextureError.GetTheTextureInARow(), this->m_VOAAMBasic->m_MatRc, -1, Mat(), 0, this->m_MatDeltaC, GEMM_2_T); + + // damp -- C + for(unsigned int i = 0; i < k_values.size(); i++) + { + // make damped c prediction + cv::scaleAdd(this->m_MatDeltaC, k_values[i], this->m_MatCurrentC, this->m_MatEstimatedC); + + // make sure m_MatEstimatedC are constrained + this->m_VOAAMBasic->VO_AppearanceParameterConstraint(this->m_MatEstimatedC); + this->VO_CParamTParam2FittingShape( this->m_MatEstimatedC, + this->m_MatCurrentT, + this->m_VOModelNormalizedTexture, + this->m_VOEstimatedShape, + estScale, + estRotateAngles, + estCOG); + if ( !VO_ShapeModel::VO_IsShapeInsideImage(this->m_VOEstimatedShape, this->m_ImageProcessing) ) + continue; + else + this->m_E = this->VO_CalcErrorImage(this->m_ImageProcessing, + this->m_VOEstimatedShape, + this->m_VOModelNormalizedTexture, + this->m_VOEstimatedTextureError); + + if (this->m_E < this->m_E_previous) + { + this->m_MatEstimatedC.copyTo(this->m_MatCurrentC); + this->m_VOFittingShape.clone(this->m_VOEstimatedShape); + this->m_VOTextureError.clone(this->m_VOEstimatedTextureError); + this->m_E_previous = this->m_E; + cBetter = true; + this->m_fScale = estScale; + this->m_vRotateAngles = estRotateAngles; + this->m_MatCenterOfGravity = estCOG.clone(); + break; + } + } + + /** Second pose, t parameters. refer to equation (9.3) + * Cootes "Statistical Model of Appearance for Computer Vision" */ + cv::gemm(this->m_VOTextureError.GetTheTextureInARow(), this->m_VOAAMBasic->m_MatRt, -1, Mat(), 0, this->m_MatDeltaT, GEMM_2_T); + + // damp -- T + for(unsigned int i = 0; i < k_values.size(); i++) + { + // make damped c/pose prediction + cv::scaleAdd(this->m_MatDeltaT, k_values[i], this->m_MatCurrentT, this->m_MatEstimatedT); + this->VO_CParamTParam2FittingShape( this->m_MatCurrentC, + this->m_MatEstimatedT, + this->m_VOModelNormalizedTexture, + this->m_VOEstimatedShape, + estScale, + estRotateAngles, + estCOG); + if ( !VO_ShapeModel::VO_IsShapeInsideImage(this->m_VOEstimatedShape, this->m_ImageProcessing) ) + continue; + else + this->m_E = this->VO_CalcErrorImage(this->m_ImageProcessing, + this->m_VOEstimatedShape, + this->m_VOModelNormalizedTexture, + this->m_VOEstimatedTextureError); + + if (this->m_E < this->m_E_previous) + { + // Since m_fScale, m_vRotateAngles and m_MatCenterOfGravity have been updated, + // m_MatCurrentT should be assigned to 0 now! + this->m_MatCurrentT = Mat_::zeros(this->m_MatCurrentT.size()); +// this->m_MatEstimatedT.copyTo(this->m_MatCurrentT); + this->m_VOFittingShape.clone(this->m_VOEstimatedShape); + this->m_VOTextureError.clone(this->m_VOEstimatedTextureError); + this->m_E_previous = this->m_E; + poseBetter = true; + this->m_fScale = estScale; + this->m_vRotateAngles = estRotateAngles; + this->m_MatCenterOfGravity = estCOG.clone(); + break; + } + } + + if( cBetter || poseBetter) + { + if(record) + { + oMeshInfo.push_back(DrawMeshInfo(1,1,this->m_VOFittingShape,this->m_VOAAMBasic)); + } + } + else + break; + + }while( ( fabs(this->m_E) > FLT_EPSILON ) && (this->m_iIteration < epoch)/* && (cv::norm(this->m_MatDeltaC) > FLT_EPSILON) */ ); + +t = ((double)cvGetTickCount() - t )/ (cvGetTickFrequency()*1000.); +cout << "Basic fitting time cost: " << t << " millisec" << endl; + + return safeDoubleToFloat(t); +} + + +/** + * @author JIA Pei + * @version 2010-05-20 + * @brief Basic AAM Fitting, for dynamic image sequence + * @param iImg Input - image to be fitted + * @param ioShape Input and Output - the fitted shape + * @param oImg Output - the fitted image + * @param epoch Input - the iteration epoch +*/ +float VO_FittingAAMBasic::VO_BasicAAMFitting(const Mat& iImg, + VO_Shape& ioShape, + Mat& oImg, + unsigned int epoch) +{ + this->m_VOFittingShape.clone(ioShape); +double t = (double)cvGetTickCount(); + + this->SetProcessingImage(iImg, this->m_VOAAMBasic); + this->m_iIteration = 0; + + // Get m_MatModelAlignedShapeParam and m_fScale, m_vRotateAngles, m_MatCenterOfGravity + this->m_VOAAMBasic->VO_CalcAllParams4AnyShapeWithConstrain( this->m_VOFittingShape, + this->m_MatModelAlignedShapeParam, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); + + // Get m_MatModelNormalizedTextureParam + VO_TextureModel::VO_LoadOneTextureFromShape(this->m_VOFittingShape, + this->m_ImageProcessing, + this->m_vTriangle2D, + this->m_vPointWarpInfo, + this->m_VOFittingTexture ); + // estimate the texture model parameters + this->m_VOAAMBasic->VO_CalcAllParams4AnyTexture(this->m_VOFittingTexture, this->m_MatModelNormalizedTextureParam); + + // Calculate m_MatCurrentC + this->m_VOAAMBasic->VO_SParamTParamProjectToCParam( this->m_MatModelAlignedShapeParam, + this->m_MatModelNormalizedTextureParam, + this->m_MatCurrentC ); + // Set m_MatCurrentT, m_MatDeltaT, m_MatEstimatedT, m_MatDeltaC, m_MatEstimatedC, etc. + this->m_MatCurrentT = Mat_::zeros(this->m_MatCurrentT.size()); + this->m_MatDeltaT = Mat_::zeros(this->m_MatDeltaT.size()); + this->m_MatEstimatedT = Mat_::zeros(this->m_MatEstimatedT.size()); + this->m_MatDeltaC = Mat_::zeros(this->m_MatDeltaC.size()); + this->m_MatEstimatedC = Mat_::zeros(this->m_MatEstimatedC.size()); + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // explained by JIA Pei. 2010-05-20 + // For the first round, this->m_VOFittingShape should not change after calling "VO_CParamTParam2FittingShape" + // But this is not the case. why? + // Before calling VO_CParamTParam2FittingShape, this->m_VOFittingShape is calculated by + // a) assigning m_VOTemplateAlignedShape + // b) align to the real-size face using detected eyes and mouth + // c) constrain the shape within the image + // d) constrain the shape parameters and calculate those rigid transform parameters + // cout << this->m_VOFittingShape << endl; + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // Estimate m_VOFittingShape and m_VOFittingTexture + this->VO_CParamTParam2FittingShape( this->m_MatCurrentC, + this->m_MatCurrentT, + this->m_VOModelNormalizedTexture, + this->m_VOFittingShape, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity ); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); // Remember to call ConstrainShapeInImage() whenever you update m_VOFittingShape + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // When calling VO_CParamTParam2FittingShape, this->m_VOFittingShape is calculated by + // a) c parameters to reconstruct shape parameters + // b) shape parameters to reconstruct shape + // c) align to the real-size face by global shape normalization + // cout << this->m_VOFittingShape << endl; + ////////////////////////////////////////////////////////////////////////////////////////////////////// + + this->m_E_previous = this->m_E = this->VO_CalcErrorImage(this->m_ImageProcessing, + this->m_VOFittingShape, + this->m_VOModelNormalizedTexture, + this->m_VOTextureError); + + do + { + float estScale = this->m_fScale; + vector estRotateAngles = this->m_vRotateAngles; + Mat_ estCOG = this->m_MatCenterOfGravity.clone(); + bool cBetter = false; + bool poseBetter = false; + + /**First shape parameters, c parameters. refer to equation (9.3) + * Cootes "Statistical Model of Appearance for Computer Vision" */ + cv::gemm(this->m_VOTextureError.GetTheTextureInARow(), this->m_VOAAMBasic->m_MatRc, -1, Mat(), 0.0, this->m_MatDeltaC, GEMM_2_T); + + // damp -- C + for(unsigned int i = 0; i < k_values.size(); i++) + { + // make damped c prediction + cv::scaleAdd(this->m_MatDeltaC, k_values[i], this->m_MatCurrentC, this->m_MatEstimatedC); + + // make sure m_MatEstimatedC are constrained + this->m_VOAAMBasic->VO_AppearanceParameterConstraint(this->m_MatEstimatedC); + this->VO_CParamTParam2FittingShape( this->m_MatEstimatedC, + this->m_MatCurrentT, + this->m_VOModelNormalizedTexture, + this->m_VOEstimatedShape, + estScale, + estRotateAngles, + estCOG); + if ( !VO_ShapeModel::VO_IsShapeInsideImage(this->m_VOEstimatedShape, this->m_ImageProcessing) ) + continue; + else + this->m_E = this->VO_CalcErrorImage(this->m_ImageProcessing, + this->m_VOEstimatedShape, + this->m_VOModelNormalizedTexture, + this->m_VOEstimatedTextureError); + + if (this->m_E < this->m_E_previous) + { + this->m_MatEstimatedC.copyTo(this->m_MatCurrentC); + this->m_VOFittingShape.clone(this->m_VOEstimatedShape); + this->m_VOTextureError.clone(this->m_VOEstimatedTextureError); + this->m_E_previous = this->m_E; + cBetter = true; + this->m_fScale = estScale; + this->m_vRotateAngles = estRotateAngles; + this->m_MatCenterOfGravity = estCOG.clone(); + break; + } + } + + /** Second pose, t parameters. refer to equation (9.3) + * Cootes "Statistical Model of Appearance for Computer Vision" */ + cv::gemm(this->m_VOTextureError.GetTheTextureInARow(), this->m_VOAAMBasic->m_MatRt, -1, Mat(), 0, this->m_MatDeltaT, GEMM_2_T); + + // damp -- T + for(unsigned int i = 0; i < k_values.size(); i++) + { + // make damped c/pose prediction + cv::scaleAdd(this->m_MatDeltaT, k_values[i], this->m_MatCurrentT, this->m_MatEstimatedT); + this->VO_CParamTParam2FittingShape( this->m_MatCurrentC, + this->m_MatEstimatedT, + this->m_VOModelNormalizedTexture, + this->m_VOEstimatedShape, + estScale, + estRotateAngles, + estCOG); + if ( !VO_ShapeModel::VO_IsShapeInsideImage(this->m_VOEstimatedShape, this->m_ImageProcessing) ) + continue; + else + this->m_E = this->VO_CalcErrorImage(this->m_ImageProcessing, + this->m_VOEstimatedShape, + this->m_VOModelNormalizedTexture, + this->m_VOEstimatedTextureError); + + if (this->m_E < this->m_E_previous) + { + // Since m_fScale, m_vRotateAngles and m_MatCenterOfGravity have been updated, + // m_MatCurrentT should be assigned to 0 now! + this->m_MatCurrentT = Mat_::zeros(this->m_MatCurrentT.size()); + // this->m_MatEstimatedT.copyTo(this->m_MatCurrentT); + this->m_VOFittingShape.clone(this->m_VOEstimatedShape); + this->m_VOTextureError.clone(this->m_VOEstimatedTextureError); + this->m_E_previous = this->m_E; + poseBetter = true; + this->m_fScale = estScale; + this->m_vRotateAngles = estRotateAngles; + this->m_MatCenterOfGravity = estCOG.clone(); + break; + } + } + + if( cBetter || poseBetter) + { + ioShape.clone(this->m_VOFittingShape); + } + else + break; + + ++this->m_iIteration; + + }while( ( fabs(this->m_E) > FLT_EPSILON ) && (this->m_iIteration < epoch)/* && (cv::norm(this->m_MatDeltaC) > FLT_EPSILON) */ ); + +t = ((double)cvGetTickCount() - t )/ (cvGetTickFrequency()*1000.); +cout << "Basic fitting time cost: " << t << " millisec" << endl; +this->m_fFittingTime = safeDoubleToFloat(t); + + VO_Fitting2DSM::VO_DrawMesh(ioShape, this->m_VOAAMBasic, oImg); + + return safeDoubleToFloat(t); +} + + +/** + * @author JIA Pei + * @version 2010-05-20 + * @brief Direct AAM Fitting, for static images, so that we record the whole fitting process + * @param iImg Input - image to be fitted + * @param epoch Input - the iteration epoch + * @param oImages Output - the fitted shape +*/ +float VO_FittingAAMBasic::VO_DirectAAMFitting(const Mat& iImg, + vector& oMeshInfo, + unsigned int epoch, + bool record) +{ +double t = (double)cvGetTickCount(); + +t = ((double)cvGetTickCount() - t )/ (cvGetTickFrequency()*1000.); +cout << "Direct fitting time cost: " << t << " millisec" << endl; + + return safeDoubleToFloat(t); +} + + +/** + * @author JIA Pei + * @version 2010-05-20 + * @brief Direct AAM Fitting, for dynamic image sequence + * @param iImg Input - image to be fitted + * @param ioShape Input and Output - the fitted shape + * @param oImg Output - the fitted image + * @param epoch Input - the iteration epoch +*/ +float VO_FittingAAMBasic::VO_DirectAAMFitting(const Mat& iImg, + VO_Shape& ioShape, + Mat& oImg, + unsigned int epoch) +{ +double t = (double)cvGetTickCount(); + +t = ((double)cvGetTickCount() - t )/ (cvGetTickFrequency()*1000.); +cout << "Direct fitting time cost: " << t << " millisec" << endl; + + return safeDoubleToFloat(t); +} + diff --git a/modules/smfitting/src/VO_FittingAAMForwardIA.cpp b/modules/smfitting/src/VO_FittingAAMForwardIA.cpp new file mode 100644 index 0000000..780661e --- /dev/null +++ b/modules/smfitting/src/VO_FittingAAMForwardIA.cpp @@ -0,0 +1,140 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include +#include +#include + +#include "VO_FittingAAMForwardIA.h" + + +/** Constructor */ +VO_FittingAAMForwardIA::VO_FittingAAMForwardIA() +{ + this->init(); +} + +/** Destructor */ +VO_FittingAAMForwardIA::~VO_FittingAAMForwardIA() +{ + if(this->m_VOAAMForwardIA) delete this->m_VOAAMForwardIA; this->m_VOAAMForwardIA = NULL; +} + + +/** Initialization */ +void VO_FittingAAMForwardIA::init() +{ + VO_Fitting2DSM::init(); + this->m_VOAAMForwardIA = new VO_AAMForwardIA(); + + string fn = "/home/jiapei/Desktop/forwardiaaamtime.txt"; +} + + +/** + * @author JIA Pei + * @version 2010-05-18 + * @brief Load all AAM data from a specified folder for later fitting, to member variable m_VOAAMForwardIA + * @param fd Input - the folder that AAM to be loaded from +*/ +void VO_FittingAAMForwardIA::VO_LoadParameters4Fitting(const string& fd) +{ + this->m_VOAAMForwardIA->VO_LoadParameters4Fitting(fd); + + // VO_Fitting2DSM + this->m_VOTemplateAlignedShape = this->m_VOAAMForwardIA->m_VOAlignedMeanShape; + this->m_VOTemplateNormalizedTexture = this->m_VOAAMForwardIA->m_VONormalizedMeanTexture; + this->m_vTriangle2D = this->m_VOAAMForwardIA->m_vNormalizedTriangle2D; + this->m_vShape2DInfo = this->m_VOAAMForwardIA->m_vShape2DInfo; + this->m_FaceParts = this->m_VOAAMForwardIA->m_FaceParts; + this->m_vPointWarpInfo = this->m_VOAAMForwardIA->m_vNormalizedPointWarpInfo; + + // VO_FittingAAMForwardIA +} + + +float VO_FittingAAMForwardIA::VO_FAIAAAMFitting(const Mat& iImg, + vector& oMeshInfo, + unsigned int epoch, + bool record) +{ +double t = (double)cvGetTickCount(); + +t = ((double)cvGetTickCount() - t )/ (cvGetTickFrequency()*1000.); +cout << "FAIA time cost: " << t << " millisec" << endl; + + return safeDoubleToFloat(t); +} + + +float VO_FittingAAMForwardIA::VO_FAIAAAMFitting(const Mat& iImg, + VO_Shape& ioShape, + Mat& oImg, + unsigned int epoch) +{ +double t = (double)cvGetTickCount(); + +t = ((double)cvGetTickCount() - t )/ (cvGetTickFrequency()*1000.); +cout << "FAIA fitting time cost: " << t << " millisec" << endl; +this->m_fFittingTime = safeDoubleToFloat(t); + + return safeDoubleToFloat(t); +} + diff --git a/modules/smfitting/src/VO_FittingAAMInverseIA.cpp b/modules/smfitting/src/VO_FittingAAMInverseIA.cpp new file mode 100644 index 0000000..40eaa29 --- /dev/null +++ b/modules/smfitting/src/VO_FittingAAMInverseIA.cpp @@ -0,0 +1,751 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include +#include +#include + +#include "VO_FittingAAMInverseIA.h" +#include "VO_AAMBasic.h" + + +/** Default Constructor */ +VO_FittingAAMInverseIA::VO_FittingAAMInverseIA() +{ + this->init(); +} + +/** Destructor */ +VO_FittingAAMInverseIA::~VO_FittingAAMInverseIA() +{ + if(this->m_VOAAMInverseIA) delete this->m_VOAAMInverseIA; this->m_VOAAMInverseIA = NULL; +} + +/** Initialization */ +void VO_FittingAAMInverseIA::init() +{ + VO_Fitting2DSM::init(); + this->m_VOAAMInverseIA = new VO_AAMInverseIA(); + this->m_E = 0.0f; + this->m_E_previous = 0.0f; +} + + +void VO_FittingAAMInverseIA::VO_PParamQParam2ModelAlignedShape( const Mat_& p, + const Mat_& q, + VO_Shape& oAlignedShape) +{ + // generate shape and texture from p parameters + this->m_VOAAMInverseIA->VO_SParamBackProjectToAlignedShape(p, oAlignedShape); + VO_Shape::GlobalShapeNormalization2D(oAlignedShape, oAlignedShape, q); +} + + +void VO_FittingAAMInverseIA::VO_PParamQParam2FittingShape( const Mat_& p, + const Mat_& q, + VO_Shape& oShape, + float& scale, + vector& rotateAngles, + Mat_& matCOG, + unsigned int mtd) +{ + switch(mtd) + { + case VO_Fitting2DSM::USEGLOBALSHAPENORMALIZATION: + { + this->VO_PParamQParam2ModelAlignedShape(p, q, this->m_VOModelAlignedShape); + VO_Shape::GlobalShapeNormalization2D(this->m_VOModelAlignedShape, oShape, scale, -rotateAngles, matCOG); + } + break; + case VO_Fitting2DSM::USESIMILARITYTRANSFORM: + default: + { + this->m_VOAAMInverseIA->VO_SParamBackProjectToAlignedShape(p, this->m_VOModelAlignedShape); + float updatescale = 1.0; + vector updateangles(1); + Mat_ updatetranslation = Mat_::zeros(2, 1); + VO_Shape::GlobalShapeNormalization2SimilarityTrans(q, updatescale, updateangles, updatetranslation ); + scale *= updatescale; + rotateAngles[0] = -rotateAngles[0]+updateangles[0]; + matCOG(0,0) += updatetranslation(0,0); + matCOG(1,0) += updatetranslation(1,0); + + // shape adjustment by t parameters - pose transform + VO_Shape::GlobalShapeNormalization2D(this->m_VOModelAlignedShape, oShape, scale, rotateAngles, matCOG); + } + break; + } +} + + +/** + * @author JIA Pei + * @version 2010-05-18 + * @brief Load all AAM data from a specified folder for later fitting, to member variable m_VOAAMInverseIA + * @param fd Input - the folder that AAM to be loaded from +*/ +void VO_FittingAAMInverseIA::VO_LoadParameters4Fitting(const string& fd) +{ + this->m_VOAAMInverseIA->VO_LoadParameters4Fitting(fd); + + // VO_Fitting2DSM + this->m_VOTemplateAlignedShape = this->m_VOAAMInverseIA->m_VOAlignedMeanShape; + this->m_VOTemplateNormalizedTexture = this->m_VOAAMInverseIA->m_VONormalizedMeanTexture; + this->m_vTriangle2D = this->m_VOAAMInverseIA->m_vNormalizedTriangle2D; + this->m_vShape2DInfo = this->m_VOAAMInverseIA->m_vShape2DInfo; + this->m_FaceParts = this->m_VOAAMInverseIA->m_FaceParts; + this->m_vPointWarpInfo = this->m_VOAAMInverseIA->m_vNormalizedPointWarpInfo; + + // VO_FittingAAMInverseIA + this->m_MatCurrentP = Mat_::zeros(1, this->m_VOAAMInverseIA->m_iNbOfShapeEigens); + this->m_MatEstimatedP = Mat_::zeros(1, this->m_VOAAMInverseIA->m_iNbOfShapeEigens); + this->m_MatDeltaP = Mat_::zeros(1, this->m_VOAAMInverseIA->m_iNbOfShapeEigens); + this->m_MatCurrentQ = Mat_::zeros(1, 4); + this->m_MatEstimatedQ = Mat_::zeros(1, 4); + this->m_MatDeltaQ = Mat_::zeros(1, 4); + this->m_MatDeltaPQ = Mat_::zeros(1, this->m_VOAAMInverseIA->m_iNbOfShapeEigens+4); +} + + +/** + * @author JIA Pei + * @version 2010-05-20 + * @brief Additive ICIA AAM Fitting, for static images, so that we record the whole fitting process + * @param iImg Input - image to be fitted + * @param oImages Output - the fitted shape + * @param epoch Input - the iteration epoch + * @note Refer to "AAM Revisited, page 34, figure 13", particularly, those steps. +*/ +float VO_FittingAAMInverseIA::VO_IAIAAAMFitting(const Mat& iImg, + vector& oMeshInfo, + unsigned int epoch, + bool record) +{ + this->m_VOEstimatedShape.clone(this->m_VOFittingShape); +double t = (double)cvGetTickCount(); + + this->SetProcessingImage(iImg, this->m_VOAAMInverseIA); + + this->m_iIteration = 0; +if(record) +{ + oMeshInfo.push_back(DrawMeshInfo(1,1,this->m_VOFittingShape,this->m_VOAAMInverseIA)); +} + + // Get m_MatModelAlignedShapeParam and m_fScale, m_vRotateAngles, m_MatCenterOfGravity + this->m_VOAAMInverseIA->VO_CalcAllParams4AnyShapeWithConstrain(this->m_VOFittingShape, + this->m_MatCurrentP, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); +if(record) +{ + oMeshInfo.push_back(DrawMeshInfo(1,1,this->m_VOFittingShape,this->m_VOAAMInverseIA)); +} + + // Set m_MatEstimatedP, m_MatDeltaP, m_MatCurrentQ, m_MatEstimatedQ, m_MatDeltaQ, m_MatDeltaPQ, etc. + this->m_MatEstimatedP = Mat_::zeros(this->m_MatEstimatedP.size()); + this->m_MatDeltaP = Mat_::zeros(this->m_MatDeltaP.size()); + this->m_MatEstimatedQ = Mat_::zeros(this->m_MatEstimatedQ.size()); + this->m_MatCurrentQ = Mat_::zeros(this->m_MatCurrentQ.size()); + this->m_MatDeltaQ = Mat_::zeros(this->m_MatDeltaQ.size()); + this->m_MatDeltaPQ = Mat_::zeros(this->m_MatDeltaPQ.size()); + + // Step (1) Warp I with W(x;p) followed by N(x;q) to compute I(N(W(x;p);q)) + this->VO_PParamQParam2FittingShape( this->m_MatCurrentP, + this->m_MatCurrentQ, + this->m_VOFittingShape, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity ); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); +if(record) +{ + oMeshInfo.push_back(DrawMeshInfo(1,1,this->m_VOFittingShape,this->m_VOAAMInverseIA)); +} + + // Step (2) Compute the error image I(N(W(x;p);q))-A0(x) + this->m_E_previous = this->m_E = this->VO_CalcErrorImage(this->m_ImageProcessing, + this->m_VOFittingShape, + this->m_VOTemplateNormalizedTexture, + this->m_VOTextureError); + + do + { + ++this->m_iIteration; + float estScale = this->m_fScale; + vector estRotateAngles = this->m_vRotateAngles; + Mat_ estCOG = this->m_MatCenterOfGravity.clone(); + + // Step (7) -- a bit modification + cv::gemm(this->m_VOTextureError.GetTheTextureInARow(), this->m_VOAAMInverseIA->m_MatICIAPreMatrix, -1, Mat(), 0, this->m_MatDeltaPQ, GEMM_2_T); + + // Step (8) -- a bit modification. Get DeltaP DeltaQ respectively + this->m_MatDeltaQ = this->m_MatDeltaPQ(Rect( 0, 0, this->m_MatDeltaQ.cols, 1)); + this->m_MatDeltaP = this->m_MatDeltaPQ(Rect( this->m_MatDeltaQ.cols, 0, this->m_MatDeltaP.cols, 1)); + + // Step (9) -- Additive + cv::add(this->m_MatCurrentQ, this->m_MatDeltaQ, this->m_MatEstimatedQ); + cv::add(this->m_MatCurrentP, this->m_MatDeltaP, this->m_MatEstimatedP); + + this->m_VOAAMInverseIA->VO_ShapeParameterConstraint(this->m_MatEstimatedP); + this->VO_PParamQParam2FittingShape( this->m_MatEstimatedP, + this->m_MatEstimatedQ, + this->m_VOEstimatedShape, + estScale, + estRotateAngles, + estCOG ); + this->m_VOEstimatedShape.ConstrainShapeInImage(this->m_ImageProcessing); + this->m_E = this->VO_CalcErrorImage(this->m_ImageProcessing, + this->m_VOEstimatedShape, + this->m_VOTemplateNormalizedTexture, + this->m_VOEstimatedTextureError); + + if (this->m_E < this->m_E_previous) + { + // Unlike what's happening in Basic AAM, + // since m_fScale, m_vRotateAngles and m_MatCenterOfGravity have not been updated in ICIA, + // m_MatCurrentT should not be assigned to 0 now! +// this->m_MatCurrentQ = Mat_::zeros(this->m_MatCurrentQ.size()); + this->m_MatEstimatedQ.copyTo(this->m_MatCurrentQ); + this->m_MatEstimatedP.copyTo(this->m_MatCurrentP); + this->m_VOFittingShape.clone(this->m_VOEstimatedShape); + this->m_VOTextureError.clone(this->m_VOEstimatedTextureError); + this->m_E_previous = this->m_E; +if(record) +{ + oMeshInfo.push_back(DrawMeshInfo(1,1,this->m_VOFittingShape,this->m_VOAAMInverseIA)); +} + } + else + break; + + }while( ( fabs(this->m_E) > FLT_EPSILON ) && ( this->m_iIteration < epoch ) ); + + // Recalculate all parameters finally, this is also optional. + this->m_VOAAMInverseIA->VO_CalcAllParams4AnyShapeWithConstrain( this->m_VOFittingShape, + this->m_MatCurrentP, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity ); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); + + // Step (10) (Option step), Post-computation. Get m_MatModelNormalizedTextureParam + VO_TextureModel::VO_LoadOneTextureFromShape(this->m_VOFittingShape, + this->m_ImageProcessing, + this->m_vTriangle2D, + this->m_vPointWarpInfo, + this->m_VOFittingTexture ); + // estimate the texture model parameters + this->m_VOAAMInverseIA->VO_CalcAllParams4AnyTexture(this->m_VOFittingTexture, this->m_MatModelNormalizedTextureParam); + +t = ((double)cvGetTickCount() - t )/ (cvGetTickFrequency()*1000.); +cout << "IAIA AAM fitting time cost: " << t << " millisec" << endl; + + return safeDoubleToFloat(t); +} + + +/** + * @author JIA Pei + * @version 2010-05-20 + * @brief Additive ICIA AAM Fitting, for dynamic image sequence + * @param iImg Input - image to be fitted + * @param ioShape Input and Output - the fitted shape + * @param oImg Output - the fitted image + * @param epoch Input - the iteration epoch +*/ +float VO_FittingAAMInverseIA::VO_IAIAAAMFitting(const Mat& iImg, + VO_Shape& ioShape, + Mat& oImg, + unsigned int epoch) +{ + this->m_VOFittingShape.clone(ioShape); + this->m_VOEstimatedShape.clone(this->m_VOFittingShape); +double t = (double)cvGetTickCount(); + + this->SetProcessingImage(iImg, this->m_VOAAMInverseIA); + this->m_iIteration = 0; + + // Get m_MatModelAlignedShapeParam and m_fScale, m_vRotateAngles, m_MatCenterOfGravity + this->m_VOAAMInverseIA->VO_CalcAllParams4AnyShapeWithConstrain( this->m_VOFittingShape, + this->m_MatCurrentP, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); + + // Set m_MatEstimatedP, m_MatDeltaP, m_MatCurrentQ, m_MatEstimatedQ, m_MatDeltaQ, m_MatDeltaPQ, etc. + this->m_MatEstimatedP = Mat_::zeros(this->m_MatEstimatedP.size()); + this->m_MatDeltaP = Mat_::zeros(this->m_MatDeltaP.size()); + this->m_MatEstimatedQ = Mat_::zeros(this->m_MatEstimatedQ.size()); + this->m_MatCurrentQ = Mat_::zeros(this->m_MatCurrentQ.size()); + this->m_MatDeltaQ = Mat_::zeros(this->m_MatDeltaQ.size()); + this->m_MatDeltaPQ = Mat_::zeros(this->m_MatDeltaPQ.size()); + + // Step (1) Warp I with W(x;p) followed by N(x;q) to compute I(N(W(x;p);q)) + this->VO_PParamQParam2FittingShape( this->m_MatCurrentP, + this->m_MatCurrentQ, + this->m_VOFittingShape, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity ); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); + + // Step (2) Compute the error image I(N(W(x;p);q))-A0(x) + this->m_E_previous = this->m_E = this->VO_CalcErrorImage( this->m_ImageProcessing, + this->m_VOFittingShape, + this->m_VOTemplateNormalizedTexture, + this->m_VOTextureError); + + do + { + ++this->m_iIteration; + float estScale = this->m_fScale; + vector estRotateAngles = this->m_vRotateAngles; + Mat_ estCOG = this->m_MatCenterOfGravity.clone(); + + // Step (7) -- a bit modification + cv::gemm(this->m_VOTextureError.GetTheTextureInARow(), this->m_VOAAMInverseIA->m_MatICIAPreMatrix, -1, Mat(), 0, this->m_MatDeltaPQ, GEMM_2_T); + + // Step (8) -- a bit modification. Get DeltaP DeltaQ respectively + this->m_MatDeltaQ = this->m_MatDeltaPQ(Rect( 0, 0, this->m_MatDeltaQ.cols, 1)); + this->m_MatDeltaP = this->m_MatDeltaPQ(Rect( this->m_MatDeltaQ.cols, 0, this->m_MatDeltaP.cols, 1)); + + // Step (9) -- Additive + cv::add(this->m_MatCurrentQ, this->m_MatDeltaQ, this->m_MatEstimatedQ); + cv::add(this->m_MatCurrentP, this->m_MatDeltaP, this->m_MatEstimatedP); + + this->m_VOAAMInverseIA->VO_ShapeParameterConstraint(this->m_MatEstimatedP); + this->VO_PParamQParam2FittingShape( this->m_MatEstimatedP, + this->m_MatEstimatedQ, + this->m_VOEstimatedShape, + estScale, + estRotateAngles, + estCOG ); + this->m_VOEstimatedShape.ConstrainShapeInImage(this->m_ImageProcessing); + this->m_E = this->VO_CalcErrorImage(this->m_ImageProcessing, + this->m_VOEstimatedShape, + this->m_VOTemplateNormalizedTexture, + this->m_VOEstimatedTextureError); + + if (this->m_E < this->m_E_previous) + { + // Unlike what's happening in Basic AAM, + // since m_fScale, m_vRotateAngles and m_MatCenterOfGravity have not been updated in ICIA, + // m_MatCurrentT should not be assigned to 0 now! +// this->m_MatCurrentQ = Mat_::zeros(this->m_MatCurrentQ.size()); + this->m_MatEstimatedQ.copyTo(this->m_MatCurrentQ); + this->m_MatEstimatedP.copyTo(this->m_MatCurrentP); + this->m_VOFittingShape.clone(this->m_VOEstimatedShape); + this->m_VOTextureError.clone(this->m_VOEstimatedTextureError); + this->m_E_previous = this->m_E; + } + else + break; + + }while( ( fabs(this->m_E) > FLT_EPSILON ) && ( this->m_iIteration < epoch ) ); + + VO_Fitting2DSM::VO_DrawMesh(this->m_VOFittingShape, this->m_VOAAMInverseIA, oImg); + + // Recalculate all parameters finally, this is also optional. + this->m_VOAAMInverseIA->VO_CalcAllParams4AnyShapeWithConstrain( this->m_VOFittingShape, + this->m_MatCurrentP, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity ); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); + + // Step (10) (Option step), Post-computation. Get m_MatModelNormalizedTextureParam + VO_TextureModel::VO_LoadOneTextureFromShape(this->m_VOFittingShape, + this->m_ImageProcessing, + this->m_vTriangle2D, + this->m_vPointWarpInfo, + this->m_VOFittingTexture ); + // estimate the texture model parameters + this->m_VOAAMInverseIA->VO_CalcAllParams4AnyTexture(this->m_VOFittingTexture, this->m_MatModelNormalizedTextureParam); + + ioShape.clone(this->m_VOFittingShape); + +t = ((double)cvGetTickCount() - t )/ (cvGetTickFrequency()*1000.); +cout << "IAIA AAM fitting time cost: " << t << " millisec" << endl; +this->m_fFittingTime = safeDoubleToFloat(t); + + return safeDoubleToFloat(t); +} + + +/** + * @author JIA Pei + * @version 2010-05-20 + * @brief CMU ICIA AAM Fitting, for static images, so that we record the whole fitting process + * @param iImg Input - image to be fitted + * @param oImages Output - the fitted shape + * @param epoch Input - the iteration epoch +*/ +float VO_FittingAAMInverseIA::VO_ICIAAAMFitting(const Mat& iImg, + vector& oMeshInfo, + unsigned int epoch, + bool record) +{ + this->m_VOEstimatedShape.clone(this->m_VOFittingShape); +double t = (double)cvGetTickCount(); + + this->SetProcessingImage(iImg, this->m_VOAAMInverseIA); + this->m_iIteration = 0; + +if(record) +{ + oMeshInfo.push_back(DrawMeshInfo(1,1,this->m_VOFittingShape,this->m_VOAAMInverseIA)); +} + + // Get m_MatCurrentP and m_MatCurrentQ + this->m_VOAAMInverseIA->VO_CalcAllParams4AnyShapeWithConstrain(this->m_VOFittingShape, + this->m_MatCurrentP, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); +if(record) +{ + oMeshInfo.push_back(DrawMeshInfo(1,1,this->m_VOFittingShape,this->m_VOAAMInverseIA)); +} + + this->m_MatDeltaP = Mat_::zeros(this->m_MatDeltaP.size()); + this->m_MatDeltaQ = Mat_::zeros(this->m_MatDeltaQ.size()); + this->m_MatCurrentQ = Mat_::zeros(this->m_MatCurrentQ.size()); + this->m_MatDeltaPQ = Mat_::zeros(this->m_MatDeltaPQ.size()); + + // Step (1) Warp I with W(x;p) followed by N(x;q) to compute I(N(W(x;p);q)) + this->VO_PParamQParam2FittingShape( this->m_MatCurrentP, + this->m_MatCurrentQ, + this->m_VOFittingShape, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity ); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); +if(record) +{ + oMeshInfo.push_back(DrawMeshInfo(1,1,this->m_VOFittingShape,this->m_VOAAMInverseIA)); +} + + // Step (2) Compute the error image I(N(W(x;p);q))-A0(x) + this->m_E_previous = this->m_E = this->VO_CalcErrorImage(this->m_ImageProcessing, + this->m_VOFittingShape, + this->m_VOTemplateNormalizedTexture, + this->m_VOTextureError); + + do + { + ++this->m_iIteration; + + // Step (7) -- a bit modification + cv::gemm(this->m_VOTextureError.GetTheTextureInARow(), this->m_VOAAMInverseIA->m_MatICIAPreMatrix, -1, Mat(), 0, this->m_MatDeltaPQ, GEMM_2_T); + + // Step (8) -- a bit modification. Get DeltaP DeltaQ respectively + this->m_MatDeltaQ = this->m_MatDeltaPQ(Rect( 0, 0, this->m_MatDeltaQ.cols, 1)); + this->m_MatDeltaP = this->m_MatDeltaPQ(Rect( this->m_MatDeltaQ.cols, 0, this->m_MatDeltaP.cols, 1)); + + // Step (9) -- CMU Inverse Compositional + this->VO_CMUInverseCompositional( this->m_MatDeltaP, this->m_MatDeltaQ, this->m_VOFittingShape, this->m_VOEstimatedShape ); + + // Ensure Inverse Compositional still satisfies global shape constraints + this->m_VOAAMInverseIA->VO_CalcAllParams4AnyShapeWithConstrain( this->m_VOEstimatedShape, + this->m_MatEstimatedP, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity); + this->m_VOEstimatedShape.ConstrainShapeInImage(this->m_ImageProcessing); + + this->m_E = this->VO_CalcErrorImage(this->m_ImageProcessing, + this->m_VOEstimatedShape, + this->m_VOTemplateNormalizedTexture, + this->m_VOEstimatedTextureError); + + if (this->m_E < this->m_E_previous) + { + // Unlike what's happening in Basic AAM, + // since m_fScale, m_vRotateAngles and m_MatCenterOfGravity have not been updated in ICIA, + // m_MatCurrentT should not be assigned to 0 now! +// this->m_MatCurrentQ = Mat_::zeros(this->m_MatCurrentQ.size()); + this->m_VOFittingShape.clone(this->m_VOEstimatedShape); + this->m_VOAAMInverseIA->VO_CalcAllParams4AnyShapeWithConstrain( this->m_VOFittingShape, + this->m_MatCurrentP, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); + this->m_VOTextureError.clone(this->m_VOEstimatedTextureError); + this->m_E_previous = this->m_E; +if(record) +{ + oMeshInfo.push_back(DrawMeshInfo(1,1,this->m_VOFittingShape,this->m_VOAAMInverseIA)); +} + } + else + break; + + }while( ( fabs(this->m_E) > FLT_EPSILON ) && ( this->m_iIteration < epoch ) ); + + // Recalculate all parameters finally, this is also optional. + this->m_VOAAMInverseIA->VO_CalcAllParams4AnyShapeWithConstrain( this->m_VOFittingShape, + this->m_MatCurrentP, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity ); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); + + // Step (10) (Option step), Post-computation. Get m_MatModelNormalizedTextureParam + VO_TextureModel::VO_LoadOneTextureFromShape(this->m_VOFittingShape, + this->m_ImageProcessing, + this->m_vTriangle2D, + this->m_vPointWarpInfo, + this->m_VOFittingTexture ); + // estimate the texture model parameters + this->m_VOAAMInverseIA->VO_CalcAllParams4AnyTexture(this->m_VOFittingTexture, this->m_MatModelNormalizedTextureParam); + +t = ((double)cvGetTickCount() - t )/ (cvGetTickFrequency()*1000.); +cout << "ICIA AAM fitting time cost: " << t << " millisec" << endl; + + return safeDoubleToFloat(t); +} + + +/** + * @author JIA Pei + * @version 2010-05-20 + * @brief CMU ICIA AAM Fitting, for dynamic image sequence + * @param iImg Input - image to be fitted + * @param ioShape Input and Output - the fitted shape + * @param oImg Output - the fitted image + * @param epoch Input - the iteration epoch +*/ +float VO_FittingAAMInverseIA::VO_ICIAAAMFitting(const Mat& iImg, + VO_Shape& ioShape, + Mat& oImg, + unsigned int epoch) +{ + this->m_VOFittingShape.clone(ioShape); + this->m_VOEstimatedShape.clone(this->m_VOFittingShape); +double t = (double)cvGetTickCount(); + + this->SetProcessingImage(iImg, this->m_VOAAMInverseIA); + this->m_iIteration = 0; + + // Get m_MatCurrentP and m_MatCurrentQ + this->m_VOAAMInverseIA->VO_CalcAllParams4AnyShapeWithConstrain( this->m_VOFittingShape, + this->m_MatCurrentP, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); + + this->m_MatDeltaP = Mat_::zeros(this->m_MatDeltaP.size()); + this->m_MatDeltaQ = Mat_::zeros(this->m_MatDeltaQ.size()); + this->m_MatCurrentQ = Mat_::zeros(this->m_MatCurrentQ.size()); + this->m_MatDeltaPQ = Mat_::zeros(this->m_MatDeltaPQ.size()); + + // Step (1) Warp I with W(x;p) followed by N(x;q) to compute I(N(W(x;p);q)) + this->VO_PParamQParam2FittingShape( this->m_MatCurrentP, + this->m_MatCurrentQ, + this->m_VOFittingShape, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity ); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); + + // Step (2) Compute the error image I(N(W(x;p);q))-A0(x) + this->m_E_previous = this->m_E = this->VO_CalcErrorImage(this->m_ImageProcessing, + this->m_VOFittingShape, + this->m_VOTemplateNormalizedTexture, + this->m_VOTextureError); + + do + { + ++this->m_iIteration; + + // Step (7) -- a bit modification + cv::gemm(this->m_VOTextureError.GetTheTextureInARow(), this->m_VOAAMInverseIA->m_MatICIAPreMatrix, -1, Mat(), 0, this->m_MatDeltaPQ, GEMM_2_T); + + // Step (8) -- a bit modification. Get DeltaP DeltaQ respectively + this->m_MatDeltaQ = this->m_MatDeltaPQ(Rect( 0, 0, this->m_MatDeltaQ.cols, 1)); + this->m_MatDeltaP = this->m_MatDeltaPQ(Rect( this->m_MatDeltaQ.cols, 0, this->m_MatDeltaP.cols, 1)); + + // Step (9) -- CMU Inverse Compositional + this->VO_CMUInverseCompositional( this->m_MatDeltaP, this->m_MatDeltaQ, this->m_VOFittingShape, this->m_VOEstimatedShape ); + + // Ensure Inverse Compositional still satisfies global shape constraints + this->m_VOAAMInverseIA->VO_CalcAllParams4AnyShapeWithConstrain( this->m_VOEstimatedShape, + this->m_MatEstimatedP, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity); + this->m_VOEstimatedShape.ConstrainShapeInImage(this->m_ImageProcessing); + + this->m_E = this->VO_CalcErrorImage(this->m_ImageProcessing, + this->m_VOEstimatedShape, + this->m_VOTemplateNormalizedTexture, + this->m_VOEstimatedTextureError); + + if (this->m_E < this->m_E_previous) + { + // Unlike what's happening in Basic AAM, + // since m_fScale, m_vRotateAngles and m_MatCenterOfGravity have not been updated in ICIA, + // m_MatCurrentT should not be assigned to 0 now! +// this->m_MatCurrentQ = Mat_::zeros(this->m_MatCurrentQ.size()); + this->m_VOFittingShape.clone(this->m_VOEstimatedShape); + this->m_VOAAMInverseIA->VO_CalcAllParams4AnyShapeWithConstrain( this->m_VOFittingShape, + this->m_MatCurrentP, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); + this->m_VOTextureError.clone(this->m_VOEstimatedTextureError); + this->m_E_previous = this->m_E; + + } + else + break; + + }while( ( fabs(this->m_E) > FLT_EPSILON ) && ( this->m_iIteration < epoch ) ); + + VO_Fitting2DSM::VO_DrawMesh(this->m_VOFittingShape, this->m_VOAAMInverseIA, oImg); + + // Recalculate all parameters finally, this is also optional. + this->m_VOAAMInverseIA->VO_CalcAllParams4AnyShapeWithConstrain( this->m_VOFittingShape, + this->m_MatCurrentP, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity ); + + // Step (10) (Option step), Post-computation. Get m_MatModelNormalizedTextureParam + VO_TextureModel::VO_LoadOneTextureFromShape(this->m_VOFittingShape, + this->m_ImageProcessing, + this->m_vTriangle2D, + this->m_vPointWarpInfo, + this->m_VOFittingTexture ); + // estimate the texture model parameters + this->m_VOAAMInverseIA->VO_CalcAllParams4AnyTexture(this->m_VOFittingTexture, this->m_MatModelNormalizedTextureParam); + ioShape.clone(this->m_VOFittingShape); + +t = ((double)cvGetTickCount() - t )/ (cvGetTickFrequency()*1000.); +cout << "ICIA AAM fitting time cost: " << t << " millisec" << endl; +this->m_fFittingTime = safeDoubleToFloat(t); + + return safeDoubleToFloat(t); +} + + +/** + * @author Yao Wei + * @brief CMU Inverse Compositional !! + * @param - matDeltaP Input -- deltap + * @param - matDeltaQ Input -- deltaq + * @param - s Input -- the shape + * @param - estShape Output -- newly estimated shape by Inverse compositional + */ +void VO_FittingAAMInverseIA::VO_CMUInverseCompositional(const Mat_& matDeltaP, + const Mat_& matDeltaQ, + const VO_Shape& s, + VO_Shape& estShape) +{ + VO_Shape S0; + this->VO_PParamQParam2ModelAlignedShape( matDeltaP, matDeltaQ, S0); +// cvConvertScale(dpq, __inv_pq, -1); +// __shape.CalcShape(__inv_pq, __update_s0); // __update_s0 = N.W(s0, -delta_p, -delta_q) + + //Secondly: Composing the Incremental Warp with the Current Warp Estimate. + Point2f res, tmp; + int count = 0; + vector vertexIdxes; + + for(unsigned int i = 0; i < this->m_VOAAMInverseIA->m_iNbOfPoints; i++) + { + res.x = 0.0; res.y = 0.0; + count = 0; + //The only problem with this approach is which triangle do we use? + //In general there will be several triangles that share the i-th vertex. + for(unsigned j = 0; j < this->m_VOAAMInverseIA->m_iNbOfTriangles; j++) // see Figure (11) + { + if ( this->m_vTriangle2D[j].HasNode(i) ) + { + vertexIdxes = this->m_vTriangle2D[j].GetVertexIndexes(); + + VO_WarpingPoint::WarpOnePoint( S0.GetA2DPoint(i), + this->m_vTriangle2D[j], + tmp, + s.GetA2DPoint(vertexIdxes[0]), + s.GetA2DPoint(vertexIdxes[1]), + s.GetA2DPoint(vertexIdxes[2]) ); + res.x += tmp.x; + res.y += tmp.y; + count++; + } + } + // average the result so as to smooth the warp at each vertex + if(count == 0) + cerr << "There must be something wrong when CMU Inverse Compositional !" << endl; + res.x /= count; + res.y /= count; + estShape.SetA2DPoint(res, i); + } +} + diff --git a/modules/smfitting/src/VO_FittingAFM.cpp b/modules/smfitting/src/VO_FittingAFM.cpp new file mode 100644 index 0000000..b24a769 --- /dev/null +++ b/modules/smfitting/src/VO_FittingAFM.cpp @@ -0,0 +1,152 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include +#include +#include + +#include "VO_FittingAFM.h" + + +/** Constructor */ +VO_FittingAFM::VO_FittingAFM() +{ + this->init(); +} + + +/** Destructor */ +VO_FittingAFM::~VO_FittingAFM() +{ + if(this->m_VOAFM) delete this->m_VOAFM; this->m_VOAFM = NULL; +} + + +/** Initialization */ +void VO_FittingAFM::init() +{ + VO_Fitting2DSM::init(); + this->m_VOAFM = new VO_AFM(); + + string fn = "/home/jiapei/Desktop/forwardiaaamtime.txt"; +} + + +/** Load ICIA AAM fitting training results */ +void VO_FittingAFM::VO_LoadParameters4Fitting(const string& fd) +{ + this->m_VOAFM->VO_LoadParameters4Fitting(fd); + + // VO_Fitting2DSM + this->m_VOTemplateAlignedShape = this->m_VOAFM->m_VOAlignedMeanShape; + this->m_VOTemplateNormalizedTexture = this->m_VOAFM->m_VONormalizedMeanTexture; + this->m_vTriangle2D = this->m_VOAFM->m_vNormalizedTriangle2D; + this->m_vShape2DInfo = this->m_VOAFM->m_vShape2DInfo; + this->m_FaceParts = this->m_VOAFM->m_FaceParts; + this->m_vPointWarpInfo = this->m_VOAFM->m_vNormalizedPointWarpInfo; +} + + +float VO_FittingAFM::VO_AFMFitting( const Mat& iImg, + vector& oMeshInfo, + unsigned int afmType, + unsigned int epoch, + bool record) +{ +double t = (double)cvGetTickCount(); + + switch (afmType) + { + case VO_AXM::AFM: + break; + case VO_AXM::CLM: + break; + } + +t = ((double)cvGetTickCount() - t )/ (cvGetTickFrequency()*1000.); +cout << "Basic fitting time cost: " << t << " millisec" << endl; + + return safeDoubleToFloat(t); +} + + +float VO_FittingAFM::VO_AFMFitting( const Mat& iImg, + VO_Shape& ioShape, + Mat& oImg, + unsigned int afmType, + unsigned int epoch) +{ +double t = (double)cvGetTickCount(); + + switch (afmType) + { + case VO_AXM::AFM: + break; + case VO_AXM::CLM: + break; + } + +t = ((double)cvGetTickCount() - t )/ (cvGetTickFrequency()*1000.); +cout << "Basic fitting time cost: " << t << " millisec" << endl; +this->m_fFittingTime = safeDoubleToFloat(t); + + return safeDoubleToFloat(t); +} + diff --git a/modules/smfitting/src/VO_FittingASMLTCs.cpp b/modules/smfitting/src/VO_FittingASMLTCs.cpp new file mode 100644 index 0000000..9414f72 --- /dev/null +++ b/modules/smfitting/src/VO_FittingASMLTCs.cpp @@ -0,0 +1,530 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include +#include +#include + +#include "VO_FittingASMLTCs.h" + + +/** Constructor */ +VO_FittingASMLTCs::VO_FittingASMLTCs() +{ + this->init(); +} + + +/** Destructor */ +VO_FittingASMLTCs::~VO_FittingASMLTCs() +{ + if(this->m_VOASMLTC) delete this->m_VOASMLTC; this->m_VOASMLTC = NULL; +} + + +/** Initialization */ +void VO_FittingASMLTCs::init() +{ + VO_Fitting2DSM::init(); + this->m_VOASMLTC = new VO_ASMLTCs(); + this->m_iFittingMethod = VO_AXM::ASM_LTC; +} + + +/** + * @author JIA Pei + * @version 2010-05-18 + * @brief Load all AAM data from a specified folder for later fitting, to member variable m_VOASMLTC + * @param fd Input - the folder that AAM to be loaded from + */ +void VO_FittingASMLTCs::VO_LoadParameters4Fitting(const string& fd) +{ + this->m_VOASMLTC->VO_LoadParameters4Fitting(fd); + + // VO_Fitting2DSM + this->m_VOTemplateAlignedShape = this->m_VOASMLTC->m_VOAlignedMeanShape; + this->m_VOTemplateNormalizedTexture = this->m_VOASMLTC->m_VONormalizedMeanTexture; + this->m_vTriangle2D = this->m_VOASMLTC->m_vNormalizedTriangle2D; + this->m_vShape2DInfo = this->m_VOASMLTC->m_vShape2DInfo; + this->m_FaceParts = this->m_VOASMLTC->m_FaceParts; + this->m_vPointWarpInfo = this->m_VOASMLTC->m_vNormalizedPointWarpInfo; +} + +/** + * @author JIA Pei + * @version 2010-05-20 + * @brief Additive ASM ND Profiles Fitting, for static images, so that we record the whole fitting process + * @param iImg Input - image to be fitted + * @param oImages Output - the fitting process + * @param trmethod Input - texture representation method + * @param epoch Input - the iteration epoch + * @param pyramidlevel Input - pyramid level, 1, 2, 3 or 4 at most + * @note Refer to "AAM Revisited, page 34, figure 13", particularly, those steps. +*/ +float VO_FittingASMLTCs::VO_ASMLTCFitting( const Mat& iImg, + vector& oMeshInfo, + unsigned int trmethod, + unsigned int epoch, + unsigned int pyramidlevel, + bool record) +{ +double t = (double)cvGetTickCount(); + + this->m_iNbOfPyramidLevels = pyramidlevel; + this->SetProcessingImage(iImg, this->m_VOASMLTC); + this->m_iIteration = 0; + + if(record) + { + oMeshInfo.push_back(DrawMeshInfo(1,1,this->m_VOFittingShape,this->m_VOASMLTC)); + } + + // Get m_MatModelAlignedShapeParam and m_fScale, m_vRotateAngles, m_MatCenterOfGravity + this->m_VOASMLTC->VO_CalcAllParams4AnyShapeWithConstrain( this->m_VOFittingShape, + this->m_MatModelAlignedShapeParam, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity ); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); + if(record) + { + oMeshInfo.push_back(DrawMeshInfo(1,1,this->m_VOFittingShape,this->m_VOASMLTC)); + } + + // Explained by YAO Wei, 2008-2-9. + // Scale this->m_VOFittingShape, so face width is a constant StdFaceWidth. + //this->m_fScale2 = this->m_VOASMLTC->m_VOReferenceShape.GetWidth() / this->m_VOFittingShape.GetWidth(); + this->m_fScale2 = this->m_VOASMLTC->m_VOReferenceShape.GetCentralizedShapeSize() / this->m_VOFittingShape.GetCentralizedShapeSize(); + this->m_VOFittingShape *= this->m_fScale2; + + int w = (int)(iImg.cols*this->m_fScale2); + int h = (int)(iImg.rows*this->m_fScale2); + Mat SearchImage = Mat(Size( w, h ), this->m_ImageProcessing.type(), this->m_ImageProcessing.channels()); + + float PyrScale = pow(2.0f, (float) (this->m_iNbOfPyramidLevels-1.0f) ); + this->m_VOFittingShape /= PyrScale; + + const int nQualifyingDisplacements = (int)(this->m_VOASMLTC->m_iNbOfPoints * VO_Fitting2DSM::pClose); + + // for each level in the image pyramid + for (int iLev = this->m_iNbOfPyramidLevels-1; iLev >= 0; iLev--) + { + // Set image roi, instead of cvCreateImage a new image to speed up + Mat siROI = SearchImage(Rect(0, 0, (int)(w/PyrScale), (int)(h/PyrScale) ) ); + cv::resize(this->m_ImageProcessing, siROI, siROI.size()); + + if(record) + { + oMeshInfo.push_back(DrawMeshInfo(PyrScale,this->m_fScale2,this->m_VOFittingShape,this->m_VOASMLTC)); + } + + this->m_VOEstimatedShape = this->m_VOFittingShape; + this->PyramidFit( this->m_VOEstimatedShape, + SearchImage, + oMeshInfo, + iLev, + VO_Fitting2DSM::pClose, + epoch, + record); + this->m_VOFittingShape = this->m_VOEstimatedShape; + + if (iLev != 0) + { + PyrScale /= 2.0f; + this->m_VOFittingShape *= 2.0f; + } + } + + // Explained by YAO Wei, 2008-02-09. + // this->m_fScale2 back to original size + this->m_VOFittingShape /= this->m_fScale2; + +t = ((double)cvGetTickCount() - t )/ (cvGetTickFrequency()*1000.); +printf("MRASM fitting time cost: %.2f millisec\n", t); + + return safeDoubleToFloat(t); +} + + +/** + * @author JIA Pei + * @version 2010-05-20 + * @brief Additive LTC ASM Fitting, for dynamic image sequence + * @param iImg Input - image to be fitted + * @param ioShape Input and output - the shape + * @param oImg Output - the fitted image + * @param trmethod Input - texture representation method + * @param epoch Input - the iteration epoch + * @param pyramidlevel Input - pyramid level, 1, 2, 3 or 4 at most + * @note Refer to "AAM Revisited, page 34, figure 13", particularly, those steps. +*/ +float VO_FittingASMLTCs::VO_ASMLTCFitting( const Mat& iImg, + VO_Shape& ioShape, + Mat& oImg, + unsigned int trmethod, + unsigned int epoch, + unsigned int pyramidlevel) +{ + this->m_VOFittingShape.clone(ioShape); +double t = (double)cvGetTickCount(); + + this->m_iNbOfPyramidLevels = pyramidlevel; + this->SetProcessingImage(iImg, this->m_VOASMLTC); + this->m_iIteration = 0; + + // Get m_MatModelAlignedShapeParam and m_fScale, m_vRotateAngles, m_MatCenterOfGravity + this->m_VOASMLTC->VO_CalcAllParams4AnyShapeWithConstrain( this->m_VOFittingShape, + this->m_MatModelAlignedShapeParam, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); + + // Explained by YAO Wei, 2008-2-9. + // Scale this->m_VOShape, so face width is a constant StdFaceWidth. + //this->m_fScale2 = this->m_VOASMLTC->m_VOReferenceShape.GetWidth() / this->m_VOFittingShape.GetWidth(); + this->m_fScale2 = this->m_VOASMLTC->m_VOReferenceShape.GetCentralizedShapeSize() / this->m_VOFittingShape.GetCentralizedShapeSize(); + this->m_VOFittingShape *= this->m_fScale2; + + int w = (int)(iImg.cols*this->m_fScale2); + int h = (int)(iImg.rows*this->m_fScale2); + Mat SearchImage = Mat(Size( w, h ), this->m_ImageProcessing.type(), this->m_ImageProcessing.channels() ); + + float PyrScale = pow(2.0f, (float) (this->m_iNbOfPyramidLevels-1.0f) ); + this->m_VOFittingShape /= PyrScale; + + const int nQualifyingDisplacements = (int)(this->m_VOASMLTC->m_iNbOfPoints * VO_Fitting2DSM::pClose); + + // for each level in the image pyramid + for (int iLev = this->m_iNbOfPyramidLevels-1; iLev >= 0; iLev--) + { + // Set image roi, instead of cvCreateImage a new image to speed up + Mat siROI = SearchImage(Rect(0, 0, (int)(w/PyrScale), (int)(h/PyrScale) ) ); + cv::resize(this->m_ImageProcessing, siROI, siROI.size()); + + int nGoodLandmarks = 0; + + this->m_VOEstimatedShape = this->m_VOFittingShape; + this->PyramidFit( this->m_VOEstimatedShape, + SearchImage, + iLev, + VO_Fitting2DSM::pClose, + epoch); + this->m_VOFittingShape = this->m_VOEstimatedShape; + + if (iLev != 0) + { + PyrScale /= 2.0f; + this->m_VOFittingShape *= 2.0f; + } + } + + // Explained by YAO Wei, 2008-02-09. + // this->m_fScale2 back to original size + this->m_VOFittingShape /= this->m_fScale2; + + ioShape.clone(this->m_VOFittingShape); + VO_Fitting2DSM::VO_DrawMesh(ioShape, this->m_VOASMLTC, oImg); + +t = ((double)cvGetTickCount() - t )/ (cvGetTickFrequency()*1000.); +printf("MRASM fitting time cost: %.2f millisec\n", t); +this->m_fFittingTime = safeDoubleToFloat(t); + + return safeDoubleToFloat(t); + +} + + +/** +Find the best offset for one point +*/ +float VO_FittingASMLTCs::VO_FindBestMatchingLTC(const VO_ASMLTCs* asmmodel, + const Mat& iImg, + const VO_Shape& iShape, + const vector& iShapeInfo, + const Mat_& iMean, + const Mat_& iCovInverse, + unsigned int ptIdx, + unsigned int offSetTolerance, + Point2f& ioLocation, + unsigned int LTCType) +{ + float BestFit = FLT_MAX; + //what was going on here. + int nBestOffsetX = INT_MAX, nBestOffsetY = INT_MAX; // might be + or - + float Fit; + float dist = 0.0f; + ioLocation = iShape.GetA2DPoint ( ptIdx ); + float xx = ioLocation.x - (float)cvRound(ioLocation.x); + float yy = ioLocation.y - (float)cvRound(ioLocation.y); + + Mat tmpFeatures; + // Find the best in just one direction + for (int x = -(int)offSetTolerance; x <= (int)offSetTolerance; ++x) + { + for (int y = -(int)offSetTolerance; y <= (int)offSetTolerance; ++y) + { + VO_ASMLTCs::VO_LoadLTC4OneAnnotatedPoint( iImg, + iShape, + ptIdx, + asmmodel->m_localImageSize, + asmmodel->m_pVOfeatures, + x, + y); + tmpFeatures = asmmodel->m_pVOfeatures->m_MatFeatures; + cv::normalize(tmpFeatures, tmpFeatures); + Fit = (float) cv::Mahalanobis(tmpFeatures, iMean, iCovInverse ); + + if(Fit < BestFit) + { + nBestOffsetX = x; + nBestOffsetY = y; + dist = safeDoubleToFloat(sqrt( pow( (double)(nBestOffsetX - xx), 2.0) + pow( (double)(nBestOffsetY - yy), 2.0) ) ); + BestFit = Fit; + } + else if( Fit == BestFit ) // pick up one closer to (0,0) + { + if( sqrt(pow( (double)x, 2.0 ) + pow( (double)y, 2.0 )) + < sqrt(pow( (double)nBestOffsetX, 2.0 ) + pow( (double)nBestOffsetY, 2.0 )) ) + { + nBestOffsetX = x; + nBestOffsetY = y; + dist = safeDoubleToFloat(sqrt( pow( (double)(nBestOffsetX - xx), 2.0) + pow( (double)(nBestOffsetY - yy), 2.0) ) ); + } + } + } + } + + ioLocation.x += nBestOffsetX; + ioLocation.y += nBestOffsetY; + + return dist; +} + + +int VO_FittingASMLTCs::UpdateShape(const VO_ASMLTCs* asmmodel, + const Mat& iImg, + VO_Shape& ioShape, + const vector& iShapeInfo, + const vector< Mat_ >& iMeans, + const vector< Mat_ >& iCovInverses, + unsigned int offSetTolerance) +{ + int nGoodLandmarks = 0; + float dist = 0.0f; + unsigned int NbOfPoints = ioShape.GetNbOfPoints(); + unsigned int NbOfShapeDim = ioShape.GetNbOfDim(); + Point2f pt; + + + // Take care of the image patch first. + for (unsigned int i = 0; i < NbOfPoints; i++) + { + dist = VO_FittingASMLTCs::VO_FindBestMatchingLTC( asmmodel, + iImg, + ioShape, + iShapeInfo, + iMeans[i], + iCovInverses[i], + i, + offSetTolerance, + pt, + VO_Features::DIRECT); + + // set OutShape(iPoint) to best offset from current position + // one dimensional profile: must move point along the whisker + ioShape.SetA2DPoint(pt, i); + + //if (abs(nBestOffset[0]) <= offSetTolerance/2) + if (fabs(dist) <= 1.41421357f) + nGoodLandmarks++; + } + + return nGoodLandmarks; +} + + +//----------------------------------------------------------------------------- +// Pyramid ASM Fitting Algorithm at certain level +// +// An iterative approach to improving the fit of the instance, this->m_VOShape, to an image +// proceeds as follows: +// 1. Examine a region of the image around each point Point-ith to find the best +// nearby match for the point Point'-ith. ---> UpdateShape +// 2. Update the parameters(s, sigma, tx, ty; b) to best fit the new found points +// X. ---> ConformShapeToModel +// 3. Repeat until convergence. +// +// For more details, ref to [Cootes & Taylor, 2004]. +//----------------------------------------------------------------------------- +/** + * @author JIA Pei + * @version 2010-05-20 + * @brief Find the best offset for one point + * @param ioShape Input and output - the input and output shape + * @param iImg Input - image to be fitted + * @param oImages Output - the output images + * @param iLev Input - current pyramid level + * @param PClose Input - percentage of converged points. Say, 0.9 means if 90% of the points + * are judged as converged, the iteration of this pyramid can stop + * @param epoch Input - the maximum iteration times + * @note Refer to "AAM Revisited, page 34, figure 13", particularly, those steps. +*/ +void VO_FittingASMLTCs::PyramidFit( VO_Shape& ioShape, + const Mat& iImg, + vector& oMeshInfo, + unsigned int iLev, + float PClose, + unsigned int epoch, + bool record) +{ + VO_Shape tempShape = ioShape; + int nGoodLandmarks = 0; + float PyrScale = pow(2.0f, (float) (iLev) ); + + const int nQualifyingDisplacements = (int)(this->m_VOASMLTC->m_iNbOfPoints * PClose); + + for(unsigned int iter = 0; iter < epoch; iter++) + { + // estimate the best this->m_VOEstimatedShape by profile matching the landmarks in this->m_VOShape + nGoodLandmarks = VO_FittingASMLTCs::UpdateShape(this->m_VOASMLTC, + iImg, + tempShape, + this->m_vShape2DInfo, + this->m_VOASMLTC->m_vvLTCMeans[iLev], + this->m_VOASMLTC->m_vvCVMInverseOfLTCCov[iLev], + 3); + + // conform ioShape to the shape model + this->m_VOASMLTC->VO_CalcAllParams4AnyShapeWithConstrain( tempShape, + this->m_MatModelAlignedShapeParam, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity ); + tempShape.ConstrainShapeInImage(iImg); + + if(record) + { + // If we get better fitting result, record this fitting result + oMeshInfo.push_back(DrawMeshInfo(PyrScale,this->m_fScale2,tempShape,this->m_VOASMLTC)); + } + + if(nGoodLandmarks > nQualifyingDisplacements) + break; + } + + ioShape=tempShape; + +} + + +//----------------------------------------------------------------------------- +// Pyramid ASM Fitting Algorithm at certain level +// +// An iterative approach to improving the fit of the instance, this->m_VOShape, to an image +// proceeds as follows: +// 1. Examine a region of the image around each point Point-ith to find the best +// nearby match for the point Point'-ith. ---> UpdateShape +// 2. Update the parameters(s, sigma, tx, ty; b) to best fit the new found points +// X. ---> ConformShapeToModel +// 3. Repeat until convergence. +// +// For more details, ref to [Cootes & Taylor, 2004]. +//----------------------------------------------------------------------------- +void VO_FittingASMLTCs::PyramidFit(VO_Shape& ioShape, + const Mat& iImg, + unsigned int iLev, + float PClose, + unsigned int epoch) +{ + VO_Shape tempShape = ioShape; + int nGoodLandmarks = 0; + float PyrScale = pow(2.0f, (float) (iLev-1.0f) ); + + const int nQualifyingDisplacements = (int)(this->m_VOASMLTC->m_iNbOfPoints * PClose); + + for(unsigned int iter = 0; iter < epoch; iter++) + { + // estimate the best this->m_VOEstimatedShape by profile matching the landmarks in this->m_VOShape + nGoodLandmarks = this->UpdateShape( this->m_VOASMLTC, + iImg, + ioShape, + this->m_vShape2DInfo, + this->m_VOASMLTC->m_vvLTCMeans[iLev], + this->m_VOASMLTC->m_vvCVMInverseOfLTCCov[iLev], + 3); + + // conform ioShape to the shape model + this->m_VOASMLTC->VO_CalcAllParams4AnyShapeWithConstrain( tempShape, + this->m_MatModelAlignedShapeParam, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity ); + tempShape.ConstrainShapeInImage(iImg); + + // the fitting result is good enough to stop the iteration + if(nGoodLandmarks > nQualifyingDisplacements) + break; + } + + ioShape=this->m_VOEstimatedShape; + +} + diff --git a/modules/smfitting/src/VO_FittingASMNDProfiles.cpp b/modules/smfitting/src/VO_FittingASMNDProfiles.cpp new file mode 100644 index 0000000..ef39960 --- /dev/null +++ b/modules/smfitting/src/VO_FittingASMNDProfiles.cpp @@ -0,0 +1,849 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include +#include +#include + + +#include "VO_FittingASMNDProfiles.h" + + +/** Constructor */ +VO_FittingASMNDProfiles::VO_FittingASMNDProfiles() +{ + this->init(); +} + + +/** Destructor */ +VO_FittingASMNDProfiles::~VO_FittingASMNDProfiles() +{ + if(this->m_VOASMNDProfile) delete this->m_VOASMNDProfile; this->m_VOASMNDProfile = NULL; +} + + +/** Initialization */ +void VO_FittingASMNDProfiles::init() +{ + VO_Fitting2DSM::init(); + this->m_VOASMNDProfile = new VO_ASMNDProfiles(); + this->m_iFittingMethod = VO_AXM::ASM_PROFILEND; +} + + +/** + * @author JIA Pei + * @version 2010-05-18 + * @brief Load all AAM data from a specified folder for later fitting, to member variable m_VOASMNDProfile + * @param fd Input - the folder that AAM to be loaded from +*/ +void VO_FittingASMNDProfiles::VO_LoadParameters4Fitting(const string& fd) +{ + this->m_VOASMNDProfile->VO_LoadParameters4Fitting(fd); + + // VO_Fitting2DSM + this->m_VOTemplateAlignedShape = this->m_VOASMNDProfile->m_VOAlignedMeanShape; + // this->m_VOTemplateNormalizedTexture = this->m_VOASMNDProfile->m_VONormalizedMeanTexture; + this->m_vTriangle2D = this->m_VOASMNDProfile->m_vNormalizedTriangle2D; + this->m_vShape2DInfo = this->m_VOASMNDProfile->m_vShape2DInfo; + this->m_FaceParts = this->m_VOASMNDProfile->m_FaceParts; + // this->m_vPointWarpInfo = this->m_VOASMNDProfile->m_vNormalizedPointWarpInfo; +} + + +/** + * @author JIA Pei, YAO Wei + * @version 2010-05-20 + * @brief Additive ASM ND Profiles Fitting, for static images, so that we record the whole fitting process + * @param iImg Input - image to be fitted + * @param oMeshInfo Output - the fitting process + * @param dim Input - profile dimension, 1, 2, 4 or 8 + * @param epoch Input - the iteration epoch + * @param pyramidlevel Input - pyramid level, 1, 2, 3 or 4 at most + * @note Refer to "AAM Revisited, page 34, figure 13", particularly, those steps. +*/ +float VO_FittingASMNDProfiles::VO_ASMNDProfileFitting( const Mat& iImg, + vector& oMeshInfo, + unsigned int epoch, + unsigned int pyramidlevel, + bool record, + const vector& fitTechs) +{ + double t = (double)cvGetTickCount(); + + //overwrites pyramid levels + this->m_iNbOfPyramidLevels = pyramidlevel; + this->SetProcessingImage(iImg, this->m_VOASMNDProfile); + this->m_iIteration = 0; + + if(record) + { + //Save initial shape placement + DrawMeshInfo temp(1,1,this->m_VOFittingShape,this->m_VOASMNDProfile); + oMeshInfo.push_back(temp); + } + + // Get m_MatModelAlignedShapeParam and m_fScale, m_vRotateAngles, m_MatCenterOfGravity + this->m_VOASMNDProfile->VO_CalcAllParams4AnyShapeWithConstrain( this->m_VOFittingShape, + this->m_MatModelAlignedShapeParam, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); + if(record) + { + //Save shape after shape constraints are applied + oMeshInfo.push_back(DrawMeshInfo(1,1,this->m_VOFittingShape,this->m_VOASMNDProfile)); + } + + // Explained by YAO Wei, 2008-2-9. + // Scale this->m_VOFittingShape, so face width is a constant StdFaceWidth. + //this->m_fScale2 = this->m_VOASMNDProfile->m_VOReferenceShape.GetWidth() / this->m_VOFittingShape.GetWidth(); + this->m_fScale2 = this->m_VOASMNDProfile->m_VOReferenceShape.GetCentralizedShapeSize() / this->m_VOFittingShape.GetCentralizedShapeSize(); + this->m_VOFittingShape *= this->m_fScale2; + + int w = (int)(iImg.cols*this->m_fScale2); + int h = (int)(iImg.rows*this->m_fScale2); + Mat SearchImage = Mat(Size( w, h ), this->m_ImageProcessing.type(), this->m_ImageProcessing.channels() ); + + float PyrScale = pow(2.0f, (float) (this->m_iNbOfPyramidLevels-1.0f) ); + this->m_VOFittingShape /= PyrScale; + + const int nQualifyingDisplacements = (int)(this->m_VOASMNDProfile->m_iNbOfPoints * VO_Fitting2DSM::pClose); + + // for each level in the image pyramid + for (int iLev = this->m_iNbOfPyramidLevels-1; iLev >= 0; iLev--) + { + // Set image roi, instead of cvCreateImage a new image to speed up + Mat siROI = SearchImage(Rect(0, 0, (int)(w/PyrScale), (int)(h/PyrScale) ) ); + cv::resize(this->m_ImageProcessing, siROI, siROI.size()); + + if(record) + { + //Save results after scaled for pyramiding (if any) + oMeshInfo.push_back(DrawMeshInfo(PyrScale,this->m_fScale2,this->m_VOFittingShape,this->m_VOASMNDProfile)); + } + + this->m_VOEstimatedShape = this->m_VOFittingShape; + //Why does find require non-const iterators??? :( + //static vector::iterator anyPyramds = find(fitTechs.begin(),fitTechs.end(),VO_Fitting2DSM::HYBRIDPYRAMID); + bool usePyramidFit = fitTechs[0] == VO_Fitting2DSM::HYBRIDPYRAMID; //anyPyramds != fitTechs.end(); + if(this->m_VOASMNDProfile->GetNbOfChannels() == 2 && usePyramidFit){ + //The pyramid fits save the shape after each fit+constraint application + this->StagedPyramidFit( this->m_VOEstimatedShape, + SearchImage, + oMeshInfo, + iLev, + VO_Fitting2DSM::pClose, + epoch, + record); + }else + { + this->PyramidFit( this->m_VOEstimatedShape, + SearchImage, + oMeshInfo, + iLev, + VO_Fitting2DSM::pClose, + epoch, + record, + fitTechs); + } + this->m_VOFittingShape = this->m_VOEstimatedShape; + + if (iLev != 0) + { + PyrScale /= 2.0f; + this->m_VOFittingShape *= 2.0f; + } + } + + // Explained by YAO Wei, 2008-02-09. + // this->m_fScale2 back to original size + this->m_VOFittingShape /= this->m_fScale2; + +t = ((double)cvGetTickCount() - t )/ (cvGetTickFrequency()*1000.); +printf("MRASM fitting time cost: %.2f millisec\n", t); + + return safeDoubleToFloat(t); +} + + +/** + * @author JIA Pei, YAO Wei + * @version 2010-05-20 + * @brief Additive ASM ND Profiles Fitting, for dynamic image sequence + * @param iImg Input - image to be fitted + * @param ioShape Input and output - the shape + * @param oImg Output - the fitted image + * @param dim Input - profile dimension, 1, 2, 4 or 8 + * @param epoch Input - the iteration epoch + * @param pyramidlevel Input - pyramid level, 1, 2, 3 or 4 at most + * @note Refer to "AAM Revisited, page 34, figure 13", particularly, those steps. +*/ +float VO_FittingASMNDProfiles::VO_ASMNDProfileFitting( const Mat& iImg, + VO_Shape& ioShape, + Mat& oImg, + unsigned int epoch, + unsigned int pyramidlevel, + unsigned int dim) +{ + this->m_VOFittingShape.clone(ioShape); +double t = (double)cvGetTickCount(); + + this->m_iNbOfPyramidLevels = pyramidlevel; + this->SetProcessingImage(iImg, this->m_VOASMNDProfile); + this->m_iIteration = 0; + + // Get m_MatModelAlignedShapeParam and m_fScale, m_vRotateAngles, m_MatCenterOfGravity + this->m_VOASMNDProfile->VO_CalcAllParams4AnyShapeWithConstrain( this->m_VOFittingShape, + this->m_MatModelAlignedShapeParam, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity); + this->m_VOFittingShape.ConstrainShapeInImage(this->m_ImageProcessing); + + // Explained by YAO Wei, 2008-2-9. + // Scale this->m_VOFittingShape, so face width is a constant StdFaceWidth. + //this->m_fScale2 = this->m_VOASMNDProfile->m_VOReferenceShape.GetWidth() / this->m_VOFittingShape.GetWidth(); + this->m_fScale2 = this->m_VOASMNDProfile->m_VOReferenceShape.GetCentralizedShapeSize() / this->m_VOFittingShape.GetCentralizedShapeSize(); + this->m_VOFittingShape *= this->m_fScale2; + + int w = (int)(iImg.cols*this->m_fScale2); + int h = (int)(iImg.rows*this->m_fScale2); + Mat SearchImage = Mat(Size( w, h ), this->m_ImageProcessing.type(), this->m_ImageProcessing.channels() ); + + float PyrScale = pow(2.0f, (float) (this->m_iNbOfPyramidLevels-1.0f) ); + this->m_VOFittingShape /= PyrScale; + + const int nQualifyingDisplacements = (int)(this->m_VOASMNDProfile->m_iNbOfPoints * VO_Fitting2DSM::pClose); + + // for each level in the image pyramid + for (int iLev = this->m_iNbOfPyramidLevels-1; iLev >= 0; iLev--) + { + // Set image roi, instead of cvCreateImage a new image to speed up + Mat siROI = SearchImage(Rect(0, 0, (int)(w/PyrScale), (int)(h/PyrScale) ) ); + cv::resize(this->m_ImageProcessing, siROI, siROI.size()); + + this->m_VOEstimatedShape = this->m_VOFittingShape; + this->PyramidFit( this->m_VOEstimatedShape, + SearchImage, + iLev, + VO_Fitting2DSM::pClose, + epoch, + dim); + this->m_VOFittingShape = this->m_VOEstimatedShape; + + if (iLev != 0) + { + PyrScale /= 2.0f; + this->m_VOFittingShape *= 2.0f; + } + } + + // Explained by YAO Wei, 2008-02-09. + // this->m_fScale2 back to original size + this->m_VOFittingShape /= this->m_fScale2; + + ioShape.clone(this->m_VOFittingShape); + VO_Fitting2DSM::VO_DrawMesh(ioShape, this->m_VOASMNDProfile, oImg); + +t = ((double)cvGetTickCount() - t )/ (cvGetTickFrequency()*1000.); +printf("MRASM fitting time cost: %.2f millisec\n", t); +this->m_fFittingTime = safeDoubleToFloat(t); + + return safeDoubleToFloat(t); +} + + +/** + * @author JIA Pei, YAO Wei + * @version 2010-05-20 + * @brief Find the best offset for one point + * @param iImg Input - image to be fitted + * @param iShape Input - the input shape + * @param iShapeInfo Input - the shape information + * @param iMean Input - mean profile + * @param iCovInverse Input - covariance inverse + * @param ptIdx Input - point index + * @param ProfileLength Input - number of profiles per pixel, number of pixel for a single profile * dim of profiles + * @param offSetTolerance Input - offset tolerance, which is used to determine whether this point is converged or not + * @param DeltaX Output - update in X direction + * @param DeltaY Output - update in Y direction + * @param dim Input - profile dim + * @return int return the offset of the best fit from the profile center + * @note Refer to "AAM Revisited, page 34, figure 13", particularly, those steps. +*/ +int VO_FittingASMNDProfiles::VO_FindBestMatchingProfile1D( const Mat& iImg, + const Point2f& ThisPoint, + const Mat_& iMean, + const Mat_& iCovInverse, + const unsigned int ProfileLength, + const unsigned int offSetTolerance, + const float DeltaX, + const float DeltaY) +{ + float BestFit = FLT_MAX; + int nBestOffset = 0; // might be + or - + float Fit; + + VO_Profile tempProfile; + // Do one dim a time. + VO_Profile::VO_Get1DProfileInMat4OneLandmark ( iImg, + ThisPoint, + tempProfile, + DeltaX, + DeltaY, + ProfileLength+2*offSetTolerance); + VO_Profile tempSubProfile; + VO_Profile tempSubProfile_depth; + + // Find the best in just one direction + for (int i = -(int)offSetTolerance; i <= (int)offSetTolerance; ++i) + { + tempSubProfile = tempProfile.GetSubProfile(i + offSetTolerance, ProfileLength, 0); + tempSubProfile.Normalize(); + + Fit = (float) cv::Mahalanobis(tempSubProfile.m_MatProf, iMean, iCovInverse ); + + // Explained by YAO Wei, 2008-2-9. + // Test for a new best fit. We test using "<=" instead of just "<" + // so if there is an exact match then ixBest=0 i.e. no change. +// if((i <= 0 ) ? Fit <= BestFit: Fit < BestFit) + if(Fit < BestFit) + { + nBestOffset = i; + BestFit = Fit; + } + } + + // Find the additional best in the 2nd direction + + return nBestOffset; +} + +/** + * @author JIA Pei, YAO Wei + * @author Colin Bellmore + * @version 2012-06-12 + * @brief Find the best offset for one point, in two channels + * @param iImg Input - image to be fitted + * @param This Point Input - the xy location of the point + * @param iMean Input - mean profile + * @param iCovInverse Input - covariance inverse + * @param ProfileLength Input - number of profiles per pixel, number of pixel for a single profile * dim of profiles + * @param offSetTolerance Input - offset tolerance, which is used to determine whether this point is converged or not + * @param DeltaX Output - update in X direction + * @param DeltaY Output - update in Y direction + * @param dir Input - profile direction + * @return int return the offset of the best fit from the profile center + * @note Refer to "AAM Revisited, page 34, figure 13", particularly, those steps. +*/ +int VO_FittingASMNDProfiles::VO_FindBestMatchingProfile2D( const Mat& iImg, + const Point2f& ThisPoint, + const VO_Profile iMean, + const vector< Mat_ > iCovInverse, + const unsigned int ProfileLength, + const unsigned int offSetTolerance, + const float DeltaX, + const float DeltaY, + const int dir) +{ + float BestFit = FLT_MAX; + int nBestOffset = 0; // might be + or - + float Fit_final; + float Fit_c1; + float Fit_c2; + + VO_Profile tempProfile; + + // Do one dim a time, returns depth profile in second dim. + VO_Profile::VO_Get2DProfileInMat4OneLandmark ( iImg, + ThisPoint, + tempProfile, + DeltaX, + DeltaY, + ProfileLength+2*offSetTolerance); + VO_Profile tempSubProfile_c1; + VO_Profile tempSubProfile_c2; + + // Find the best in just one direction + for (int i = -(int)offSetTolerance; i <= (int)offSetTolerance; ++i) + { + tempSubProfile_c1 = tempProfile.GetSubProfile(i + offSetTolerance, ProfileLength, 0); + tempSubProfile_c2 = tempProfile.GetSubProfile(i + offSetTolerance, ProfileLength, 1); + + tempSubProfile_c1.Normalize(); + tempSubProfile_c2.Normalize(); + + //dir offsets to access mean and cov of normal profiles + Fit_c1 = (float) cv::Mahalanobis(tempSubProfile_c1.m_MatProf, iMean.Get1DimProfile(dir+0), iCovInverse[dir+0] ); + Fit_c2 = (float) cv::Mahalanobis(tempSubProfile_c2.m_MatProf, iMean.Get1DimProfile(dir+1), iCovInverse[dir+1] ); + + //pick the better fit, smaller distances are better. + if(Fit_c1 < Fit_c2){ + Fit_final = Fit_c1; + }else{ + Fit_final = Fit_c2; + } + + + // Explained by YAO Wei, 2008-2-9. + // Test for a new best fit. We test using "<=" instead of just "<" + // so if there is an exact match then ixBest=0 i.e. no change. +// if((i <= 0 ) ? Fit <= BestFit: Fit < BestFit) + if(Fit_final < BestFit) + { + nBestOffset = i; + BestFit = Fit_final; + } + } + + // Find the additional best in the 2nd direction + + return nBestOffset; + +} + +int VO_FittingASMNDProfiles::FindBestOffset( + const Mat& inputMat, + const Mat& firstChannelImg, + const Mat& secondChannelImg, + const Mat& thirdChannelImg, + const Point2f& ThisPoint, + const vector< VO_Profile >& iMean, + const vector< vector< Mat_ > >& iCovInverse, + const unsigned int offSetTolerance, + VO_Fitting2DSM::MULTICHANNELTECH fitTech, + const int ptIndex, + const int ptDir, + const Point2f& dirDistPt ) +{ + + unsigned int ProfileLength = iMean[0].GetProfileLength(); + switch(fitTech){ + case VO_Fitting2DSM::FIRSTCHANNELONLY: + { + //profiles are stored differently in two channel fitting (0=gray, 2=n.gray) + return VO_FittingASMNDProfiles::VO_FindBestMatchingProfile1D( + firstChannelImg, + ThisPoint, + iMean[ptIndex].Get1DimProfile(ptDir), + iCovInverse[ptIndex][ptDir], + ProfileLength, + offSetTolerance, + dirDistPt.x, + dirDistPt.y); + } + case VO_Fitting2DSM::SECONDCHANNELONLY: + { + //assumes iImg was split to become gray only + //profiles are stored differently in two channel fitting (1=depth, 3=n.depth) + return VO_FittingASMNDProfiles::VO_FindBestMatchingProfile1D( + secondChannelImg, + ThisPoint, + iMean[ptIndex].Get1DimProfile(ptDir+1), + iCovInverse[ptIndex][ptDir], + ProfileLength, + offSetTolerance, + dirDistPt.x, + dirDistPt.y); + } + case VO_Fitting2DSM::THIRDCHANNELONLY: + { + //requires 3 channel profile model as well + return VO_FittingASMNDProfiles::VO_FindBestMatchingProfile1D( + thirdChannelImg, + ThisPoint, + iMean[ptIndex].Get1DimProfile(ptDir+2), + iCovInverse[ptIndex][ptDir], + ProfileLength, + offSetTolerance, + dirDistPt.x, + dirDistPt.y); + } + case VO_Fitting2DSM::FULLHYBRID: + { + + //assume 2 channels in 2d profiles (gray, depth, normal gray, normal depth) + return VO_FittingASMNDProfiles::VO_FindBestMatchingProfile2D( + inputMat, + ThisPoint, + iMean[ptIndex], + iCovInverse[ptIndex], + ProfileLength, + offSetTolerance, + dirDistPt.x, + dirDistPt.y, + (this->m_VOASMNDProfile->m_iNbOfChannels)*ptDir); + //dir is determined by half the profiles. + } + default: + std::cerr << "Unsupported fitting method" << std::endl; + exit(EXIT_FAILURE); + } + +} +/** + * @author YAO Wei, JIA Pei + * @version 2010-05-20 + * @brief Find the best offset for one point + * @param asmmodel Input - the ASM model + * @param iImg Input - image to be fitted + * @param ioShape Input and output - the input and output shape + * @param iShapeInfo Input - the shape information + * @param iMean Input - mean profile + * @param iCovInverse Input - covariance inverse + * @param Lev Input - current pyramid level + * @param offSetTolerance Input - offset tolerance, which is used to determine whether this point is convergede or not + * Sometimes, the trained data is of 4D profiles, but the user may only use 1D to test. + * @note Refer to "AAM Revisited, page 34, figure 13", particularly, those steps. +*/ +unsigned int VO_FittingASMNDProfiles::UpdateShape( const VO_ASMNDProfiles* const asmmodel, + const Mat& inputMat, + VO_Shape& ioShape, + const vector& iShapeInfo, + const vector< VO_Profile >& iMean, + const vector< vector< Mat_ > >& iCovInverse, + const unsigned int offSetTolerance, + const vector& fitTechs) +{ + unsigned int nGoodLandmarks = 0; + int bestOffsetIndex[2]; + unsigned int NbOfPoints = ioShape.GetNbOfPoints(); + unsigned int NbOfShapeDim = ioShape.GetNbOfDim(); + + Point2f pt; + vector multiChannelMat; + + unsigned int NbOfChannels = asmmodel->GetNbOfChannels(); + assert(NbOfChannels >= 1); + if(NbOfChannels > 1){ + cv::split(inputMat,multiChannelMat); + assert(multiChannelMat.size() >= 3); + }else if (NbOfChannels == 1){ + multiChannelMat[0] = inputMat; + } + + + Point2f deltaPt; + Point2f normPt; + Point2f tangentPt; + float sqrtsum; + Point2f bestOffsetPt; + + for (unsigned int i = 0; i < NbOfPoints; i++) + { + ///Calculate profile norm direction + /** Here, this is not compatible with 3D */ + Point2f PrevPoint = ioShape.GetA2DPoint ( iShapeInfo[i].GetFrom() ); + Point2f ThisPoint = ioShape.GetA2DPoint ( i ); + Point2f NextPoint = ioShape.GetA2DPoint ( iShapeInfo[i].GetTo() ); + + // left side (connected from side) + deltaPt = ThisPoint - PrevPoint; + sqrtsum = sqrt ( (deltaPt.x*deltaPt.x) + (deltaPt.y*deltaPt.y) ); + if ( sqrtsum < FLT_EPSILON ) sqrtsum = 1.0f; + deltaPt = deltaPt / sqrtsum; // Normalize + // Firstly, normX normY record left side norm. + normPt.x = -deltaPt.y; + normPt.y = deltaPt.x; + + // right side (connected to side) + deltaPt = NextPoint - ThisPoint; + sqrtsum = sqrt ( (deltaPt.x*deltaPt.x) + (deltaPt.y*deltaPt.y) ); + if ( sqrtsum < FLT_EPSILON ) sqrtsum = 1.0f; + deltaPt = deltaPt / sqrtsum; // Normalize + // Secondly, normX normY will average both left side and right side norm. + normPt.x += -deltaPt.y; + normPt.y += deltaPt.x; + + // Average left right side + sqrtsum = sqrt ( (normPt.x*normPt.x) + (normPt.y*normPt.y) ); + if ( sqrtsum < FLT_EPSILON ) sqrtsum = 1.0f; + normPt = normPt / sqrtsum; // Final Normalize + tangentPt.x = -normPt.y; + tangentPt.y = normPt.x; // Final tangent + + ///////////////////////////////////////////////////////////////////////////// + bestOffsetIndex[0] = FindBestOffset(inputMat,multiChannelMat[0],multiChannelMat[1],multiChannelMat[2], + ThisPoint, iMean, iCovInverse, offSetTolerance, fitTechs[i],i,0,normPt); + //Update point from norm profiles before doing tangent profiles + bestOffsetPt = bestOffsetIndex[0] * normPt; + ThisPoint += bestOffsetPt; + unsigned int chanCount = this->m_VOASMNDProfile->m_iNbOfChannels; + unsigned int profCount = this->m_VOASMNDProfile->m_iNbOfProfileDim; + if(profCount > 1){ + bestOffsetIndex[1] = FindBestOffset(inputMat,multiChannelMat[0],multiChannelMat[1],multiChannelMat[2], + ThisPoint, iMean, iCovInverse, offSetTolerance, fitTechs[i],i,1,tangentPt); + + // set OutShape(iPoint) to best offset from current position + // one dimensional profile: must move point along the whisker + bestOffsetPt = bestOffsetIndex[1] * tangentPt; + } + pt = ThisPoint + bestOffsetPt; + ioShape.SetA2DPoint(pt, i); + + if (abs(bestOffsetIndex[0]) <= 1 && abs(bestOffsetIndex[1]) <= 1) + nGoodLandmarks++; + } + + return nGoodLandmarks; +} + + +//----------------------------------------------------------------------------- +// Pyramid ASM Fitting Algorithm at certain level +// +// An iterative approach to improving the fit of the instance, this->m_VOFittingShape, to an image +// proceeds as follows: +// 1. Examine a region of the image around each point Point-ith to find the best +// nearby match for the point Point'-ith. ---> UpdateShape +// 2. Update the parameters(s, sigma, tx, ty; b) to best fit the new found points +// X. ---> ConformShapeToModel +// 3. Repeat until convergence. +// +// For more details, ref to [Cootes & Taylor, 2004]. +//----------------------------------------------------------------------------- +/** + * @author YAO Wei, JIA Pei + * @version 2010-05-20 + * @brief Find the best offset for one point + * @param ioShape Input and output - the input and output shape + * @param iImg Input - image to be fitted + * @param oImages Output - the output images + * @param iLev Input - current pyramid level + * @param PClose Input - percentage of converged points. Say, 0.9 means if 90% of the points + * are judged as converged, the iteration of this pyramid can stop + * @param epoch Input - the maximum iteration times + * @note Refer to "AAM Revisited, page 34, figure 13", particularly, those steps. +*/ +void VO_FittingASMNDProfiles::PyramidFit( VO_Shape& ioShape, + const Mat& iImg, + vector& oMeshInfo, + const unsigned int iLev, + const float PClose, + const unsigned int epoch, + const bool record, + const vector& fitTechs) +{ + + float PyrScale = pow(2.0f, (float) (iLev) ); + + const int nQualifyingDisplacements = (int)(this->m_VOASMNDProfile->m_iNbOfPoints * PClose); + + Fit(iImg,PyrScale,nQualifyingDisplacements,iLev,epoch,fitTechs,record,ioShape,oMeshInfo); + +} + + +/** + * @author YAO Wei, JIA Pei + * @version 2010-05-20 + * @brief Find the best offset for one point + * @param iImg Input - image to be fitted + * @param ioShape Input and output - the input and output shape + * @param iShapeInfo Input - the shape information + * @param iLev Input - current pyramid level + * @param PClose Input - percentage of converged points. Say, 0.9 means if 90% of the points + * are judged as converged, the iteration of this pyramid can stop + * @param epoch Input - the maximum iteration times + * @param profdim Input - dimension used during fitting. For example, the trained data could be 4D, but the user may only use 1D + * @note Refer to "AAM Revisited, page 34, figure 13", particularly, those steps. +*/ +void VO_FittingASMNDProfiles::PyramidFit( VO_Shape& ioShape, + const Mat& iImg, + const unsigned int iLev, + const float PClose, + const unsigned int epoch, + const unsigned int profdim) +{ + //realistically, this method can never be hit anymore + + VO_Shape tempShape = ioShape; + int nGoodLandmarks = 0; + float PyrScale = pow(2.0f, (float) (iLev-1.0f) ); + + const int nQualifyingDisplacements = (int)(this->m_VOASMNDProfile->m_iNbOfPoints * PClose); + vector allFirst; + fill(allFirst.begin(),allFirst.end(),VO_Fitting2DSM::FIRSTCHANNELONLY); + + for(unsigned int iter = 0; iter < epoch; iter++) + { + this->m_iIteration++; + // estimate the best ioShape by profile matching the landmarks in this->m_VOFittingShape + nGoodLandmarks = VO_FittingASMNDProfiles::UpdateShape( this->m_VOASMNDProfile, + iImg, + tempShape, + this->m_vShape2DInfo, + this->m_VOASMNDProfile->m_vvMeanNormalizedProfile[iLev], + this->m_VOASMNDProfile->m_vvvCVMInverseOfSg[iLev], + 3, + allFirst);//assume single channel? + + // conform ioShape to the shape model + this->m_VOASMNDProfile->VO_CalcAllParams4AnyShapeWithConstrain( tempShape, + this->m_MatModelAlignedShapeParam, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity ); + tempShape.ConstrainShapeInImage(iImg); + + // the fitting result is good enough to stop the iteration + if(nGoodLandmarks > nQualifyingDisplacements) + break; + } + ioShape = tempShape; +} +/** + * @author Colin B + * @brief Update shape with profiles and apply shape constraints until num epochs or distance + * @params Far too many. :( + */ +void VO_FittingASMNDProfiles::Fit( const Mat& iImage, + const float PyrScale, + const int nQualifyingDisplacements, + const unsigned int iLev, + const unsigned int epoch, + const vector& fitTechs, + const bool record, + VO_Shape& ioShape, + vector& oMeshInfo) +{ + int nGoodLandmarks = 0; + for(unsigned int iter = 0; iter < epoch; iter++) + { + this->m_iIteration++; + + // estimate the best ioShape by profile matching the landmarks in this->m_VOFittingShape + nGoodLandmarks = VO_FittingASMNDProfiles::UpdateShape( this->m_VOASMNDProfile, + iImage, + ioShape, + this->m_vShape2DInfo, + this->m_VOASMNDProfile->m_vvMeanNormalizedProfile[iLev], + this->m_VOASMNDProfile->m_vvvCVMInverseOfSg[iLev], + 3, + fitTechs); + if(record) + { + // Record after shape profiles have fit + oMeshInfo.push_back(DrawMeshInfo(PyrScale,this->m_fScale2,ioShape,this->m_VOASMNDProfile)); + } + // conform ioShape to the shape model + this->m_VOASMNDProfile->VO_CalcAllParams4AnyShapeWithConstrain( ioShape, + this->m_MatModelAlignedShapeParam, + this->m_fScale, + this->m_vRotateAngles, + this->m_MatCenterOfGravity ); + ioShape.ConstrainShapeInImage(iImage); + if(record) + { + // Record after shape constraint is applied and shape profiles have fit + oMeshInfo.push_back(DrawMeshInfo(PyrScale,this->m_fScale2,ioShape,this->m_VOASMNDProfile)); + } + + // the fitting result is good enough to stop the iteration + if(nGoodLandmarks > nQualifyingDisplacements) + break; + } +} + +/** + * @author YAO Wei, JIA Pei + * @author Colin B + * @version 2012-05-20 + * @brief Find the best offset for one point + * @param iImg Input - image to be fitted + * @param ioShape Input and output - the input and output shape + * @param iShapeInfo Input - the shape information + * @param iLev Input - current pyramid level + * @param PClose Input - percentage of converged points. Say, 0.9 means if 90% of the points + * are judged as converged, the iteration of this pyramid can stop + * @param epoch Input - the maximum iteration times + * @note Refer to "AAM Revisited, page 34, figure 13", particularly, those steps. +*/ +void VO_FittingASMNDProfiles::StagedPyramidFit( VO_Shape& ioShape, + const Mat& iImg, + vector& oMeshInfo, + unsigned int iLev, + float PClose, + unsigned int epoch, + bool record) +{ + + float PyrScale = pow(2.0f, (float) (iLev) ); + + const int nQualifyingDisplacements = (int)(this->m_VOASMNDProfile->m_iNbOfPoints * PClose); + //Add more or rearrange order here. + vector allFirstChannel(this->m_VOASMNDProfile->m_iNbOfPoints,VO_Fitting2DSM::FIRSTCHANNELONLY); + vector allSecondChannel(this->m_VOASMNDProfile->m_iNbOfPoints,VO_Fitting2DSM::SECONDCHANNELONLY); + vector allHybrid(this->m_VOASMNDProfile->m_iNbOfPoints,VO_Fitting2DSM::FULLHYBRID); + + //Depth fitting + Fit(iImg,PyrScale,nQualifyingDisplacements,iLev,epoch,allSecondChannel,record,ioShape,oMeshInfo); + + //Color fitting + Fit(iImg,PyrScale,nQualifyingDisplacements,iLev,epoch,allFirstChannel,record,ioShape,oMeshInfo); + + //2-Channel fitting + Fit(iImg,PyrScale,nQualifyingDisplacements,iLev,epoch,allHybrid,record,ioShape,oMeshInfo); + +} + diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000..d89934d --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,5 @@ +add_subdirectory(test_integraltransform) +add_subdirectory(test_featureextraction) +add_subdirectory(test_ensembletraining) +add_subdirectory(test_smbuilding) +add_subdirectory(test_smfitting) diff --git a/tests/test_ensembletraining/CMakeLists.txt b/tests/test_ensembletraining/CMakeLists.txt new file mode 100644 index 0000000..dc24488 --- /dev/null +++ b/tests/test_ensembletraining/CMakeLists.txt @@ -0,0 +1,54 @@ + +if (BUILD_EXAMPLES) + project(ensembletraining_exe) + + if(CMAKE_COMPILER_IS_GNUCXX) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function") + endif() + + include_directories( + "${CMAKE_SOURCE_DIR}/modules/ensembletraining/include" + "${CMAKE_SOURCE_DIR}/modules/common/include" + "${CMAKE_SOURCE_DIR}/modules/featureextraction/include" + ) + + # --------------------------------------------- + # Define executable targets + # --------------------------------------------- + MACRO(VO_DEFINE_EXAMPLE name srcs) + set(the_target "test_${name}") + add_executable(${the_target} ${srcs}) + set_target_properties(${the_target} PROPERTIES + OUTPUT_NAME "test_${name}" + PROJECT_LABEL "(EXAMPLE) test_${name}") + add_dependencies(${the_target} ensembletraining boost_regex boost_filesystem boost_system + opencv_core opencv_flann opencv_imgproc opencv_highgui + opencv_ml opencv_video opencv_objdetect opencv_features2d + opencv_calib3d opencv_legacy opencv_contrib) + target_link_libraries(${the_target} ${VOSM_LINKER_LIBS} ensembletraining utils featureextraction integraltransform + boost_regex boost_filesystem boost_system opencv_core + opencv_flann opencv_imgproc opencv_highgui opencv_ml opencv_video opencv_objdetect + opencv_features2d opencv_calib3d opencv_legacy opencv_contrib) + + if(WIN32) + install(TARGETS ${the_target} + RUNTIME DESTINATION "tests" COMPONENT main) + endif() + install(TARGETS ${the_target} RUNTIME DESTINATION bin COMPONENT main) + ENDMACRO(VO_DEFINE_EXAMPLE) + + file(GLOB cpp_samples RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cpp *.c) + + foreach(sample_filename ${cpp_samples}) + get_filename_component(sample ${sample_filename} NAME_WE) + VO_DEFINE_EXAMPLE(${sample} ${sample_filename}) + endforeach() +endif(BUILD_EXAMPLES) + +if (INSTALL_C_EXAMPLES AND NOT WIN32) + file(GLOB C_SAMPLES *.c *.cpp *.jpg *.png *.data makefile.* build_all.sh *.dsp *.cmd ) + install(FILES ${C_SAMPLES} + DESTINATION share/vosm/tests + PERMISSIONS OWNER_READ GROUP_READ WORLD_READ) +endif () + diff --git a/tests/test_ensembletraining/ensembletraining.cpp b/tests/test_ensembletraining/ensembletraining.cpp new file mode 100644 index 0000000..dbc70ea --- /dev/null +++ b/tests/test_ensembletraining/ensembletraining.cpp @@ -0,0 +1,301 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-02-16 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +// FIXME: to be finished +// Actually, current OpenCV affords a module named "traincascade", which is robust and has good performance +// However, some header files of VOSM's ensembletraining module are required for successful VOSM building! + +#include +#include + +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "opencv/ml.h" +#include "VO_ScanFilesInDir.h" +#include "VO_BaggingRandomForestClassifier.h" +#include "VO_BoostingSingleEnsembleClassifier.h" +#include "VO_BoostingCascadeClassifier.h" +#include "VO_LBPFeatures.h" +#include "VO_HaarFeatures.h" + +using namespace std; +using namespace cv; + + + +void usage_build() +{ + cout << "Usage: ensembletraining [options] save_directory pos_image_directory neg_image_directory ensemble_type boosting_type feature_type stage mintruepositive maxwrongclassification false" << endl + << "options: " << endl + << " -o output directory (default './') " << endl + << " -p positive training image directory (containing training positive images of the same size ) " << endl + << " -n negative training image directory " << endl + << " -a additive model type (BOOSTING or BAGGING. default BOOSTING ) " << endl + << " -b boosting type (SINGLECASCADE, PARALLELCASCADE, CASCADEDECISIONTREE or BFSCASCADE. If using BOOSTING additive model, you must specify this vale. default SINGLECASCADE )" << endl + << " -f multi-scale feature type (DIRECT, LBP, HAAR_BASIC, HAAR_CORE, HAAR_ALL, GABOR, etc. default LBP ) " << endl + << " -s boosting stage " << endl + << " -t minimum true positive (default 0.95) " << endl + << " -c maximum wrong classification, including true negative plus false positive (default 0.5) " << endl; + + exit(0); +} + + +void parse_option( int argc, + char **argv, + string& outputDir, + vector& posImageFNs, + vector& negImageFNs, + unsigned int& ensembleType, + unsigned int& boostingType, + string& featureType, + unsigned int& stage, + float& minFalsePositive, + float& maxWrongClassification) +{ + char *arg = NULL; + int optindex; + + /* parse options */ + optindex = 0; + while (++optindex < argc) + { + if(argv[optindex][0] != '-') break; + if(++optindex >= argc) usage_build(); + + switch(argv[optindex-1][1]) + { + case 'o': + outputDir = argv[optindex]; + break; + case 'p': + { + if ( ! MakeDirectory( argv[optindex] ) ) + { + cerr << "positive image path does not exist!" << endl; + exit(EXIT_FAILURE); + } + posImageFNs = VO_IO::ScanNSortImagesInDirectory ( argv[optindex] ); + } + break; + case 'n': + { + if ( ! MakeDirectory( argv[optindex] ) ) + { + cerr << "negative image path does not exist!" << endl; + exit(EXIT_FAILURE); + } + negImageFNs = VO_IO::ScanNSortImagesInDirectory ( argv[optindex] ); + } + break; + case 'a': + { + if(strcmp(argv[optindex], "BOOSTING") == 0) + ensembleType = VO_AdditiveStrongerClassifier::BOOSTING; + else if(strcmp(argv[optindex], "BAGGING") == 0) + ensembleType = VO_AdditiveStrongerClassifier::BAGGING; + } + break; + case 'b': + { + if(strcmp(argv[optindex], "SINGLECASCADE") == 0) + boostingType = VO_BoostingCascadeClassifier::SINGLECASCADE; + else if(strcmp(argv[optindex], "PARALLELCASCADE") == 0) + boostingType = VO_BoostingCascadeClassifier::PARALLELCASCADE; + else if(strcmp(argv[optindex], "CASCADEDECISIONTREE") == 0) + boostingType = VO_BoostingCascadeClassifier::CASCADEDECISIONTREE; + else if(strcmp(argv[optindex], "BFSCASCADE") == 0) + boostingType = VO_BoostingCascadeClassifier::BFSCASCADE; + } + break; + case 'f': + { + featureType = argv[optindex]; + } + break; + case 's': + { + stage = atoi(argv[optindex]); + } + break; + case 't': + { + minFalsePositive = safeDoubleToFloat(atof(argv[optindex])); + } + break; + case 'c': + { + maxWrongClassification = safeDoubleToFloat(atof(argv[optindex])); + } + break; + default: + { + cerr << "unknown options" << endl; + usage_build(); + } + break; + } + } + + if (posImageFNs.size() == 0) + { + cerr << " No positive images loaded" << endl; + usage_build(); + exit(EXIT_FAILURE); + } + if (negImageFNs.size() == 0) + { + cerr << " No negative image loaded" << endl; + usage_build(); + exit(EXIT_FAILURE); + } +} + + +int main(int argc, char **argv) +{ + string outputDir = "./"; + vector posImageFNs; + vector negImageFNs; + unsigned int ensembleType = VO_AdditiveStrongerClassifier::BOOSTING; + unsigned int boostingType = VO_BoostingCascadeClassifier::SINGLECASCADE; + string featureType = "LBP"; + unsigned int numStages =16; + float minTruePositive = 0.95f;; + float maxWrongClassification = 0.5f; + + + const int precalcValBufSize = 128; + const int precalcIdxBufSize = 128; + + parse_option( argc, + argv, + outputDir, + posImageFNs, + negImageFNs, + ensembleType, + boostingType, + featureType, + numStages, + minTruePositive, + maxWrongClassification); + + VO_Features* featureParams = NULL; + + if(featureType == "LBP") + { + featureParams = new VO_LBPFeatures(); + } + else if(featureType == "HAAR_BASIC") + { + featureParams = new VO_HaarFeatures(); + } + else if(featureType == "HAAR_CORE") + { + featureParams = new VO_HaarFeatures(); + } + else if(featureType == "HAAR_ALL") + { + featureParams = new VO_HaarFeatures(); + } + + if(ensembleType == VO_AdditiveStrongerClassifier::BOOSTING) + { + switch(boostingType) + { + case VO_BoostingCascadeClassifier::PARALLELCASCADE: + { + VO_BoostingCascadeClassifier boostcascadeclassifier; + } + break; + case VO_BoostingCascadeClassifier::CASCADEDECISIONTREE: + { + VO_BoostingCascadeClassifier boostcascadeclassifier; + } + break; + case VO_BoostingCascadeClassifier::BFSCASCADE: + { + VO_BoostingCascadeClassifier boostcascadeclassifier; + } + break; + case VO_BoostingCascadeClassifier::SINGLECASCADE: + { + VO_BoostingCascadeClassifier boostcascadeclassifier; + boostcascadeclassifier.train( "./", + posImageFNs, + negImageFNs, + precalcValBufSize, + precalcIdxBufSize, + numStages, + minTruePositive, + maxWrongClassification, + featureParams); + } + default: + break; + } + } + else if (boostingType == VO_AdditiveStrongerClassifier::BAGGING) + { + VO_BaggingRandomForestClassifier baggingrandomforestclassifier; + } + + return 0; +} diff --git a/tests/test_ensembletraining/test_ensembletraining.vcxproj b/tests/test_ensembletraining/test_ensembletraining.vcxproj new file mode 100644 index 0000000..a074063 --- /dev/null +++ b/tests/test_ensembletraining/test_ensembletraining.vcxproj @@ -0,0 +1,88 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {B357DFE4-CFAB-4361-9C72-EEA806EECEB4} + test_ensembletraining + + + + Application + true + MultiByte + v110 + + + Application + false + true + MultiByte + v110 + + + + + + + + + + + + + C:\OpenCV2.4.5\opencv\include;$(IncludePath) + C:\OpenCV2.4.5\opencv\build\x86\vc11\bin;C:\OpenCV2.4.5\opencv\build\x86\vc11\lib;$(LibraryPath) + + + + Level3 + Disabled + ../../modules/ensembletraining/include;../../modules/common/include;../../modules/featureextraction/include;%(AdditionalIncludeDirectories);C:\OpenCV2.4.5\opencv\build\include;C:\boost\boost_1_53_0 + NOMINMAX;_MBCS;%(PreprocessorDefinitions) + + + true + ensembletraining.lib;featureextraction.lib;common.lib;opencv_core245d.lib;opencv_objdetect245d.lib;opencv_ml245d.lib;opencv_imgproc245d.lib;%(AdditionalDependencies) + ../../Debug;C:\OpenCV2.4.5\opencv\build\x86\vc11\lib;C:\boost\boost_1_53_0\stage\lib;%(AdditionalLibraryDirectories) + + + + + Level3 + MaxSpeed + true + true + + + true + true + true + + + + + + + + {6dc6813e-859a-4759-94cd-a1f1a6466b7b} + + + {838e7724-4e9b-4489-b26c-b419547473d2} + + + {01a5e120-9d02-44a1-a7e1-f6190f212aa5} + + + + + + \ No newline at end of file diff --git a/tests/test_ensembletraining/vo_ensembletraining.vcxproj.filters b/tests/test_ensembletraining/vo_ensembletraining.vcxproj.filters new file mode 100644 index 0000000..a85110e --- /dev/null +++ b/tests/test_ensembletraining/vo_ensembletraining.vcxproj.filters @@ -0,0 +1,22 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + \ No newline at end of file diff --git a/tests/test_featureextraction/CMakeLists.txt b/tests/test_featureextraction/CMakeLists.txt new file mode 100644 index 0000000..fc8864a --- /dev/null +++ b/tests/test_featureextraction/CMakeLists.txt @@ -0,0 +1,52 @@ + +if (BUILD_EXAMPLES) + project(featureextraction_samples) + + if(CMAKE_COMPILER_IS_GNUCXX) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function") + endif() + + include_directories( + "${CMAKE_SOURCE_DIR}/modules/featureextraction/include" + "${CMAKE_SOURCE_DIR}/modules/integraltransform/include" + ) + + # --------------------------------------------- + # Define executable targets + # --------------------------------------------- + MACRO(VO_DEFINE_EXAMPLE name srcs) + set(the_target "test_${name}") + add_executable(${the_target} ${srcs}) + set_target_properties(${the_target} PROPERTIES + OUTPUT_NAME "test_${name}" + PROJECT_LABEL "(EXAMPLE) test_${name}") + add_dependencies(${the_target} featureextraction integraltransform + opencv_core opencv_flann opencv_imgproc opencv_highgui + opencv_ml opencv_video opencv_objdetect opencv_features2d + opencv_calib3d opencv_legacy opencv_contrib) + target_link_libraries(${the_target} ${VOSM_LINKER_LIBS} featureextraction integraltransform + opencv_core opencv_flann opencv_imgproc opencv_highgui opencv_ml opencv_video + opencv_objdetect opencv_features2d opencv_calib3d opencv_legacy opencv_contrib) + + if(WIN32) + install(TARGETS ${the_target} + RUNTIME DESTINATION "tests" COMPONENT main) + endif() + install(TARGETS ${the_target} RUNTIME DESTINATION bin COMPONENT main) + ENDMACRO(VO_DEFINE_EXAMPLE) + + file(GLOB cpp_samples RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cpp *.c) + + foreach(sample_filename ${cpp_samples}) + get_filename_component(sample ${sample_filename} NAME_WE) + VO_DEFINE_EXAMPLE(${sample} ${sample_filename}) + endforeach() +endif(BUILD_EXAMPLES) + +if (INSTALL_C_EXAMPLES AND NOT WIN32) + file(GLOB C_SAMPLES *.h *.c *.cpp *.jpg *.png *.data makefile.* build_all.sh *.dsp *.cmd ) + install(FILES ${C_SAMPLES} + DESTINATION share/vosm/tests + PERMISSIONS OWNER_READ GROUP_READ WORLD_READ) +endif () + diff --git a/tests/test_featureextraction/featureextraction.cpp b/tests/test_featureextraction/featureextraction.cpp new file mode 100644 index 0000000..1041b17 --- /dev/null +++ b/tests/test_featureextraction/featureextraction.cpp @@ -0,0 +1,95 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-08 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include "opencv/cv.h" +#include "opencv/highgui.h" +#include "VO_Features.h" +#include "VO_HaarFeatures.h" +#include "VO_LBPFeatures.h" +#include "VO_GaborFeatures.h" +#include "VO_DaubechiesFeatures.h" + + +using namespace std; +using namespace cv; + +int main( int argc, char* argv[] ) +{ + Mat image = cv::imread("/usr/local/share/opencv/samples/c/lena.jpg", 0); + Mat resizedImage; + cv::resize(image, resizedImage, Size(32, 32) ); + + // Explained by JIA Pei. + // lena.jpg is of size 512*512, too big for multi-scale feature extraction + // so, we resize it to 32*32 + VO_LBPFeatures volbpf; + // In my implementation, you must produce the feature info before you actually generate all features + volbpf.VO_GenerateAllFeatureInfo(resizedImage.size(), VO_Features::LBP); + unsigned int nbOfFeatures = volbpf.GetNbOfFeatures(); + volbpf.VO_GenerateAllFeatures(resizedImage); + +// VO_HaarFeatures vohf; +// // In my implementation, you must produce the feature info before you actually generate all features +// vohf.VO_GenerateAllFeatureInfo(resizedImage.size(), VO_HaarFeatures::BASIC); +// unsigned int nbOfFeatures = vohf.GetNbOfFeatures(); +// vohf.VO_GenerateAllFeatures(resizedImage); + + return 0; +} diff --git a/tests/test_featureextraction/test_featureextraction.vcxproj b/tests/test_featureextraction/test_featureextraction.vcxproj new file mode 100644 index 0000000..6a7e17c --- /dev/null +++ b/tests/test_featureextraction/test_featureextraction.vcxproj @@ -0,0 +1,89 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {5D77FD6C-4547-42C0-88AC-85C16614AB68} + test_featureextraction + + + + Application + true + MultiByte + v110 + + + Application + false + true + MultiByte + v110 + + + + + + + + + + + + + C:\OpenCV2.4.5\opencv\include;$(IncludePath) + C:\OpenCV2.4.5\opencv\build\x86\vc11\bin;C:\OpenCV2.4.5\opencv\build\x86\vc11\lib;$(LibraryPath) + true + + + + Level3 + Disabled + ../../modules/featureextraction/include;../../modules/integraltransform/include;C:\OpenCV2.4.5\opencv\build\include;%(AdditionalIncludeDirectories) + + + true + featureextraction.lib;integraltransform.lib;opencv_core245d.lib;opencv_contrib245d.lib;opencv_highgui245d.lib;opencv_imgproc245d.lib;opencv_video245d.lib;opencv_objdetect245d.lib;opencv_ml245d.lib;opencv_features2d245d.lib;opencv_calib3d245d.lib;opencv_legacy245d.lib;opencv_flann245d.lib;%(AdditionalDependencies) + ../../Debug;C:\OpenCV2.4.5\opencv\build\x86\vc11\lib;C:\boost\boost_1_53_0\stage\lib;%(AdditionalLibraryDirectories) + + + false + false + + + + + Level3 + MaxSpeed + true + true + + + true + true + true + + + + + + + + {01a5e120-9d02-44a1-a7e1-f6190f212aa5} + + + {900f9166-cd10-49d8-a339-23e0a035f3e9} + + + + + + \ No newline at end of file diff --git a/tests/test_featureextraction/vo_featureextraction.vcxproj.filters b/tests/test_featureextraction/vo_featureextraction.vcxproj.filters new file mode 100644 index 0000000..1167a26 --- /dev/null +++ b/tests/test_featureextraction/vo_featureextraction.vcxproj.filters @@ -0,0 +1,22 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + \ No newline at end of file diff --git a/tests/test_integraltransform/CMakeLists.txt b/tests/test_integraltransform/CMakeLists.txt new file mode 100644 index 0000000..0fec53c --- /dev/null +++ b/tests/test_integraltransform/CMakeLists.txt @@ -0,0 +1,50 @@ + +if (BUILD_EXAMPLES) + project(integraltransform_exe) + + if(CMAKE_COMPILER_IS_GNUCXX) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function") + endif() + + include_directories( + "${CMAKE_SOURCE_DIR}/modules/integraltransform/include" + ) + + # --------------------------------------------- + # Define executable targets + # --------------------------------------------- + MACRO(VO_DEFINE_EXAMPLE name srcs) + set(the_target "test_${name}") + add_executable(${the_target} ${srcs}) + set_target_properties(${the_target} PROPERTIES + OUTPUT_NAME "test_${name}" + PROJECT_LABEL "(EXAMPLE) test_${name}") + add_dependencies(${the_target} integraltransform opencv_core opencv_flann opencv_imgproc opencv_highgui + opencv_ml opencv_video opencv_objdetect opencv_features2d + opencv_calib3d opencv_legacy opencv_contrib) + target_link_libraries(${the_target} ${VOSM_LINKER_LIBS} integraltransform opencv_core + opencv_flann opencv_imgproc opencv_highgui opencv_ml opencv_video opencv_objdetect + opencv_features2d opencv_calib3d opencv_legacy opencv_contrib) + + if(WIN32) + install(TARGETS ${the_target} + RUNTIME DESTINATION "tests" COMPONENT main) + endif() + install(TARGETS ${the_target} RUNTIME DESTINATION bin COMPONENT main) + ENDMACRO(VO_DEFINE_EXAMPLE) + + file(GLOB cpp_samples RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cpp *.c) + + foreach(sample_filename ${cpp_samples}) + get_filename_component(sample ${sample_filename} NAME_WE) + VO_DEFINE_EXAMPLE(${sample} ${sample_filename}) + endforeach() +endif(BUILD_EXAMPLES) + +if (INSTALL_C_EXAMPLES AND NOT WIN32) + file(GLOB C_SAMPLES *.h *.c *.cpp *.jpg *.png *.data makefile.* build_all.sh *.dsp *.cmd ) + install(FILES ${C_SAMPLES} + DESTINATION share/vosm/tests + PERMISSIONS OWNER_READ GROUP_READ WORLD_READ) +endif () + diff --git a/tests/test_integraltransform/dft.jpg b/tests/test_integraltransform/dft.jpg new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_integraltransform/gabored.jpg b/tests/test_integraltransform/gabored.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b695206d7b4bcb3cb2b13635313f4f2eca5b1638 GIT binary patch literal 333 zcmex=iF;N$`UAd82aiwDH{e}I9TgMk@nDx)C7|04|YK*L!#U-U> z$dGXcJ4ZK_{h;?$4{I* zb?NeztJkjIxOwa0qsLF4K70P+<*SdMK7aZ8?fZ|Pzy9B1-~jo@p5b5p>;E?a7N})? literal 0 HcmV?d00001 diff --git a/tests/test_integraltransform/gaborkernel.jpg b/tests/test_integraltransform/gaborkernel.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1397830db09c710dd212b85a70c9282f8c3325c4 GIT binary patch literal 333 zcmex=iF;N$`UAd82aiwDH{e}I9T185pEBcmY0|04|YK*L$VW&&-107fQe7FITP zpk8jEf~^8TJDHh*Hn9S&0jjPA$}_MCvI;30I#U-U> z$dGXcJ4ZK_{h;?$4{I* zb?NeztJkjIxOwa0qsLF4K70P+<*SdMK7aZ8?fZ|Pzy9B1-~a`iJ;T5H*Z*$<02ZWW AeEiF;N$`UAd82aiwDH{e}I9TgMk@nDx)C7|04|YK*L!#U-U> z$dGXcJ4ZK_{h;?$4{I* zb?NeztJkjIxOwa0qsLF4K70P+<*SdMK7aZ8?fZ|Pzy9B1-~jo@p5b5p>;E?a7N})? literal 0 HcmV?d00001 diff --git a/tests/test_integraltransform/integraltransform.cpp b/tests/test_integraltransform/integraltransform.cpp new file mode 100644 index 0000000..0d4d447 --- /dev/null +++ b/tests/test_integraltransform/integraltransform.cpp @@ -0,0 +1,115 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-03-05 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include +#include + +#include "opencv/cv.h" + +#include "VO_Gabor.h" +#include "VO_CohenDaubechiesFeauveau.h" +#include "VO_Coiflets.h" +#include "VO_Daubechies.h" +#include "VO_Haar.h" +#include "VO_Symlets.h" + + +using namespace std; +using namespace cv; + + + +int main(int argc, char **argv) +{ + fstream fs("test.txt", ios::out); + VO_CohenDaubechiesFeauveau voCDF; + VO_Coiflets vocoiflets; + VO_Daubechies vodaubechies; + VO_Haar vohaar; + VO_Symlets vosymlets; + VO_Gabor vogabor; + vogabor.VO_PrepareGaborKernel( 4, + 2.0f, + 0.0f, + 0.0f, + 4, + 1.0f); + vogabor.GetWindowFunc()->VO_DisplayWindowFuncKernel("gaborkernel.jpg"); + Mat iImg = imread ( "/usr/local/share/opencv/samples/c/lena.jpg", 0 ); + cout << iImg.channels() << endl; + + Mat_ inputImg; + iImg.copyTo(inputImg); + inputImg.convertTo(inputImg, CV_32FC1); + Mat_ waveletImg; + cv::dft(inputImg, waveletImg ); + imwrite("dft.jpg", waveletImg); + cv::idft(waveletImg, inputImg, DFT_SCALE); + imwrite("idft.jpg", inputImg); + Mat oImg = Mat::zeros(iImg.size(), iImg.type()); +// vogabor.VO_ForwardTransform(inputImg, waveletImg); + vogabor.VO_ForwardTransform(inputImg, Point(256, 256), waveletImg); + imwrite("gabored.jpg", waveletImg); + vogabor.VO_BackwardTransform(waveletImg, inputImg); + imwrite("igabored.jpg", inputImg); + + return 0; +} + diff --git a/tests/test_integraltransform/test.txt b/tests/test_integraltransform/test.txt new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_integraltransform/test_integraltransform.vcxproj b/tests/test_integraltransform/test_integraltransform.vcxproj new file mode 100644 index 0000000..d1bb243 --- /dev/null +++ b/tests/test_integraltransform/test_integraltransform.vcxproj @@ -0,0 +1,84 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {8B707CE2-543C-4398-9A30-9982EE43D249} + test_integraltransform + + + + Application + true + MultiByte + v110 + + + Application + false + true + MultiByte + v110 + + + + + + + + + + + + + C:\OpenCV2.4.5\opencv\include;$(IncludePath) + C:\data\vosm-0.3.3\Debug;$(LibraryPath) + + + + Level3 + Disabled + ../../modules/integraltransform/include;%(AdditionalIncludeDirectories);C:\OpenCV2.4.5\opencv\build\include + + + true + integraltransform.lib;opencv_core245d.lib;opencv_highgui245d.lib;opencv_imgproc245d.lib;opencv_video245d.lib;opencv_objdetect245d.lib;opencv_ml245d.lib;opencv_features2d245d.lib;opencv_calib3d245d.lib;opencv_legacy245d.lib;%(AdditionalDependencies) + ../../Debug;C:\OpenCV2.4.5\opencv\build\x86\vc11\lib;C:\boost\boost_1_53_0\stage\lib;%(AdditionalLibraryDirectories) + + + true + + + + + Level3 + MaxSpeed + true + true + + + true + true + true + + + + + + + + {900f9166-cd10-49d8-a339-23e0a035f3e9} + + + + + + \ No newline at end of file diff --git a/tests/test_integraltransform/test_integraltransform.vcxproj.filters b/tests/test_integraltransform/test_integraltransform.vcxproj.filters new file mode 100644 index 0000000..02c1135 --- /dev/null +++ b/tests/test_integraltransform/test_integraltransform.vcxproj.filters @@ -0,0 +1,22 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + \ No newline at end of file diff --git a/tests/test_smbuilding/CMakeLists.txt b/tests/test_smbuilding/CMakeLists.txt new file mode 100644 index 0000000..5503f79 --- /dev/null +++ b/tests/test_smbuilding/CMakeLists.txt @@ -0,0 +1,60 @@ + +if (BUILD_EXAMPLES) + project(smbuilding_exe) + + if(CMAKE_COMPILER_IS_GNUCXX) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function") + endif() + + include_directories( + "${CMAKE_SOURCE_DIR}/modules/smbuilding/include" + "${CMAKE_SOURCE_DIR}/modules/common/include" + "${CMAKE_SOURCE_DIR}/modules/cvcommon/include" + "${CMAKE_SOURCE_DIR}/modules/featureextraction/include" + "${CMAKE_SOURCE_DIR}/modules/integraltransform/include" + ) + + # --------------------------------------------- + # Define executable targets + # --------------------------------------------- + MACRO(VO_DEFINE_EXAMPLE name srcs) + set(the_target "test_${name}") + add_executable(${the_target} ${srcs}) + set_target_properties(${the_target} PROPERTIES + OUTPUT_NAME "test_${name}" + PROJECT_LABEL "(EXAMPLE) test_${name}") + add_dependencies(${the_target} smbuilding featureextraction integraltransform cvcommon utils + boost_regex boost_filesystem boost_system + opencv_core opencv_flann opencv_imgproc opencv_highgui + opencv_ml opencv_video opencv_objdetect opencv_features2d + opencv_calib3d opencv_legacy opencv_contrib) + target_link_libraries(${the_target} ${VOSM_LINKER_LIBS} smbuilding featureextraction integraltransform cvcommon utils + boost_regex boost_filesystem boost_system + opencv_core opencv_flann opencv_imgproc opencv_highgui opencv_ml opencv_video opencv_objdetect + opencv_features2d opencv_calib3d opencv_legacy opencv_contrib) + + if(WIN32) + install(TARGETS ${the_target} + RUNTIME DESTINATION "tests" COMPONENT main) + endif() + install(TARGETS ${the_target} RUNTIME DESTINATION bin COMPONENT main) + ENDMACRO(VO_DEFINE_EXAMPLE) + + file(GLOB cpp_samples RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cpp *.c) + + foreach(sample_filename ${cpp_samples}) + get_filename_component(sample ${sample_filename} NAME_WE) + VO_DEFINE_EXAMPLE(${sample} ${sample_filename}) + endforeach() +endif(BUILD_EXAMPLES) + +if (INSTALL_C_EXAMPLES AND NOT WIN32) + file(GLOB C_SAMPLES *.h *.c *.cpp *.jpg *.png *.data makefile.* build_all.sh *.dsp *.cmd ) + install(FILES ${C_SAMPLES} + DESTINATION share/vosm/tests + PERMISSIONS OWNER_READ GROUP_READ WORLD_READ) +# install(TARGETS ${sample} +# DESTINATION bin +# PERMISSIONS OWNER_ALL GROUP_ALL WORLD_ALL) +endif () + diff --git a/tests/test_smbuilding/smb.h b/tests/test_smbuilding/smb.h new file mode 100644 index 0000000..609b19d --- /dev/null +++ b/tests/test_smbuilding/smb.h @@ -0,0 +1,84 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + + +#include "VO_Common.h" +#include "VO_Shape.h" +#include "VO_ShapeModel.h" +#include "VO_TextureModel.h" +#include "VO_AAMBasic.h" +#include "VO_AAMForwardIA.h" +#include "VO_AAMInverseIA.h" +#include "VO_AXM.h" +#include "VO_AFM.h" +#include "VO_ASMLTCs.h" +#include "VO_ASMNDProfiles.h" +#include "VO_ScanFilesInDir.h" +#include "VO_AnnotationDBIO.h" + +enum { + SHAPEMODEL = 0, + TEXTUREMODEL = 1, + APPEARANCEMODEL = 2, + INVERSEIMAGEALIGNMENT = 3, + AFM = 4, + ASMLTC = 5, + ASMNDPROFILE = 6 +}; diff --git a/tests/test_smbuilding/smbuilding.cpp b/tests/test_smbuilding/smbuilding.cpp new file mode 100644 index 0000000..87a3467 --- /dev/null +++ b/tests/test_smbuilding/smbuilding.cpp @@ -0,0 +1,371 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include +#include + +#include +#include + +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "smb.h" + +using namespace std; +using namespace cv; + + +void usage_build() +{ + cout << "Usage: test_smbuilding [options] save_directory annotation_directory image_directory shapeinfo_path database channels type levels percentage " << endl + << "options: " << endl + << " -o output directory (default './') " << endl + << " -a annotation directory (required) " << endl + << " -i image directory (required) " << endl + << " -s path of the file shapeinfo (required) " << endl + << " -d training database (required, IMM, AGING, BIOID, XM2VTS, FRANCK, EMOUNT, JIAPEI, BOSPHORUS ) " << endl + << " -c channels (1, 2, or 3, default 3) " << endl + << " -t statistical model type (SM, TM, AM, IA, FM, SMLTC, SMNDPROFILE. default SMNDPROFILE ) " << endl + << " -l level of parymid (default 4) " << endl + << " -p percentage of shape, texture and appearance PCA (default 0.95) " << endl; + + exit(0); +} + + +void parse_option( int argc, + char **argv, + string& outputDir, + vector& annotationFNs, + vector& imageFNs, + string& shapeinfoFN, + unsigned int& database, + unsigned int& channels, + unsigned int& type, + unsigned int& levels, + double& percentage) +{ + char *arg = NULL; + int optindex; + + /* parse options */ + optindex = 0; + while (++optindex < argc) + { + if(argv[optindex][0] != '-') break; + if(++optindex >= argc) usage_build(); + + switch(argv[optindex-1][1]) + { + case 'o': + outputDir = argv[optindex]; + break; + case 'a': + { + if ( ! MakeDirectory( argv[optindex] ) ) + { + cerr << "landmark path does not exist!" << endl; + exit(EXIT_FAILURE); + } + annotationFNs = VO_IO::ScanNSortAnnotationInDirectory ( argv[optindex] ); + } + break; + case 'i': + { + if ( ! MakeDirectory( argv[optindex] ) ) + { + cerr << "image path does not exist!" << endl; + exit(EXIT_FAILURE); + } + imageFNs = VO_IO::ScanNSortImagesInDirectory ( argv[optindex] ); + } + break; + case 's': + { + + if ( ! boost::filesystem::is_regular( argv[optindex] ) ) + { + cerr << "shapeinfo file does not exist!" << endl; + exit(EXIT_FAILURE); + } + + shapeinfoFN = argv[optindex]; + } + break; + case 'd': + { + if(strcmp(argv[optindex], "PUT") == 0) + database = CAnnotationDBIO::PUT; + else if(strcmp(argv[optindex], "IMM") == 0) + database = CAnnotationDBIO::IMM; + else if(strcmp(argv[optindex], "AGING") == 0) + database = CAnnotationDBIO::AGING; + else if(strcmp(argv[optindex], "BIOID") == 0) + database = CAnnotationDBIO::BIOID; + else if(strcmp(argv[optindex], "XM2VTS") == 0) + database = CAnnotationDBIO::XM2VTS; + else if(strcmp(argv[optindex], "FRANCK") == 0) + database = CAnnotationDBIO::FRANCK; + else if(strcmp(argv[optindex], "EMOUNT") == 0) + database = CAnnotationDBIO::EMOUNT; + else if(strcmp(argv[optindex], "JIAPEI") == 0) + database = CAnnotationDBIO::JIAPEI; + else if(strcmp(argv[optindex], "BOSPHORUS") == 0) + database = CAnnotationDBIO::BOSPHORUS; + } + break; + case 'c': + channels = atoi(argv[optindex]); + break; + case 't': + { + if(strcmp(argv[optindex], "SM") == 0) + type = SHAPEMODEL; + else if(strcmp(argv[optindex], "TM") == 0) + type = TEXTUREMODEL; + else if(strcmp(argv[optindex], "AM") == 0) + type = APPEARANCEMODEL; + else if(strcmp(argv[optindex], "IA") == 0) + type = INVERSEIMAGEALIGNMENT; + else if(strcmp(argv[optindex], "FM") == 0) + type = AFM ; + else if(strcmp(argv[optindex], "SMLTC") == 0) + type = ASMLTC; + else if(strcmp(argv[optindex], "SMNDPROFILE") == 0) + type = ASMNDPROFILE; + } + break; + case 'l': + levels = atoi(argv[optindex]); + break; + case 'p': + percentage = atof(argv[optindex]); + break; + default: + { + cerr << "unknown options" << endl; + usage_build(); + } + break; + } + } + + if (annotationFNs.size() == 0) + { + cerr << " No landmark loaded" << endl; + usage_build(); + exit(EXIT_FAILURE); + } + else if (imageFNs.size() == 0) + { + cerr << " No image loaded" << endl; + usage_build(); + exit(EXIT_FAILURE); + } + else if (annotationFNs.size() != imageFNs.size()) + { + cerr << " The number of images should be equal to the number of landmarks" << endl; + usage_build(); + exit(EXIT_FAILURE); + } +} + + +int main(int argc, char **argv) +{ + string outputDir = "./"; + vector AllImgFiles4Training; + vector AllLandmarkFiles4Training; + string shapeinfoFileName; + unsigned int database = CAnnotationDBIO::EMOUNT; + unsigned int channels = 3; + unsigned int type = ASMNDPROFILE; + unsigned int levels = 4; + double percentage = 0.95; + + parse_option( argc, + argv, + outputDir, + AllLandmarkFiles4Training, + AllImgFiles4Training, + shapeinfoFileName, + database, + channels, + type, + levels, + percentage); + + switch(type) + { + case SHAPEMODEL: + { + VO_ShapeModel shapeModel; + shapeModel.VO_BuildShapeModel( AllLandmarkFiles4Training, + shapeinfoFileName, + database, + safeDoubleToFloat(percentage), + false); + shapeModel.VO_Save(outputDir); + } + break; + case TEXTUREMODEL: + { + VO_TextureModel textureModel; + textureModel.VO_BuildTextureModel( AllLandmarkFiles4Training, + AllImgFiles4Training, + shapeinfoFileName, + database, + channels, + VO_Features::DIRECT, + safeDoubleToFloat(percentage), + safeDoubleToFloat(percentage), + false ); + textureModel.VO_Save(outputDir); + } + break; + case APPEARANCEMODEL: + { + VO_AAMBasic aamBasicModel; + aamBasicModel.VO_BuildAppearanceModel( AllLandmarkFiles4Training, + AllImgFiles4Training, + shapeinfoFileName, + database, + channels, + levels, + VO_Features::DIRECT, + safeDoubleToFloat(percentage), + safeDoubleToFloat(percentage), + safeDoubleToFloat(percentage), + false ); + aamBasicModel.VO_Save(outputDir); + } + break; + case INVERSEIMAGEALIGNMENT: + { + VO_AAMInverseIA aamInverseIAModel; + aamInverseIAModel.VO_BuildAAMICIA( AllLandmarkFiles4Training, + AllImgFiles4Training, + shapeinfoFileName, + database, + channels, + levels, + VO_Features::DIRECT, + safeDoubleToFloat(percentage), + safeDoubleToFloat(percentage), + false ); + aamInverseIAModel.VO_Save(outputDir); + } + break; + case AFM: + { + VO_AFM featureModel; + featureModel.VO_BuildFeatureModel( AllLandmarkFiles4Training, + AllImgFiles4Training, + shapeinfoFileName, + database, + channels, + levels, + VO_Features::DIRECT, + safeDoubleToFloat(percentage), + false, + VO_DiscreteWavelet::HAAR, + Size(16, 16) ); + featureModel.VO_Save(outputDir); + } + break; + case ASMLTC: + { + VO_ASMLTCs asmLTCModel; + asmLTCModel.VO_BuildASMLTCs(AllLandmarkFiles4Training, + AllImgFiles4Training, + shapeinfoFileName, + database, + channels, + levels, + VO_Features::DIRECT, + safeDoubleToFloat(percentage), + false, + VO_Features::DIRECT, + Size(16, 16) ); + asmLTCModel.VO_Save(outputDir); + } + break; + case ASMNDPROFILE: + { + VO_ASMNDProfiles asmNDProfilesModel; + asmNDProfilesModel.VO_BuildASMNDProfiles( AllLandmarkFiles4Training, + AllImgFiles4Training, + shapeinfoFileName, + database, + channels, + levels, + 2, // profdim + 8, //kk + VO_Features::DIRECT, //trm + safeDoubleToFloat(percentage), //tpsShape + false); //useKnownTriangles + asmNDProfilesModel.VO_Save(outputDir); + } + break; + } + + return 0; +} diff --git a/tests/test_smbuilding/test_smbuilding.vcxproj b/tests/test_smbuilding/test_smbuilding.vcxproj new file mode 100644 index 0000000..3f30f16 --- /dev/null +++ b/tests/test_smbuilding/test_smbuilding.vcxproj @@ -0,0 +1,85 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {34E3A0E0-A9AA-4338-A68C-5B086E505117} + test_smbuilding + + + + Application + true + MultiByte + v110 + + + Application + false + true + MultiByte + v110 + + + + + + + + + + + + + C:\OpenCV2.4.5\opencv\include;$(IncludePath) + C:\OpenCV2.4.5\opencv\build\x86\vc11\bin;C:\OpenCV2.4.5\opencv\build\x86\vc11\lib;$(LibraryPath) + + + + Level3 + Disabled + C:\OpenCV2.4.5\opencv\build\include;../../modules/smbuilding/include;../../modules/common/include;../../modules/featureextraction/include;../../modules/integraltransform/include;../../modules/cvcommon/include;C:\boost\boost_1_53_0;%(AdditionalIncludeDirectories) + NOMINMAX;_MBCS;%(PreprocessorDefinitions) + + + true + ../../Debug;C:\OpenCV2.4.5\opencv\build\x86\vc11\lib;C:\boost\boost_1_53_0\stage\lib;%(AdditionalLibraryDirectories) + common.lib;cvcommon.lib;smbuilding.lib;featureextraction.lib;integraltransform.lib;opencv_core245d.lib;opencv_imgproc245d.lib;opencv_highgui245d.lib;opencv_legacy245d.lib;%(AdditionalDependencies) + + + + + Level3 + MaxSpeed + true + true + + + true + true + true + + + + + + + + + + + {6b3e57cc-3ca2-4bd6-88d5-ed99990d8ddb} + + + + + + \ No newline at end of file diff --git a/tests/test_smbuilding/test_smbuilding.vcxproj.filters b/tests/test_smbuilding/test_smbuilding.vcxproj.filters new file mode 100644 index 0000000..e734bf9 --- /dev/null +++ b/tests/test_smbuilding/test_smbuilding.vcxproj.filters @@ -0,0 +1,27 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Header Files + + + + + Source Files + + + \ No newline at end of file diff --git a/tests/test_smfitting/smf.h b/tests/test_smfitting/smf.h new file mode 100644 index 0000000..154b309 --- /dev/null +++ b/tests/test_smfitting/smf.h @@ -0,0 +1,75 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include "VO_Common.h" +#include "VO_FaceDetectionAlgs.h" +#include "VO_TrackingAlgs.h" +#include "VO_RecognitionAlgs.h" +#include "VO_Fitting2DSM.h" +#include "VO_FittingAAMBasic.h" +#include "VO_FittingAAMForwardIA.h" +#include "VO_FittingAAMInverseIA.h" +#include "VO_FittingAFM.h" +#include "VO_FittingASMLTCs.h" +#include "VO_FittingASMNDProfiles.h" +#include "VO_ScanFilesInDir.h" +#include "VO_AnnotationDBIO.h" + + diff --git a/tests/test_smfitting/smfitting.cpp b/tests/test_smfitting/smfitting.cpp new file mode 100644 index 0000000..b4323ba --- /dev/null +++ b/tests/test_smfitting/smfitting.cpp @@ -0,0 +1,1078 @@ +/**************************************************************************** +* * +* IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. * +* * +* By downloading, copying, installing or using the software you agree to * +* this license. If you do not agree to this license, do not download, * +* install, copy or use the software. * +* * +* License Agreement * +* For Vision Open Statistical Models * +* * +* Copyright (C): 2006~2012 by JIA Pei, all rights reserved. * +* * +* VOSM is free software under the terms of the GNU Lesser General Public * +* License (GNU LGPL) as published by the Free Software Foundation; either * +* version 3.0 of the License, or (at your option) any later version. * +* You can use it, modify it, redistribute it, etc; and redistribution and * +* use in source and binary forms, with or without modification, are * +* permitted provided that the following conditions are met: * +* * +* a) Redistribution's of source code must retain this whole paragraph of * +* copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph. * +* * +* b) Redistribution's in binary form must reproduce this whole paragraph * +* of copyright notice, including this list of conditions and all the * +* following contents in this copyright paragraph, and/or other materials * +* provided with the distribution. * +* * +* c) The name of the copyright holders may not be used to endorse or * +* promote products derived from this software without specific prior * +* written permission. * +* * +* Any publications based on this code must cite the following five papers,* +* technical reports and on-line materials. * +* 1) P. JIA, 2D Statistical Models, Technical Report of Vision Open * +* Working Group, 2st Edition, October 21, 2010. * +* http://www.visionopen.com/members/jiapei/publications/pei_sm2dreport2010.pdf* +* 2) P. JIA. Audio-visual based HMI for an Intelligent Wheelchair. * +* PhD thesis, University of Essex, February, 2011. * +* http://www.visionopen.com/members/jiapei/publications/pei_phdthesis2010.pdf* +* 3) T. Cootes and C. Taylor. Statistical models of appearance for * +* computer vision. Technical report, Imaging Science and Biomedical * +* Engineering, University of Manchester, March 8, 2004. * +* http://www.isbe.man.ac.uk/~bim/Models/app_models.pdf * +* 4) I. Matthews and S. Baker. Active appearance models revisited. * +* International Journal of Computer Vision, 60(2):135--164, November 2004.* +* http://www.ri.cmu.edu/pub_files/pub4/matthews_iain_2004_2/matthews_iain_2004_2.pdf* +* 5) M. B. Stegmann, Active Appearance Models: Theory, Extensions and * +* Cases, 2000. * +* http://www2.imm.dtu.dk/~aam/main/ * +* * +* Version: 0.4 * +* Author: JIA Pei * +* Contact: jp4work@gmail.com * +* URL: http://www.visionopen.com * +* Create Date: 2010-11-04 * +* Revise Date: 2012-03-22 * +*****************************************************************************/ + +#include +#include +#include +#include +#include "float.h" + +#include +#include + +#include "yaml-cpp/yaml.h" + +#include "opencv/cv.h" +#include "opencv/highgui.h" + +#include "smf.h" +#include "VO_FaceKeyPoint.h" + +using namespace std; +using namespace cv; + +//implemented in file +enum dualChannelCanidate{ VJC = 0, MINMAX = 1, GT = 2, MINVJC = 3}; + + +void usage_build() +{ + cout << "Usage: test_smfitting [options] trained_data type testing_images testing_annotations database staticORdynamic recording" << endl + << "options: " << endl + << " -o trained data directory (required) " << endl + << " -t fitting method to be used (ASM_PROFILEND, ASM_LTC, AAM_BASIC, AAM_CMUICIA, AAM_IAIA. default ASM_PROFILEND ) " << endl + << " -i testing image directory containing at least one image (required) " << endl + << " -a testing annotation directory (can be ignored) " << endl + << " -d testing database -- if annotation directory is specified, database should also be specified for further evaluation on fitting performance (can be ignored) " << endl + << " -s static image sequence or dynamic image sequence (default value true) " << endl + << " -r recording the fitting results or not (default value false) " << endl + << " -c enables two-channel fitting technique(gray = channel 1 only, depth = channel 2 only, pyr = hybrid pyramid, sub = hybrid subset, default = full 2-channel) " << endl + << " -l canidate location finding technique for two-channel fitting (min-max = search channel 2, GT = taken from annotations, VJc = OpenCV's Adaboost, min-VJc = Hybrid" << endl + << " -y yaml config file" << endl + << " -w use webcam number" << endl + << endl << endl; + + cout << "Note: current smfitting doesn't defaultly afford 1D Profile ASM. " << endl + << "If you would like to try 1D Profile ASM for static images in current smfitting, " + << "you have to manually change the code in function VO_Fitting2DSM::VO_StartFitting " + << "in file VO_Fitting2DSM.cpp, say, around line 306 of file VO_Fitting2DSM.cpp " + << "change the 5th parameter from '2' to '1' of function " + << "dynamic_cast(this)->VO_ASMNDProfileFitting. " << endl << endl + << "If you would like to try 1D Profile ASM for dynamic image sequence in current smfitting, " + << "you have to manually change the code in function main() " + << "in file smfitting.cpp, say around line 666 of file smfitting.cpp " + << "change the 5th parameter from '2' to '1' of function " + << "dynamic_cast(fitting2dsm)->VO_ASMNDProfileFitting " + << endl << endl; + + cout << "Face Detection: current smfitting use Adaboost technology to detect face as well as face components " + << "for face location initialization. Refer to CFaceDetectionAlgs in main(). " + << "Default Adaboost detectors installed with OpenCV installation are used in current smfitting " + << "You may manually change the Adaboost detectors according to your own cascade file paths " + << endl << endl; + + cout << "Face Tracking: current smfitting deals with image sequences. " + << "If static images are to be tested, there is no point to carry out tracking because " + << "for every image, Adaboost detection will be carried out. " + << "If dynamic image sequences are to be tested, current smfitting only affords Camshift tracking strategy. " + << "Please Refer to CTrackingAlgs() in main(), the default setting of function CTrackingAlgs() is Camshift algorithm " + << endl << endl; + + cout<< "Vision Open doesn't afford the video IO or webcam IO, although the author has done his own IO for all kinds. " + << "Users are highly encouraged to use their own video file or webcam IO and use VOSM in their own real-time applications. " + << endl << endl; + + exit(-1); +} + +struct VOSM_Fitting_Args{ + string trainedData; + VO_AXM::TYPE type; + vector imageFNs; + vector annotationFNs; + unsigned int channels; + VO_Fitting2DSM::MULTICHANNELTECH fitTech; + vector fitTechs; + dualChannelCanidate canidateLoc; + CAnnotationDBIO::DB database; + bool staticImgs; + bool recordResults; + int webcamNumber; + string webcamName; +}; + + +void parse_option( char argType, + std::string argVal, + VOSM_Fitting_Args& output + ) +{ + + switch(argType) + { + case 'o': + output.trainedData = argVal; + break; + case 't': + { + if(argVal == "ASM_PROFILEND") + output.type = VO_AXM::ASM_PROFILEND; + else if(argVal == "ASM_LTC") + output.type = VO_AXM::ASM_LTC; + else if(argVal == "AAM_BASIC") + output.type = VO_AXM::AAM_BASIC; + else if(argVal == "AAM_CMUICIA") + output.type = VO_AXM::AAM_CMUICIA; + else if(argVal == "AAM_IAIA") + output.type = VO_AXM::AAM_IAIA; + else + { + cerr << "Wrong fitting type parameters!" << endl; + exit(EXIT_FAILURE); + } + } + break; + case 'i': + { + if ( ! MakeDirectory( argVal ) ) + { + cerr << "image path does not exist!" << endl; + exit(EXIT_FAILURE); + } + output.imageFNs = VO_IO::ScanNSortImagesInDirectory ( argVal ); + } + break; + case 'a': + { + if ( ! MakeDirectory( argVal ) ) + { + cerr << "landmark path does not exist!" << endl; + exit(EXIT_FAILURE); + } + output.annotationFNs = VO_IO::ScanNSortAnnotationInDirectory ( argVal ); + } + break; + case 'd': + { + if(argVal == "PUT") + output.database = CAnnotationDBIO::PUT; + else if(argVal == "IMM") + output.database = CAnnotationDBIO::IMM; + else if(argVal == "AGING") + output.database = CAnnotationDBIO::AGING; + else if(argVal == "BIOID") + output.database = CAnnotationDBIO::BIOID; + else if(argVal == "XM2VTS") + output.database = CAnnotationDBIO::XM2VTS; + else if(argVal == "FRANCK") + output.database = CAnnotationDBIO::FRANCK; + else if(argVal == "EMOUNT") + output.database = CAnnotationDBIO::EMOUNT; + else if(argVal == "BOSPHORUS") + output.database = CAnnotationDBIO::BOSPHORUS; + else if(argVal == "JIAPEI") + output.database = CAnnotationDBIO::JIAPEI; + else + { + cerr << "Wrong database parameters!" << endl; + exit(EXIT_FAILURE); + } + } + break; + case 's': + { + if(argVal == "false") + output.staticImgs = false; + else if(argVal == "true") + output.staticImgs = true; + else + { + cerr << "Wrong StaticOrNot parameter!" << endl; + exit(EXIT_FAILURE); + } + + } + break; + case 'c': + { + //Uses assumpions for BS database + output.channels = 2; + if(argVal == "gray") + output.fitTech = VO_Fitting2DSM::FIRSTCHANNELONLY; + else if(argVal == "first") + output.fitTech = VO_Fitting2DSM::FIRSTCHANNELONLY; + else if(argVal == "depth") + output.fitTech = VO_Fitting2DSM::SECONDCHANNELONLY; + else if(argVal == "second") + output.fitTech = VO_Fitting2DSM::SECONDCHANNELONLY; + else if(argVal == "third") + output.fitTech = VO_Fitting2DSM::THIRDCHANNELONLY; + else if(argVal == "pyr") + output.fitTech = VO_Fitting2DSM::HYBRIDPYRAMID; + else if(argVal == "sub") + output.fitTech = VO_Fitting2DSM::HYBRIDSUBSET; + else + { + output.fitTech = VO_Fitting2DSM::FULLHYBRID; + } + } + break; + case 'l': + { + if(argVal == "min-max") + output.canidateLoc = MINMAX; + else if(argVal == "GT") + output.canidateLoc = GT; + else if(argVal == "VJc") + output.canidateLoc = VJC; + else + { + output.canidateLoc = MINVJC; + } + } + break; + case 'r': + { + if(argVal == "false") + output.recordResults = false; + else if(argVal == "true") + output.recordResults = true; + else + { + cerr << "Wrong recordResults parameter!" << endl; + exit(EXIT_FAILURE); + } + } + break; + case 'y': + { + // Recursivly calls parse_option for certain values + // so, clearly don't call parse_option('y',...) from here) + YAML::Node config = YAML::LoadFile(argVal); + + if(config["config-version"]) + std::cout << "config-version=" << config["version"].as() << "\n"; + + // Verbose lets us skip verbose checks for existence + if(config["verbose"].as()){ + + parse_option('i', config["test-image-directory"].as(),output); + parse_option('c',config["fitting"]["channel-tech"].as(),output); + parse_option('a',config["annotation"]["directory"].as(),output); + parse_option('l',config["fitting"]["starting-location"].as(),output); + parse_option('d',config["annotation"]["format"].as(),output); + parse_option('t',config["fitting"]["method"].as(),output); + parse_option('w',config["webcam"].as(),output); + + output.recordResults = config["fitting"]["record"].as(); + output.staticImgs = config["fitting"]["static-images"].as(); + output.trainedData = config["trained-model-directory"].as(); + output.channels = config["fitting"]["channels"].as();//right now, this is only way to specify 3 channels + + }else{ + + if(config["test-image-directory"]) + parse_option('i', config["test-image-directory"].as(),output); + if(config["fitting"]["method"]) + parse_option('t', config["fitting"]["method"].as(),output); + if(config["fitting"]["channel-tech"]) + parse_option('c',config["fitting"]["channel-tech"].as(),output); + if(config["annotation"]["directory"]) + parse_option('a',config["annotation"]["directory"].as(),output); + if(config["fitting"]["starting-location"]) + parse_option('l',config["fitting"]["starting-location"].as(),output); + if(config["annotation"]["format"]) + parse_option('d',config["annotation"]["format"].as(),output); + if(config["webcam"]) + parse_option('w',config["webcam"].as(),output); + + if(config["fitting"]["record"]) + output.recordResults = config["fitting"]["record"].as(); + if(config["fitting"]["static-images"]) + output.staticImgs = config["fitting"]["static-images"].as(); + if(config["trained-model-directory"]) + output.trainedData = config["trained-model-directory"].as(); + if(config["fitting"]["channels"]) + output.channels = config["fitting"]["channels"].as(); + } + + //The only way to specify a subset is with the yaml file + if(output.fitTech == VO_Fitting2DSM::HYBRIDSUBSET){ + if( !config["fitting"]["techniques"]){ + cerr << "Complete subset specification required" << endl; + exit(EXIT_FAILURE); + } + //We don't know how many points are in the model yet, so we can't verify the completeness + for(YAML::iterator iter = config["fitting"]["techniques"].begin(); iter != config["fitting"]["techniques"].end(); iter++){ + //convert from strings to enums + if( iter->as() == "default") + output.fitTechs.push_back(VO_Fitting2DSM::FULLHYBRID); + else if( iter->as() == "gray") + output.fitTechs.push_back(VO_Fitting2DSM::FIRSTCHANNELONLY); + else if( iter->as() == "first") + output.fitTechs.push_back(VO_Fitting2DSM::FIRSTCHANNELONLY); + else if( iter->as() == "depth") + output.fitTechs.push_back(VO_Fitting2DSM::SECONDCHANNELONLY); + else if( iter->as() == "second") + output.fitTechs.push_back(VO_Fitting2DSM::SECONDCHANNELONLY); + else if( iter->as() == "third") + output.fitTechs.push_back(VO_Fitting2DSM::THIRDCHANNELONLY); + else if( iter->as() == "pyr") + output.fitTechs.push_back(VO_Fitting2DSM::HYBRIDPYRAMID); + else if( iter->as() == "sub") + output.fitTechs.push_back(VO_Fitting2DSM::HYBRIDSUBSET); + else{ + cerr << "Incorrect subset specification = " << iter->as() << endl; + exit(EXIT_FAILURE); + } + } + } + } + break; + case 'w': + { + //assume that device numbers will be between 0-9, and names will be 2+ characters + if(argVal.length() == 1){ + output.webcamNumber = atoi(argVal.c_str()); + output.webcamName = ""; + }else{ + output.webcamNumber = -1; + output.webcamName = argVal; + } + + + } + break; + default: + { + cerr << "unknown options" << endl; + usage_build(); + } + break; + } +} + +void cmd_line_parse( int argc, + char **argv, + VOSM_Fitting_Args& fittingArgs) +{ + char *arg = NULL; + int optindex, handleoptions=1; + ; + /* parse options */ + optindex = 0; + while (++optindex < argc) + { + if(argv[optindex][0] != '-') break; + if(++optindex >= argc) usage_build(); + + parse_option( argv[optindex-1][1], + argv[optindex], + fittingArgs); + } + + if (fittingArgs.imageFNs.size() == 0) + { + cerr << " No image loaded" << endl; + usage_build(); + exit(EXIT_FAILURE); + } + if (fittingArgs.annotationFNs.size() != 0 && fittingArgs.annotationFNs.size() != fittingArgs.imageFNs.size() ) + { + cerr << " If annotations are loaded, then, the number of landmarks should be equal to the number of images " << endl; + usage_build(); + exit(EXIT_FAILURE); + } +} +void PrintSummary(const int nb, const int detectionTimes, const Mat& nbOfIterations, const Mat& times, bool doEvaluation, + const vector& ptsErrorAvg, const Mat& deviations, const Mat& ptsErrorFreq){ + + cout << "Detection Times = " << detectionTimes << endl; + float sumIter = safeDoubleToFloat(cv::sum(nbOfIterations).val[0]); + cout << "Average Number of Iterations = " << (sumIter / detectionTimes) << endl; + float sumTime = safeDoubleToFloat(cv::sum(times).val[0]); + cout << "Average Detection time (in ms) = " << (sumTime / detectionTimes) << endl; + Scalar avgDev, stdDev; + if(doEvaluation) + { + cout << "Average Pt Distance = " << std::accumulate(ptsErrorAvg.begin(),ptsErrorAvg.end(),0.00)/detectionTimes << endl; + + cv::meanStdDev(deviations, avgDev, stdDev); + cout << "Average Deviation of Errors = " << avgDev.val[0] << endl; + cout << "Standard Deviation of Errors = " << stdDev.val[0] << endl << endl; + vector avgErrorFreq(nb, 0.0f); + for(int j = 0; j < nb; j++) + { + Mat_ col = ptsErrorFreq.col(j); + avgErrorFreq[j] = safeDoubleToFloat(cv::mean(col).val[0]); + cout << avgErrorFreq[j] << " percentage of points are in " << j << " pixels" << endl; + } + } +} + +//Difficult to abstract, so it stays closeby. +void SaveSequentialResultsInFolder(const Mat& img, const VO_Shape& refShape, vector& meshInfos, const string& fdname){ + + MakeDirectory(fdname); + + Mat tempMat; + vector ptErrorFreq; + float deviation = 0.0f; + vector unsatisfiedPtList; + vector ptErrorPerPoint; + for(unsigned int i = 0;i( &(ostringstream() << i) )->str(), + deviation, + ptErrorPerPoint, + ptErrorFreq, + (meshInfos[i].drawPts / meshInfos[i].f2Scale)); + //draw the image + VO_Fitting2DSM::VO_DrawMeshOnImage(meshInfos[i],img,tempMat); + //Save the Image + string fn = fdname +"/"+ static_cast( &(ostringstream() << i) )->str() +".jpg"; + imwrite(fn.c_str(), tempMat); + } +} + +void drawFaceBoxes(cv::Rect* faceRect, cv::Mat* bgrimg, IplImage* depth, cv::Point leftEye, cv::Point rightEye, cv::Point nose){ + + //dark blue rectangle for nose search + cv::Rect noseRect; + noseRect.x = faceRect->x + 1 + (faceRect->width/4); + noseRect.y = faceRect->y + (1 + faceRect->height/8); + noseRect.height = ((faceRect->y) + 7*(faceRect->height)/8) - noseRect.y; + noseRect.width = (faceRect->x + 3*(faceRect->width/4)) - noseRect.x; + //sloppy repeat! + cv::rectangle (*(bgrimg),noseRect, colors[2], 1, 8, 0); + + + //decide which side it is on by dividing up the nose region + if((nose.x - noseRect.x) < (3*noseRect.width)/8){ + //left green + cv::circle(*(bgrimg),cv::Point(nose.x,nose.y),5,colors[3],3,8,0); + + }else if((nose.x - noseRect.x) < (5*noseRect.width)/8){ + //mid blue + cv::circle(*(bgrimg),cv::Point(nose.x,nose.y),5,colors[4],3,8,0); + + }else{ + //right red + cv::circle(*(bgrimg),cv::Point(nose.x,nose.y),5,colors[5],3,8,0); + + //only do expression on front facing? + } + //right eye search box is blue, based on nose position. + cv::Rect rEyeRect(noseRect.x,noseRect.y,noseRect.width/2,noseRect.height/4); + cv::rectangle (*(bgrimg), rEyeRect, colors[1], 1, 8, 0); + //right eye is blue + cv::circle(*(bgrimg),cv::Point(rightEye.x,rightEye.y),2,colors[1],2,8,0); + + + //left eye red + cv::Rect lEyeRect(noseRect.x + noseRect.width/2,noseRect.y,noseRect.width/2,noseRect.height/4); + cv::rectangle (*(bgrimg), lEyeRect, colors[0], 1, 8, 0); + //left is red + cv::circle(*(bgrimg),cv::Point(leftEye.x,leftEye.y),2,colors[0],2,8,0); +} + +unsigned short findNose(const Rect& faceRect, const Mat& depthMat8, Point2i& nosePoint){ + + //nose variables + unsigned short minThresh = 65535; + unsigned short maxThresh = 0; + int oneD_index = 0; + + //limited search area to center of facebox (1/4 of the whole) + for(int rowi = (faceRect.height/2); rowi < 7*(faceRect.height)/8;rowi++){ + for(int coli = (faceRect.width/4); coli < 3*(faceRect.width/4);coli++){ + //rowi and coli are internal indexes, they are converted to image co-ordinates during extraction + oneD_index = ((faceRect.y)+rowi)*depthMat8.cols + coli+faceRect.x; + //Compare based on mapped value + if(depthMat8.data[oneD_index] > minThresh && depthMat8.data[oneD_index] != 0){ + minThresh = depthMat8.data[oneD_index]; + } + //max + if(depthMat8.data[oneD_index] > maxThresh){ + maxThresh = depthMat8.data[oneD_index]; + nosePoint.y = faceRect.y + rowi; + nosePoint.x = faceRect.x + coli; + } + } + } + return maxThresh; +} + +//This is always off by a few pixels because of the shape of the eyes on the face +//co-ordinates are relative to cropped head image +void findEyes(const Rect& faceRect, const Mat& depthMat8, Point2i& ptRightEye, Point2i& ptLeftEye){ + int oneD_index = 0; + + unsigned char min = 255; + int pxCount = 0; + Rect noseRect; + noseRect.x = faceRect.x + 1 + (faceRect.width/4); + noseRect.y = faceRect.y + (1 + faceRect.height/8); + noseRect.height = ((faceRect.y) + 7*(faceRect.height)/8) - noseRect.y; + noseRect.width = (faceRect.x + 3*(faceRect.width/4)) - noseRect.x; + cv::Rect rEyeRect(noseRect.x,noseRect.y,noseRect.width/2,noseRect.height/4); + + //right eye, use a smaller area + for(int ecoli = rEyeRect.x;ecoli < (rEyeRect.x + rEyeRect.width);ecoli++){ + //use eye boundries directly + for(int erowi = rEyeRect.y;erowi < (rEyeRect.y + rEyeRect.height);erowi++){ + oneD_index = (erowi*depthMat8.cols) + ecoli; + //check if it is within 10mm of the nose depth + if(depthMat8.data[oneD_index] <= min ){ + min = depthMat8.data[oneD_index]; + if(ptRightEye.y == erowi){ + ++pxCount; + } + else{ + pxCount = 0; + } + //including pxCount offset might be helpful if applied to both eyes + ptRightEye.x = ecoli; + ptRightEye.y = erowi; + } + } + }//end of right eye search + + min = 255; + //left eye, start where the right eye search left off, red box + + Rect lEyeRect(noseRect.x + noseRect.width/2,noseRect.y,noseRect.width/2,noseRect.height/4); + + //CAUTION: NOT SYMMETRIC WITH PREVIOUS SEARCH + for(int erowi = lEyeRect.y;erowi < (lEyeRect.y + lEyeRect.height);erowi++){ + for(int ecoli = lEyeRect.x;ecoli < (lEyeRect.x + lEyeRect.width);ecoli++){ + oneD_index = (erowi*depthMat8.cols) + ecoli; + if(depthMat8.data[oneD_index] <= min){ + min = depthMat8.data[oneD_index]; + ptLeftEye.x = ecoli; + ptLeftEye.y = erowi; + } + } + }//end of left eye search +} + +cv::Point2f partCenter(const VO_FaceParts& faceParts, const VO_Shape& refShape,unsigned int facePartEnum){ + + const VO_FacePart& facepart = faceParts.VO_GetOneFacePart(facePartEnum); + vector facepart_indexes = facepart.GetIndexes(); + cv::Point2f partSum(0,0); + for(unsigned int index = 0; index < facepart_indexes.size();++index){ + partSum += refShape.GetA2DPoint(facepart_indexes[index]); + } + return partSum / facepart_indexes.size(); +} + +struct VOSM_Fitting_Results{ + + vector oMeshInfo; + int iterations; + float fittingTime; + VO_Shape finalShape; + Point2f test_points[3]; + Point2f truth_points[3]; + +}; + +bool RunFitting(const VOSM_Fitting_Args& fittingArgs,const Mat& iImage, const bool validRefShape, + const VO_Shape& refShape, VO_Fitting2DSM* const fitting2dsm, CFaceDetectionAlgs& fd, + const int nbOfPyramidLevels, VOSM_Fitting_Results& output, const bool warpBeforeFitting = true){ + + vector multiChannelMat; + Mat vjDetImage, drawImage, fittedImage; + Point2f ptLeftEyeCenter, ptRightEyeCenter, ptMouthCenter; + //These are only valid if doEvaluation is true. + Point2f gt_mouthCenter; + Point2f gt_creye; + Point2f gt_cleye; + + Rect faceRect; + + //Assume the first channel is for face detection + if(fittingArgs.channels > 1){ + cv::split(iImage,multiChannelMat); + } + if((fittingArgs.canidateLoc == VJC || fittingArgs.canidateLoc == MINVJC)){ + vjDetImage = multiChannelMat[0]; + } + else{ + iImage.copyTo(vjDetImage); + } + //iImage.copyTo(fittedImage); + // Size(240,240) + fd.FullFaceDetection( vjDetImage, NULL, true, true, true, true, 1.0, + Size(80,80), + Size( min(vjDetImage.rows,vjDetImage.cols), min(vjDetImage.rows,vjDetImage.cols) ) ); + if( fd.IsFaceDetected() ) + { + fd.CalcFaceKeyPoints(); + float tmpScaleX = static_cast(iImage.cols)/static_cast(vjDetImage.cols); + float tmpScaleY = static_cast(iImage.rows)/static_cast(vjDetImage.rows); + faceRect = fd.GetDetectedFaceWindow(); + ptLeftEyeCenter = fd.GetDetectedFaceKeyPoint(VO_KeyPoint::LEFTEYECENTER); + ptRightEyeCenter = fd.GetDetectedFaceKeyPoint(VO_KeyPoint::RIGHTEYECENTER); + ptMouthCenter = fd.GetDetectedFaceKeyPoint(VO_KeyPoint::MOUTHCENTER); + ptLeftEyeCenter.x *= tmpScaleX; + ptLeftEyeCenter.y *= tmpScaleY; + ptRightEyeCenter.x *= tmpScaleX; + ptRightEyeCenter.y *= tmpScaleY; + ptMouthCenter.x *= tmpScaleX; + ptMouthCenter.y *= tmpScaleY; + faceRect.x *= static_cast(tmpScaleX); + faceRect.y *= static_cast(tmpScaleY); + faceRect.height *= static_cast(tmpScaleY); + faceRect.width *= static_cast(tmpScaleX); + }else if(fittingArgs.canidateLoc == MINVJC || fittingArgs.canidateLoc == VJC) + { + std::cout << "Face detection failed, trying next face." << std::endl; + return false; + } + + if(validRefShape){ + const VO_FaceParts& facePts = fitting2dsm->GetFaceParts(); + //Find the average of whatever points were given in the shape_info file + gt_mouthCenter = partCenter(facePts,refShape,VO_FacePart::MOUTHCORNERPOINTS); + gt_creye = partCenter(facePts,refShape,VO_FacePart::RIGHTEYE); + gt_cleye = partCenter(facePts,refShape,VO_FacePart::LEFTEYE); + } + + //These methods assume a certain strcuture of the face that may not always be true. + // These are not real "face detectors" + if(fittingArgs.channels > 1 && + (fittingArgs.canidateLoc == MINMAX || fittingArgs.canidateLoc == MINVJC)) + { + //Reduce search area to the middle of the face + faceRect.width = (2*iImage.cols)/4; + faceRect.height = (2*iImage.rows)/4; + faceRect.x = (1*iImage.cols)/4; + faceRect.y = (1*iImage.rows)/4; + + //search only gives pixel resolution, whereas facedetection gives sub-pixel resolution + Point2i nosePoint; + findNose(faceRect,multiChannelMat[1],nosePoint); + + Point2i rightEyeLoc, leftEyeLoc; + if(fittingArgs.canidateLoc == MINMAX) + { + findEyes(faceRect,multiChannelMat[1],rightEyeLoc, leftEyeLoc); + ptRightEyeCenter = rightEyeLoc; + ptLeftEyeCenter = leftEyeLoc; + } + ptMouthCenter = nosePoint; + //Again, adjusted based on nose distance from mouth & resolution of images used in testing + // will not hold true for more cases + ptMouthCenter.y = float(nosePoint.y) + (iImage.rows/8); + }//end of custom candidate finding + + if(fittingArgs.canidateLoc == GT) + { + ptRightEyeCenter = gt_creye; + ptLeftEyeCenter = gt_cleye; + ptMouthCenter = gt_mouthCenter; + } + + // Explained by JIA Pei, you can save to see the detection results. + iImage.copyTo(drawImage); + + cv::rectangle(drawImage, Point(static_cast(ptLeftEyeCenter.x)-1, static_cast(ptLeftEyeCenter.y)-1), + Point(static_cast(ptLeftEyeCenter.x)+1, static_cast(ptLeftEyeCenter.y)+1), + colors[5], 2, 8, 0); + cv::rectangle(drawImage, Point(static_cast(ptRightEyeCenter.x)-1, static_cast(ptRightEyeCenter.y)-1), + Point(static_cast(ptRightEyeCenter.x)+1, static_cast(ptRightEyeCenter.y)+1), + colors[6], 2, 8, 0); + cv::rectangle(drawImage, Point(static_cast(ptMouthCenter.x)-1, static_cast(ptMouthCenter.y)-1), + Point(static_cast(ptMouthCenter.x)+1, static_cast(ptMouthCenter.y)+1), + colors[7], 2, 8, 0); + //imwrite("drawImage.jpg", drawImage); + //imwrite("resizedImage.jpg", resizedImage); + + fitting2dsm->VO_StartFitting( iImage, + output.oMeshInfo, + fittingArgs.type, + ptLeftEyeCenter, + ptRightEyeCenter, + ptMouthCenter, + VO_Fitting2DSM::EPOCH, // at most, how many iterations will be carried out + nbOfPyramidLevels, // read from file AXM\AXM.txt + fittingArgs.recordResults, + fittingArgs.fitTechs, + warpBeforeFitting); + + output.iterations = fitting2dsm->GetNbOfIterations(); + output.finalShape = fitting2dsm->VO_GetFittedShape(); + output.fittingTime = fitting2dsm->GetFittingTime(); + output.test_points[0] = ptLeftEyeCenter; + output.test_points[1] = ptRightEyeCenter; + output.test_points[2] = ptMouthCenter; + output.truth_points[0] = gt_cleye; + output.truth_points[1] = gt_creye; + output.truth_points[2] = gt_mouthCenter; + + return true; +} + +int main(int argc, char **argv) +{ + VOSM_Fitting_Args fittingArgs; + //default to single channel support + fittingArgs.fitTech = VO_Fitting2DSM::FIRSTCHANNELONLY; + fittingArgs.webcamNumber = -1; + fittingArgs.database = CAnnotationDBIO::JIAPEI; + + cmd_line_parse( argc, + argv, + fittingArgs); + + VO_Fitting2DSM* fitting2dsm = NULL; + + unsigned int nbOfPyramidLevels = 0; + + // :( + switch(fittingArgs.type) + { + case VO_AXM::AAM_BASIC: + case VO_AXM::AAM_DIRECT: + fitting2dsm = new VO_FittingAAMBasic(); + dynamic_cast(fitting2dsm)->VO_LoadParameters4Fitting(fittingArgs.trainedData); + nbOfPyramidLevels = (&(*(VO_AXM*)(&*((*(VO_FittingAAMBasic*)(fitting2dsm)).m_VOAAMBasic))))->GetNbOfPyramidLevels(); + break; + case VO_AXM::CLM: + case VO_AXM::AFM: + fitting2dsm = new VO_FittingAFM(); + dynamic_cast(fitting2dsm)->VO_LoadParameters4Fitting(fittingArgs.trainedData); + nbOfPyramidLevels = (&(*(VO_AXM*)(&*((*(VO_FittingAFM*)(fitting2dsm)).m_VOAFM))))->GetNbOfPyramidLevels(); + break; + case VO_AXM::AAM_IAIA: + case VO_AXM::AAM_CMUICIA: + fitting2dsm = new VO_FittingAAMInverseIA(); + dynamic_cast(fitting2dsm)->VO_LoadParameters4Fitting(fittingArgs.trainedData); + nbOfPyramidLevels = (&(*(VO_AXM*)(&*((*(VO_FittingAAMInverseIA*)(fitting2dsm)).m_VOAAMInverseIA))))->GetNbOfPyramidLevels(); + break; + case VO_AXM::AAM_FAIA: + fitting2dsm = new VO_FittingAAMForwardIA(); + dynamic_cast(fitting2dsm)->VO_LoadParameters4Fitting(fittingArgs.trainedData); + nbOfPyramidLevels = (&(*(VO_AXM*)(&*((*(VO_FittingAAMForwardIA*)(fitting2dsm)).m_VOAAMForwardIA))))->GetNbOfPyramidLevels(); + break; + case VO_AXM::ASM_LTC: + fitting2dsm = new VO_FittingASMLTCs(); + dynamic_cast(fitting2dsm)->VO_LoadParameters4Fitting(fittingArgs.trainedData); + nbOfPyramidLevels = (&(*(VO_AXM*)(&*((*(VO_FittingASMLTCs*)(fitting2dsm)).m_VOASMLTC))))->GetNbOfPyramidLevels(); + break; + case VO_AXM::ASM_PROFILEND: + fitting2dsm = new VO_FittingASMNDProfiles(); + ((*(VO_TextureModel*)(&(*(VO_AXM*)(&*((*(VO_FittingASMNDProfiles*)(fitting2dsm)).m_VOASMNDProfile)))))).SetNbOfChannels(fittingArgs.channels); + dynamic_cast(fitting2dsm)->VO_LoadParameters4Fitting(fittingArgs.trainedData); + nbOfPyramidLevels = (&(*(VO_AXM*)(&*((*(VO_FittingASMNDProfiles*)(fitting2dsm)).m_VOASMNDProfile))))->GetNbOfPyramidLevels(); + break; + } + + unsigned int numModelPts = (*((VO_ShapeModel*)(&(*((VO_TextureModel*)(&(*((VO_AXM*)((*((VO_FittingASMNDProfiles*)(fitting2dsm))).m_VOASMNDProfile))))))))).GetNbOfPoints(); + if(fittingArgs.fitTechs.empty()){ + fittingArgs.fitTechs.resize(numModelPts); + fill(fittingArgs.fitTechs.begin(),fittingArgs.fitTechs.end(),fittingArgs.fitTech); + }else if(fittingArgs.fitTechs.size() != numModelPts){ + std::cout << "Cannot fit with subset list of incorrect length" << std::endl; + exit(EXIT_FAILURE); + } + + vector oShapes; + static const int nb = 20; //number of pixels away in frequency format for error reporting + bool doEvaluation = false; + unsigned int nbOfTestingSamples = fittingArgs.imageFNs.size(); + Mat_ nbOfIterations = Mat_::zeros(1, nbOfTestingSamples); + Mat_ deviations; + Mat_ ptsErrorFreq; + Mat_ times = Mat_::zeros(1, nbOfTestingSamples); + vector ptsErrorAvg; + if (fittingArgs.annotationFNs.size() == nbOfTestingSamples && fittingArgs.recordResults) + { + doEvaluation = true; + deviations = Mat_::zeros(1, nbOfTestingSamples); + ptsErrorFreq = Mat_::zeros(nbOfTestingSamples, nb); + }else{ + std::cout << "Cannot do evaluation with mismatched landmark files" << std::endl; + exit(EXIT_FAILURE); + } + CAnnotationDBIO::VO_LoadShapeTrainingData( fittingArgs.annotationFNs, fittingArgs.database, oShapes); + + CFaceDetectionAlgs fd; + + //this static pathing is silly, should use cross-platform env var(OPENCV_PATH) expander. + fd.SetConfiguration("C:/OpenCV2.4.5/opencv/data/lbpcascades/lbpcascade_frontalface.xml", + "C:/OpenCV2.4.5/opencv/data/haarcascades/haarcascade_profileface.xml", + "C:/OpenCV2.4.5/opencv/data/haarcascades/haarcascade_mcs_lefteye.xml", + "C:/OpenCV2.4.5/opencv/data/haarcascades/haarcascade_mcs_righteye.xml", + "C:/OpenCV2.4.5/opencv/data/haarcascades/haarcascade_mcs_nose.xml", + "C:/OpenCV2.4.5/opencv/data/haarcascades/haarcascade_mcs_mouth.xml", + VO_AdditiveStrongerClassifier::BOOSTING, + CFaceDetectionAlgs::FRONTAL ); + + Mat iImage; + + unsigned int detectionTimes = 0; + + // For static images from standard face fittingArgs.databases + // (Detection only, no tracking) + ASM/AAM + if(fittingArgs.staticImgs) + { + for(unsigned int i = 0; i < fittingArgs.imageFNs.size(); i++) + { + iImage = imread(fittingArgs.imageFNs[i]); + VOSM_Fitting_Results fittingResults; + + //If face can't be found, go to the next face + if(RunFitting(fittingArgs,iImage,doEvaluation,oShapes[i],fitting2dsm,fd,nbOfPyramidLevels, + fittingResults)) + { + detectionTimes++; + nbOfIterations(0,i) = fittingResults.iterations; + times(0,i) = fittingResults.fittingTime; + } + else + continue; + + //Now report results + size_t found1 = fittingArgs.imageFNs[i].find_last_of("/\\"); + size_t found2 = fittingArgs.imageFNs[i].find_last_of("."); + string prefix = fittingArgs.imageFNs[i].substr(found1+1, found2-1-found1); + + if(fittingArgs.recordResults) + { + // Inputs: Shapes, an Image, a method to draw meshes from given shapes onto image + // Outputs: Result file like Save Shape Results, image file + SaveSequentialResultsInFolder(iImage,oShapes[i],fittingResults.oMeshInfo,prefix); + + string fn = prefix+".jpg"; + if(fittingResults.oMeshInfo.size() > 0) + imwrite(fn.c_str(), fitting2dsm->GetFittedImage()); + } + // Computes errors vs annotations + if(doEvaluation) + { + + vector ptErrorFreq; + float deviation = 0.0f; + vector unsatisfiedPtList; + vector ptErrorPerPoint(fittingResults.finalShape.GetNbOfPoints(),0.0f); + unsatisfiedPtList.clear(); + CRecognitionAlgs::CalcShapeFittingEffect( oShapes[i], + fittingResults.finalShape, + deviation, + ptErrorFreq, + nb, + &ptErrorPerPoint); + deviations(0,i) = deviation; + for(int j = 0; j < nb; j++) + ptsErrorFreq(i, j) = ptErrorFreq[j]; + + CRecognitionAlgs::SaveFittingResults( "./", + prefix, + deviation, + ptErrorPerPoint, + ptErrorFreq, + fittingResults.finalShape, + fittingResults.truth_points, + fittingResults.test_points, + times(0,i)); + + ptsErrorAvg.push_back(std::accumulate(ptErrorPerPoint.begin(),ptErrorPerPoint.end(),0.0f)); + } + + } //end image fitting of loop + + cout << endl << endl; + PrintSummary(nb,detectionTimes,nbOfIterations,times,doEvaluation,ptsErrorAvg,deviations, ptsErrorFreq); + + } + else + { + // For dynamic image sequences + // (Detection or Tracking) + ASM/AAM + CTrackingAlgs* trackAlg = new CTrackingAlgs(); + bool isTracked = false; + detectionTimes = 0; + if(!fittingArgs.webcamName.empty() || fittingArgs.webcamNumber >= 0){ + VOSM_Fitting_Results liveTrackingResults; + VideoCapture cap; + if(fittingArgs.webcamName.empty()) + cap.open(fittingArgs.webcamNumber); + else + cap.open(fittingArgs.webcamName); + + if(!cap.isOpened()) // check if we succeeded + return -1; + + //should read until you kill the program + while(cap.read(iImage)) + { + if(!isTracked) + { + detectionTimes += RunFitting(fittingArgs,iImage,false,NULL,fitting2dsm,fd,nbOfPyramidLevels, + liveTrackingResults); + + // Whenever the face is re-detected, initialize the tracker and set isTracked = true; + Rect rect1 = liveTrackingResults.finalShape.GetShapeBoundRect(); + trackAlg->UpdateTracker(iImage, rect1); + isTracked = true; + } + else + { + + RunFitting(fittingArgs,iImage,false,NULL, fitting2dsm, fd, nbOfPyramidLevels, + liveTrackingResults,false); + // Explained by JIA Pei. For every consequent image, whose previous image is regarded as tracked, + // we have to double-check whether current image is still a tracked one. + isTracked = CRecognitionAlgs::EvaluateFaceTrackedByProbabilityImage( + trackAlg, + iImage, + liveTrackingResults.finalShape, + Size(80,80), + Size( min(iImage.rows,iImage.cols), min(iImage.rows,iImage.cols) ) ); + } + if(fittingArgs.recordResults) + { + string fn = detectionTimes+".jpg"; + imwrite(fn.c_str(), fitting2dsm->GetFittedImage()); + } + } + cout << "detection times = " << detectionTimes << endl; + exit(EXIT_SUCCESS); + } + + for(unsigned int i = 0; i < fittingArgs.imageFNs.size(); i++) + { + VOSM_Fitting_Results fittingResults; + iImage = imread(fittingArgs.imageFNs[i]); + + size_t found1 = fittingArgs.imageFNs[i].find_last_of("/\\"); + size_t found2 = fittingArgs.imageFNs[i].find_last_of("."); + string prefix = fittingArgs.imageFNs[i].substr(found1+1, found2-1-found1); + + if(!isTracked) + { + detectionTimes += RunFitting(fittingArgs,iImage,doEvaluation,oShapes[i],fitting2dsm,fd,nbOfPyramidLevels, + fittingResults); + + // Whenever the face is re-detected, initialize the tracker and set isTracked = true; + Rect rect1 = fittingResults.finalShape.GetShapeBoundRect(); + trackAlg->UpdateTracker(iImage, rect1); + isTracked = true; + } + else + { + + RunFitting(fittingArgs,iImage,doEvaluation,oShapes[i], fitting2dsm, fd, nbOfPyramidLevels, + fittingResults,false); + // Explained by JIA Pei. For every consequent image, whose previous image is regarded as tracked, + // we have to double-check whether current image is still a tracked one. +// isTracked = true; + isTracked = CRecognitionAlgs::EvaluateFaceTrackedByProbabilityImage( + trackAlg, + iImage, + fittingResults.finalShape, + Size(80,80), + Size( min(iImage.rows,iImage.cols), min(iImage.rows,iImage.cols) ) ); + } + + if(fittingArgs.recordResults) + { + string fn = prefix+".jpg"; + imwrite(fn.c_str(), fitting2dsm->GetFittedImage()); + } + + // For evaluation + if(doEvaluation) + { + vector ptErrorFreq; + float deviation = 0.0f; + vector unsatisfiedPtList; + unsatisfiedPtList.clear(); + CRecognitionAlgs::CalcShapeFittingEffect( oShapes[i], + fittingResults.finalShape, + deviation, + ptErrorFreq, + nb); + deviations(0,i) = deviation; + for(int j = 0; j < nb; j++) + ptsErrorFreq(i, j) = ptErrorFreq[j]; + CRecognitionAlgs::SaveShapeRecogResults( "./", + prefix, + deviation, + ptErrorFreq); + } + } + + cout << endl << endl; + PrintSummary(nb,detectionTimes,nbOfIterations,times,doEvaluation,ptsErrorAvg,deviations, ptsErrorFreq); + + delete trackAlg; + } + + delete fitting2dsm; + + return 0; +} diff --git a/tests/test_smfitting/subset.yaml b/tests/test_smfitting/subset.yaml new file mode 100644 index 0000000..bc072bf --- /dev/null +++ b/tests/test_smfitting/subset.yaml @@ -0,0 +1,21 @@ +# Matching command line arguments +# -t ASM_PROFILEND -s true -r true -d + +--- +version: VOSM 0.3.3 +verbose: No +fitting: + # -t ASM_PROFILEND + method: ASM_PROFILEND + # -s true + static-images: Yes + # -r true + record: Yes + #list must the same length and order as the point annotations + techniques: [gray, gray, gray, gray, default, default, default, default, default, default, default, gray, gray, gray, gray, default] + +annotation: + # -d JIAPEI + format: JIAPEI + + \ No newline at end of file diff --git a/tests/test_smfitting/test_smfitting.vcxproj b/tests/test_smfitting/test_smfitting.vcxproj new file mode 100644 index 0000000..973591d --- /dev/null +++ b/tests/test_smfitting/test_smfitting.vcxproj @@ -0,0 +1,109 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {9CF05F3E-48D3-4EB0-B03A-1E0BE5E9E949} + test_smfitting + + + + Application + true + MultiByte + v110 + + + Application + false + true + MultiByte + v110 + + + + + + + + + + + + + C:\OpenCV2.4.5\opencv\include;$(IncludePath) + C:\OpenCV2.4.5\opencv\build\x86\vc11\bin;C:\OpenCV2.4.5\opencv\build\x86\vc11\lib;$(LibraryPath) + + + + Level3 + Disabled + C:\OpenCV2.4.5\opencv\build\include;../../modules/smfitting/include;../../modules/smbuilding/include;../../modules/featureextraction/include;../../modules/common/include;../../modules/comalgs/include;../../modules/cvcommon/include;../../modules/ensembletraining/include;../../modules/integraltransform/include;C:\boost\boost_1_53_0;C:\data\yaml-cpp-0.5.1\include;%(AdditionalIncludeDirectories) + NOMINMAX;%(PreprocessorDefinitions) + + + true + ../../Debug;C:\OpenCV2.4.5\opencv\build\x86\vc11\lib;C:\boost\boost_1_53_0\stage\lib;C:\data\yaml-cpp-0.5.1\build\Debug;%(AdditionalLibraryDirectories) + common.lib;cvcommon.lib;smbuilding.lib;smfitting.lib;comalgs.lib;ensembletraining.lib;integraltransform.lib;featureextraction.lib;opencv_core245d.lib;opencv_highgui245d.lib;opencv_imgproc245d.lib;opencv_video245d.lib;opencv_objdetect245d.lib;opencv_ml245d.lib;opencv_features2d245d.lib;opencv_calib3d245d.lib;opencv_legacy245d.lib;libyaml-cppmdd.lib;%(AdditionalDependencies) + + + true + + + + + Level3 + MaxSpeed + true + true + + + true + true + true + + + + + + + + + + + {5f3443c8-6018-489e-9972-86c08813580b} + + + {6dc6813e-859a-4759-94cd-a1f1a6466b7b} + + + {4e713018-b49c-44ee-bf6a-4aaecbd0d148} + + + {838e7724-4e9b-4489-b26c-b419547473d2} + + + {01a5e120-9d02-44a1-a7e1-f6190f212aa5} + + + {900f9166-cd10-49d8-a339-23e0a035f3e9} + + + {6b3e57cc-3ca2-4bd6-88d5-ed99990d8ddb} + + + {57706470-a0d4-455d-ab60-626eae943596} + + + + + + \ No newline at end of file diff --git a/tests/test_smfitting/test_smfitting.vcxproj.filters b/tests/test_smfitting/test_smfitting.vcxproj.filters new file mode 100644 index 0000000..581c210 --- /dev/null +++ b/tests/test_smfitting/test_smfitting.vcxproj.filters @@ -0,0 +1,27 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Header Files + + + + + Source Files + + + \ No newline at end of file diff --git a/vosm.pc.cmake.in b/vosm.pc.cmake.in new file mode 100644 index 0000000..e9b4653 --- /dev/null +++ b/vosm.pc.cmake.in @@ -0,0 +1,13 @@ +# Package Information for pkg-config + +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@/vosm + +Name: vosm +Description: Vision Open Statistical Models +Version: @VERSION@ +Libs: -L${libdir} -lcomalgs@VOSM_DLLVERSION@ -lcvcommon@VOSM_DLLVERSION@ -lensembletraining@VOSM_DLLVERSION@ -lfeatureextraction@VOSM_DLLVERSION@ -lintegraltransform@VOSM_DLLVERSION@ -lsmbuilding@VOSM_DLLVERSION@ +-lsmfitting@VOSM_DLLVERSION@ -lcommon@VOSM_DLLVERSION@ +Cflags: -I${includedir} diff --git a/vosm.sln b/vosm.sln new file mode 100644 index 0000000..22759c7 --- /dev/null +++ b/vosm.sln @@ -0,0 +1,132 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2012 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "modules", "modules", "{7D219998-5DFC-4A57-BD2B-CB6078CBF0DE}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "comalgs", "modules\comalgs\comalgs.vcxproj", "{5F3443C8-6018-489E-9972-86C08813580B}" + ProjectSection(ProjectDependencies) = postProject + {838E7724-4E9B-4489-B26C-B419547473D2} = {838E7724-4E9B-4489-B26C-B419547473D2} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "common", "modules\common\common.vcxproj", "{6DC6813E-859A-4759-94CD-A1F1A6466B7B}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cvcommon", "modules\cvcommon\cvcommon.vcxproj", "{4E713018-B49C-44EE-BF6A-4AAECBD0D148}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ensembletraining", "modules\ensembletraining\ensembletraining.vcxproj", "{838E7724-4E9B-4489-B26C-B419547473D2}" + ProjectSection(ProjectDependencies) = postProject + {01A5E120-9D02-44A1-A7E1-F6190F212AA5} = {01A5E120-9D02-44A1-A7E1-F6190F212AA5} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "featureextraction", "modules\featureextraction\featureextraction.vcxproj", "{01A5E120-9D02-44A1-A7E1-F6190F212AA5}" + ProjectSection(ProjectDependencies) = postProject + {900F9166-CD10-49D8-A339-23E0A035F3E9} = {900F9166-CD10-49D8-A339-23E0A035F3E9} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "integraltransform", "modules\integraltransform\integraltransform.vcxproj", "{900F9166-CD10-49D8-A339-23E0A035F3E9}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "smbuilding", "modules\smbuilding\smbuilding.vcxproj", "{6B3E57CC-3CA2-4BD6-88D5-ED99990D8DDB}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "smfitting", "modules\smfitting\smfitting.vcxproj", "{57706470-A0D4-455D-AB60-626EAE943596}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{5A9CF4E6-98D1-41BA-B419-2FA23DBB730B}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_ensembletraining", "tests\test_ensembletraining\test_ensembletraining.vcxproj", "{B357DFE4-CFAB-4361-9C72-EEA806EECEB4}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_featureextraction", "tests\test_featureextraction\test_featureextraction.vcxproj", "{5D77FD6C-4547-42C0-88AC-85C16614AB68}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_integraltransform", "tests\test_integraltransform\test_integraltransform.vcxproj", "{8B707CE2-543C-4398-9A30-9982EE43D249}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_smbuilding", "tests\test_smbuilding\test_smbuilding.vcxproj", "{34E3A0E0-A9AA-4338-A68C-5B086E505117}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_smfitting", "tests\test_smfitting\test_smfitting.vcxproj", "{9CF05F3E-48D3-4EB0-B03A-1E0BE5E9E949}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{1DD55F31-5765-4626-8E72-5EEF3DC30BFD}" + ProjectSection(SolutionItems) = preProject + Performance1.psess = Performance1.psess + EndProjectSection +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Release|Win32 = Release|Win32 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {5F3443C8-6018-489E-9972-86C08813580B}.Debug|Win32.ActiveCfg = Debug|Win32 + {5F3443C8-6018-489E-9972-86C08813580B}.Debug|Win32.Build.0 = Debug|Win32 + {5F3443C8-6018-489E-9972-86C08813580B}.Release|Win32.ActiveCfg = Release|Win32 + {5F3443C8-6018-489E-9972-86C08813580B}.Release|Win32.Build.0 = Release|Win32 + {6DC6813E-859A-4759-94CD-A1F1A6466B7B}.Debug|Win32.ActiveCfg = Debug|Win32 + {6DC6813E-859A-4759-94CD-A1F1A6466B7B}.Debug|Win32.Build.0 = Debug|Win32 + {6DC6813E-859A-4759-94CD-A1F1A6466B7B}.Release|Win32.ActiveCfg = Release|Win32 + {6DC6813E-859A-4759-94CD-A1F1A6466B7B}.Release|Win32.Build.0 = Release|Win32 + {4E713018-B49C-44EE-BF6A-4AAECBD0D148}.Debug|Win32.ActiveCfg = Debug|Win32 + {4E713018-B49C-44EE-BF6A-4AAECBD0D148}.Debug|Win32.Build.0 = Debug|Win32 + {4E713018-B49C-44EE-BF6A-4AAECBD0D148}.Release|Win32.ActiveCfg = Release|Win32 + {4E713018-B49C-44EE-BF6A-4AAECBD0D148}.Release|Win32.Build.0 = Release|Win32 + {838E7724-4E9B-4489-B26C-B419547473D2}.Debug|Win32.ActiveCfg = Debug|Win32 + {838E7724-4E9B-4489-B26C-B419547473D2}.Debug|Win32.Build.0 = Debug|Win32 + {838E7724-4E9B-4489-B26C-B419547473D2}.Release|Win32.ActiveCfg = Release|Win32 + {838E7724-4E9B-4489-B26C-B419547473D2}.Release|Win32.Build.0 = Release|Win32 + {01A5E120-9D02-44A1-A7E1-F6190F212AA5}.Debug|Win32.ActiveCfg = Debug|Win32 + {01A5E120-9D02-44A1-A7E1-F6190F212AA5}.Debug|Win32.Build.0 = Debug|Win32 + {01A5E120-9D02-44A1-A7E1-F6190F212AA5}.Release|Win32.ActiveCfg = Release|Win32 + {01A5E120-9D02-44A1-A7E1-F6190F212AA5}.Release|Win32.Build.0 = Release|Win32 + {900F9166-CD10-49D8-A339-23E0A035F3E9}.Debug|Win32.ActiveCfg = Debug|Win32 + {900F9166-CD10-49D8-A339-23E0A035F3E9}.Debug|Win32.Build.0 = Debug|Win32 + {900F9166-CD10-49D8-A339-23E0A035F3E9}.Release|Win32.ActiveCfg = Release|Win32 + {900F9166-CD10-49D8-A339-23E0A035F3E9}.Release|Win32.Build.0 = Release|Win32 + {6B3E57CC-3CA2-4BD6-88D5-ED99990D8DDB}.Debug|Win32.ActiveCfg = Debug|Win32 + {6B3E57CC-3CA2-4BD6-88D5-ED99990D8DDB}.Debug|Win32.Build.0 = Debug|Win32 + {6B3E57CC-3CA2-4BD6-88D5-ED99990D8DDB}.Release|Win32.ActiveCfg = Release|Win32 + {6B3E57CC-3CA2-4BD6-88D5-ED99990D8DDB}.Release|Win32.Build.0 = Release|Win32 + {57706470-A0D4-455D-AB60-626EAE943596}.Debug|Win32.ActiveCfg = Debug|Win32 + {57706470-A0D4-455D-AB60-626EAE943596}.Debug|Win32.Build.0 = Debug|Win32 + {57706470-A0D4-455D-AB60-626EAE943596}.Release|Win32.ActiveCfg = Release|Win32 + {57706470-A0D4-455D-AB60-626EAE943596}.Release|Win32.Build.0 = Release|Win32 + {B357DFE4-CFAB-4361-9C72-EEA806EECEB4}.Debug|Win32.ActiveCfg = Debug|Win32 + {B357DFE4-CFAB-4361-9C72-EEA806EECEB4}.Debug|Win32.Build.0 = Debug|Win32 + {B357DFE4-CFAB-4361-9C72-EEA806EECEB4}.Release|Win32.ActiveCfg = Release|Win32 + {B357DFE4-CFAB-4361-9C72-EEA806EECEB4}.Release|Win32.Build.0 = Release|Win32 + {5D77FD6C-4547-42C0-88AC-85C16614AB68}.Debug|Win32.ActiveCfg = Debug|Win32 + {5D77FD6C-4547-42C0-88AC-85C16614AB68}.Debug|Win32.Build.0 = Debug|Win32 + {5D77FD6C-4547-42C0-88AC-85C16614AB68}.Release|Win32.ActiveCfg = Release|Win32 + {5D77FD6C-4547-42C0-88AC-85C16614AB68}.Release|Win32.Build.0 = Release|Win32 + {8B707CE2-543C-4398-9A30-9982EE43D249}.Debug|Win32.ActiveCfg = Debug|Win32 + {8B707CE2-543C-4398-9A30-9982EE43D249}.Debug|Win32.Build.0 = Debug|Win32 + {8B707CE2-543C-4398-9A30-9982EE43D249}.Release|Win32.ActiveCfg = Release|Win32 + {8B707CE2-543C-4398-9A30-9982EE43D249}.Release|Win32.Build.0 = Release|Win32 + {34E3A0E0-A9AA-4338-A68C-5B086E505117}.Debug|Win32.ActiveCfg = Debug|Win32 + {34E3A0E0-A9AA-4338-A68C-5B086E505117}.Debug|Win32.Build.0 = Debug|Win32 + {34E3A0E0-A9AA-4338-A68C-5B086E505117}.Release|Win32.ActiveCfg = Release|Win32 + {34E3A0E0-A9AA-4338-A68C-5B086E505117}.Release|Win32.Build.0 = Release|Win32 + {9CF05F3E-48D3-4EB0-B03A-1E0BE5E9E949}.Debug|Win32.ActiveCfg = Debug|Win32 + {9CF05F3E-48D3-4EB0-B03A-1E0BE5E9E949}.Debug|Win32.Build.0 = Debug|Win32 + {9CF05F3E-48D3-4EB0-B03A-1E0BE5E9E949}.Release|Win32.ActiveCfg = Release|Win32 + {9CF05F3E-48D3-4EB0-B03A-1E0BE5E9E949}.Release|Win32.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {5F3443C8-6018-489E-9972-86C08813580B} = {7D219998-5DFC-4A57-BD2B-CB6078CBF0DE} + {6DC6813E-859A-4759-94CD-A1F1A6466B7B} = {7D219998-5DFC-4A57-BD2B-CB6078CBF0DE} + {4E713018-B49C-44EE-BF6A-4AAECBD0D148} = {7D219998-5DFC-4A57-BD2B-CB6078CBF0DE} + {838E7724-4E9B-4489-B26C-B419547473D2} = {7D219998-5DFC-4A57-BD2B-CB6078CBF0DE} + {01A5E120-9D02-44A1-A7E1-F6190F212AA5} = {7D219998-5DFC-4A57-BD2B-CB6078CBF0DE} + {900F9166-CD10-49D8-A339-23E0A035F3E9} = {7D219998-5DFC-4A57-BD2B-CB6078CBF0DE} + {6B3E57CC-3CA2-4BD6-88D5-ED99990D8DDB} = {7D219998-5DFC-4A57-BD2B-CB6078CBF0DE} + {57706470-A0D4-455D-AB60-626EAE943596} = {7D219998-5DFC-4A57-BD2B-CB6078CBF0DE} + {B357DFE4-CFAB-4361-9C72-EEA806EECEB4} = {5A9CF4E6-98D1-41BA-B419-2FA23DBB730B} + {5D77FD6C-4547-42C0-88AC-85C16614AB68} = {5A9CF4E6-98D1-41BA-B419-2FA23DBB730B} + {8B707CE2-543C-4398-9A30-9982EE43D249} = {5A9CF4E6-98D1-41BA-B419-2FA23DBB730B} + {34E3A0E0-A9AA-4338-A68C-5B086E505117} = {5A9CF4E6-98D1-41BA-B419-2FA23DBB730B} + {9CF05F3E-48D3-4EB0-B03A-1E0BE5E9E949} = {5A9CF4E6-98D1-41BA-B419-2FA23DBB730B} + EndGlobalSection + GlobalSection(Performance) = preSolution + HasPerformanceSessions = true + EndGlobalSection + GlobalSection(SubversionScc) = preSolution + Svn-Managed = True + Manager = AnkhSVN - Subversion Support for Visual Studio + EndGlobalSection +EndGlobal diff --git a/vosmConfig.cmake.in b/vosmConfig.cmake.in new file mode 100644 index 0000000..9848b2e --- /dev/null +++ b/vosmConfig.cmake.in @@ -0,0 +1,83 @@ +# =================================================================================== +# The VOSM CMake configuration file +# +# ** File generated automatically, do not modify ** +# +# Usage from an external project: +# In your CMakeLists.txt, add these lines: +# +# FIND_PACKAGE(VOSM REQUIRED ) +# TARGET_LINK_LIBRARIES(MY_TARGET_NAME ${VOSM_LIBS}) +# +# This file will define the following variables: +# - VOSM_LIBS : The list of libraries to links against. +# - VOSM_LIB_DIR : The directory where lib files are. Calling LINK_DIRECTORIES +# with this path is NOT needed. +# - VOSM_INCLUDE_DIRS : The VOSM include directories. +# - VOSM_VERSION : The version of this VOSM build. Example: "1.2.0" +# - VOSM_VERSION_MAJOR : Major version part of VOSM_VERSION. Example: "1" +# - VOSM_VERSION_MINOR : Minor version part of VOSM_VERSION. Example: "2" +# - VOSM_VERSION_PATCH : Patch version part of VOSM_VERSION. Example: "0" +# +# =================================================================================== + + +# Extract the directory where *this* file has been installed (determined at cmake run-time) +# This variable may or may not be used below, depending on the parsing of vosmConfig.cmake +get_filename_component(THIS_VOSM_CONFIG_PATH "${CMAKE_CURRENT_LIST_FILE}" PATH) + +# ====================================================== +# Include directories to add to the user project: +# ====================================================== +INCLUDE_DIRECTORIES(@CMAKE_INCLUDE_DIRS_CONFIGCMAKE@) +# Provide the include directories to the caller +SET(VOSM_INCLUDE_DIRS @CMAKE_INCLUDE_DIRS_CONFIGCMAKE@) + +# ====================================================== +# Link directories to add to the user project: +# ====================================================== +LINK_DIRECTORIES("@CMAKE_LIB_DIRS_CONFIGCMAKE@") +# Provide the libs directory anyway, it may be needed in some cases. +SET(VOSM_LIB_DIR "@CMAKE_LIB_DIRS_CONFIGCMAKE@") + +# ==================================================================== +# Link libraries: e.g. cxcore110.so, cv110.lib / cv110d.lib, etc... +# ==================================================================== +set(VOSM_LIB_COMPONENTS cxcore cv ml highgui cvaux) +SET(VOSM_LIBS "") +foreach(__CVLIB ${VOSM_LIB_COMPONENTS}) + # CMake>=2.6 supports the notation "debug XXd optimized XX" + if (CMAKE_MAJOR_VERSION GREATER 2 OR CMAKE_MINOR_VERSION GREATER 4) + # Modern CMake: + SET(VOSM_LIBS ${VOSM_LIBS} debug ${__CVLIB}@VOSM_DLLVERSION@@VOSM_DEBUG_POSTFIX@ optimized ${__CVLIB}@VOSM_DLLVERSION@) + else(CMAKE_MAJOR_VERSION GREATER 2 OR CMAKE_MINOR_VERSION GREATER 4) + # Old CMake: + SET(VOSM_LIBS ${VOSM_LIBS} ${__CVLIB}@VOSM_DLLVERSION@) + endif(CMAKE_MAJOR_VERSION GREATER 2 OR CMAKE_MINOR_VERSION GREATER 4) +endforeach(__CVLIB) + +# For VOSM built as static libs, we need the user to link against +# many more dependencies: +IF (NOT @VOSM_BUILD_SHARED_LIB@) + # Under static libs, the user of VOSM needs access to the 3rdparty libs as well: + LINK_DIRECTORIES("@CMAKE_BINARY_DIR@/3rdparty/lib") + + SET(VOSM_LIBS + @VOSM_LINKER_LIBS@ + @IPP_LIBS@ + @JPEG_LIBRARIES@ + @PNG_LIBRARIES@ + @TIFF_LIBRARIES@ + @JASPER_LIBRARIES@ + @HIGHGUI_LIBRARIES@ + ${VOSM_LIBS}) # The standard libs BEFORE (VOSM_LIBS) since these can have "debug","optimized". +ENDIF(NOT @VOSM_BUILD_SHARED_LIB@) + + +# ====================================================== +# Version variables: +# ====================================================== +SET(VOSM_VERSION @VOSM_VERSION@) +SET(VOSM_VERSION_MAJOR @VOSM_VERSION_MAJOR@) +SET(VOSM_VERSION_MINOR @VOSM_VERSION_MINOR@) +SET(VOSM_VERSION_PATCH @VOSM_VERSION_PATCH@) diff --git a/vosmModule.cmake b/vosmModule.cmake new file mode 100644 index 0000000..85778a8 --- /dev/null +++ b/vosmModule.cmake @@ -0,0 +1,61 @@ +# this is template for a VOSM module +macro(define_vosm_module name) + + project(${name}) + add_definitions(-DCVAPI_EXPORTS) + + include_directories("${CMAKE_CURRENT_SOURCE_DIR}/include" + "${CMAKE_CURRENT_SOURCE_DIR}/src" + "${CMAKE_CURRENT_BINARY_DIR}") + + file(GLOB lib_srcs "src/*.cpp") + file(GLOB lib_hdrs "include/*.h*") + source_group("Src" FILES ${lib_srcs} ${lib_hdrs}) + + set(the_target "${name}") + + add_library(${the_target} ${lib_srcs} ${lib_hdrs} ) + + + # For dynamic link numbering convenions + set_target_properties(${the_target} PROPERTIES + VERSION ${VOSM_VERSION} + SOVERSION ${VOSM_SOVERSION} + OUTPUT_NAME "${the_target}${VOSM_DLLVERSION}" + ) + + # Additional target properties + set_target_properties(${the_target} PROPERTIES + DEBUG_POSTFIX "${VOSM_DEBUG_POSTFIX}" + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib/" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin/" + INSTALL_NAME_DIR "${CMAKE_INSTALL_PREFIX}/lib" + ) + + # Add the required libraries for linking: + target_link_libraries(${the_target} ${VOSM_LINKER_LIBS} ${IPP_LIBS} ) + + if(MSVC) + if(CMAKE_CROSSCOMPILING) + set_target_properties(${the_target} PROPERTIES + LINK_FLAGS "/NODEFAULTLIB:secchk" + ) + endif() + set_target_properties(${the_target} PROPERTIES + LINK_FLAGS "/NODEFAULTLIB:libc" + ) + endif() + + # Dependencies of this target: + add_dependencies(${the_target} ${ARGN}) + + install(TARGETS ${the_target} + RUNTIME DESTINATION bin COMPONENT main + LIBRARY DESTINATION lib COMPONENT main + ARCHIVE DESTINATION lib COMPONENT main) + + install(FILES ${lib_hdrs} + DESTINATION include/vosm/${name} + COMPONENT main) + +endmacro()