VOSM Code

/****************************************************************************
* 
* Copyright (c) 2006 by JIA Pei, all rights reserved.
* Copyright (c) 2006 by Vision Open: http://www.visionopen.com/
*
* Author:      	JIA Pei
* Contact:     	jp4work@gmail.com
* URL:		http://www.visionopen.com/members/jiapei.php
* The author administrates Vision Open -- http://www.visionopen.com
*
* This program is free software; you can redistribute it and/or modify it 
* under the terms of the GNU General Public License as published by the 
* Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but 
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 
* or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 
* for more details.
* 
* You should have received a copy of the GNU General Public License along 
* with this program; if not, write to the Free Software Foundation, Inc., 
* 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
* 
* This software is partly based on the following open source: 
*  
*		- Boost 
*		- OpenCV 
* 
* This software is using IMM Face Database, which can be downloaded from
* http://www2.imm.dtu.dk/~aam/.
*
* M. B. Stegmann, B. K. Ersb{\o}ll, and R. Larsen. FAME - a flexible appearance 
* modelling environment. IEEE Trans. on Medical Imaging, 22(10):1319-1331, 2003
*
****************************************************************************/

// $Id: lv_aambuilding.h,v 1.1.1.1 2006-09-03 17:49:04 JIA Pei Exp $ 

#ifndef __lv_aambuilding__
#define __lv_aambuilding__


#include <vector>

#include "cv.h"
#include "highgui.h"
#include "ml.h"

#include "lv_aamedge.h"
#include "lv_aamtriangle2d.h"
#include "lv_aamshape2d.h"
#include "lv_aamtexture2d3c.h"


using namespace std;


/**

    @author     JIA Pei

    @version    2006-09-01

    @brief      Longer Vision AAM model building.

    @Note       It's convenient to code by STL, while OpenCV is not compatible
                with STL yet.
    
*/
class lv_aambuilding
{
public:

    // For Shape. Data Type: OpenCV -- CV_64FC1

    // The shape samples // Actually, the same as "m_vAAMAlignedShapes", but not scale to unit
    CvMat*                          m_CVMShape;       // In the form of xyxyxyxy...240*116 

    // Aligned shape samples
    CvMat*                          m_CVMAlignedShapes;                         // 240*116

    // Mean shape // Actually, "m_CVMMeanShape" should be the same as 
    // "m_oAAMMeanShape", and "m_CVMMeanShape" is used for cvCalcPCA
    CvMat*                          m_CVMMeanShape;                             // 1*116

    // Original Eigenvectors of m_CVMAlignedShapes
    CvMat*                          m_CVMAlignedShapesEigenVectors;             // 116*116
    
    // Original Eigenvalues of m_CVMAlignedShapes
    CvMat*                          m_CVMAlignedShapesEigenValues;              // 1*116

    // Truncated Eigenvectors of m_CVMAlignedShapes
    CvMat*                          m_CVMTruncatedAlignedShapesEigenVectors;    // 20*116

    // Truncated Eigenvalues of m_CVMAlignedShapes
    CvMat*                          m_CVMTruncatedAlignedShapesEigenValues;     // 1*20





    // For Connection of Shape and Texture. Data Type: OpenCV -- CV_32FC2

    // "m_CVMPoints" the same as "m_CVMShape", comes from "m_oAAMReferenceShape", for texture!
    // m_CVMPoints is moved to left top most origin.
    CvMat*                          m_CVMPoints;      // In the form of CvPoint, CvPoint...1*58

    // Convex Hull of the points and sub division of the plane
    CvMat*                          m_CVMConvexHull;
    CvSubdiv2D*                     m_CVSubdiv;






    // For Texture. Data Type: OpenCV -- CV_64FC1

    // The texture samples
    CvMat*                          m_CVMTexture;                       // 240*(31461 * 3 (B, G, R))

    // Mean Texture // Actually, "m_CVMMeanTexture" should be the same as 
    // "m_oAAMMeanTexture", and "m_CVMMeanTexture" is used for cvCalcPCA
    CvMat*                          m_CVMMeanTexture;                   // 1**(31461 * 3 (B, G, R))

    // Original Eigenvectors of m_MAlignedTexture
    CvMat*                          m_CVMTexturesEigenVectors;          // 240*94383

    // Original Eigenvalues of m_MAlignedTexture
    CvMat*                          m_CVMTexturesEigenValues;           // 1*240

    // Truncated Eigenvectors of m_MAlignedTexture
    CvMat*                          m_CVMTruncatedTexturesEigenVectors; // 116*94383

    // Truncated Eigenvalues of m_MAlignedTexture
    CvMat*                          m_CVMTruncatedTexturesEigenValues;  // 1*116






    // For Concatenated AAM. Data Type: OpenCV -- CV_64FC1

    // Shape weights, for scaling to texture scale.
    CvMat*                          m_CVMWeightsScale2Texture;                  // 20*20

    // The Concatenated samples
    CvMat*                          m_CVMConcatenated;                          // 240*136 

    // Mean Concatenated // Actually, "m_CVMMeanConcatenated" should be the same as 
    // "m_oAAMMeanTexture", and "m_CVMMeanConcatenated" is used for cvCalcPCA
    CvMat*                          m_CVMMeanConcatenated;                      // 1*136

    // Original Eigenvectors of m_CVMConcatenated
    CvMat*                          m_CVMConcatenatedEigenVectors;              // 136*136 

    // Original Eigenvalues of m_CVMConcatenated
    CvMat*                          m_CVMConcatenatedEigenValues;               // 1*136

    // Truncated Eigenvectors of m_CVMConcatenated
    CvMat*                          m_CVMTruncatedConcatenatedEigenVectors;     // 15*136

    // Truncated Eigenvalues of m_CVMConcatenated
    CvMat*                          m_CVMTruncatedConcatenatedEigenValues;      // 1*15





    // Q_s: matrix for shape model
    // The shape part of the combined eigenvectors
    
    CvMat*                          m_CVMQs;                                    // 15*20

    // Q_g: matrix for texture model
    // The texture part of the combined eigenvectors
    CvMat*                          m_CVMQg;                                    // 15*116

    CvMat*                          m_CVDeltaC;                                 // 136*240
    CvMat*                          m_CVDeltaG;                                 // 240*94383
    CvMat*                          m_CVR;                                      // 136*94383

    // m_R_s
    CvMat*                          m_R_s;                                      // 

    // m_R_g
    CvMat*                          m_R_g;


    CvMat*                          GradTPartialWPB;
    CvMat*                          GradTPartialWPG;
    CvMat*                          GradTPartialWPR;

    IplImage*                       FaceTemplate;   
    
    

    // The file vectors to store ASF file ( for shapes) and JPG file ( for textures).
    vector<string>                  m_vasfFiles;
    vector<string>                  m_vimgFiles;

    // Number of Samples : 240
    int                             m_iNbOfSamples;

    // Number of points to describe per shape: 58
    int                             m_iNbOfPoints;

    // Number of texture to describe per image : 31461 * 3 (B, G, R)
    int                             m_iNbOfTextures;

    // The shape samples' average size : 582.425
    double                          m_dAverageSize;

    // will load all 240 pictures
    vector<IplImage*>               m_vImages;

    // The same as m_CVMAlignedShapes, m_CVMMeanShape, already normalized to scale 1
    vector<lv_aamshape2d>           m_vAAMAlignedShapes;
    lv_aamshape2d                   m_oAAMMeanShape;

    // The shape that to be used as reference shape!!!
    // scale back to the original size// and translate back to the original position
    lv_aamshape2d                   m_oAAMReferenceShape;
   
    // a vector of lv_aamedge; should be 152 edges
    vector<lv_aamedge>              m_vAAMEdge;

    // a vector of lv_aamtriangle2d; should be 95 triangles
    vector<lv_aamtriangle2d>        m_vAAMTriangle2D;

    // Use to specify which texture (31461) is in which triangle (95 triangles)
    vector<lv_aamwarping>           m_vTextureTriangle;         // 31461

    // Steepest Descent Images (blue, green, red, gray) * 31461 * 20
    vector< vector< vector<double> > > m_SteepestDescentImages4All;   // 4*31461*20

    // Hessian Matrix   4*20*20
    vector< vector< vector<double> > > m_HessianMatrixInverse;

    // The same as m_CVMTexture, m_CVMMeanTexture
    vector<lv_aamtexture2d3c>       m_vAAMTextures;
    lv_aamtexture2d3c               m_oAAMMeanTexture;

    // Unlike m_oAAMReferenceShape, m_oAAMRefrencetexture cannot be used directly
    // m_oAAMRefrencetexture is just used for the reference to adjust m_oAAMMeanTexture
    // back to the original size instead of normalized size!!!!!
    lv_aamtexture2d3c               m_oAAMReferenceTexture;

    // The same as m_CVMConcatenated, m_CVMMeanConcatenated
    vector<vector<double> >         m_vAAMConcatenated;
    //vector<double>                m_oAAMMeanConcatenated;




    // constructors
    lv_aambuilding();
    lv_aambuilding(const vector<string> &asfFiles, const vector<string> &imgFiles, string build2File);
    
    // destructor
    ~lv_aambuilding();





    void                            LV_BuildAAM(const vector<string> &asfFiles, const vector<string> &imgFiles, string build2File);

    // AAM Shape Model First

    // Build AAM Shape model
    void                            LV_BuildAAMShapeModel();

    // Load shapes and change to absolute coordinators
    void                            LV_LoadShapes2Absolute(const vector<string> &asfFiles);

    // Align all shapes
    void                            LV_AlignShapes();

    // Returns the mean shape of all shapes.
    void                            LV_MeanShape(lv_aamshape2d &meanShape); 





    // AAM Texture Model Second

    // Build AAM Texture model
    void                            LV_BuildAAMTextureModel();

    // Build template mesh
    void                            LV_BuildTemplateMesh();

    // Build all the edges ( Delaunay Triangulation edges)
    void                            LV_BuildEdges();

    // Is the current point pt in the convex hull
    static bool                     LV_IsPointInConvexHull(CvPoint2D32f pt, CvMat* ch, bool includinghull = true);

    // To calculate all the triangles in the mesh
    void                            LV_BuildTriangles();

    // Is the current edge (ind1, ind2) already in the AAM model edges?
    bool                            LV_IsCurrentEdgeAlreadyInAAM(int ind1, int ind2);

    // Help to build up triangles in the mesh
    bool                            LV_VectorIsNotInFormerTriangles(const vector<int> v);

    // Assign triangles to all textures, specify which texture is in which triangle.
    void                            LV_TextureTriangles ();

    // Load and normalize texture, for all images
    void                            LV_LoadTexture2Normalized();

    // imageNo specify which image is used now.
    static vector<double>           LV_GetSubPixelInto3Channels(double x, double y, const IplImage* image);

    static vector<unsigned char>    LV_GetPixelInto3Channels(int x, int y, const IplImage* image);

    // Normalize all textures.
    void                            LV_NormalizeTexture( );

    // Zero Mean Unit Length one texture.
    static void                     LV_ZeroMeanUnitLengthOneTexture(lv_aamtexture2d3c &onetexture);
    static void                     LV_ZeroMeanUnitLengthOneTexture(CvMat* onetexture);

    // Returns the mean texture of all textures.
    void                            LV_MeanTexture(lv_aamtexture2d3c &meanTexture);








    // Pre-computation Of Inverse Compositional Image Alignment AAM Fitting
    void                            LV_TemplateFaceGradient();
    void                            LV_SteepestDescentImages();
    void                            LV_HessianMatrix();



    // Concatenated AAM Third

    void                            LV_BuildConcatenatedAAM();

    void                            LV_ShapeTexture2Concatenated( lv_aamshape2d iShape, lv_aamtexture2d3c iTexture, vector<double>& b );

    void                            LV_ShapeProject2TruncatedEigenSpace(const lv_aamshape2d& iShape, CvMat* projectedTruncatedShape);

    void                            LV_TextureProject2TruncatedEigenSpace(const lv_aamtexture2d3c& iTexture, CvMat* projectedTruncatedTexture);






    // 2006-10-12
    // Off-line build prediction matrix for fitting.
    void                            BuildRegressionMatrices();

    vector<vector<double> >         LV_Concatenated2Parameters( );

    vector<double>                  LV_ConcatenatedProject2TruncatedEigenSpace(const vector<double> iConcatenated);




    // Read ASF file
    void                            LV_LoadParamtersFromFile(string fn);

    // Write the trained data to file.
    void                            LV_WriteParameters2File(string fn);

};

#endif // __lv_aambuilding__