00001
00002 /***************************************************************************
00003  *  sift.cpp - Feature-based classifier using OpenCV structures
00004  *
00005  *  Created: Mon Mar 15 15:47:11 2008
00006  *  Copyright 2008 Stefan Schiffer [stefanschiffer.de]
00007  *
00008  ****************************************************************************/
00009
00010 /*  This program is free software; you can redistribute it and/or modify
00011  *  it under the terms of the GNU General Public License as published by
00012  *  the Free Software Foundation; either version 2 of the License, or
00013  *  (at your option) any later version. A runtime exception applies to
00014  *  this software (see LICENSE.GPL_WRE file mentioned below for details).
00015  *
00016  *  This program is distributed in the hope that it will be useful,
00017  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
00018  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00019  *  GNU Library General Public License for more details.
00020  *
00021  *  Read the full text in the LICENSE.GPL_WRE file in the doc directory.
00022  */
00023
00024 #include <iostream>
00025 #include <vector>
00026
00027 #include <classifiers/sift.h>
00028 //#ifdef SIFT_TIMETRACKER
00029 #include <utils/time/clock.h>
00030 #include <utils/time/tracker.h>
00031 //#endif
00032
00033 extern "C" {
00034 #include <sift/sift.h>
00035 #include <sift/imgfeatures.h>
00036 #include <sift/kdtree.h>
00037 #include <sift/utils.h>
00038 #include <sift/xform.h>
00039 }
00040
00041 #include <core/exception.h>
00042 #include <core/exceptions/software.h>
00043 #include <fvutils/color/colorspaces.h>
00044 #include <fvutils/color/conversions.h>
00045
00046 #include <opencv/cv.h>
00047 #include <opencv/cxcore.h>
00048 #include <opencv/highgui.h>
00049
00050 using namespace fawkes;
00051 
00052 /** @class SiftClassifier <classifiers/sift.h>
00053  * SIFT classifier.
00054  *
00055  * This class provides a classifier that uses OpenCV to detect objects in a given
00056  * image by matching features using SIFT. The objects are reported back as regions
00057  * of interest.  Each ROI contains an object.
00058  *
00059  * This code is based on the sift package provided by Rob Hess.
00060  * at http://web.engr.oregonstate.edu/~hess/
00061  *
00062  * @author Stefan Schiffer
00063  */
00064 
00065 /** Constructor.
00066  * @param object_file file that contains the object to detect
00067  * @param pixel_width width of images that will be processed
00068  * @param pixel_height height of images that will be processed
00069  * @param kdtree_bbf_max_nn_chks maximum number of keypoint NN candidates to check during BBF search
00070  * @param nn_sq_dist_ratio_thr threshold on squared ratio of distances between NN and 2nd NN
00071  * @param flags flags, not used yet.
00072  */
00073 SiftClassifier::SiftClassifier( const char * object_file,
00074                                       unsigned int pixel_width, unsigned int pixel_height,
00075                                       int kdtree_bbf_max_nn_chks, float nn_sq_dist_ratio_thr, int flags)
00076   : Classifier("SiftClassifier")
00077 {
00078   __kdtree_bbf_max_nn_chks = kdtree_bbf_max_nn_chks;
00079   __nn_sq_dist_ratio_thr = nn_sq_dist_ratio_thr;
00080   __flags = flags;
00081
00082
00083   //#ifdef SIFT_TIMETRACKER
00084   __tt = new TimeTracker();
00085   __loop_count = 0;
00086   __ttc_objconv = __tt->add_class("ObjectConvert");
00087   __ttc_objfeat = __tt->add_class("ObjectFeatures");
00088   __ttc_imgconv = __tt->add_class("ImageConvert");
00089   __ttc_imgfeat = __tt->add_class("ImageFeatures");
00090   __ttc_matchin = __tt->add_class("Matching");
00091   __ttc_roimerg = __tt->add_class("MergeROIs");
00092   //#endif
00093
00094   //#ifdef SIFT_TIMETRACKER
00095   __tt->ping_start(__ttc_objconv);
00096   //#endif
00097   __obj_img = cvLoadImage( object_file, 1 );
00098   if ( ! __obj_img ) {
00099     throw Exception("Could not load object file");
00100   }
00101   //#ifdef SIFT_TIMETRACKER
00102   __tt->ping_end(__ttc_objconv);
00103   //#endif
00104
00105   //#ifdef SIFT_TIMETRACKER
00106   __tt->ping_start(__ttc_objfeat);
00107   //#endif
00108   __obj_num_features = 0;
00109   __obj_num_features = sift_features( __obj_img, &__obj_features );
00110   if ( ! __obj_num_features > 0 ) {
00111     throw Exception("Could not compute object features");
00112   }
00113   std::cout << "SiftClassifier(classify): computed '" << __obj_num_features << "' features from object" << std::endl;
00114   //cvReleaseImage(&__obj_img);
00115   //#ifdef SIFT_TIMETRACKER
00116   __tt->ping_end(__ttc_objfeat);
00117   //#endif
00118
00119   // create space for OpenCV image
00120   __image = cvCreateImage(cvSize(pixel_width, pixel_height), IPL_DEPTH_8U, 3);
00121
00122 }
00123
00124 
00125 /** Destructor. */
00126 SiftClassifier::~SiftClassifier()
00127 {
00128   //
00129   cvReleaseImage(&__obj_img);
00130   cvReleaseImage(&__image);
00131 }
00132
00133
00134 std::list< ROI > *
00135 SiftClassifier::classify()
00136 {
00137   //#ifdef SIFT_TIMETRACKER
00138   __tt->ping_start(0);
00139   //#endif
00140
00141   // list of ROIs to return
00142   std::list< ROI > *rv = new std::list< ROI >();
00143
00144   struct feature * feat;
00145   struct feature** nbrs;
00146   struct kd_node* kd_root;
00147   CvPoint pt1, pt2;
00148
00149   // for ROI calculation
00150   CvPoint ftpt;
00151   std::vector< CvPoint > ftlist;
00152   //= new std::vector< CvPoint >();
00153   int x_min = _width;
00154   int y_min = _height;
00155   int x_max = 0;
00156   int y_max = 0;
00157
00158   double d0, d1;// = 0.0;
00159   int k, m = 0;
00160
00161   //#ifdef SIFT_TIMETRACKER
00162   __tt->ping_start(__ttc_imgconv);
00163   //#endif
00164   //std::cout << "SiftClassifier(classify): convert frame to IplImage" << std::endl;
00165   convert(YUV422_PLANAR, BGR, _src, (unsigned char *)__image->imageData, _width, _height);
00166   //#ifdef SIFT_TIMETRACKER
00167   __tt->ping_end(__ttc_imgconv);
00168   //#endif
00169
00170   //#ifdef SIFT_TIMETRACKER
00171   __tt->ping_start(__ttc_imgfeat);
00172   //#endif
00173   //std::cout << "SiftClassifier(classify): compute features on current frame " << std::endl;
00174   int num_img_ft = sift_features( __image, &__img_features );
00175   kd_root = kdtree_build( __img_features, num_img_ft );
00176   //#ifdef SIFT_TIMETRACKER
00177   __tt->ping_end(__ttc_imgfeat);
00178   //#endif
00179
00180   if( ! kd_root ) {
00181     std::cerr << "SiftClassifier(classify): KD-Root NULL!" << std::endl;
00182   }
00183
00184   //#ifdef SIFT_TIMETRACKER
00185   __tt->ping_start(__ttc_matchin);
00186   //#endif
00187   std::cout << "SiftClassifier(classify): matching ..." << std::endl;
00188   for( int i = 0; i < __obj_num_features; ++i ) {
00189     //std::cout << "SiftClassifier(classify): ... feature '" << i << "'" << std::endl;
00190     feat = __obj_features + i;
00191     k = kdtree_bbf_knn( kd_root, feat, 2, &nbrs, __kdtree_bbf_max_nn_chks );
00192     if( k == 2 )
00193       {
00194         d0 = descr_dist_sq( feat, nbrs[0] );
00195         d1 = descr_dist_sq( feat, nbrs[1] );
00196         if( d0 < d1 * __nn_sq_dist_ratio_thr )
00197           {
00198             pt1 = cvPoint( cvRound( feat->x ), cvRound( feat->y ) );
00199             pt2 = cvPoint( cvRound( nbrs[0]->x ), cvRound( nbrs[0]->y ) );
00200             m++;
00201             __obj_features[i].fwd_match = nbrs[0];
00202             // save matched feature points
00203             ftpt = cvPoint( cvRound( nbrs[0]->x), cvRound( nbrs[0]->y ) );
00204             ftlist.push_back(ftpt);
00205             // save matched features as ROIs
00206             ROI r( pt2.x-5, pt2.y-5, 11, 11, _width, _height);
00207             rv->push_back(r);
00208           }
00209        }
00210      free( nbrs );
00211   }
00212   std::cout << "SiftClassifier(classify): found '" << m << "' matches" << std::endl;
00213   kdtree_release( kd_root );
00214   //#ifdef SIFT_TIMETRACKER
00215   __tt->ping_end(__ttc_matchin);
00216   //#endif
00217
00218   //#ifdef SIFT_TIMETRACKER
00219   __tt->ping_start(__ttc_roimerg);
00220   //#endif
00221   std::cout << "SiftClassifier(classify): computing ROI" << std::endl;
00222   //for ( int i = 0; i < m; ++i) {
00223   for ( std::vector< CvPoint >::size_type i = 0; i < ftlist.size(); ++i) {
00224     if( ftlist[i].x < x_min )
00225       x_min = ftlist[i].x;
00226     if( ftlist[i].y < y_min )
00227       y_min = ftlist[i].y;
00228     if( ftlist[i].x > x_max )
00229       x_max = ftlist[i].x;
00230     if( ftlist[i].y > y_max )
00231       y_max = ftlist[i].y;
00232   }
00233   if( m != 0 ) {
00234     ROI r(x_min, y_min, x_max-x_min, y_max-y_min, _width, _height);
00235     rv->push_back(r);
00236   }
00237   //#ifdef SIFT_TIMETRACKER
00238   __tt->ping_end(__ttc_roimerg);
00239   //#endif
00240
00241   //#ifdef SIFT_TIMETRACKER
00242   __tt->ping_end(0);
00243   //#endif
00244
00245   //#ifdef SIFT_TIMETRACKER
00246   __tt->print_to_stdout();
00247   //#endif
00248
00249   std::cout << "SiftClassifier(classify): done ... returning '" << rv->size() << "' ROIs." << std::endl;
00250   return rv;
00251 }