pyvision.face.FilterEyeLocator

1 # PyVision License 2 # 3 # Copyright (c) 2006-2008 David S. Bolme 4 # All rights reserved. 5 # 6 # Redistribution and use in source and binary forms, with or without 7 # modification, are permitted provided that the following conditions 8 # are met: 9 # 10 # 1. Redistributions of source code must retain the above copyright 11 # notice, this list of conditions and the following disclaimer. 12 # 13 # 2. Redistributions in binary form must reproduce the above copyright 14 # notice, this list of conditions and the following disclaimer in the 15 # documentation and/or other materials provided with the distribution. 16 # 17 # 3. Neither name of copyright holders nor the names of its contributors 18 # may be used to endorse or promote products derived from this software 19 # without specific prior written permission. 20 # 21 # 22 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 # ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR 26 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 27 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 28 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 29 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 30 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 31 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 32 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 34 ''' 35 This is a simplified correlation filter implementation used to locate eyes 36 using ASEF correlation filters. This file contains two classes: 37 OpenCVFilterEyeLocator and FilterEyeLocator. The first is the bare minimum 38 required to locate eye and only requires opencv. The second need is a wrapper 39 that includes a nice pyvision compatible interface. This class is not integrated with 40 PyVision. PyVision supplies an interface to this class which cleans 41 up the interface and provides a bridge to many of the PyVision data 42 structures. 43 ''' 44 45 import cv 46 import math 47 import struct 48 import array 49 import os.path 50 import pyvision as pv 51 import numpy as np 52 import sys 53 54 __author__ = "David S. Bolme - Colorado State Univeristy" 55 __version__ = "$Revision: 729 $" 56 57 if pv.WARN_COMMERCIAL_USE: 58 warning = ''' 59 WARNING: A patent protection is anticipated for ASEF and 60 similar filters by the Colorado State University 61 Research Foundation (CSURF). 62 63 This module, "FilterEyeLocator.py", my not be 64 suitable for commercial use. 65 66 Commercial and government users should contact 67 CSURF for additional details: 68 http://www.csurf.org/tto/pdfs/ncs_forms/09-017_csurf_ncs.pdf 69 ''' 70 sys.stderr.write(warning) 71 72 #TODO: Unknown error - This may be related version 1.0.0 of opencv 73 #Traceback (most recent call last): 74 # File "/home/dbolme/ASEFFilters/python/csu/tools/face_scan.py", line 117, in ? 75 # results = processFaces(im,face_detect,locate_eyes) 76 # File "/home/dbolme/ASEFFilters/python/csu/tools/face_scan.py", line 57, in processFaces 77 # eye1, eye2, corr1, corr2 = locate_eyes.locateEyes(cv_im) 78 # File "/home/dbolme/ASEFFilters/python/csu/face/FilterEyeLocator.py", line 185, in locateEyes 79 # leye = cv.cvMinMaxLoc(self.left_roi)[3] 80 #IndexError: list index out of range 81 82 #TODO: Add a quality estimate 83

84 -def saveFilterEyeLocator(filename, el, comment="",copyright_str=""):

85 ''' 86 File Format 87 - Line 1: CFEL 88 - Line 2: <comment> 89 - Line 3: <copyright> 90 - Line 4: ROWS COLS 91 - Line 5: LEFT_RECT 92 - Line 6: RIGHT_RECT 93 - Line 7: BYTE_ORDER(0x41424344 or 'ABCD') 94 - Line 8: <binary data: two single precision floating point arrays of 4*WIDTH*HEIGHT bytes) 95 ''' 96 r,c = el.left_filter.rows,el.left_filter.cols 97 98 f = open(filename,'wb') 99 f.write("CFEL\n") 100 f.write(comment.strip()+"\n") 101 f.write(copyright_str.strip()+"\n") 102 f.write("%d %d\n"%(r,c)) 103 f.write("%d %d %d %d\n"%(el.left_rect.x,el.left_rect.y,el.left_rect.width,el.left_rect.height)) 104 f.write("%d %d %d %d\n"%(el.right_rect.x,el.right_rect.y,el.right_rect.width,el.right_rect.height)) 105 f.write("%s\n"%struct.pack("i",0x41424344)) 106 107 assert len(el.left_filter.imageData) == 4*r*c 108 f.write(el.left_filter.imageData) 109 110 assert len(el.right_filter.imageData) == 4*r*c 111 f.write(el.right_filter.imageData)

112 113

114 -def loadFilterEyeLocator(filename,ilog=None):

115 ''' 116 Loads the eye locator from a file.' 117 ''' 118 119 # open the file 120 f = open(filename,'rb') 121 122 # Check the first line 123 line = f.readline().strip() 124 assert line == "CFEL" 125 126 # read past the comment and copyright. 127 f.readline() 128 f.readline() 129 130 # get the width and the height 131 r,c = f.readline().split() 132 r,c = int(r),int(c) 133 134 # read in the left bounding rectangle 135 x,y,w,h = f.readline().split() 136 left_rect = (int(x),int(y),int(w),int(h)) 137 138 # read in the right bounding rectangle 139 x,y,w,h = f.readline().split() 140 right_rect = (int(x),int(y),int(w),int(h)) 141 142 # read the magic number 143 magic_number = f.readline().strip() 144 assert len(magic_number) == 4 145 magic_number = struct.unpack('i',magic_number)[0] 146 147 # Read in the filter data 148 lf = array.array('f') 149 rf = array.array('f') 150 151 lf.fromfile(f,r*c) 152 rf.fromfile(f,r*c) 153 154 # Test the magic number and byteswap if necessary. 155 if magic_number == 0x41424344: 156 pass 157 elif magic_number == 0x44434241: 158 lf.byteswap() 159 rf.byteswap() 160 else: 161 raise ValueError("Bad Magic Number: Unknown byte ordering in file") 162 163 # Create the left and right filters 164 left_filter = cv.CreateMat(r,c,cv.CV_32F) 165 right_filter = cv.CreateMat(r,c,cv.CV_32F) 166 167 # Copy data into the left and right filters 168 cv.SetData(left_filter, lf.tostring()) 169 cv.SetData(right_filter, rf.tostring()) 170 171 tmp = pv.OpenCVToNumpy(left_filter) 172 t1 = tmp.mean() 173 t2 = tmp.std() 174 cv.Scale(left_filter,left_filter,1.0/t2,-t1*1.0/t2) 175 176 tmp = pv.OpenCVToNumpy(right_filter) 177 t1 = tmp.mean() 178 t2 = tmp.std() 179 cv.Scale(right_filter,right_filter,1.0/t2,-t1*1.0/t2) 180 181 #tmp = pv.OpenCVToNumpy(left_filter) 182 #print tmp.mean(),tmp.std() 183 184 if ilog != None: 185 #lf = cv.cvCreateMat(r,c,cv.CV_8U) 186 #rf = cv.cvCreateMat(r,c,cv.CV_8U) 187 188 lf = pv.OpenCVToNumpy(left_filter) 189 rf = pv.OpenCVToNumpy(right_filter) 190 191 lf = np.fft.fftshift(lf).transpose() 192 rf = np.fft.fftshift(rf).transpose() 193 194 ilog.log(pv.Image(lf),label="LeftEyeFilter") 195 ilog.log(pv.Image(rf),label="RightEyeFilter") 196 197 # Return the eye locator 198 return OpenCVFilterEyeLocator(left_filter,right_filter,left_rect,right_rect)

199 200

201 -class OpenCVFilterEyeLocator:

202 ''' 203 This class is used for someone only interested in locating the eyes in an 204 image using correlation filters. This class does not include any support 205 for training correlation filters. For training see ASEF. This class 206 is written only using OpenCV and is much faster than the ASEF class. 207 208 For details see the paper: 209 210 David S. Bolme, Bruce A. Draper, and J. Ross Beveridge. Average of 211 Synthetic Exact Filters. Submitted to Computer Vision and Pattern 212 Recoginition. 2009. 213 214 The class uses two ASEF filters to find the eyes. The eyes are located by 215 first computing the correlation of the face tile with each filter. The 216 max value from the correlation plain is returned as the eye coordinate. 217 Also returned is the full correlation output from the image. 218 219 The images are normalized by computing log transforming the pixel values 220 221 To improve performance, this class is not thread safe. The class reuses 222 data arrays allocated for each call to use this class for multiple threads 223 you should create an instance for each threads. Also note that each method 224 call may overwrite arrays returned by this application. So if you need 225 the returned data to persist be sure to create a copy. 226 227 - Left and right eyes are in relation to the location in the image. 228 ''' 229 230

231 - def __init__(self,left_filter,right_filter, left_rect, right_rect):

232 ''' 233 @param left_filter: is in the Fourier domain where the left eye 234 corresponds to the real output and the right eye corresponds to 235 the imaginary output 236 ''' 237 # Check the input to this function 238 r,c = left_filter.rows,left_filter.cols 239 240 assert left_filter.width == right_filter.width 241 assert left_filter.height == right_filter.height 242 assert left_filter.channels == 1 243 assert right_filter.channels == 1 244 245 # Create the arrays needed for the computation 246 self.left_filter = cv.CreateMat(r,c,cv.CV_32F) 247 self.right_filter = cv.CreateMat(r,c,cv.CV_32F) 248 self.left_filter_dft = cv.CreateMat(r,c,cv.CV_32F) 249 self.right_filter_dft = cv.CreateMat(r,c,cv.CV_32F) 250 self.image = cv.CreateMat(r,c,cv.CV_32F) 251 self.left_corr = cv.CreateMat(r,c,cv.CV_32F) 252 self.right_corr = cv.CreateMat(r,c,cv.CV_32F) 253 254 # Populate the spatial filters 255 cv.ConvertScale(left_filter, self.left_filter) 256 cv.ConvertScale(right_filter, self.right_filter) 257 258 # Compute the filters in the Fourier domain 259 cv.DFT(self.left_filter, self.left_filter_dft, cv.CV_DXT_FORWARD) 260 cv.DFT(self.right_filter, self.right_filter_dft, cv.CV_DXT_FORWARD) 261 262 # Set up correlation region of interest 263 self.left_rect = left_rect 264 self.right_rect = right_rect 265 266 self.left_roi = cv.GetSubRect(self.left_corr,self.left_rect) 267 self.right_roi = cv.GetSubRect(self.right_corr,self.right_rect) 268 269 # Create the look up table for the log transform 270 self.lut = cv.CreateMat(256,1,cv.CV_32F) 271 272 for i in range(256): 273 self.lut[i,0] = math.log(i+1)

274 275

276 - def locateEyes(self,image_tile):

277 ''' 278 @param image_tile: is an 32-bit gray scale opencv image tile of a face 279 that is the same size as the filter 280 @type image_tile: 8-bit gray scale opencv image 281 282 @returns: a tuple consisting of the location of the left and right eyes 283 (opencv 2D points), and the complex correlation plain output 284 285 @raises AssertionError: is raised if the image is not 8-bit or not the 286 same size as the filter 287 ''' 288 self.correlate(image_tile) 289 290 leye = cv.MinMaxLoc(self.left_roi)[3] 291 leye = (self.left_rect[0]+leye[0],self.left_rect[1]+leye[1]) 292 293 reye = cv.MinMaxLoc(self.right_roi)[3] 294 reye = (self.right_rect[0]+reye[0],self.right_rect[1]+reye[1]) 295 296 return leye,reye,self.left_corr,self.right_corr

297 298

299 - def _preprocess(self,image_tile):

300 ''' 301 preprocess an image tile. 302 ''' 303 # TODO: This function has problems in opencv 2.2. There appears to be a bug. 304 image_tile = pv.OpenCVToNumpy(image_tile) 305 self.image = pv.NumpyToOpenCV(np.log(image_tile + 1.0).astype(np.float32)) 306 307 return self.image

308 309

310 - def correlate(self,image_tile):

311 ''' 312 Correlate the image with the left and right filters. 313 ''' 314 self._preprocess(image_tile) 315 316 cv.DFT(self.image, self.image, cv.CV_DXT_FORWARD) 317 cv.MulSpectrums( self.image, self.left_filter_dft, self.left_corr, cv.CV_DXT_MUL_CONJ ) 318 cv.MulSpectrums( self.image, self.right_filter_dft, self.right_corr, cv.CV_DXT_MUL_CONJ ) 319 320 cv.DFT(self.left_corr,self.left_corr,cv.CV_DXT_INV_SCALE) 321 cv.DFT(self.right_corr,self.right_corr,cv.CV_DXT_INV_SCALE) 322 323 return self.left_corr,self.right_corr

324 325

326 -class FilterEyeLocator:

327 ''' 328 This class provides a PyVision interface to the ASEF eye locator. 329 ''' 330

331 - def __init__(self,filename=None,ilog=None):

332 ''' 333 Load the eye detector from the file. 334 ''' 335 if filename == None: 336 filename = os.path.join(pv.__path__[0],"config","EyeLocatorASEF128x128.fel") 337 338 self.fel = loadFilterEyeLocator(filename,ilog=ilog) 339 340 self.bwtile = cv.CreateMat(128,128,cv.CV_8U)

341 342

343 - def __call__(self,im,face_rects,ilog=None):

344 return self.locateEyes(im,face_rects,ilog=ilog)

345 346

347 - def locateEyes(self,im,face_rects,ilog=None):

348 ''' 349 Finds the eyes in the image. 350 351 @param im: full sized image 352 @param face_rects: list of rectangle which are the output from the cascade face detector. 353 ''' 354 cvim = im.asOpenCVBW() 355 356 faces = [] 357 358 for rect in face_rects: 359 faceim = cv.GetSubRect(cvim, rect.asOpenCV()) 360 cv.Resize(faceim,self.bwtile) 361 362 affine = pv.AffineFromRect(rect,(128,128)) 363 364 #cv.cvCvtColor( self.cvtile, self.bwtile, cv.CV_BGR2GRAY ) 365 366 leye,reye,lcp,rcp = self.fel.locateEyes(self.bwtile) 367 le = pv.Point(leye) 368 re = pv.Point(reye) 369 370 leye = affine.invertPoint(le) 371 reye = affine.invertPoint(re) 372 373 faces.append([rect,leye,reye]) 374 375 if ilog != None: 376 ilog.log(pv.Image(self.bwtile),label="FaceDetection") 377 lcp = pv.OpenCVToNumpy(lcp).transpose() 378 lcp = lcp*(lcp > 0.0) 379 rcp = pv.OpenCVToNumpy(rcp).transpose() 380 rcp = rcp*(rcp > 0.0) 381 ilog.log(pv.Image(lcp),label="Left_Corr") 382 ilog.log(pv.Image(rcp),label="Right_Corr") 383 tmp = pv.Image(self.bwtile) 384 tmp.annotatePoint(le) 385 tmp.annotatePoint(re) 386 ilog.log(tmp,"EyeLocations") 387 388 return faces

389 390 391 ############################################################################# 392 # Unit Tests 393 ############################################################################# 394 import unittest 395 import pyvision.face.CascadeDetector as cd 396 from pyvision.analysis.FaceAnalysis.FaceDatabase import ScrapShotsDatabase 397 from pyvision.analysis.FaceAnalysis.EyeDetectionTest import EyeDetectionTest 398

399 -class _TestFilterEyeLocator(unittest.TestCase):

400

401 - def test_ASEFEyeLocalization(self):

402 ''' 403 This trains the FaceFinder on the scraps database. 404 ''' 405 # Load a face database 406 ssdb = ScrapShotsDatabase() 407 408 # Create a face detector 409 face_detector = cd.CascadeDetector() 410 411 # Create an eye locator 412 eye_locator = FilterEyeLocator() 413 414 # Create an eye detection test 415 edt = EyeDetectionTest(name='asef_scraps') 416 417 #print "Testing..." 418 for face_id in ssdb.keys(): 419 face = ssdb[face_id] 420 im = face.image 421 422 # Detect the faces 423 faces = face_detector.detect(im) 424 425 # Detect the eyes 426 pred_eyes = eye_locator(im,faces) 427 428 truth_eyes = [[face.left_eye,face.right_eye]] 429 pred_eyes = [ [leye,reye] for _,leye,reye in pred_eyes] 430 431 # Add to eye detection test 432 edt.addSample(truth_eyes, pred_eyes, im=im, annotate=False) 433 434 edt.createSummary() 435 self.assertAlmostEqual( edt.face_rate , 0.97109826589595372, places = 3 ) # Updated numbers for OpenCV 2.0 436 self.assertAlmostEqual( edt.both25_rate , 0.82658959537572252, places = 3 ) 437 self.assertAlmostEqual( edt.both10_rate , 0.47976878612716761, places = 3 ) 438 self.assertAlmostEqual( edt.both05_rate , 0.30635838150289019, places = 3 )

439

Source Code for Module pyvision.face.FilterEyeLocator