pyvision.face.CascadeDetector

1 # PyVision License 2 # 3 # Copyright (c) 2006-2008 David S. Bolme 4 # All rights reserved. 5 # 6 # Redistribution and use in source and binary forms, with or without 7 # modification, are permitted provided that the following conditions 8 # are met: 9 # 10 # 1. Redistributions of source code must retain the above copyright 11 # notice, this list of conditions and the following disclaimer. 12 # 13 # 2. Redistributions in binary form must reproduce the above copyright 14 # notice, this list of conditions and the following disclaimer in the 15 # documentation and/or other materials provided with the distribution. 16 # 17 # 3. Neither name of copyright holders nor the names of its contributors 18 # may be used to endorse or promote products derived from this software 19 # without specific prior written permission. 20 # 21 # 22 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 # ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR 26 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 27 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 28 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 29 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 30 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 31 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 32 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 34 import cv 35 36 import os 37 import subprocess 38 import random 39 import tempfile 40 import unittest 41 42 import pyvision as pv 43 44 from pyvision.analysis.face import EyesFile 45 46 from pyvision.analysis.FaceAnalysis.FaceDetectionTest import FaceDetectionTest 47 import pickle 48 #from pyvision.optimize.GeneticAlgorithm import GeneticAlgorithm,ChoiceVariable 49 import time 50 #from pyvision.analysis.Table import Table 51

52 -class CascadeNotFound(Exception):

53 pass

54 -class HaarTrainingError(Exception):

55 pass

56 57 58 DEFAULT_CASCADE=os.path.join(pv.__path__[0],"config","haarcascade_frontalface_alt.xml") 59 OPENCV_CASCADE=os.path.join(pv.__path__[0],"config","haarcascade_frontalface_alt.xml") 60 CELEB1_CASCADE=os.path.join(pv.__path__[0],"config","facedetector_celebdb1.xml") 61 CELEB2_CASCADE=os.path.join(pv.__path__[0],"config","facedetector_celebdb2.xml") 62 FULLBODY_CASCADE=os.path.join(pv.__path__[0],"config","haarcascade_fullbody.xml") 63 UPPERBODY_CASCADE=os.path.join(pv.__path__[0],"config","haarcascade_upperbody.xml") 64 LOWERBODY_CASCADE=os.path.join(pv.__path__[0],"config","haarcascade_lowerbody.xml") 65 UPPERBODY_MCS_CASCADE=os.path.join(pv.__path__[0],"config","haarcascade_mcs_upperbody.xml") 66 67 DEFAULT_NEGATIVE=os.path.join(pv.__path__[0],"data","nonface") 68 69 # These are the average left and right eye locations relative to the face detection rectangle for the 70 # haarcascade_frontalface_alt cascade file. Estimated using the first 1000 images from FERET. 71 # To find the expected left eye location for a 64X64 detection rectangle: 64*AVE_LEFT_EYE 72 AVE_LEFT_EYE = pv.Point(0.300655,0.381525,0.000000) 73 AVE_RIGHT_EYE = pv.Point(0.708847,0.379736,0.000000) 74

75 -class CascadeDetector:

76 ''' This class is a wrapper around the OpenCV cascade detectior. ''' 77

78 - def __init__(self, cascade_name=DEFAULT_CASCADE,orig_size=None,min_size=(60,60), image_scale=1.3, haar_scale=1.2, min_neighbors=2, haar_flags=0):

79 ''' Init the detector and create the cascade classifier ''' 80 81 self.cascade_name = cascade_name 82 self.min_size = min_size 83 self.image_scale = image_scale 84 self.haar_scale = haar_scale 85 self.min_neighbors = min_neighbors 86 self.haar_flags = haar_flags 87 88 if cascade_name != None: 89 if not os.path.isfile(cascade_name): 90 raise CascadeNotFound("Could not find file: "+cascade_name) 91 # Save data for later pickling 92 if orig_size == None: 93 orig_size = (1,1) 94 else: 95 orig_size = (orig_size[0],orig_size[1]) 96 97 self.cascade_data = open(cascade_name).read() 98 self.cascade = cv.Load( cascade_name ) 99 self.storage = cv.CreateMemStorage(0) 100 self.trained = True

101 102

103 - def __call__(self,im):

104 ''' This function is the same as detect. ''' 105 return self.detect(im)

106

107 - def __getstate__(self):

108 ''' Function required to save and load the state from pickel. ''' 109 state = {} 110 111 for key,value in self.__dict__.iteritems(): 112 if key in ['cascade','storage']: 113 continue 114 115 state[key] = value 116 117 return state

118

119 - def __setstate__(self,state):

120 ''' Function required to save and load the state from pickel. ''' 121 # Modeled after SVM pickling 122 123 for key,value in state.iteritems(): 124 self.__dict__[key] = value 125 126 127 filename = tempfile.mktemp() 128 open(filename,'w').write(self.cascade_data) 129 self.cascade = cv.Load( filename ) 130 self.storage = cv.CreateMemStorage(0) 131 os.remove(filename)

132

133 - def _resizeImage(self, image, scale=None, size=None):

134 ''' Resize an image by a scale or a size. Internal use only.''' 135 if scale != None and type(scale) in (int,float): 136 size = (int(image.width*scale),int(image.height*scale)) 137 elif size != None and type(size) in [list,tuple]: 138 size = (int(size[0]),int(size[1])) 139 else: 140 pass 141 depth = image.depth 142 channels = image.nChannels 143 resized = cv.CreateImage( (size[0],size[1]), depth, channels ) 144 cv.Resize( image, resized, cv.CV_INTER_LINEAR ) 145 return resized

146

147 - def detect(self, im):

148 ''' Runs the cascade classifer on an image. ''' 149 image = im.asOpenCV() 150 151 min_size = (self.min_size[0],self.min_size[1]) 152 153 # Create a resized gray scale image 154 if image.nChannels == 3: 155 gray = cv.CreateImage( (image.width,image.height), image.depth, 1 ) 156 cv.CvtColor( image, gray, cv.CV_BGR2GRAY ); 157 image = gray 158 159 160 image = self._resizeImage(image,self.image_scale) 161 162 # Equalize the image 163 cv.EqualizeHist( image, image ) 164 165 # Detect faces 166 faces = cv.HaarDetectObjects( image, self.cascade, self.storage, 167 self.haar_scale, self.min_neighbors, self.haar_flags, min_size ); 168 169 # Transform and return the points 170 result = [] 171 for r in faces: 172 rect = pv.Rect(r[0][0]/self.image_scale, r[0][1]/self.image_scale, r[0][2]/self.image_scale, r[0][3]/self.image_scale) 173 result.append(rect) 174 175 return result

176

177 -def trainHaarClassifier(pos_rects, 178 neg_images, 179 tile_size=(20,20), 180 nneg=2000, 181 nstages=20, 182 mem = 1500, 183 maxtreesplits = 0, 184 mode='BASIC', 185 minhitrate=0.9990, 186 maxfalsealarm=0.50, 187 max_run_time=72*3600, 188 verbose=False, 189 createsamples='/usr/local/bin/opencv-createsamples', 190 haartraining='/usr/local/bin/opencv-haartraining', 191 ):

192 ''' 193 Train the detector. 194 ''' 195 # Create a directory for training. 196 training_dir = tempfile.mktemp() 197 os.makedirs(training_dir, 0700) 198 199 random_name = "haar_" 200 for _ in range(8): 201 random_name += random.choice('abcdefghijklmnopqrstuvwxyz') 202 cascade_name = random_name+"_cascade" 203 pos_name = random_name+"_pos.txt" 204 pos_vec_name = random_name+"_pos.vec" 205 neg_name = random_name+"_neg.txt" 206 207 # Add positives to the positives file. 208 pos_filename = os.path.join(training_dir,pos_name) 209 pos_file = open(pos_filename,'w') 210 num_pos = 0 211 for im_name,rects in pos_rects: 212 num_pos += len(rects) 213 if len(rects) > 0: pos_file.write("%s %d "%(im_name,len(rects))) 214 for rect in rects: 215 pos_file.write("%d %d %d %d "%(rect.x,rect.y,rect.w,rect.h)) 216 num_pos += 1 217 pos_file.write("\n") 218 pos_file.close() 219 220 # Add negatives to the negitives file. 221 neg_filename = os.path.join(training_dir,neg_name) 222 neg_file = open(neg_filename,'w') 223 for im_name in neg_images: 224 neg_file.write("%s\n"%im_name) 225 neg_file.close() 226 227 # Create positives vec. 228 proc = subprocess.Popen( 229 (createsamples, 230 '-info',pos_name, 231 '-vec',pos_vec_name, 232 '-num',str(num_pos), 233 '-w',str(tile_size[0]), 234 '-h',str(tile_size[1]), 235 ), 236 cwd=training_dir 237 ,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.STDOUT 238 ) 239 240 proc.wait() 241 if verbose: 242 print proc.stdout.read() 243 244 # Run haar training 245 success = False 246 247 start_time = time.time() 248 249 if verbose: 250 proc = subprocess.Popen( 251 (haartraining, 252 '-data',cascade_name, 253 '-vec',pos_vec_name, 254 '-bg',neg_name, 255 '-nstages',str(nstages), 256 '-mode','ALL', 257 '-npos',str(num_pos), 258 '-nneg',str(nneg), 259 '-mem',str(mem), 260 '-w',str(tile_size[0]), 261 '-h',str(tile_size[1]), 262 '-maxtreesplits',str(maxtreesplits), 263 '-minhitrate',str(minhitrate), 264 '-maxfalsealarm',str(maxfalsealarm), 265 '-mode',mode, 266 ), 267 cwd=training_dir 268 #,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.STDOUT 269 ) 270 else: 271 proc = subprocess.Popen( 272 (haartraining, 273 '-data',cascade_name, 274 '-vec',pos_vec_name, 275 '-bg',neg_name, 276 '-nstages',str(nstages), 277 '-mode','ALL', 278 '-npos',str(num_pos), 279 '-nneg',str(nneg), 280 '-mem',str(mem), 281 '-w',str(tile_size[0]), 282 '-h',str(tile_size[1]), 283 '-maxtreesplits',str(maxtreesplits), 284 '-minhitrate',str(minhitrate), 285 '-maxfalsealarm',str(maxfalsealarm), 286 '-mode',mode, 287 ), 288 cwd=training_dir 289 ,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.STDOUT 290 ) 291 292 while True: 293 if proc.poll() != None: 294 break 295 if time.time() - start_time > max_run_time: 296 print "Haar Training time exceeded. Killing process..." 297 os.kill(proc.pid,6) #6 = abort, 3=quit, 9=kill 298 proc.wait() 299 break 300 #out = proc.stdout.read() 301 #if verbose: 302 # print out 303 time.sleep(1) 304 305 if proc.returncode == 0: 306 if verbose: print "Cascade successful." 307 success = True 308 309 310 else: 311 print "Problem with return code:",proc.returncode 312 #levels = os.listdir(os.path.join(training_dir,cascade_name)) 313 #nlevels = len(levels) 314 315 316 # Load the detector if training was successful. 317 detector = None 318 if success: 319 detector = CascadeDetector(os.path.join(training_dir,cascade_name+'.xml')) 320 else: 321 # TODO: loading a partial cascade. Maybe training should be removed from this module all together 322 print "Cascade Failure. Could not create classifier." 323 324 # Clean up the temporary files 325 os.system("rm -rf %s"%training_dir) 326 327 time.sleep(5) 328 return detector

329 330 331 332 SCRAPS_FACE_DATA = os.path.join(pv.__path__[0],"data","csuScrapShots") 333 NONFACE_DATA = os.path.join(pv.__path__[0],"data","NonFace") 334 BAD_CASCADE=os.path.join(pv.__path__[0],"config","not_there.xml") 335 336

337 -class _TestCascadeDetector(unittest.TestCase):

338 ''' Unit tests for the Cascade Detector ''' 339 340

341 - def test_detect_bad_file(self):

342 ''' 343 If the cascade file does not exist, opencv can crash without warning. 344 This makes sure a test is run to make sure the cascade is there. 345 ''' 346 347 self.assertRaises(CascadeNotFound,CascadeDetector,BAD_CASCADE)

348

349 - def test_face_detection_pickle(self):

350 fd = CascadeDetector(OPENCV_CASCADE) 351 352 fdt = FaceDetectionTest(name='scraps') 353 354 data_buffer = pickle.dumps(fd) 355 fd = pickle.loads(data_buffer) 356 357 self.eyes = EyesFile(os.path.join(SCRAPS_FACE_DATA,"coords.txt")) 358 for filename in self.eyes.files(): 359 img = pv.Image(os.path.join(SCRAPS_FACE_DATA, filename + ".pgm")) 360 rects = fd(img) 361 truth = self.eyes.getFaces(img.filename) 362 fdt.addSample(truth,rects,im=img) 363 364 self.assertAlmostEqual( fdt.pos_rate , 0.98265895953757221, places = 2 ) # TODO: Version 2 performance is better

365 366

367 - def test_detect_scraps_opencv(self):

368 369 fd = CascadeDetector(OPENCV_CASCADE) 370 fdt = FaceDetectionTest(name='scraps') 371 372 self.eyes = EyesFile(os.path.join(SCRAPS_FACE_DATA,"coords.txt")) 373 for filename in self.eyes.files(): 374 img = pv.Image(os.path.join(SCRAPS_FACE_DATA, filename + ".pgm")) 375 rects = fd(img) 376 truth = self.eyes.getFaces(img.filename) 377 fdt.addSample(truth,rects,im=img) 378 379 self.assertAlmostEqual( fdt.pos_rate , 0.98265895953757221, places = 2 ) # TODO: Version 2 performance is better

380

381 - def test_detect_scraps_celeb1(self):

382 fd = CascadeDetector(CELEB1_CASCADE) 383 fdt = FaceDetectionTest(name='scraps') 384 385 self.eyes = EyesFile(os.path.join(SCRAPS_FACE_DATA,"coords.txt")) 386 for filename in self.eyes.files(): 387 img = pv.Image(os.path.join(SCRAPS_FACE_DATA, filename + ".pgm")) 388 rects = fd(img) 389 truth = self.eyes.getFaces(img.filename) 390 fdt.addSample(truth,rects,im=img) 391 392 self.assertAlmostEqual( fdt.pos_rate , 0.76878612716763006, places = 2 ) # TODO: Version 2 performance is better

393

394 - def test_detect_scraps_celeb2(self):

395 396 fd = CascadeDetector(CELEB2_CASCADE) 397 fdt = FaceDetectionTest(name='scraps') 398 399 self.eyes = EyesFile(os.path.join(SCRAPS_FACE_DATA,"coords.txt")) 400 for filename in self.eyes.files(): 401 img = pv.Image(os.path.join(SCRAPS_FACE_DATA, filename + ".pgm")) 402 rects = fd(img) 403 truth = self.eyes.getFaces(img.filename) 404 fdt.addSample(truth,rects,im=img) 405 406 self.assertAlmostEqual( fdt.pos_rate , 0.925, places = 2 )

407 408

409 - def donttest_detector_train(self): # TODO: Cascade training fails for Version OpenCV 2.0

410 411 positives = [] 412 self.eyes = EyesFile(os.path.join(SCRAPS_FACE_DATA,"coords.txt")) 413 n = len(self.eyes.files()) 414 for filename in self.eyes.files()[:n/2]: 415 img = pv.Image(os.path.join(SCRAPS_FACE_DATA, filename + ".pgm")) 416 faces = self.eyes.getFaces(img.filename) 417 positives.append([os.path.join(SCRAPS_FACE_DATA,img.filename),faces]) 418 419 neg_files = [] 420 for im_name in os.listdir(NONFACE_DATA): 421 if im_name[-4:] != ".jpg": continue 422 neg_files.append(os.path.join(NONFACE_DATA,im_name)) 423 424 fd = trainHaarClassifier(positives,neg_files,nstages=6,maxtreesplits=0,max_run_time=300) 425 fdt = FaceDetectionTest(name='scraps') 426 427 self.eyes = EyesFile(os.path.join(SCRAPS_FACE_DATA,"coords.txt")) 428 for filename in self.eyes.files(): 429 img = pv.Image(os.path.join(SCRAPS_FACE_DATA, filename + ".pgm")) 430 rects = fd(img) 431 truth = self.eyes.getFaces(img.filename) 432 fdt.addSample(truth,rects,im=img) 433 434 self.assertAlmostEqual( fdt.pos_rate , 0.9942196531791907 )

435

Source Code for Module pyvision.face.CascadeDetector