pyvision.analysis.FaceAnalysis.FRGC2004

1 # PyVision License 2 # 3 # Copyright (c) 2006-2008 David S. Bolme 4 # All rights reserved. 5 # 6 # Redistribution and use in source and binary forms, with or without 7 # modification, are permitted provided that the following conditions 8 # are met: 9 # 10 # 1. Redistributions of source code must retain the above copyright 11 # notice, this list of conditions and the following disclaimer. 12 # 13 # 2. Redistributions in binary form must reproduce the above copyright 14 # notice, this list of conditions and the following disclaimer in the 15 # documentation and/or other materials provided with the distribution. 16 # 17 # 3. Neither name of copyright holders nor the names of its contributors 18 # may be used to endorse or promote products derived from this software 19 # without specific prior written permission. 20 # 21 # 22 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 # ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR 26 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 27 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 28 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 29 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 30 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 31 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 32 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 34 35 ''' 36 Created on May 15, 2009 37 38 @author: bolme 39 ''' 40 41 #import BEE 42 from FaceDatabase import FaceDatabase 43 #import os.path 44 import os 45 import copy 46 import pyvision as pv 47 import xml.etree.cElementTree as et 48 import shutil 49 import time 50 #from pyvision.analysis.FaceAnalysis.BEE import BEEDistanceMatrix 51 52 REDUCED_LEYE = pv.Point(128+64,256-20) 53 REDUCED_REYE = pv.Point(256+128-64,256-20) 54 REDUCED_SIZE = (512,512) 55 56 ''' 57 This is a training set that does not include any overlap with the good bad and ugly. 58 ''' 59 GOOD_BAD_UGLY_TRAINING = ['nd1S04201','nd1S04207','nd1S04211','nd1S04212','nd1S04213','nd1S04217','nd1S04219','nd1S04222', 60 'nd1S04226','nd1S04227','nd1S04228','nd1S04229','nd1S04243','nd1S04256','nd1S04265','nd1S04273', 61 'nd1S04274','nd1S04279','nd1S04282','nd1S04288','nd1S04295','nd1S04300','nd1S04305','nd1S04308', 62 'nd1S04315','nd1S04316','nd1S04320','nd1S04322','nd1S04323','nd1S04326','nd1S04331','nd1S04335', 63 'nd1S04337','nd1S04339','nd1S04344','nd1S04352','nd1S04358','nd1S04360','nd1S04361','nd1S04365', 64 'nd1S04366','nd1S04367','nd1S04368','nd1S04369','nd1S04371','nd1S04372','nd1S04374','nd1S04376', 65 'nd1S04378','nd1S04380','nd1S04381','nd1S04382','nd1S04386','nd1S04388','nd1S04392','nd1S04395', 66 'nd1S04402','nd1S04403','nd1S04409','nd1S04410','nd1S04411','nd1S04412','nd1S04414','nd1S04415', 67 'nd1S04418','nd1S04423','nd1S04424','nd1S04425','nd1S04428','nd1S04430','nd1S04431','nd1S04432', 68 'nd1S04433','nd1S04435','nd1S04437','nd1S04442','nd1S04444','nd1S04454','nd1S04460','nd1S04467', 69 'nd1S04471','nd1S04479','nd1S04487','nd1S04489','nd1S04495','nd1S04500','nd1S04513','nd1S04515', 70 'nd1S04516','nd1S04519','nd1S04520','nd1S04522','nd1S04523','nd1S04524','nd1S04525','nd1S04527', 71 'nd1S04529','nd1S04530','nd1S04533','nd1S04539','nd1S04540','nd1S04545','nd1S04548','nd1S04549', 72 'nd1S04551','nd1S04558','nd1S04559','nd1S04561','nd1S04563','nd1S04564','nd1S04572','nd1S04573', 73 'nd1S04577','nd1S04579','nd1S04582','nd1S04584','nd1S04585','nd1S04589','nd1S04598','nd1S04599', 74 'nd1S04600','nd1S04610','nd1S04618','nd1S04619','nd1S04624','nd1S04637','nd1S04638','nd1S04641', 75 'nd1S04644','nd1S04657','nd1S04675',] 76

77 -class FRGCMetadata:

78 - def __init__(self):

79 pass

80 81

82 -class FRGC_Exp1(FaceDatabase):

83 ''' 84 FRGC Experiment 1 is a controlled to controlled scenario 85 ''' 86

87 - def __init__(self,location):

88 pass

89

90 -class FRGC_Exp4(FaceDatabase):

91 ''' 92 FRGC Experiment 4 is a controled to uncontrolled scenario 93 94 Note: The left and right eye in the metadata is relative to the 95 person. For the face structure this is reversed and follows 96 the pyvision convention which is relative to the image. 97 ''' 98

99 - def __init__(self,location):

100 self.location = location 101 102 self.data_path = os.path.join(location,"nd1") 103 self.metadata_path = os.path.join(location,"BEE_DIST","FRGC2.0","metadata","FRGC_2.0_Metadata.xml") 104 105 self.readSigsets() 106 self.readEyeData()

107 108

109 - def readSigsets(self):

110 self.orig_sigset_path = os.path.join(self.location,"BEE_DIST","FRGC2.0","signature_sets","experiments","FRGC_Exp_2.0.4_Orig.xml") 111 self.query_sigset_path = os.path.join(self.location,"BEE_DIST","FRGC2.0","signature_sets","experiments","FRGC_Exp_2.0.4_Query.xml") 112 self.target_sigset_path = os.path.join(self.location,"BEE_DIST","FRGC2.0","signature_sets","experiments","FRGC_Exp_2.0.4_Target.xml") 113 self.training_sigset_path = os.path.join(self.location,"BEE_DIST","FRGC2.0","signature_sets","experiments","FRGC_Exp_2.0.4_Training.xml") 114 115 self.orig_sigset = pv.parseSigSet(self.orig_sigset_path) 116 self.query_sigset = pv.parseSigSet(self.query_sigset_path) 117 self.target_sigset = pv.parseSigSet(self.target_sigset_path) 118 self.training_sigset = pv.parseSigSet(self.training_sigset_path) 119 120 self.orig_sigset_map = dict([ (data[0]['name'],[key,data]) for key,data in self.orig_sigset ]) 121 self.query_sigset_map = dict([ (data[0]['name'],[key,data]) for key,data in self.query_sigset ]) 122 self.target_sigset_map = dict([ (data[0]['name'],[key,data]) for key,data in self.target_sigset ]) 123 self.training_sigset_map = dict([ (data[0]['name'],[key,data]) for key,data in self.training_sigset ]) 124 125 self.orig_keys = [ data[0]['name'] for key,data in self.orig_sigset ] 126 self.query_keys = [ data[0]['name'] for key,data in self.query_sigset ] 127 self.target_keys = [ data[0]['name'] for key,data in self.target_sigset ] 128 self.training_keys = [ data[0]['name'] for key,data in self.training_sigset ] 129 130 # Check that everything adds up. 131 assert len(set(self.orig_keys)) == len(self.orig_keys) 132 assert len(set(self.query_keys + self.target_keys + self.training_keys)) == len(self.orig_keys) 133 assert len(set(self.query_keys + self.target_keys + self.training_keys)) == len(self.query_keys) + len(self.target_keys) + len(self.training_keys)

134

135 - def readEyeData(self):

136 f = open(self.metadata_path,'rb') 137 xml = et.parse(f) 138 139 self.metadata = {} 140 141 for recording in xml.findall('Recording'): 142 md = FRGCMetadata() 143 md.rec_id = recording.get('recording_id') 144 md.sub_id = recording.get('subject_id') 145 md.capture_date = recording.get('capturedate') 146 147 md.left_eye = None 148 md.right_eye = None 149 md.nose = None 150 md.mouth = None 151 152 for leye_center in recording.findall('LeftEyeCenter'): 153 x,y = float(leye_center.get('x')),float(leye_center.get('y')) 154 md.left_eye = pv.Point(x,y) 155 156 for leye_center in recording.findall('RightEyeCenter'): 157 x,y = float(leye_center.get('x')),float(leye_center.get('y')) 158 md.right_eye = pv.Point(x,y) 159 160 for leye_center in recording.findall('Nose'): 161 x,y = float(leye_center.get('x')),float(leye_center.get('y')) 162 md.nose = pv.Point(x,y) 163 164 for leye_center in recording.findall('Mouth'): 165 x,y = float(leye_center.get('x')),float(leye_center.get('y')) 166 md.mouth = pv.Point(x,y) 167 168 self.metadata[md.rec_id] = md

169 170

171 - def keys(self):

172 return copy.copy(self.orig_keys)

173

174 - def query(self):

175 return copy.copy(self.query_keys)

176

177 - def target(self):

178 return copy.copy(self.target_keys)

179

180 - def training(self):

181 return copy.copy(self.training_keys)

182

183 - def getMetadata(self,key):

184 meta = self.metadata[key] 185 sig = self.orig_sigset_map[key] 186 return key,sig[0],sig[1][0]['file-name'],meta.right_eye,meta.left_eye,meta.nose,meta.mouth

187 188

189 - def __getitem__(self,key):

190 entry = self.orig_sigset_map[key] 191 192 face_obj = FaceDatabase.FaceObject() 193 194 face_obj.key = key 195 face_obj.person_id = entry[0] 196 face_obj.entry = entry 197 face_obj.metadata = self.metadata[key] 198 199 # The left_eye and right_eye in the metadata is relative to the subject 200 # This needs to be reversed because pyvision used image left and image right. 201 face_obj.left_eye = face_obj.metadata.right_eye 202 face_obj.right_eye = face_obj.metadata.left_eye 203 204 im_name = os.path.join(self.location,entry[1][0]['file-name']) 205 im = pv.Image(im_name) 206 face_obj.image = im 207 208 return face_obj

209 210

211 -class FRGC_V1(FaceDatabase):

212 ''' 213 FRGC Experiment 4 is a controled to uncontrolled scenario 214 215 Note: The left and right eye in the metadata is relative to the 216 person. For the face structure this is reversed and follows 217 the pyvision convention which is relative to the image. 218 ''' 219

220 - def __init__(self,location):

221 self.location = location 222 223 self.data_path = os.path.join(location,"nd1") 224 self.metadata_path = os.path.join(location,"BEE_DIST","FRGC1.0","metadata","FRGC_1.0_Metadata.xml") 225 226 self.readSigsets() 227 self.readEyeData()

228 229

230 - def readSigsets(self):

231 self.orig_sigset_path = os.path.join(self.location,"BEE_DIST","FRGC1.0","signature_sets","all.xml") 232 233 self.orig_sigset = pv.parseSigSet(self.orig_sigset_path) 234 #print self.orig_sigset 235 236 self.orig_sigset = filter(lambda x: len(x[1]) > 0, self.orig_sigset) 237 238 self.orig_sigset_map = dict([ (data[0]['name'],[key,data]) for key,data in self.orig_sigset ]) 239 240 self.orig_keys = [ data[0]['name'] for key,data in self.orig_sigset ]

241 242

243 - def readEyeData(self):

244 f = open(self.metadata_path,'rb') 245 xml = et.parse(f) 246 247 self.metadata = {} 248 249 for recording in xml.findall('Recording'): 250 md = FRGCMetadata() 251 md.rec_id = recording.get('recording_id') 252 md.sub_id = recording.get('subject_id') 253 md.capture_date = recording.get('capturedate') 254 255 md.left_eye = None 256 md.right_eye = None 257 md.nose = None 258 md.mouth = None 259 260 for leye_center in recording.findall('LeftEyeCenter'): 261 x,y = float(leye_center.get('x')),float(leye_center.get('y')) 262 md.left_eye = pv.Point(x,y) 263 264 for leye_center in recording.findall('RightEyeCenter'): 265 x,y = float(leye_center.get('x')),float(leye_center.get('y')) 266 md.right_eye = pv.Point(x,y) 267 268 for leye_center in recording.findall('Nose'): 269 x,y = float(leye_center.get('x')),float(leye_center.get('y')) 270 md.nose = pv.Point(x,y) 271 272 for leye_center in recording.findall('Mouth'): 273 x,y = float(leye_center.get('x')),float(leye_center.get('y')) 274 md.mouth = pv.Point(x,y) 275 276 self.metadata[md.rec_id] = md

277 278

279 - def keys(self):

280 return copy.copy(self.orig_keys)

281

282 - def getMetadata(self,key):

283 meta = self.metadata[key] 284 sig = self.orig_sigset_map[key] 285 return key,sig[0],sig[1][0]['file-name'],meta.right_eye,meta.left_eye,meta.nose,meta.mouth

286 287

288 - def __getitem__(self,key):

289 entry = self.orig_sigset_map[key] 290 291 face_obj = FaceDatabase.FaceObject() 292 293 face_obj.key = key 294 face_obj.person_id = entry[0] 295 face_obj.entry = entry 296 face_obj.metadata = self.metadata[key] 297 298 # The left_eye and right_eye in the metadata is relative to the subject 299 # This needs to be reversed because pyvision used image left and image right. 300 face_obj.left_eye = face_obj.metadata.right_eye 301 face_obj.right_eye = face_obj.metadata.left_eye 302 303 im_name = os.path.join(self.location,entry[1][0]['file-name']) 304 im = pv.Image(im_name) 305 face_obj.image = im 306 307 return face_obj

308 309

310 -class FRGC_Exp4_Reduced(FaceDatabase):

311 ''' 312 FRGC Experiment 4 is a controled to uncontrolled scenario 313 314 Note: The left and right eye in the metadata is relative to the 315 person. For the face structure this is reversed and follows 316 the pyvision convention which is relative to the image. 317 ''' 318

319 - def __init__(self,location):

320 self.location = location 321 322 self.data_path = os.path.join(location,"nd1") 323 324 self.readSigsets()

325 326

327 - def readSigsets(self):

328 self.sigset_dir = os.path.join(self.location,"sigsets") 329 330 self.orig_sigset_path = os.path.join(self.location,"sigsets","FRGC_Exp_2.0.4_Orig.xml") 331 self.query_sigset_path = os.path.join(self.location,"sigsets","FRGC_Exp_2.0.4_Query.xml") 332 self.target_sigset_path = os.path.join(self.location,"sigsets","FRGC_Exp_2.0.4_Target.xml") 333 self.training_sigset_path = os.path.join(self.location,"sigsets","FRGC_Exp_2.0.4_Training.xml") 334 335 self.orig_sigset = pv.parseSigSet(self.orig_sigset_path) 336 self.query_sigset = pv.parseSigSet(self.query_sigset_path) 337 self.target_sigset = pv.parseSigSet(self.target_sigset_path) 338 self.training_sigset = pv.parseSigSet(self.training_sigset_path) 339 340 self.orig_sigset_map = dict([ (data[0]['name'],[key,data]) for key,data in self.orig_sigset ]) 341 self.query_sigset_map = dict([ (data[0]['name'],[key,data]) for key,data in self.query_sigset ]) 342 self.target_sigset_map = dict([ (data[0]['name'],[key,data]) for key,data in self.target_sigset ]) 343 self.training_sigset_map = dict([ (data[0]['name'],[key,data]) for key,data in self.training_sigset ]) 344 345 self.orig_keys = [ data[0]['name'] for key,data in self.orig_sigset ] 346 self.query_keys = [ data[0]['name'] for key,data in self.query_sigset ] 347 self.target_keys = [ data[0]['name'] for key,data in self.target_sigset ] 348 self.training_keys = [ data[0]['name'] for key,data in self.training_sigset ] 349 350 # Check that everything adds up. 351 assert len(set(self.orig_keys)) == len(self.orig_keys) 352 assert len(set(self.query_keys + self.target_keys + self.training_keys)) == len(self.orig_keys) 353 assert len(set(self.query_keys + self.target_keys + self.training_keys)) == len(self.query_keys) + len(self.target_keys) + len(self.training_keys)

354 355

356 - def keys(self):

357 return copy.copy(self.orig_keys)

358

359 - def query(self):

360 return copy.copy(self.query_keys)

361

362 - def target(self):

363 return copy.copy(self.target_keys)

364

365 - def training(self):

366 return copy.copy(self.training_keys)

367

368 - def __getitem__(self,key):

369 entry = self.orig_sigset_map[key] 370 371 face_obj = FaceDatabase.FaceObject() 372 373 face_obj.key = key 374 face_obj.person_id = entry[0] 375 face_obj.entry = entry 376 377 # The left_eye and right_eye in the metadata is relative to the subject 378 # This needs to be reversed because pyvision used image left and image right. 379 face_obj.left_eye = REDUCED_LEYE 380 face_obj.right_eye = REDUCED_REYE 381 382 im_name = os.path.join(self.location,'recordings',key+".jpg") 383 im = pv.Image(im_name) 384 face_obj.image = im 385 386 return face_obj

387 388

389 -def reduce_exp4(source_dir,dest_dir):

390 '''''' 391 print "Creating directories." 392 try: 393 os.makedirs(os.path.join(dest_dir,'recordings')) 394 except: 395 pass 396 397 try: 398 os.makedirs(os.path.join(dest_dir,'sigsets')) 399 except: 400 pass 401 402 print "Loading FRGC Information." 403 frgc = FRGC_Exp4(source_dir) 404 405 print "Processing Images." 406 keys = frgc.keys() 407 for i in xrange(len(keys)): 408 key = keys[i] 409 face = frgc[key] 410 print "Processing %d of %d:"%(i+1,len(keys)), key,face.person_id 411 412 affine = pv.AffineFromPoints(face.left_eye,face.right_eye,REDUCED_LEYE,REDUCED_REYE,REDUCED_SIZE) 413 tile = affine.transformImage(face.image) 414 tile.asPIL().save(os.path.join(dest_dir,'recordings',key+".jpg"),quality=95) 415 416 #if i > 10: 417 # break 418 419 print "Copying sigsets." 420 shutil.copy(frgc.orig_sigset_path, os.path.join(dest_dir,'sigsets')) 421 shutil.copy(frgc.query_sigset_path, os.path.join(dest_dir,'sigsets')) 422 shutil.copy(frgc.target_sigset_path, os.path.join(dest_dir,'sigsets')) 423 shutil.copy(frgc.training_sigset_path, os.path.join(dest_dir,'sigsets')) 424 425 print "Copying metadata." 426 shutil.copy(frgc.metadata_path, os.path.join(dest_dir,'sigsets'))

427 428

429 -def FRGCExp4Test(database, algorithm, face_detector=None, eye_locator=None, n=None,verbose=10.0,ilog=None):

430 ''' 431 Run the FRGC Experiment 4 Test 432 433 On completion this will produce a BEE distance matrix. 434 ''' 435 message_time = time.time() 436 timer = pv.Timer() 437 438 # Produce face records for each image in the query set 439 query_keys = database.query() 440 if n != None: 441 query_keys = query_keys[:n] 442 query_recs = [] 443 timer.mark("QueryStart") 444 i = 0 445 for key in query_keys: 446 i += 1 447 face = database[key] 448 449 face_rec = algorithm.getFaceRecord(face.image,None,face.left_eye,face.right_eye) 450 query_recs.append(face_rec) 451 if verbose: 452 if time.time() - message_time > verbose: 453 message_time = time.time() 454 print "Processed query image %d of %d"%(i,len(query_keys)) 455 456 timer.mark("QueryStop",notes="Processed %d images."%len(query_keys)) 457 458 459 # Produce face records for each image in the target set 460 message_time = time.time() 461 target_keys = database.target() 462 if n != None: 463 target_keys = target_keys[:n] 464 target_recs = [] 465 timer.mark("TargetStart") 466 i = 0 467 for key in target_keys: 468 i += 1 469 face = database[key] 470 471 face_rec = algorithm.getFaceRecord(face.image,None,face.left_eye,face.right_eye) 472 target_recs.append(face_rec) 473 if verbose: 474 if time.time() - message_time > verbose: 475 message_time = time.time() 476 print "Processed target image %d of %d"%(i,len(target_keys)) 477 478 timer.mark("TargetStop",notes="Processed %d images."%len(target_keys)) 479 480 print "Finished processing FaceRecs (%d query, %d target)"%(len(query_keys),len(target_keys)) 481 482 # Compute the matrix 483 print "Computing similarity matrix..." 484 timer.mark("SimilarityStart") 485 mat = algorithm.similarityMatrix(query_recs,target_recs) 486 timer.mark("SimilarityStop",notes="Processed %d comparisons."%(mat.shape[0]*mat.shape[1],)) 487 488 print "Completing task..." 489 print mat.shape 490 491 bee_mat = pv.BEEDistanceMatrix(mat,"FRGC_Exp_2.0.4_Query.xml", "FRGC_Exp_2.0.4_Target.xml", sigset_dir=database.sigset_dir, is_distance=False) 492 493 if ilog != None: 494 ilog(timer) 495 496 return bee_mat, timer

497

Source Code for Module pyvision.analysis.FaceAnalysis.FRGC2004