Source Code for Module pyvision.surveillance.VideoStreamProcessor

  1  ''' 
  2  Created on Mar 18, 2011 
  3  @author: svohara 
  4  ''' 
  5  # PyVision License 
  6  # 
  7  # Copyright (c) 2006-2008 Stephen O'Hara 
  8  # All rights reserved. 
  9  # 
 10  # Redistribution and use in source and binary forms, with or without 
 11  # modification, are permitted provided that the following conditions 
 12  # are met: 
 13  #  
 14  # 1. Redistributions of source code must retain the above copyright 
 15  # notice, this list of conditions and the following disclaimer. 
 16  #  
 17  # 2. Redistributions in binary form must reproduce the above copyright 
 18  # notice, this list of conditions and the following disclaimer in the 
 19  # documentation and/or other materials provided with the distribution. 
 20  #  
 21  # 3. Neither name of copyright holders nor the names of its contributors 
 22  # may be used to endorse or promote products derived from this software 
 23  # without specific prior written permission. 
 24  #  
 25  #  
 26  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
 27  # ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
 28  # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 
 29  # A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR 
 30  # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 
 31  # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
 32  # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
 33  # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 
 34  # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 
 35  # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 
 36  # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
 37  import pyvision as pv 
 38  import cv 
 39   
 40  ''' 
 41  This module implements various Video Stream Processors, or VSPs for short. 
 42  A VSP is designed to encapsulate a per-frame operation that can be 
 43  applied to a video stream. Examples include displaying the image while 
 44  overlaying the frame number (SimpleVSP), writing the output of a video 
 45  stream to a video file (VideoWriterVSP), and performing motion detection 
 46  on each video frame (MotionDetectionVSP). 
 47   
 48  The general idea is to chain together a VSP sequence, and then attach 
 49  the head of the chain to a video's play method. We hope that users will 
 50  create and/or contribute many useful subclasses of AbstractVSP. 
 51   
 52  For example: 
 53  import pyvision as pv 
 54  vsp_write = pv.VideoWriterVSP('tmp.avi',size=(640,480)) 
 55  vsp_disp = pv.SimpleVSP(window="Display", nextModule=vsp_write) 
 56  vid = pv.Video(sourceFile) 
 57  vid.play(window=None, delay=25, onNewFrame=vsp_disp) 
 58  ''' 
 59   
 60  VSP_SWALLOW_IMG = -1 #special return onNewFrame to indicate that the frame should be skipped, and not passed downstream 
 61   
 62 -class AbstractVSP(): 
 63      '''AbstractVSP is the abstract class definition of a 
 64      Video Stream Processor (VSP) object. VSP's are designed to be chained 
 65      together to accomplish processing on a video stream. 
 66      ''' 
 67       
 68 -    def __init__(self, window=None, nextModule=None): 
 69          ''' Constructor 
 70          @param window: The window name to use when displaying this VSP's 
 71          output. Specify None to suppress showing the output, but note that 
 72          if you modify the current image with annotations, those will be 
 73          persisted "downstream" to later processors. 
 74          @param nextModule: A Video Stream Processor object that should be 
 75          invoked on every frame after this processor has finished. 
 76          ''' 
 77          self._windowName = window 
 78          self._nextModule = nextModule 
 79       
 80 -    def __call__(self, img, fn, **kwargs): 
 81          rc = self._onNewFrame(img, fn, **kwargs) 
 82          if type(rc) == list or type(rc) == tuple: 
 83              #then we should have (newImg, newFn) 
 84              (newImg, fn) = rc  #we overwrite the fn parameter that downstream modules will see 
 85          else: 
 86              #rc should be just newImg to pass on to next Module 
 87              newImg = rc 
 88               
 89          if newImg == VSP_SWALLOW_IMG: 
 90              #special return indicates that the nextModule should 
 91              # not be called at this iteration...the current input image 
 92              # should be swallowed with no output 
 93              pass 
 94          else: 
 95              if self._nextModule != None: 
 96                  if newImg != None: 
 97                      #we have a new image to replace the current one instream 
 98                      kwargs['orig_img']=img #add a new keyword arg to allow access to origininal img 
 99                      self._nextModule(newImg, fn, **kwargs) 
100                  else: 
101                      self._nextModule(img, fn, **kwargs) 
102               
103 -    def _onNewFrame(self, img, fn, **kwargs): 
104          ''' Override this abstract method with the processing your object 
105          performs on a per-frame basis. It is recommended that you do not 
106          directly call this method. Rather, the VSP is a callable object, 
107          and so the __call__ method takes care of invoking this method as 
108          well as calling the next module, if any. 
109          ''' 
110          raise NotImplemented 
111           
112 -class FrameNumberVSP(AbstractVSP): 
113      '''A simple VSP object simply displays the input video frame with 
114      some simple annotation to show the frame number in upper left corner. 
115      NOTE: The vid.play(...) method will automatically add a frame number 
116      annotation to the source image, which can be problematic for downstream 
117      processing. Instead, call vid.play(...,annotate=False) to suppress 
118      the frame number display, and then use this FrameNumberVSP as a final 
119      step to put the frame number on the video after any processing has 
120      occurred. 
121      '''         
122 -    def __init__(self, display_pad=4, window=None, nextModule=None): 
123          ''' 
124          Constructor 
125          @param display_pad: Pads the frame number with leading zeros 
126          in order to have at least this many digits. 
127          ''' 
128          self.pad = display_pad 
129          AbstractVSP.__init__(self, window, nextModule) 
130           
131 -    def _onNewFrame(self, img, fn, **kwargs): 
132          pt = pv.Point(10, 10) 
133          img.annotateLabel(label="Frame: %s"%str(fn+1).zfill(self.pad), point=pt, color="white", background="black") 
134          if self._windowName != None: img.show(window=self._windowName, delay=1) 
135          return img 
136    
137  #TODO: There seems to be a bug in the video writing output when writing 
138  # frames from some source video objects in some output sizes. The symptom 
139  # appears as an output video that is "slanted" and grainy.         
140 -class VideoWriterVSP(AbstractVSP): 
141      ''' 
142      A video stream processor that outputs to a new movie file. 
143      If you want to display the frame number in the output, chain this VSP 
144      after a SimpleVSP object in the series. 
145      ''' 
146 -    def __init__(self, filename, window="Input", nextModule=None, fourCC_str="XVID", fps=15, size=None, bw=False,  
147                   no_annotations = False): 
148          ''' 
149          Constructor 
150          @param filename: The full output filename. Include the extension, such as .avi. 
151          @param window: The window name to use when displaying this VSP's 
152          output. Specify None to suppress showing the output, but note that 
153          if you modify the current image with annotations, those will be 
154          persisted "downstream" to later processors. 
155          @param nextModule: A Video Stream Processor object that should be 
156          invoked on every frame after this processor has finished. 
157          @param fourCC_str:  The "Four CC" string that is used to specify the encoder. 
158          @param fps: Frames per second. Not all codecs allow you to specify arbitrary frame rates, however. 
159          @param size: A tuple (w,h) representing the size of the output frames. 
160          @param bw: Specify true if you wish for a black-and-white only output. 
161          @param no_annotations: set to True to output the original, non-annotated version of the image 
162          ''' 
163          cvFourCC = cv.CV_FOURCC(*fourCC_str) 
164          if bw: 
165              colorFlag = cv.CV_LOAD_IMAGE_GRAYSCALE 
166          else: 
167              colorFlag = cv.CV_LOAD_IMAGE_UNCHANGED 
168          self._bw = bw 
169          self._out = cv.CreateVideoWriter(filename, cvFourCC, fps, size, colorFlag) 
170          self._no_annotations = no_annotations 
171          AbstractVSP.__init__(self, window=window, nextModule=nextModule) 
172           
173 -    def addFrame(self, img): 
174          ''' 
175          @param img: A pyvision img to write out to the video.         
176          ''' 
177          if self._no_annotations: 
178              img2 = img 
179          else: 
180              img2 = pv.Image(img.asAnnotated()) 
181               
182          if self._bw: 
183              cv.WriteFrame(self._out, img2.asOpenCVBW())     
184          else: 
185              cv.WriteFrame(self._out, img2.asOpenCV())         
186              
187 -    def _onNewFrame(self, img, fn, **kwargs): 
188          self.addFrame(img) 
189          return img 
190   
191 -class ResizerVSP(AbstractVSP): 
192      '''This VSP resizes each frame of video. Subsequent VSPs in a chain 
193      will see the resized image instead of the original. 
194      ''' 
195 -    def __init__(self, new_size=(320,240), window="Resized Image", nextModule=None): 
196          self._newSize = new_size 
197          AbstractVSP.__init__(self, window=window, nextModule=nextModule) 
198       
199 -    def _onNewFrame(self, img, fn, **kwargs): 
200          img = img.resize(self._newSize) 
201          if self._windowName != None: img.show(window=self._windowName, delay=1) 
202          return img 
203           
204 -class FrameSkipperVSP(AbstractVSP): 
205      ''' 
206      This is a video stream processor that is used to skip every k frames 
207      in a source video. You might put this vsp as the first step in processing 
208      if you need to adjust a 60fps video, for example, to skip every other frame 
209      so that downstream processing sees 30fps input. 
210       
211      Downstream modules will see a renumbered video stream. For example, if every-other 
212      frame was being skipped, the nextModule would still see its frame number input as 0,1,2,3,... 
213      even though in reality it is receiving frames 0,2,4,... from the source video. 
214      ''' 
215 -    def __init__(self, skip_param=0, nextModule=None): 
216          ''' 
217          Constructor 
218          @param skip_param: If 0, then no frames are skipped. Otherwise a frame 
219          is skipped if (frame_number + 1) modulo skip_param == 0. For example, with 
220          skip_param of 2, then frames 1,3,5,7,... will be dropped. 
221          ''' 
222          self.skip_param = skip_param 
223          if skip_param == 1: 
224              print "Warning, you specified a skip_param of 1 for the frame skipper VSP." 
225              print "This means ALL frames will be suppressed." 
226               
227          pv.AbstractVSP.__init__(self, window=None, nextModule=nextModule) 
228                
229 -    def _onNewFrame(self, img, fn, **kwargs): 
230          if self.skip_param == 0: 
231              #special case, do nothing 
232              return img 
233           
234          if ( (fn+1) % self.skip_param ) == 0: 
235              return VSP_SWALLOW_IMG 
236          else: 
237              newFn = int( round( (1 - (1.0/self.skip_param))*fn) ) 
238              return (img, newFn) #let this one through, provide new frame number 
239               
240           
241 -class MotionDetectionVSP(AbstractVSP): 
242      ''' This VSP uses an existing motion detection object to apply motion 
243      detection to each frame of video. 
244      ''' 
245 -    def __init__(self, md_object, window="Motion Detection", nextModule=None): 
246          ''' Constructor 
247          @param md_object: The pyvision motion detection object to be used by 
248          this VSP 
249          @param window: The name of the output window. Use None to suppress output. 
250          @param nextModule: The next VSP, if any, to be called by this VSP. 
251          ''' 
252          self._md = md_object 
253          AbstractVSP.__init__(self, window=window, nextModule=nextModule) 
254           
255 -    def _onNewFrame(self, img, fn, **kwargs): 
256          ''' Performs motion detection using this object's md object, 
257          displays the foreground pixels to a window. 
258          ''' 
259          md = self._md 
260          rc = md.detect(img) 
261          if rc > -1: 
262              md.annotateFrame(img, rect_color="yellow", contour_color=None, flow_color=None) 
263              if self._windowName != None: img.show(window=self._windowName, delay=1) 
264              #img_fg = md.getForegroundPixels() 
265              #img_fg.show("Foreground") 
266          return img 
267               
268 -class PeopleDetectionVSP(AbstractVSP): 
269      ''' This Video Stream Processor applies the OpenCV HOG people detector 
270      to each frame of video, annotating the detections with red rectangles. 
271      ''' 
272 -    def _onNewFrame(self, img, fn, **kwargs): 
273          rects = self._detectPeople(img) 
274          for r in rects: img.annotateRect(r) 
275          if self._windowName != None: img.show(window=self._windowName, delay=1) 
276          return img 
277           
278 -    def _detectPeople(self, img): 
279          cvim = img.asOpenCV()  #convert to OpenCV format before using OpenCV functions 
280          rect_list = [] 
281          try: 
282              found = list(cv.HOGDetectMultiScale(cvim, cv.CreateMemStorage(0))) 
283              rect_list = [ pv.Rect(x,y,w,h) for ((x,y),(w,h)) in found]  #python list comprehension             
284          except: 
285              #cv.HOGDetectMultiScale can throw exceptions, so return empty list 
286              return [] 
287           
288          return rect_list 
289