Controller starts a new batch of children immediately when reloading. This fixes #21

Tweaked css a little bit in the status page after looking at a lot of inactive children.

src/spawning/spawning_child.py

@@ -179,9 +179,10 @@ def read_pipe_and_die(the_pipe, server_coro):
                 # continue to listen for status pings while dying
     except socket.error:
         pass
-    # if here, perhaps the controller's process went down; we should die too
+    # if here, perhaps the controller's process went down; we should die too if
+    # we aren't already
     if not dying:
-        eventlet.greenthread.kill(server_coro, ExitChild)
+        eventlet.greenthread.kill(server_coro, KeyboardInterrupt)
 
 
 def deadman_timeout(signum, frame):
@@ -204,6 +205,15 @@ def tpool_wsgi(app):
     return tpooled_application
 
 
+def warn_controller_of_imminent_death(controller_pid):
+    # The controller responds to a SIGUSR1 by kicking off a new child process.
+    try:
+        os.kill(controller_pid, signal.SIGUSR1)
+    except OSError, e:
+        if not e.errno == errno.ESRCH:
+            raise
+
+
 def serve_from_child(sock, config, controller_pid):
     threads = config.get('threadpool_workers', 0)
     wsgi_application = spawning.util.named(config['app_factory'])(config)
@@ -249,12 +259,14 @@ def serve_from_child(sock, config, controller_pid):
             wsgi_kwargs.update({'timeout_value' : True})
             eventlet.with_timeout(max_age, eventlet.wsgi.server, *wsgi_args,
                     **wsgi_kwargs)
+            warn_controller_of_imminent_death(controller_pid)
         else:
             eventlet.wsgi.server(*wsgi_args, **wsgi_kwargs)
     except KeyboardInterrupt:
-        pass
+        # controller probably doesn't know that we got killed by a SIGINT
+        warn_controller_of_imminent_death(controller_pid)
     except ExitChild:
-        pass
+        pass  # parent killed us, it already knows we're dying
 
     ## Set a deadman timer to violently kill the process if it doesn't die after
     ## some long timeout.
@@ -267,13 +279,6 @@ def serve_from_child(sock, config, controller_pid):
     
     server = server_event.wait()
 
-    ## Let's tell our parent that we're dying
-    try:
-        os.kill(controller_pid, signal.SIGUSR1)
-    except OSError, e:
-        if not e.errno == errno.ESRCH:
-            raise
-
     last_outstanding = None
     while server.outstanding_requests:
         if last_outstanding != server.outstanding_requests:

src/spawning/spawning_controller.py

@@ -190,6 +190,8 @@ class Controller(object):
     def handle_sighup(self, *args, **kwargs):
         ''' Pass `no_restart` to prevent restarting the run loop '''
         self.kill_children()
+        self.spawn_children(number=self.num_processes)
+        # TODO: nothing seems to use no_restart, can it be removed?
         if not kwargs.get('no_restart', True):
             self.runloop()
 
@@ -345,8 +347,8 @@ def main():
             help='Start a backdoor bound to localhost:3000')
     parser.add_option('-a', '--max-age', dest='max_age', type='int',
         help='If given, the maximum amount of time (in seconds) an instance of spawning_child '
-            'is allowed to run. Once this time limit has expired a SIGHUP will be sent to '
-            'spawning_controller, causing it to restart all of the child processes.')
+            'is allowed to run. Once this time limit has expired the child will'
+            'gracefully kill itself while the server starts a replacement.')
     parser.add_option('--no-keepalive', dest='no_keepalive', action='store_true',
             help='Disable HTTP/1.1 KeepAlive')
     parser.add_option('-z', '--z-restart-args', dest='restart_args',

src/spawning/util/status.py

@@ -195,7 +195,7 @@ def fill_template(status_data):
                   status_data['configured_children_count'])
     if status_data['active_children_count'] < \
        status_data['configured_children_count']:
-        count_td.attrs['cls'] = "alert"
+        count_td.attrs['cls'] = "error"
         count_td.children.append(
             span("(", status_data['killed_children_count'], ")"))
     children_table = table(
@@ -310,13 +310,10 @@ input {
     display: inline;
 }
 tr.child.dying {
-    background: #ffeecc;
-    color: #884400;
+    font-style: italic;
+    color: #444444;
 }
-tr.child.error {
-    background: #ff6666;
-}
-.alert {
+.error {
     background: #ff4444;
 }