Initial commit
This commit is contained in:
donovan@ell-ee-dee.local
commit
375da0918d
|
|
@ -0,0 +1,19 @@
|
|||
Copyright (c) 2008, Donovan Preston
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
|
||||
This is a mash-up of Python Paste and eventlet. It implements the server_factory from Paste using the eventlet.wsgi server. It also has some nice features such as the ability to be multiprocess (not exposed yet) and code reloading based on either watching the filesystem for changes or watching the svn revision for changes.
|
||||
|
||||
The code reloading is graceful; that is to say, any requests which are currently in progress when the code reloading is initiated are handled to completion by the old processes, and new processes are started up to handle any new incoming requests.
|
||||
|
||||
Donovan Preston
|
||||
June 16, 2008
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
|
||||
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
|
||||
setup(
|
||||
name='Spawning',
|
||||
author='Donovan Preston',
|
||||
author_email='dsposx@mac.com',
|
||||
packages=find_packages(),
|
||||
version='0.1',
|
||||
entry_points={
|
||||
'paste.server_factory': [
|
||||
'main=spawning.spawning_controller:server_factory'
|
||||
]
|
||||
},
|
||||
classifiers=[
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Programming Language :: Python",
|
||||
"Operating System :: MacOS :: MacOS X",
|
||||
"Operating System :: POSIX",
|
||||
"Topic :: Internet",
|
||||
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||
"Intended Audience :: Developers",
|
||||
"Development Status :: 4 - Beta"
|
||||
]
|
||||
)
|
||||
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
"""
|
||||
"""
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
"""Watch files and send a SIGHUP signal to another process
|
||||
if any of the files change.
|
||||
"""
|
||||
|
||||
|
||||
import optparse, os, sets, signal, sys, tempfile, time
|
||||
|
||||
from eventlet import api, coros, jsonhttp
|
||||
|
||||
import simplejson
|
||||
|
||||
|
||||
def watch_forever(urls, pid, interval):
|
||||
"""
|
||||
"""
|
||||
if not urls:
|
||||
modules = list(get_sys_modules_files())
|
||||
fl = tempfile.NamedTemporaryFile()
|
||||
fl.write(simplejson.dumps({'files': modules}))
|
||||
fl.flush()
|
||||
urls = ['file://' + fl.name]
|
||||
|
||||
limiter = coros.CoroutinePool(track_events=True)
|
||||
module_mtimes = {}
|
||||
last_changed_time = None
|
||||
while True:
|
||||
for url in urls:
|
||||
limiter.execute(jsonhttp.get, url)
|
||||
|
||||
uniques = sets.Set()
|
||||
for i in range(len(urls)):
|
||||
uniques.update(limiter.wait()['files'])
|
||||
|
||||
changed = False
|
||||
for filename in uniques:
|
||||
try:
|
||||
stat = os.stat(filename)
|
||||
if stat:
|
||||
mtime = stat.st_mtime
|
||||
else:
|
||||
mtime = 0
|
||||
except (OSError, IOError):
|
||||
continue
|
||||
if filename.endswith('.pyc') and os.path.exists(filename[:-1]):
|
||||
mtime = max(os.stat(filename[:-1]).st_mtime, mtime)
|
||||
if not module_mtimes.has_key(filename):
|
||||
module_mtimes[filename] = mtime
|
||||
elif module_mtimes[filename] < mtime:
|
||||
changed = True
|
||||
last_changed_time = mtime
|
||||
module_mtimes[filename] = mtime
|
||||
print "(%s) File %r changed" % (os.getpid(), filename)
|
||||
|
||||
if not changed and last_changed_time is not None:
|
||||
last_changed_time = None
|
||||
if pid:
|
||||
print "(%s) Sending SIGHUP to %s at %s" % (
|
||||
os.getpid(), pid, time.asctime())
|
||||
os.kill(pid, signal.SIGHUP)
|
||||
else:
|
||||
os._exit(3)
|
||||
|
||||
api.sleep(interval)
|
||||
|
||||
|
||||
def get_sys_modules_files():
|
||||
for module in sys.modules.values():
|
||||
fn = getattr(module, '__file__', None)
|
||||
if fn is not None:
|
||||
yield os.path.abspath(fn)
|
||||
|
||||
|
||||
def main():
|
||||
parser = optparse.OptionParser()
|
||||
parser.add_option("-u", "--url",
|
||||
action="append", dest="urls",
|
||||
help="A url to GET for a JSON object with a key 'files' of filenames to check. "
|
||||
"If not given, use the filenames of everything in sys.modules.")
|
||||
parser.add_option("-p", "--pid",
|
||||
type="int", dest="pid",
|
||||
help="A pid to SIGHUP when a monitored file changes. "
|
||||
"If not given, just print a message to stdout and kill this process instead.")
|
||||
parser.add_option("-i", "--interval",
|
||||
type="int", dest="interval",
|
||||
help="The time to wait between scans, in seconds.", default=1)
|
||||
options, args = parser.parse_args()
|
||||
|
||||
try:
|
||||
watch_forever(options.urls, options.pid, options.interval)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
"""Watch the svn revision returned from svn info and send a SIGHUP
|
||||
to a process when the revision changes.
|
||||
"""
|
||||
|
||||
|
||||
import commands, optparse, os, signal, sys, tempfile, time
|
||||
|
||||
|
||||
def get_revision(directory):
|
||||
cmd = 'svn info'
|
||||
if directory is not None:
|
||||
cmd = '%s %s' % (cmd, directory)
|
||||
|
||||
try:
|
||||
out = commands.getoutput(cmd).split('\n')
|
||||
except IOError:
|
||||
return
|
||||
|
||||
for line in out:
|
||||
if line.startswith('Revision: '):
|
||||
return int(line[len('Revision: '):])
|
||||
|
||||
|
||||
def watch_forever(directory, pid, interval):
|
||||
"""
|
||||
"""
|
||||
revision = get_revision(directory)
|
||||
while True:
|
||||
new_revision = get_revision(directory)
|
||||
if new_revision is not None and new_revision != revision:
|
||||
revision = new_revision
|
||||
if pid:
|
||||
print "(%s) Sending SIGHUP to %s at %s" % (
|
||||
os.getpid(), pid, time.asctime())
|
||||
os.kill(pid, signal.SIGHUP)
|
||||
else:
|
||||
print "(%s) Revision changed, dying at %s" % (
|
||||
os.getpid(), time.asctime())
|
||||
os._exit(3)
|
||||
|
||||
time.sleep(interval)
|
||||
|
||||
|
||||
def main():
|
||||
parser = optparse.OptionParser()
|
||||
parser.add_option("-d", "--dir", dest='dir',
|
||||
help="The directory to do svn info in. If not given, use cwd.")
|
||||
parser.add_option("-p", "--pid",
|
||||
type="int", dest="pid",
|
||||
help="A pid to SIGHUP when the svn revision changes. "
|
||||
"If not given, just print a message to stdout and kill this process instead.")
|
||||
parser.add_option("-i", "--interval",
|
||||
type="int", dest="interval",
|
||||
help="The time to wait between scans, in seconds.", default=10)
|
||||
options, args = parser.parse_args()
|
||||
|
||||
print "(%s) svn watcher running, controller pid %s" % (os.getpid(), options.pid)
|
||||
if options.pid is None:
|
||||
options.pid = os.getpid()
|
||||
try:
|
||||
watch_forever(options.dir, int(options.pid), options.interval)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
"""spawning_child.py
|
||||
"""
|
||||
|
||||
from eventlet import api, coros, util, wsgi
|
||||
util.wrap_socket_with_coroutine_socket()
|
||||
util.wrap_pipes_with_coroutine_pipes()
|
||||
util.wrap_threading_local_with_coro_local()
|
||||
|
||||
import optparse, os, signal, socket, sys, time
|
||||
|
||||
from paste.deploy import loadwsgi
|
||||
|
||||
from spawning import reloader_dev
|
||||
|
||||
|
||||
class ExitChild(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def read_pipe_and_die(the_pipe, server_coro):
|
||||
os.read(the_pipe, 1)
|
||||
api.switch(server_coro, exc=ExitChild)
|
||||
|
||||
|
||||
def serve_from_child(sock, base_dir, config_url):
|
||||
wsgi_application = loadwsgi.loadapp(config_url, relative_to=base_dir)
|
||||
|
||||
host, port = sock.getsockname()
|
||||
print "(%s) wsgi server listening on %s:%s using %s from %s (in %s)" % (
|
||||
os.getpid(), host, port, wsgi_application, config_url, base_dir)
|
||||
|
||||
server_event = coros.event()
|
||||
try:
|
||||
wsgi.server(
|
||||
sock, wsgi_application, server_event=server_event)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
except ExitChild:
|
||||
pass
|
||||
|
||||
server = server_event.wait()
|
||||
|
||||
last_outstanding = None
|
||||
while server.outstanding_requests:
|
||||
if last_outstanding != server.outstanding_requests:
|
||||
print "(%s) %s requests remaining, waiting..." % (
|
||||
os.getpid(), server.outstanding_requests)
|
||||
last_outstanding = server.outstanding_requests
|
||||
api.sleep(0.1)
|
||||
print "(%s) Child exiting: all requests completed at %s" % (
|
||||
os.getpid(), time.asctime())
|
||||
|
||||
|
||||
def main():
|
||||
parser = optparse.OptionParser()
|
||||
parser.add_option("-d", "--dev",
|
||||
action='store_true', dest='dev',
|
||||
help='If --dev is passed, reload the server any time '
|
||||
'a loaded module changes. Otherwise, only when the svn '
|
||||
'revision of the current directory changes.')
|
||||
|
||||
options, args = parser.parse_args()
|
||||
|
||||
if len(args) < 5:
|
||||
print "Usage: %s controller_pid config_url base_dir httpd_fd death_fd" % (
|
||||
sys.argv[0], )
|
||||
sys.exit(1)
|
||||
|
||||
controller_pid, config_url, base_dir, httpd_fd, death_fd = args
|
||||
controller_pid = int(controller_pid)
|
||||
|
||||
## Set up the reloader
|
||||
if options.dev:
|
||||
api.spawn(
|
||||
reloader_dev.watch_forever, [], controller_pid, 1)
|
||||
|
||||
## The parent will catch sigint and tell us to shut down
|
||||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||||
api.spawn(read_pipe_and_die, int(death_fd), api.getcurrent())
|
||||
|
||||
sock = socket.fromfd(int(httpd_fd), socket.AF_INET, socket.SOCK_STREAM)
|
||||
serve_from_child(
|
||||
sock, base_dir, config_url)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,139 @@
|
|||
|
||||
|
||||
from eventlet import api, backdoor, coros, util, wsgi
|
||||
util.wrap_socket_with_coroutine_socket()
|
||||
util.wrap_pipes_with_coroutine_pipes()
|
||||
util.wrap_threading_local_with_coro_local()
|
||||
|
||||
|
||||
import errno, os, optparse, signal, sys, time
|
||||
|
||||
from paste.deploy import loadwsgi
|
||||
|
||||
|
||||
KEEP_GOING = True
|
||||
|
||||
|
||||
def spawn_new_children(sock, base_dir, config_url, dev):
|
||||
child_pipes = []
|
||||
parent_pid = os.getpid()
|
||||
for x in range(1):
|
||||
child_side, parent_side = os.pipe()
|
||||
if not os.fork():
|
||||
os.close(parent_side)
|
||||
args = [
|
||||
sys.executable,
|
||||
os.path.join(
|
||||
os.path.split(os.path.abspath(__file__))[0],
|
||||
'spawning_child.py'),
|
||||
str(parent_pid),
|
||||
config_url,
|
||||
base_dir,
|
||||
str(sock.fileno()),
|
||||
str(child_side)]
|
||||
|
||||
if dev:
|
||||
args.append('--dev')
|
||||
|
||||
os.execve(sys.executable, args, {'PYTHONPATH': os.environ['PYTHONPATH']})
|
||||
## Never gets here!
|
||||
|
||||
os.close(child_side)
|
||||
child_pipes.append(parent_side)
|
||||
|
||||
def sighup(_signum, _stack_frame):
|
||||
tokill = child_pipes[:]
|
||||
del child_pipes[:]
|
||||
|
||||
if KEEP_GOING:
|
||||
spawn_new_children(sock, base_dir, config_url, dev)
|
||||
|
||||
for child in tokill:
|
||||
os.write(child, ' ')
|
||||
|
||||
signal.signal(signal.SIGHUP, sighup)
|
||||
|
||||
|
||||
def reap_children():
|
||||
global KEEP_GOING
|
||||
|
||||
try:
|
||||
pid, result = os.wait()
|
||||
except OSError: # "Interrupted System Call"
|
||||
pass
|
||||
except KeyboardInterrupt:
|
||||
print "(%s) Controller exiting at %s" % (
|
||||
os.getpid(), time.asctime())
|
||||
|
||||
KEEP_GOING = False
|
||||
os.kill(os.getpid(), signal.SIGHUP)
|
||||
while True:
|
||||
## Keep waiting until all children are dead.
|
||||
try:
|
||||
pid, result = os.wait()
|
||||
except OSError, e:
|
||||
if e[0] == errno.ECHILD:
|
||||
break
|
||||
else:
|
||||
print "(%s) Child %s died with code %s." % (
|
||||
os.getpid(), pid, result)
|
||||
|
||||
|
||||
def run_controller(base_dir, config_url, dev=False):
|
||||
print "(%s) Controller starting up at %s" % (
|
||||
os.getpid(), time.asctime())
|
||||
|
||||
controller_pid = os.getpid()
|
||||
if not dev:
|
||||
## Set up the production reloader that watches the svn revision number.
|
||||
if not os.fork():
|
||||
base = os.path.split(__file__)[0]
|
||||
args = [
|
||||
sys.executable,
|
||||
os.path.join(
|
||||
base, 'reloader_svn.py'),
|
||||
'--dir=' + base,
|
||||
'--pid=' + str(controller_pid)]
|
||||
os.execve(sys.executable, args, {'PYTHONPATH': os.environ['PYTHONPATH']})
|
||||
## Never gets here!
|
||||
|
||||
ctx = loadwsgi.loadcontext(
|
||||
loadwsgi.SERVER,
|
||||
config_url, relative_to=base_dir)
|
||||
|
||||
sock = api.tcp_listener(
|
||||
(ctx.local_conf['host'], int(ctx.local_conf['port'])))
|
||||
spawn_new_children(sock, base_dir, config_url, dev)
|
||||
|
||||
while KEEP_GOING:
|
||||
reap_children()
|
||||
|
||||
|
||||
def server_factory(global_conf, host, port, *args, **kw):
|
||||
config_name = 'config:' + os.path.split(
|
||||
global_conf['__file__'])[1]
|
||||
base_dir = global_conf['here']
|
||||
def run(app):
|
||||
run_controller(base_dir, config_name, global_conf.get('debug') == 'true')
|
||||
return run
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = optparse.OptionParser()
|
||||
parser.add_option("-d", "--dev",
|
||||
action='store_true', dest='dev',
|
||||
help='If --dev is passed, reload the server any time '
|
||||
'a loaded module changes. Otherwise, only when the svn '
|
||||
'revision changes. Dev servers '
|
||||
'also run only one python process at a time.')
|
||||
|
||||
options, args = parser.parse_args()
|
||||
|
||||
if len(args) < 2:
|
||||
print "Usage: %s config_url base_dir" % (
|
||||
sys.argv[0], )
|
||||
sys.exit(1)
|
||||
|
||||
config_url, base_dir = args
|
||||
run_controller(base_dir, config_url, options.dev)
|
||||
|
||||
Loading…
Reference in New Issue