From 8fd72a09bc79feac71854cd45abfac3edd70f042 Mon Sep 17 00:00:00 2001 From: Zev Benjamin Date: Tue, 21 May 2013 15:21:53 -0400 Subject: [PATCH] Restart Django and Tornado separately from the other worker processes The amount of process downtime during a supervisord-mediated restart appears to be linear in the number of processes that are being restarted. Therefore, restarting just Django and Tornado causes less downtime than doing them at the same time as the other worker processes. (imported from commit 1fa9ef547bcd88caeec49800664e37d5f2fcb7a8) --- .../modules/humbug/files/supervisord/conf.d/humbug.conf | 7 +++++-- tools/restart-server | 8 ++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/servers/puppet/modules/humbug/files/supervisord/conf.d/humbug.conf b/servers/puppet/modules/humbug/files/supervisord/conf.d/humbug.conf index e472c3bcda..26076d3275 100644 --- a/servers/puppet/modules/humbug/files/supervisord/conf.d/humbug.conf +++ b/servers/puppet/modules/humbug/files/supervisord/conf.d/humbug.conf @@ -103,8 +103,11 @@ directory=/home/humbug/humbug/ ; create one or more 'real' group: sections to create "heterogeneous" ; process groups. -[group:humbug] -programs=humbug-django-fcgi,humbug-tornado,humbug-events-user-activity,humbug-events-subscribe-new-users,humbug-events-confirmation-emails,humbug-events-missedmessage_reminders ; each refers to 'x' in [program:x] definitions +[group:humbug-web] +programs=humbug-django-fcgi,humbug-tornado ; each refers to 'x' in [program:x] definitions + +[group:humbug-workers] +programs=humbug-events-user-activity,humbug-events-subscribe-new-users,humbug-events-confirmation-emails,humbug-events-missedmessage_reminders ; each refers to 'x' in [program:x] definitions ; The [include] section can just contain the "files" setting. This ; setting can list multiple files (separated by whitespace or diff --git a/tools/restart-server b/tools/restart-server index 04f2d80f4a..5451dc59f6 100755 --- a/tools/restart-server +++ b/tools/restart-server @@ -21,9 +21,13 @@ os.chdir("/home/humbug/humbug-deployments/current") # Send a statsd event on restarting the server subprocess.check_call(["python", "./manage.py", "send_stats", "incr", "events.server_restart", str(int(time.time()))]) -# Restart the FastCGI and related processes via supervisorctl +# Restart the FastCGI and related processes via supervisorctl. We +# minimize downtime by restarting Django and Tornado separately from +# the other worker processes. logging.info("Killing daemons") -subprocess.check_call(["supervisorctl", "restart", "humbug:*"]) +subprocess.check_call(["supervisorctl", "stop", "humbug-workers:*"]) +subprocess.check_call(["supervisorctl", "restart", "humbug-web:*"]) +subprocess.check_call(["supervisorctl", "start", "humbug-workers:*"]) logging.info("Flushing memcached") try: