From be27ec1ad4b635d1563e2bbf7a5cd643346d2124 Mon Sep 17 00:00:00 2001 From: Jessica McKellar Date: Mon, 26 Nov 2012 10:31:29 -0500 Subject: [PATCH] nagios: Change zephyr mirror liveness check to only care about aggregate statistics. Too many individual users occasionally don't update their mirrors, causing us to be permanently alerting; we have sufficient user notification at this point (plus Waseem keeping an eye on /activity) that we don't need to alert on individual users. We do, however, still care if something happens (say, Linerva going down) that causes many users' mirrors to go down. (imported from commit 392952c95739e183d4a711120e3a963671cec289) --- .../check_user_zephyr_mirror_liveness | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/servers/puppet/files/nagios_plugins/check_user_zephyr_mirror_liveness b/servers/puppet/files/nagios_plugins/check_user_zephyr_mirror_liveness index 3ea98a9b0a..be27931a06 100755 --- a/servers/puppet/files/nagios_plugins/check_user_zephyr_mirror_liveness +++ b/servers/puppet/files/nagios_plugins/check_user_zephyr_mirror_liveness @@ -10,19 +10,13 @@ Django ORM. import datetime import os import sys -import time os.environ['DJANGO_SETTINGS_MODULE'] = "humbug.settings" sys.path.append('/home/humbug/humbug') sys.path.append('/home/humbug/humbug/zephyr') -from django.contrib.auth.models import User from zephyr.models import UserActivity -USERS_TO_IGNORE = ( - 'rhkeeler@mit.edu', - ) - states = { "OK": 0, "WARNING": 1, @@ -44,15 +38,12 @@ def report(state, short_msg, too_old=None): now = datetime.datetime.utcnow() -too_old = [] -for user in UserActivity.objects.filter(query="/api/v1/get_messages", - client__name="zephyr_mirror"): - if user.user_profile.user.email in USERS_TO_IGNORE: - continue - if user.last_visit.replace(tzinfo=None) < now - datetime.timedelta(minutes=5): - too_old.append(user) +all_users = UserActivity.objects.filter(query="/api/v1/get_messages", + client__name="zephyr_mirror") +inactive_users = [user for user in all_users if user.last_visit.replace(tzinfo=None) < \ + now - datetime.timedelta(minutes=5)] -if not too_old: - report("OK", "All mirrors active") +if (len(inactive_users) / float(len(all_users))) < .15: + report("OK", "Most mirrors are active") else: - report("CRITICAL", "Some mirrors inactive", too_old) + report("CRITICAL", "Many mirrors are inactive", inactive_users)