From 8bc26aab08ad41ca667d0669b20e3dfeb2586a13 Mon Sep 17 00:00:00 2001
From: Alex Vandiver <alexmv@zulip.com>
Date: Tue, 14 Jun 2022 14:52:25 -0700
Subject: [PATCH] nagios: Switch check_user_zephyr_mirror_liveness to run via
 cron.

This check loads Django, and as such must be run as the zulip user.
Repeat the same pattern used elsewhere in nagios, of writing a state
file, which is read by `check_cron_file`.
---
 .../files/cron.d/check_user_zephyr_mirror_liveness     |  5 +++++
 puppet/zulip_ops/files/nagios4/commands.cfg            |  2 +-
 .../check_user_zephyr_mirror_liveness                  | 10 +++++++---
 puppet/zulip_ops/manifests/prod_app_frontend_once.pp   |  8 ++++++++
 4 files changed, 21 insertions(+), 4 deletions(-)
 create mode 100644 puppet/zulip_ops/files/cron.d/check_user_zephyr_mirror_liveness

diff --git a/puppet/zulip_ops/files/cron.d/check_user_zephyr_mirror_liveness b/puppet/zulip_ops/files/cron.d/check_user_zephyr_mirror_liveness
new file mode 100644
index 0000000000..1aa512a822
--- /dev/null
+++ b/puppet/zulip_ops/files/cron.d/check_user_zephyr_mirror_liveness
@@ -0,0 +1,5 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+USER=zulip
+
+* * * * * zulip /usr/lib/nagios/plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness >/dev/null
diff --git a/puppet/zulip_ops/files/nagios4/commands.cfg b/puppet/zulip_ops/files/nagios4/commands.cfg
index d41f013307..2db0debc85 100644
--- a/puppet/zulip_ops/files/nagios4/commands.cfg
+++ b/puppet/zulip_ops/files/nagios4/commands.cfg
@@ -72,7 +72,7 @@ define command{
 
 define command{
         command_name    check_user_zephyr_mirror_liveness
-        command_line    /usr/lib/nagios/plugins/check_by_ssh -p $ARG1$ -l nagios -t 30 -i /var/lib/nagios/.ssh/id_ed25519 -H $HOSTADDRESS$ -C '/usr/lib/nagios/plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness'
+        command_line    /usr/lib/nagios/plugins/check_by_ssh -p $ARG1$ -l nagios -t 30 -i /var/lib/nagios/.ssh/id_ed25519 -H $HOSTADDRESS$ -C '/usr/lib/nagios/plugins/zulip_app_frontend/check_cron_file /var/lib/nagios_state/check_user_zephyr_mirror_liveness'
 }
 
 define command{
diff --git a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness
index 79bceba994..df81332774 100755
--- a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness
+++ b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness
@@ -35,6 +35,9 @@ states: Dict[str, int] = {
     "UNKNOWN": 3,
 }
 
+state_file_path = "/var/lib/nagios_state/check_user_zephyr_mirror_liveness"
+now = timezone_now()
+
 
 def report(state: str, short_msg: str, too_old: AbstractSet[Any] = set()) -> None:
     too_old_data = ""
@@ -46,13 +49,14 @@ def report(state: str, short_msg: str, too_old: AbstractSet[Any] = set()) -> Non
             )
             for user in too_old
         )
-    print(f"{state}: {short_msg}{too_old_data}")
 
+    with open(state_file_path + ".tmp", "w") as f:
+        f.write(f"{now}|{states[state]}|{state}|{short_msg}{too_old_data}")
+    os.rename(state_file_path + ".tmp", state_file_path)
+    print(f"{state}: {short_msg}{too_old_data}")
     exit(states[state])
 
 
-now = timezone_now()
-
 all_users = UserActivity.objects.filter(
     query__in=["get_events", "/api/v1/events"], client__name="zephyr_mirror"
 )
diff --git a/puppet/zulip_ops/manifests/prod_app_frontend_once.pp b/puppet/zulip_ops/manifests/prod_app_frontend_once.pp
index 79e0c63027..83d5f8b452 100644
--- a/puppet/zulip_ops/manifests/prod_app_frontend_once.pp
+++ b/puppet/zulip_ops/manifests/prod_app_frontend_once.pp
@@ -32,4 +32,12 @@ class zulip_ops::prod_app_frontend_once {
     mode   => '0644',
     source => 'puppet:///modules/zulip_ops/cron.d/check_send_receive_time',
   }
+
+  file { '/etc/cron.d/check_user_zephyr_mirror_liveness':
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0644',
+    source => 'puppet:///modules/zulip_ops/cron.d/check_user_zephyr_mirror_liveness',
+  }
 }