analytics: Add cron job to run analytics jobs.

This adds a cron job to update the Zulip analytics counts, complete
with locking etc.

Substantially tweaked by tabbott.

analytics/management/commands/update_analytics_counts.py

@@ -1,15 +1,20 @@
 from __future__ import absolute_import
 from __future__ import print_function
 
+import os
+import sys
+from scripts.lib.zulip_tools import ENDC, WARNING
+
 from argparse import ArgumentParser
 from datetime import timedelta
 
 from django.core.management.base import BaseCommand
 from django.utils import timezone
 from django.utils.dateparse import parse_datetime
+from django.conf import settings
 
 from analytics.models import RealmCount, UserCount
-from analytics.lib.counts import COUNT_STATS, CountStat, process_count_stat
+from analytics.lib.counts import COUNT_STATS, process_count_stat
 from zerver.lib.timestamp import datetime_to_string, is_timezone_aware
 from zerver.models import UserProfile, Message
 
@@ -39,6 +44,19 @@ def add_arguments(self, parser):
 
     def handle(self, *args, **options):
         # type: (*Any, **Any) -> None
+        try:
+            os.mkdir(settings.ANALYTICS_LOCK_DIR)
+        except OSError:
+            print(WARNING + "Analytics lock %s is unavailable; exiting... " + ENDC)
+            return
+
+        try:
+            self.run_update_analytics_counts(options)
+        finally:
+            os.rmdir(settings.ANALYTICS_LOCK_DIR)
+
+    def run_update_analytics_counts(self, options):
+        # type: (Dict[str, Any]) -> None
         fill_to_time = parse_datetime(options['time'])
         if options['utc']:
             fill_to_time = fill_to_time.replace(tzinfo=timezone.utc)

puppet/zulip/files/cron.d/update-analytics-counts

@@ -0,0 +1,7 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+USER=zulip
+
+# Run update analytics tables every 5 minutes past the hour
+5 * * * * zulip python /home/zulip/deployments/current/manage.py update_analytics_counts
+

puppet/zulip/manifests/app_frontend.pp

@@ -42,4 +42,12 @@
     mode => 644,
     source => "puppet:///modules/zulip/cron.d/restart-zulip",
   }
+  # This should only be enabled on exactly 1 Zulip server in a cluster.
+  file { "/etc/cron.d/update-analytics-counts":
+    ensure => file,
+    owner  => "root",
+    group  => "root",
+    mode => 644,
+    source => "puppet:///modules/zulip/cron.d/update-analytics-counts",
+  }
 }

zproject/dev_settings.py

@@ -41,3 +41,4 @@
 
 # Enable inline open graph preview in development for now
 INLINE_URL_EMBED_PREVIEW = True
+ANALYTICS_LOCK_DIR = "var/analytics-lock-dir"

zproject/settings.py

@@ -192,6 +192,7 @@ def get_secret(key):
                     'POST_MIGRATION_CACHE_FLUSHING': False,
                     'ENABLE_FILE_LINKS': False,
                     'USE_WEBSOCKETS': True,
+                    'ANALYTICS_LOCK_DIR': "/home/zulip/deployments/analytics-lock-dir",
                     'PASSWORD_MIN_LENGTH': 6,
                     'PASSWORD_MIN_ZXCVBN_QUALITY': 0.5,
                     }