py3: Augment all mentions of system Python packages to include Python 3.

In some of these contexts, we may still be *using* the Python 2
version, but at least this should eliminate running into
`ImportError`s one by one in scripts that run outside a virtualenv,
as we update their shebangs to refer to Python 3.

Several Python libraries we use don't come in Python 3 versions on
trusty: gevent, boto, twisted, django, django-tagging, whisper.
The latter two don't come in Python 3 versions even on xenial.
So some work required before we can actually switch the code that
relies on those libraries to run as Python 3 -- probably the best
solution will be to backport them all in our apt repo.  (All but
`whisper` are packaged in zesty; `whisper` upstream just grew Python 3
support this year.)

docs/dev-setup-non-vagrant.md

@@ -50,8 +50,8 @@ Install the following non-Python dependencies:
  * memcached (and headers)
  * rabbitmq-server
  * libldap2-dev
- * python-dev
  * python3-dev
+ * python-dev
  * python-virtualenv
  * redis-server — rate limiting
  * tsearch-extras — better text search
@@ -67,8 +67,8 @@ https://github.com/zulip/zulip.git`
 ```
 sudo apt-get install closure-compiler libfreetype6-dev libffi-dev \
     memcached rabbitmq-server libldap2-dev redis-server \
-    postgresql-server-dev-all libmemcached-dev python-dev \
-    python3-dev python-virtualenv hunspell-en-us nodejs \
+    postgresql-server-dev-all libmemcached-dev python3-dev \
+    python-dev python-virtualenv hunspell-en-us nodejs \
     nodejs-legacy git yui-compressor puppet gettext postgresql
 
 # If using Ubuntu, install PGroonga from its PPA
@@ -127,7 +127,7 @@ sudo add-apt-repository ppa:tabbott/zulip
 sudo apt-get update
 sudo apt-get install closure-compiler libfreetype6-dev libffi-dev \
     memcached rabbitmq-server libldap2-dev redis-server \
-    postgresql-server-dev-all libmemcached-dev python-dev \
+    postgresql-server-dev-all libmemcached-dev python3-dev python-dev \
     hunspell-en-us nodejs nodejs-legacy git yui-compressor \
     puppet gettext tsearch-extras
 ```

puppet/zulip/manifests/postgres_appdb_base.pp

@@ -4,6 +4,7 @@ class zulip::postgres_appdb_base {
   include zulip::supervisor
 
   $appdb_packages = [# Needed to run process_fts_updates
+                     "python3-psycopg2", # TODO: use a virtualenv instead
                      "python-psycopg2", # TODO: use a virtualenv instead
                      # Needed for our full text search system
                      "postgresql-${zulip::base::postgres_version}-tsearch-extras",

puppet/zulip/manifests/postgres_common.pp

@@ -4,8 +4,11 @@ class zulip::postgres_common {
                         # tools for database monitoring
                         "ptop",
                         # Python modules used in our monitoring/worker threads
+                        # "python3-gevent", # missing on trusty
                         "python-gevent",
+                        "python3-tz", # TODO: use a virtualenv instead
                         "python-tz", # TODO: use a virtualenv instead
+                        "python3-dateutil", # TODO: use a virtualenv instead
                         "python-dateutil", # TODO: use a virtualenv instead
                         # Needed just to support adding postgres user to 'zulip' group
                         "ssl-cert",

puppet/zulip_ops/manifests/base.pp

@@ -17,9 +17,12 @@ class zulip_ops::base {
                         # For managing our current Debian packages
                         "debian-goodies",
                         # Needed for zulip-ec2-configure-network-interfaces
+                        "python3-six",
                         "python-six",
                           # This one is needed for postgres as well
+                        # "python3-boto", # missing on trusty
                         "python-boto",
+                        "python3-netifaces",
                         "python-netifaces",
                         # Popular editors
                         "vim",

puppet/zulip_ops/manifests/postgres_common.pp

@@ -4,6 +4,7 @@ class zulip_ops::postgres_common {
   $internal_postgres_packages = [# dependencies for our wal-e backup system
                                  "lzop",
                                  "pv",
+                                 "python3-pip",
                                  "python-pip",
                                  # Postgres Nagios check plugin
                                  "check-postgres",
@@ -13,8 +14,10 @@ class zulip_ops::postgres_common {
   exec {"pip_wal-e":
     command  => "/usr/bin/pip install git+git://github.com/zbenjamin/wal-e.git#egg=wal-e",
     creates  => "/usr/local/bin/wal-e",
-    require  => Package['python-pip', 'python-boto',
-                        'python-gevent', 'lzop', 'pv'],
+    require  => Package['python3-pip',
+                        # 'python3-boto', 'python3-gevent', # missing on trusty
+                        'python-pip', 'python-boto', 'python-gevent',
+                        'lzop', 'pv'],
   }
 
   cron { "pg_backup_and_purge":
@@ -25,7 +28,11 @@ class zulip_ops::postgres_common {
     minute => 0,
     target => "postgres",
     user => "postgres",
-    require => [ File["/usr/local/bin/pg_backup_and_purge.py"], Package["postgresql-${zulip::base::postgres_version}", "python-dateutil"] ]
+    require => [ File["/usr/local/bin/pg_backup_and_purge.py"],
+                 Package["postgresql-${zulip::base::postgres_version}",
+                         "python3-dateutil",
+                         "python-dateutil"
+                 ] ]
   }
 
   exec { "sysctl_p":

puppet/zulip_ops/manifests/stats.pp

@@ -3,8 +3,15 @@ class zulip_ops::stats {
   include zulip_ops::apache
   include zulip::supervisor
 
-  $stats_packages = [ "libssl-dev", "zlib1g-dev", "python-twisted", "python-django", "python-django-tagging",
-                      "python-carbon", "python-cairo", "python-graphite-web", "python-whisper", "redis-server" ]
+  $stats_packages = [ "libssl-dev", "zlib1g-dev", "redis-server",
+                      # "python3-twisted", "python3-django", # missing on trusty
+                      # "python3-django-tagging", # missing on trusty and xenial!
+                      "python-twisted", "python-django", "python-django-tagging",
+                      "python-carbon", "python-graphite-web", # can't find these anywhere! did this ever work?
+                      "python3-cairo",
+                      # "python3-whisper", # missing on trusty and xenial!
+                      "python-cairo", "python-whisper"
+                      ]
   package { $stats_packages: ensure => "installed" }
 
   file { "/root/setup_disks.sh":

puppet/zulip_ops/manifests/zmirror.pp

@@ -12,7 +12,9 @@ class zulip_ops::zmirror {
                        # Packages needed to build pyzephyr
                        "libzephyr-dev",
                        "comerr-dev",
+                       "python3-dev",
                        "python-dev",
+                       "cython3",
                        "cython",
                        ]
   package { $zmirror_packages: ensure => "installed" }

puppet/zulip_ops/manifests/zmirror_personals.pp

@@ -12,7 +12,9 @@ class zulip_ops::zmirror_personals {
                        # Packages needed to build pyzephyr
                        "libzephyr-dev",
                        "comerr-dev",
+                       "python3-dev",
                        "python-dev",
+                       "cython3",
                        "cython",
                        ]
   package { $zmirror_packages: ensure => "installed" }