diff --git a/tools/build-local-server-tarball b/tools/build-local-server-tarball
index 12b03c0d41..be4d151466 100755
--- a/tools/build-local-server-tarball
+++ b/tools/build-local-server-tarball
@@ -20,6 +20,16 @@ git archive -o $TARBALL --prefix=zulip-server/ HEAD
 git checkout-index -f -a --prefix $TMP_CHECKOUT
 cd $TMP_CHECKOUT
 
+# Use default settings so there is no chance of leaking secrets
+cp zproject/local_settings_template.py zproject/local_settings.py
+
+# Some settings need values for it to work
+cat >> zproject/local_settings.py <<EOF
+DEBUG = False
+LOCAL_DATABASE_PASSWORD = ''
+SECRET_KEY = 'not_used_here'
+EOF
+
 # update-prod-static generates the prod-static directory.
 # See COLLECTSTATIC in settings.py
 ./tools/update-prod-static
diff --git a/zproject/settings.py b/zproject/settings.py
index 31e2706056..8265f8d6b0 100644
--- a/zproject/settings.py
+++ b/zproject/settings.py
@@ -28,8 +28,10 @@ from local_settings import *
 
 SERVER_GENERATION = int(time.time())
 
-# Uncomment end of next line to test JS/CSS minification.
-DEBUG = not DEPLOYED # and platform.node() != 'your-machine'
+if not 'DEBUG' in globals():
+    # Uncomment end of next line to test JS/CSS minification.
+    DEBUG = not DEPLOYED # and platform.node() != 'your-machine'
+
 TEMPLATE_DEBUG = DEBUG
 TEST_SUITE = False
 
@@ -297,10 +299,7 @@ STATIC_URL = '/static/'
 # here so that urls.py can read it.
 PIPELINE = not DEBUG
 
-# Use prod settings if exporting static files in build-local-server-tarball
-COLLECTSTATIC = 'manage.py' in sys.argv[0] and (len(sys.argv) > 1 and sys.argv[1] == 'collectstatic')
-
-if DEBUG and not COLLECTSTATIC:
+if DEBUG:
     STATICFILES_STORAGE = 'pipeline.storage.PipelineStorage'
     STATICFILES_FINDERS = (
         'django.contrib.staticfiles.finders.AppDirectoriesFinder',