From 899cbabe23adfb1e5d3bf74e1ce730edfb5c1ed9 Mon Sep 17 00:00:00 2001 From: Tim Abbott Date: Thu, 13 Dec 2012 09:14:59 -0500 Subject: [PATCH] populate_db: Make some more bits of memory possible for Python to free. (imported from commit 2d8184d05f622475ffab1043a40251644a88c230) --- zephyr/management/commands/populate_db.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/zephyr/management/commands/populate_db.py b/zephyr/management/commands/populate_db.py index 44004def06..f3c3e906b0 100644 --- a/zephyr/management/commands/populate_db.py +++ b/zephyr/management/commands/populate_db.py @@ -244,7 +244,7 @@ def restore_saved_messages(): huddle_user_set = set() # First, determine all the objects our messages will need. print datetime.datetime.now(), "Creating realms/streams/etc..." - for line in file(settings.MESSAGE_LOG, "r").readlines(): + def process_line(line): old_message_json = line.strip() # Due to populate_db's shakespeare mode, we have a lot of @@ -262,7 +262,7 @@ def restore_saved_messages(): tmp_message['id'] = '1' duplicate_suppression_key = simplejson.dumps(tmp_message) if duplicate_suppression_key in duplicate_suppression_hash: - continue + return duplicate_suppression_hash[duplicate_suppression_key] = True old_message = simplejson.loads(old_message_json) @@ -308,7 +308,7 @@ def restore_saved_messages(): realm_set.add(old_message["domain"]) if message_type not in ["stream", "huddle", "personal"]: - continue + return sender_email = old_message["sender_email"] @@ -344,6 +344,10 @@ def restore_saved_messages(): else: raise ValueError('Bad message type') + with file(settings.MESSAGE_LOG, "r") as message_log: + for line in message_log.readlines(): + process_line(line) + stream_recipients = {} user_recipients = {} huddle_recipients = {} @@ -457,6 +461,7 @@ def restore_saved_messages(): print datetime.datetime.now(), "Importing messages, part 2..." batch_bulk_create(Message, messages_to_create, batch_size=100) + messages_to_create = [] # Finally, create all the UserMessage objects print datetime.datetime.now(), "Importing usermessages, part 1..."