CodeCow/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2026-07-12 21:04:41 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

export: Clarify message exporting code.

Browse Source 
The function to create the message partial files has been
renamed to export_partial_message_files().  It now gets its own
list of user profile ids and recipient ids from the response,
so that we can de-clutter do_export_realm().
This commit is contained in:
Steve Howell 2016-08-11 06:26:47 -07:00 committed by Tim Abbott
parent 5cd915694a
commit f29b32bbb2
1 changed files with 17 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
zerver/lib/export.py
View File

@ -334,12 +334,19 @@ def write_message_export(message_filename, output):
f.write(ujson.dumps(output, indent=4))
logging.info("Dumped to %s" % (message_filename,))
def export_messages(realm, user_profile_ids, recipient_ids,
chunk_size=1000, output_dir=None):
# type: (Realm, Set[int], Set[int], int, Path) -> None
def export_partial_message_files(realm, response, chunk_size=1000, output_dir=None):
# type: (Realm, TableData, int, Path) -> None
if output_dir is None:
output_dir = tempfile.mkdtemp(prefix="zulip-export")
def get_ids(records):
# type: (List[Record]) -> Set[int]
return set(x['id'] for x in records)
user_profile_ids = get_ids(response['zerver_userprofile'] +
response['zerver_userprofile_crossrealm'])
recipient_ids = get_ids(response['zerver_recipient'])
# Basic security rule: You can export everything sent by someone
# in your realm export (members of your realm plus Zulip realm
# bots) to a recipient object you're exporting (that is thus also
@ -594,11 +601,13 @@ def do_export_realm(realm, output_dir, threads):
logging.info("Exporting uploaded files and avatars")
export_uploads_and_avatars(realm, output_dir)
user_profile_ids = set(x["id"] for x in response['zerver_userprofile'] +
response['zerver_userprofile_crossrealm'])
recipient_ids = set(x["id"] for x in response['zerver_recipient'])
logging.info("Exporting messages")
export_messages(realm, user_profile_ids, recipient_ids, output_dir=output_dir)
# We (sort of) export zerver_message rows here. We write
# them to .partial files that are subsequently fleshed out
# by parallel processes to add in zerver_usermessage data.
# This is for performance reasons, of course. Some installations
# have millions of messages.
logging.info("Exporting .partial files messages")
export_partial_message_files(realm, response, output_dir=output_dir)
# Start parallel jobs to export the UserMessage objects.
launch_user_message_subprocesses(threads=threads, output_dir=output_dir)