From 6c96561624e96447586afa2becec2a67e26b3002 Mon Sep 17 00:00:00 2001 From: Zev Benjamin Date: Mon, 25 Nov 2013 18:41:24 -0500 Subject: [PATCH] [manual] Do search highlighting in Python rather than in the database This requires the tsearch_extras Postgres extension. To install the extension, first install postgresql-9.1-tsearch-extras on both postgres-primary and postgres-secondary (this would normally be done in a puppet apply, but there are currently some changes that can't be applied on Postgres machines). Then run the following as the postgres user on postgres-primary: $ psql -d zulip -c 'CREATE EXTENSION tsearch_extras SCHEMA zulip;' In dev environments, you must also run: $ psql -d zulip_test_template -c 'CREATE EXTENSION tsearch_extras SCHEMA zulip;' (imported from commit ad0a57c455b3b86002191ac5fb705d8f716f3296) --- zerver/views/messages.py | 54 +++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/zerver/views/messages.py b/zerver/views/messages.py index 1369bbcc23..276a79a4c6 100644 --- a/zerver/views/messages.py +++ b/zerver/views/messages.py @@ -8,6 +8,7 @@ from django.db.models import Q from zerver.decorator import authenticated_api_view, authenticated_json_post_view, \ has_request_variables, REQ, JsonableError, json_to_list, json_to_bool, \ to_non_negative_int, to_non_negative_float +from django.utils.html import escape as escape_html from django.views.decorators.csrf import csrf_exempt from zerver.lib import bugdown from zerver.lib.actions import recipient_for_emails, do_update_message_flags, \ @@ -191,13 +192,11 @@ class NarrowBuilder(object): if "postgres" in settings.DATABASES["default"]["ENGINE"]: tsquery = "plainto_tsquery('zulip.english_us_search', %s)" where = "search_tsvector @@ " + tsquery - match_content = "ts_headline('zulip.english_us_search', rendered_content, " \ - + tsquery + ", 'StartSel=\"\", StopSel=, " \ - "HighlightAll=TRUE')" + content_matches = "ts_match_locs_array('zulip.english_us_search', rendered_content, " \ + + tsquery + ")" # We HTML-escape the subject in Postgres to avoid doing a server round-trip - match_subject = "ts_headline('zulip.english_us_search', escape_html(subject), " \ - + tsquery + ", 'StartSel=\"\", StopSel=, " \ - "HighlightAll=TRUE')" + subject_matches = "ts_match_locs_array('zulip.english_us_search', escape_html(subject), " \ + + tsquery + ")" # Do quoted string matching. We really want phrase # search here so we can ignore punctuation and do @@ -209,8 +208,8 @@ class NarrowBuilder(object): query = query.filter(self.pQ(content__icontains=term) | self.pQ(subject__icontains=term)) - return query.extra(select={'match_content': match_content, - 'match_subject': match_subject}, + return query.extra(select={'content_matches': content_matches, + 'subject_matches': subject_matches}, where=[where], select_params=[operand, operand], params=[operand]) else: @@ -219,6 +218,26 @@ class NarrowBuilder(object): self.pQ(subject__icontains=word)) return query +def highlight_string(string, locs): + highlight_start = '' + highlight_stop = '' + pos = 0 + result = '' + for loc in locs: + (offset, length) = loc + result += string[pos:offset] + result += highlight_start + result += string[offset:offset + length] + result += highlight_stop + pos = offset + length + result += string[pos:] + return result + +def get_search_fields(msg, matches_container): + return dict(match_content=highlight_string(msg.rendered_content, + matches_container.content_matches), + match_subject=highlight_string(escape_html(msg.subject), + matches_container.subject_matches)) def narrow_parameter(json): # FIXME: A hack to support old mobile clients @@ -267,7 +286,8 @@ def get_old_messages_backend(request, user_profile, if include_history: prefix = "" - query = Message.objects.only("id").order_by('id') + wanted_fields = ["id"] + query = Message.objects.only(*wanted_fields).order_by('id') else: prefix = "message__" # Conceptually this query should be @@ -279,7 +299,8 @@ def get_old_messages_backend(request, user_profile, # actually fetching all the rows from the message table (since # doing so would cause Django to consume a lot of resources # rendering them). The following achieves these objectives. - query = UserMessage.objects.select_related("message").only("flags", "id", "message__id") \ + wanted_fields = ["flags", "id", "message__id"] + query = UserMessage.objects.select_related("message").only(*wanted_fields) \ .filter(user_profile=user_profile).order_by('message') # Add some metadata to our logging data for narrows @@ -302,7 +323,9 @@ def get_old_messages_backend(request, user_profile, num_extra_messages = 0 build = NarrowBuilder(user_profile, prefix) for operator, operand in narrow: - if operator == 'search': + if operator == 'search' and not is_search: + wanted_fields += [prefix + "subject", prefix + "rendered_content"] + query = query.only(*wanted_fields) is_search = True query = build(query, operator, operand) @@ -374,8 +397,7 @@ def get_old_messages_backend(request, user_profile, if user_message_flags.get(message.id) is None: user_message_flags[message.id] = ["read", "historical"] if is_search: - search_fields[message.id] = dict([('match_subject', message.match_subject), - ('match_content', message.match_content)]) + search_fields[message.id] = get_search_fields(message, message) else: user_message_flags = dict((user_message.message_id, user_message.flags_list()) for user_message in query_result) @@ -383,8 +405,7 @@ def get_old_messages_backend(request, user_profile, message_ids.append(user_message.message_id) if is_search: search_fields[user_message.message_id] = \ - dict([('match_subject', user_message.match_subject), - ('match_content', user_message.match_content)]) + get_search_fields(user_message.message, user_message) cache_transformer = lambda row: Message.build_dict_from_raw_db_row(row, apply_markdown) id_fetcher = lambda row: row['id'] @@ -626,6 +647,5 @@ def messages_in_narrow_backend(request, user_profile, query = build(query, operator, operand) return json_success({"messages": dict((msg.message.id, - {'match_subject': msg.match_subject, - 'match_content': msg.match_content}) + get_search_fields(msg.message, msg)) for msg in query.iterator())})