test_markdown: Import markdown the normal way.

This commit is contained in:
Tim Abbott 2020-06-26 17:18:01 -07:00
parent 99d49e88f7
commit 81d587d771

View File

@ -9,7 +9,6 @@ import ujson
from django.conf import settings
from django.test import TestCase, override_settings
from zerver.lib import markdown as bugdown
from zerver.lib.actions import (
do_add_alert_words,
do_remove_realm_emoji,
@ -20,6 +19,23 @@ from zerver.lib.alert_words import get_alert_word_automaton
from zerver.lib.create_user import create_user
from zerver.lib.emoji import get_emoji_url
from zerver.lib.exceptions import MarkdownRenderingException
from zerver.lib.markdown import (
MarkdownListPreprocessor,
MentionData,
clear_state_for_testing,
content_has_emoji_syntax,
convert,
fetch_tweet_data,
get_possible_mentions_info,
get_tweet_id,
image_preview_enabled,
maybe_update_markdown_engines,
possible_avatar_emails,
possible_linked_stream_names,
topic_links,
url_embed_preview_enabled,
url_to_a,
)
from zerver.lib.markdown.fenced_code import FencedBlockPreprocessor
from zerver.lib.mdiff import diff_strings
from zerver.lib.mention import possible_mentions, possible_user_group_mentions
@ -171,7 +187,7 @@ class FencedBlockPreprocessorTest(TestCase):
self.assertEqual(lines, expected)
def bugdown_convert(content: str) -> str:
return bugdown.convert(
return convert(
content=content,
message_realm=get_realm('zulip'),
)
@ -207,7 +223,7 @@ class BugdownMiscTest(ZulipTestCase):
fred4 = make_user('fred4@example.com', 'Fred Flintstone')
lst = bugdown.get_possible_mentions_info(realm.id, {'Fred Flintstone', 'cordelia LEAR', 'Not A User'})
lst = get_possible_mentions_info(realm.id, {'Fred Flintstone', 'cordelia LEAR', 'Not A User'})
set_of_names = set(map(lambda x: x['full_name'].lower(), lst))
self.assertEqual(set_of_names, {'fred flintstone', 'cordelia lear'})
@ -231,7 +247,7 @@ class BugdownMiscTest(ZulipTestCase):
hamlet = self.example_user('hamlet')
cordelia = self.example_user('cordelia')
content = '@**King Hamlet** @**Cordelia lear**'
mention_data = bugdown.MentionData(realm.id, content)
mention_data = MentionData(realm.id, content)
self.assertEqual(mention_data.get_user_ids(), {hamlet.id, cordelia.id})
self.assertEqual(mention_data.get_user_by_id(hamlet.id), dict(
email=hamlet.email,
@ -245,7 +261,7 @@ class BugdownMiscTest(ZulipTestCase):
self.assertFalse(mention_data.message_has_wildcards())
content = '@**King Hamlet** @**Cordelia lear** @**all**'
mention_data = bugdown.MentionData(realm.id, content)
mention_data = MentionData(realm.id, content)
self.assertTrue(mention_data.message_has_wildcards())
def test_invalid_katex_path(self) -> None:
@ -263,22 +279,22 @@ class MarkdownListPreprocessorTest(ZulipTestCase):
return original, expected
def test_basic_list(self) -> None:
preprocessor = bugdown.MarkdownListPreprocessor()
preprocessor = MarkdownListPreprocessor()
original, expected = self.split_message('List without a gap\n<>* One\n* Two')
self.assertEqual(preprocessor.run(original), expected)
def test_list_after_quotes(self) -> None:
preprocessor = bugdown.MarkdownListPreprocessor()
preprocessor = MarkdownListPreprocessor()
original, expected = self.split_message('```quote\nSomething\n```\n\nList without a gap\n<>* One\n* Two')
self.assertEqual(preprocessor.run(original), expected)
def test_list_in_code(self) -> None:
preprocessor = bugdown.MarkdownListPreprocessor()
preprocessor = MarkdownListPreprocessor()
original, expected = self.split_message('```\nList without a gap\n* One\n* Two\n```')
self.assertEqual(preprocessor.run(original), expected)
def test_complex_nesting_with_different_fences(self) -> None:
preprocessor = bugdown.MarkdownListPreprocessor()
preprocessor = MarkdownListPreprocessor()
msg = """```quote
In quote. We should convert a list here:<>
* one
@ -310,7 +326,7 @@ Outside. Should convert:<>
self.assertEqual(preprocessor.run(original), expected)
def test_complex_nesting_with_same_fence(self) -> None:
preprocessor = bugdown.MarkdownListPreprocessor()
preprocessor = MarkdownListPreprocessor()
msg = """```quote
In quote. We should convert a list here:<>
* one
@ -344,7 +360,7 @@ Outside. Should convert:<>
class BugdownTest(ZulipTestCase):
def setUp(self) -> None:
super().setUp()
bugdown.clear_state_for_testing()
clear_state_for_testing()
def assertEqual(self, first: Any, second: Any, msg: str = "") -> None:
if isinstance(first, str) and isinstance(second, str):
@ -422,11 +438,11 @@ class BugdownTest(ZulipTestCase):
converted = bugdown_convert(msg)
self.assertEqual(converted, '<p>Check out this file <a href="file:///Volumes/myserver/Users/Shared/pi.py">file:///Volumes/myserver/Users/Shared/pi.py</a></p>')
bugdown.clear_state_for_testing()
clear_state_for_testing()
with self.settings(ENABLE_FILE_LINKS=False):
realm = Realm.objects.create(string_id='file_links_test')
bugdown.maybe_update_markdown_engines(realm.id, False)
converted = bugdown.convert(msg, message_realm=realm)
maybe_update_markdown_engines(realm.id, False)
converted = convert(msg, message_realm=realm)
self.assertEqual(converted, '<p>Check out this file file:///Volumes/myserver/Users/Shared/pi.py</p>')
def test_inline_bitcoin(self) -> None:
@ -601,7 +617,7 @@ class BugdownTest(ZulipTestCase):
@override_settings(INLINE_IMAGE_PREVIEW=False)
def test_image_preview_enabled(self) -> None:
ret = bugdown.image_preview_enabled()
ret = image_preview_enabled()
self.assertEqual(ret, False)
settings.INLINE_IMAGE_PREVIEW = True
@ -610,23 +626,23 @@ class BugdownTest(ZulipTestCase):
message = Message(sender=sender_user_profile, sending_client=get_client("test"))
realm = message.get_realm()
ret = bugdown.image_preview_enabled()
ret = image_preview_enabled()
self.assertEqual(ret, True)
ret = bugdown.image_preview_enabled(no_previews=True)
ret = image_preview_enabled(no_previews=True)
self.assertEqual(ret, False)
ret = bugdown.image_preview_enabled(message, realm)
ret = image_preview_enabled(message, realm)
self.assertEqual(ret, True)
ret = bugdown.image_preview_enabled(message)
ret = image_preview_enabled(message)
self.assertEqual(ret, True)
ret = bugdown.image_preview_enabled(message, realm,
no_previews=True)
ret = image_preview_enabled(message, realm,
no_previews=True)
self.assertEqual(ret, False)
ret = bugdown.image_preview_enabled(message, no_previews=True)
ret = image_preview_enabled(message, no_previews=True)
self.assertEqual(ret, False)
@override_settings(INLINE_URL_EMBED_PREVIEW=False)
@ -637,23 +653,23 @@ class BugdownTest(ZulipTestCase):
realm.inline_url_embed_preview = True # off by default
realm.save(update_fields=['inline_url_embed_preview'])
ret = bugdown.url_embed_preview_enabled()
ret = url_embed_preview_enabled()
self.assertEqual(ret, False)
settings.INLINE_URL_EMBED_PREVIEW = True
ret = bugdown.url_embed_preview_enabled()
ret = url_embed_preview_enabled()
self.assertEqual(ret, True)
ret = bugdown.image_preview_enabled(no_previews=True)
ret = image_preview_enabled(no_previews=True)
self.assertEqual(ret, False)
ret = bugdown.url_embed_preview_enabled(message, realm)
ret = url_embed_preview_enabled(message, realm)
self.assertEqual(ret, True)
ret = bugdown.url_embed_preview_enabled(message)
ret = url_embed_preview_enabled(message)
self.assertEqual(ret, True)
ret = bugdown.url_embed_preview_enabled(message, no_previews=True)
ret = url_embed_preview_enabled(message, no_previews=True)
self.assertEqual(ret, False)
def test_inline_dropbox(self) -> None:
@ -708,13 +724,13 @@ class BugdownTest(ZulipTestCase):
self.assertEqual(converted, '<p>Test: <a href="https://developer.github.com/assets/images/hero-circuit-bg.png">https://developer.github.com/assets/images/hero-circuit-bg.png</a></p>\n<div class="message_inline_image"><a href="https://developer.github.com/assets/images/hero-circuit-bg.png"><img data-src-fullsize="/thumbnail?url=https%3A%2F%2Fdeveloper.github.com%2Fassets%2Fimages%2Fhero-circuit-bg.png&amp;size=full" src="/thumbnail?url=https%3A%2F%2Fdeveloper.github.com%2Fassets%2Fimages%2Fhero-circuit-bg.png&amp;size=thumbnail"></a></div>')
def test_twitter_id_extraction(self) -> None:
self.assertEqual(bugdown.get_tweet_id('http://twitter.com/#!/VizzQuotes/status/409030735191097344'), '409030735191097344')
self.assertEqual(bugdown.get_tweet_id('http://twitter.com/VizzQuotes/status/409030735191097344'), '409030735191097344')
self.assertEqual(bugdown.get_tweet_id('http://twitter.com/VizzQuotes/statuses/409030735191097344'), '409030735191097344')
self.assertEqual(bugdown.get_tweet_id('https://twitter.com/wdaher/status/1017581858'), '1017581858')
self.assertEqual(bugdown.get_tweet_id('https://twitter.com/wdaher/status/1017581858/'), '1017581858')
self.assertEqual(bugdown.get_tweet_id('https://twitter.com/windyoona/status/410766290349879296/photo/1'), '410766290349879296')
self.assertEqual(bugdown.get_tweet_id('https://twitter.com/windyoona/status/410766290349879296/'), '410766290349879296')
self.assertEqual(get_tweet_id('http://twitter.com/#!/VizzQuotes/status/409030735191097344'), '409030735191097344')
self.assertEqual(get_tweet_id('http://twitter.com/VizzQuotes/status/409030735191097344'), '409030735191097344')
self.assertEqual(get_tweet_id('http://twitter.com/VizzQuotes/statuses/409030735191097344'), '409030735191097344')
self.assertEqual(get_tweet_id('https://twitter.com/wdaher/status/1017581858'), '1017581858')
self.assertEqual(get_tweet_id('https://twitter.com/wdaher/status/1017581858/'), '1017581858')
self.assertEqual(get_tweet_id('https://twitter.com/windyoona/status/410766290349879296/photo/1'), '410766290349879296')
self.assertEqual(get_tweet_id('https://twitter.com/windyoona/status/410766290349879296/'), '410766290349879296')
def test_inline_interesting_links(self) -> None:
def make_link(url: str) -> str:
@ -852,17 +868,17 @@ class BugdownTest(ZulipTestCase):
def test_fetch_tweet_data_settings_validation(self) -> None:
with self.settings(TEST_SUITE=False, TWITTER_CONSUMER_KEY=None):
self.assertIs(None, bugdown.fetch_tweet_data('287977969287315459'))
self.assertIs(None, fetch_tweet_data('287977969287315459'))
def test_content_has_emoji(self) -> None:
self.assertFalse(bugdown.content_has_emoji_syntax('boring'))
self.assertFalse(bugdown.content_has_emoji_syntax('hello: world'))
self.assertFalse(bugdown.content_has_emoji_syntax(':foobar'))
self.assertFalse(bugdown.content_has_emoji_syntax('::: hello :::'))
self.assertFalse(content_has_emoji_syntax('boring'))
self.assertFalse(content_has_emoji_syntax('hello: world'))
self.assertFalse(content_has_emoji_syntax(':foobar'))
self.assertFalse(content_has_emoji_syntax('::: hello :::'))
self.assertTrue(bugdown.content_has_emoji_syntax('foo :whatever:'))
self.assertTrue(bugdown.content_has_emoji_syntax('\n:whatever:'))
self.assertTrue(bugdown.content_has_emoji_syntax(':smile: ::::::'))
self.assertTrue(content_has_emoji_syntax('foo :whatever:'))
self.assertTrue(content_has_emoji_syntax('\n:whatever:'))
self.assertTrue(content_has_emoji_syntax(':smile: ::::::'))
def test_realm_emoji(self) -> None:
def emoji_img(name: str, file_name: str, realm_id: int) -> str:
@ -873,7 +889,7 @@ class BugdownTest(ZulipTestCase):
# Needs to mock an actual message because that's how bugdown obtains the realm
msg = Message(sender=self.example_user('hamlet'))
converted = bugdown.convert(":green_tick:", message_realm=realm, message=msg)
converted = convert(":green_tick:", message_realm=realm, message=msg)
realm_emoji = RealmEmoji.objects.filter(realm=realm,
name='green_tick',
deactivated=False).get()
@ -881,7 +897,7 @@ class BugdownTest(ZulipTestCase):
# Deactivate realm emoji.
do_remove_realm_emoji(realm, 'green_tick')
converted = bugdown.convert(":green_tick:", message_realm=realm, message=msg)
converted = convert(":green_tick:", message_realm=realm, message=msg)
self.assertEqual(converted, '<p>:green_tick:</p>')
def test_deactivated_realm_emoji(self) -> None:
@ -890,7 +906,7 @@ class BugdownTest(ZulipTestCase):
do_remove_realm_emoji(realm, 'green_tick')
msg = Message(sender=self.example_user('hamlet'))
converted = bugdown.convert(":green_tick:", message_realm=realm, message=msg)
converted = convert(":green_tick:", message_realm=realm, message=msg)
self.assertEqual(converted, '<p>:green_tick:</p>')
def test_unicode_emoji(self) -> None:
@ -925,23 +941,23 @@ class BugdownTest(ZulipTestCase):
msg = Message(sender=self.example_user('othello'))
msg.set_topic_name("https://google.com/hello-world")
converted_topic = bugdown.topic_links(realm.id, msg.topic_name())
converted_topic = topic_links(realm.id, msg.topic_name())
self.assertEqual(converted_topic, ['https://google.com/hello-world'])
msg.set_topic_name("http://google.com/hello-world")
converted_topic = bugdown.topic_links(realm.id, msg.topic_name())
converted_topic = topic_links(realm.id, msg.topic_name())
self.assertEqual(converted_topic, ['http://google.com/hello-world'])
msg.set_topic_name("Without scheme google.com/hello-world")
converted_topic = bugdown.topic_links(realm.id, msg.topic_name())
converted_topic = topic_links(realm.id, msg.topic_name())
self.assertEqual(converted_topic, ['https://google.com/hello-world'])
msg.set_topic_name("Without scheme random.words/hello-world")
converted_topic = bugdown.topic_links(realm.id, msg.topic_name())
converted_topic = topic_links(realm.id, msg.topic_name())
self.assertEqual(converted_topic, [])
msg.set_topic_name("Try out http://ftp.debian.org, https://google.com/ and https://google.in/.")
converted_topic = bugdown.topic_links(realm.id, msg.topic_name())
converted_topic = topic_links(realm.id, msg.topic_name())
self.assertEqual(converted_topic, ['http://ftp.debian.org', 'https://google.com/', 'https://google.in/'])
def test_realm_patterns(self) -> None:
@ -962,14 +978,14 @@ class BugdownTest(ZulipTestCase):
flush_per_request_caches()
content = "We should fix #224 and #115, but not issue#124 or #1124z or [trac #15](https://trac.example.com/ticket/16) today."
converted = bugdown.convert(content, message_realm=realm, message=msg)
converted_topic = bugdown.topic_links(realm.id, msg.topic_name())
converted = convert(content, message_realm=realm, message=msg)
converted_topic = topic_links(realm.id, msg.topic_name())
self.assertEqual(converted, '<p>We should fix <a href="https://trac.example.com/ticket/224">#224</a> and <a href="https://trac.example.com/ticket/115">#115</a>, but not issue#124 or #1124z or <a href="https://trac.example.com/ticket/16">trac #15</a> today.</p>')
self.assertEqual(converted_topic, ['https://trac.example.com/ticket/444'])
msg.set_topic_name("#444 https://google.com")
converted_topic = bugdown.topic_links(realm.id, msg.topic_name())
converted_topic = topic_links(realm.id, msg.topic_name())
self.assertEqual(converted_topic, ['https://trac.example.com/ticket/444', 'https://google.com'])
RealmFilter(realm=realm, pattern=r'#(?P<id>[a-zA-Z]+-[0-9]+)',
@ -977,14 +993,14 @@ class BugdownTest(ZulipTestCase):
msg = Message(sender=self.example_user('hamlet'))
content = '#ZUL-123 was fixed and code was deployed to production, also #zul-321 was deployed to staging'
converted = bugdown.convert(content, message_realm=realm, message=msg)
converted = convert(content, message_realm=realm, message=msg)
self.assertEqual(converted, '<p><a href="https://trac.example.com/ticket/ZUL-123">#ZUL-123</a> was fixed and code was deployed to production, also <a href="https://trac.example.com/ticket/zul-321">#zul-321</a> was deployed to staging</p>')
def assert_conversion(content: str, convert: bool=True) -> None:
converted = bugdown.convert(content, message_realm=realm, message=msg)
converted_topic = bugdown.topic_links(realm.id, content)
if convert:
def assert_conversion(content: str, should_have_converted: bool=True) -> None:
converted = convert(content, message_realm=realm, message=msg)
converted_topic = topic_links(realm.id, content)
if should_have_converted:
self.assertTrue('https://trac.example.com' in converted)
self.assertEqual(len(converted_topic), 1)
self.assertTrue('https://trac.example.com' in converted_topic[0])
@ -1012,7 +1028,7 @@ class BugdownTest(ZulipTestCase):
# test nested realm patterns should avoid double matching
RealmFilter(realm=realm, pattern=r'hello#(?P<id>[0-9]+)',
url_format_string=r'https://trac.example.com/hello/%(id)s').save()
converted_topic = bugdown.topic_links(realm.id, 'hello#123 #234')
converted_topic = topic_links(realm.id, 'hello#123 #234')
self.assertEqual(converted_topic, ['https://trac.example.com/ticket/234', 'https://trac.example.com/hello/123'])
def test_maybe_update_markdown_engines(self) -> None:
@ -1023,9 +1039,10 @@ class BugdownTest(ZulipTestCase):
url_format_string=url_format_string)
realm_filter.save()
bugdown.realm_filter_data = {}
bugdown.maybe_update_markdown_engines(None, False)
all_filters = bugdown.realm_filter_data
import zerver.lib.markdown
zerver.lib.markdown.realm_filter_data = {}
maybe_update_markdown_engines(None, False)
all_filters = zerver.lib.markdown.realm_filter_data
zulip_filters = all_filters[realm.id]
self.assertEqual(len(zulip_filters), 1)
self.assertEqual(zulip_filters[0],
@ -1074,7 +1091,7 @@ class BugdownTest(ZulipTestCase):
url_format_string=r"https://trac.example.com/ticket/%(id)s").save()
boring_msg = Message(sender=self.example_user('othello'))
boring_msg.set_topic_name("no match here")
converted_boring_topic = bugdown.topic_links(realm.id, boring_msg.topic_name())
converted_boring_topic = topic_links(realm.id, boring_msg.topic_name())
self.assertEqual(converted_boring_topic, [])
def test_is_status_message(self) -> None:
@ -1848,7 +1865,7 @@ class BugdownTest(ZulipTestCase):
#**garçon** #**천국** @**Ignore Person**
'''
self.assertEqual(
bugdown.possible_linked_stream_names(content),
possible_linked_stream_names(content),
{'test here', 'Denmark', 'garçon', '천국'},
)
@ -1925,19 +1942,19 @@ class BugdownTest(ZulipTestCase):
client = get_client("zephyr_mirror")
message = Message(sending_client=client,
sender=self.mit_user("sipbtest"))
converted = bugdown.convert(msg, message_realm=realm, message=message)
converted = convert(msg, message_realm=realm, message=message)
self.assertEqual(
converted,
"<p>**test**</p>",
)
msg = "* test"
converted = bugdown.convert(msg, message_realm=realm, message=message)
converted = convert(msg, message_realm=realm, message=message)
self.assertEqual(
converted,
"<p>* test</p>",
)
msg = "https://lists.debian.org/debian-ctte/2014/02/msg00173.html"
converted = bugdown.convert(msg, message_realm=realm, message=message)
converted = convert(msg, message_realm=realm, message=message)
self.assertEqual(
converted,
'<p><a href="https://lists.debian.org/debian-ctte/2014/02/msg00173.html">https://lists.debian.org/debian-ctte/2014/02/msg00173.html</a></p>',
@ -1945,7 +1962,7 @@ class BugdownTest(ZulipTestCase):
def test_url_to_a(self) -> None:
url = 'javascript://example.com/invalidURL'
converted = bugdown.url_to_a(db_data=None, url=url, text=url)
converted = url_to_a(db_data=None, url=url, text=url)
self.assertEqual(
converted,
'javascript://example.com/invalidURL',
@ -1961,8 +1978,8 @@ class BugdownTest(ZulipTestCase):
self.assertEqual(converted, expected_output)
realm = Realm.objects.create(string_id='code_block_processor_test')
bugdown.maybe_update_markdown_engines(realm.id, True)
converted = bugdown.convert(msg, message_realm=realm, email_gateway=True)
maybe_update_markdown_engines(realm.id, True)
converted = convert(msg, message_realm=realm, email_gateway=True)
expected_output = '<p>Hello,</p>\n' + \
'<p>I am writing this message to test something. I am writing this message to test something.</p>'
self.assertEqual(converted, expected_output)
@ -1974,7 +1991,7 @@ class BugdownTest(ZulipTestCase):
msg = "http://example.com/#settings/"
self.assertEqual(
bugdown.convert(msg, message_realm=realm, message=message),
convert(msg, message_realm=realm, message=message),
'<p><a href="http://example.com/#settings/">http://example.com/#settings/</a></p>',
)
@ -1985,7 +2002,7 @@ class BugdownTest(ZulipTestCase):
msg = "http://zulip.testserver/#narrow/stream/999-hello"
self.assertEqual(
bugdown.convert(msg, message_realm=realm, message=message),
convert(msg, message_realm=realm, message=message),
'<p><a href="#narrow/stream/999-hello">http://zulip.testserver/#narrow/stream/999-hello</a></p>',
)
@ -1996,7 +2013,7 @@ class BugdownTest(ZulipTestCase):
msg = "http://zulip.testserver/#streams/all"
self.assertEqual(
bugdown.convert(msg, message_realm=realm, message=message),
convert(msg, message_realm=realm, message=message),
'<p><a href="#streams/all">http://zulip.testserver/#streams/all</a></p>',
)
@ -2007,7 +2024,7 @@ class BugdownTest(ZulipTestCase):
msg = "[hello](http://zulip.testserver/#narrow/stream/999-hello)"
self.assertEqual(
bugdown.convert(msg, message_realm=realm, message=message),
convert(msg, message_realm=realm, message=message),
'<p><a href="#narrow/stream/999-hello">hello</a></p>',
)
@ -2167,7 +2184,7 @@ class BugdownAvatarTestCase(ZulipTestCase):
smushing!avatar(hamlet@example.org) is allowed
'''
self.assertEqual(
bugdown.possible_avatar_emails(content),
possible_avatar_emails(content),
{'foo@example.com', 'bar@yo.tv', 'hamlet@example.org'},
)
@ -2177,7 +2194,7 @@ class BugdownAvatarTestCase(ZulipTestCase):
user_profile = self.example_user('hamlet')
msg = f'!avatar({user_profile.email})'
converted = bugdown.convert(msg, message=message)
converted = convert(msg, message=message)
values = {'email': user_profile.email, 'id': user_profile.id}
self.assertEqual(
converted,
@ -2189,7 +2206,7 @@ class BugdownAvatarTestCase(ZulipTestCase):
email = 'fakeuser@example.com'
msg = f'!avatar({email})'
converted = bugdown.convert(msg, message=message)
converted = convert(msg, message=message)
self.assertEqual(
converted,
'<p><img alt="{0}" class="message_body_gravatar" src="/avatar/{0}?s=30" title="{0}"></p>'.format(email))