diff --git a/zerver/tests/test_markdown.py b/zerver/tests/test_markdown.py index a393ba2926..65332ab024 100644 --- a/zerver/tests/test_markdown.py +++ b/zerver/tests/test_markdown.py @@ -9,7 +9,6 @@ import ujson from django.conf import settings from django.test import TestCase, override_settings -from zerver.lib import markdown as bugdown from zerver.lib.actions import ( do_add_alert_words, do_remove_realm_emoji, @@ -20,6 +19,23 @@ from zerver.lib.alert_words import get_alert_word_automaton from zerver.lib.create_user import create_user from zerver.lib.emoji import get_emoji_url from zerver.lib.exceptions import MarkdownRenderingException +from zerver.lib.markdown import ( + MarkdownListPreprocessor, + MentionData, + clear_state_for_testing, + content_has_emoji_syntax, + convert, + fetch_tweet_data, + get_possible_mentions_info, + get_tweet_id, + image_preview_enabled, + maybe_update_markdown_engines, + possible_avatar_emails, + possible_linked_stream_names, + topic_links, + url_embed_preview_enabled, + url_to_a, +) from zerver.lib.markdown.fenced_code import FencedBlockPreprocessor from zerver.lib.mdiff import diff_strings from zerver.lib.mention import possible_mentions, possible_user_group_mentions @@ -171,7 +187,7 @@ class FencedBlockPreprocessorTest(TestCase): self.assertEqual(lines, expected) def bugdown_convert(content: str) -> str: - return bugdown.convert( + return convert( content=content, message_realm=get_realm('zulip'), ) @@ -207,7 +223,7 @@ class BugdownMiscTest(ZulipTestCase): fred4 = make_user('fred4@example.com', 'Fred Flintstone') - lst = bugdown.get_possible_mentions_info(realm.id, {'Fred Flintstone', 'cordelia LEAR', 'Not A User'}) + lst = get_possible_mentions_info(realm.id, {'Fred Flintstone', 'cordelia LEAR', 'Not A User'}) set_of_names = set(map(lambda x: x['full_name'].lower(), lst)) self.assertEqual(set_of_names, {'fred flintstone', 'cordelia lear'}) @@ -231,7 +247,7 @@ class BugdownMiscTest(ZulipTestCase): hamlet = self.example_user('hamlet') cordelia = self.example_user('cordelia') content = '@**King Hamlet** @**Cordelia lear**' - mention_data = bugdown.MentionData(realm.id, content) + mention_data = MentionData(realm.id, content) self.assertEqual(mention_data.get_user_ids(), {hamlet.id, cordelia.id}) self.assertEqual(mention_data.get_user_by_id(hamlet.id), dict( email=hamlet.email, @@ -245,7 +261,7 @@ class BugdownMiscTest(ZulipTestCase): self.assertFalse(mention_data.message_has_wildcards()) content = '@**King Hamlet** @**Cordelia lear** @**all**' - mention_data = bugdown.MentionData(realm.id, content) + mention_data = MentionData(realm.id, content) self.assertTrue(mention_data.message_has_wildcards()) def test_invalid_katex_path(self) -> None: @@ -263,22 +279,22 @@ class MarkdownListPreprocessorTest(ZulipTestCase): return original, expected def test_basic_list(self) -> None: - preprocessor = bugdown.MarkdownListPreprocessor() + preprocessor = MarkdownListPreprocessor() original, expected = self.split_message('List without a gap\n<>* One\n* Two') self.assertEqual(preprocessor.run(original), expected) def test_list_after_quotes(self) -> None: - preprocessor = bugdown.MarkdownListPreprocessor() + preprocessor = MarkdownListPreprocessor() original, expected = self.split_message('```quote\nSomething\n```\n\nList without a gap\n<>* One\n* Two') self.assertEqual(preprocessor.run(original), expected) def test_list_in_code(self) -> None: - preprocessor = bugdown.MarkdownListPreprocessor() + preprocessor = MarkdownListPreprocessor() original, expected = self.split_message('```\nList without a gap\n* One\n* Two\n```') self.assertEqual(preprocessor.run(original), expected) def test_complex_nesting_with_different_fences(self) -> None: - preprocessor = bugdown.MarkdownListPreprocessor() + preprocessor = MarkdownListPreprocessor() msg = """```quote In quote. We should convert a list here:<> * one @@ -310,7 +326,7 @@ Outside. Should convert:<> self.assertEqual(preprocessor.run(original), expected) def test_complex_nesting_with_same_fence(self) -> None: - preprocessor = bugdown.MarkdownListPreprocessor() + preprocessor = MarkdownListPreprocessor() msg = """```quote In quote. We should convert a list here:<> * one @@ -344,7 +360,7 @@ Outside. Should convert:<> class BugdownTest(ZulipTestCase): def setUp(self) -> None: super().setUp() - bugdown.clear_state_for_testing() + clear_state_for_testing() def assertEqual(self, first: Any, second: Any, msg: str = "") -> None: if isinstance(first, str) and isinstance(second, str): @@ -422,11 +438,11 @@ class BugdownTest(ZulipTestCase): converted = bugdown_convert(msg) self.assertEqual(converted, '

Check out this file file:///Volumes/myserver/Users/Shared/pi.py

') - bugdown.clear_state_for_testing() + clear_state_for_testing() with self.settings(ENABLE_FILE_LINKS=False): realm = Realm.objects.create(string_id='file_links_test') - bugdown.maybe_update_markdown_engines(realm.id, False) - converted = bugdown.convert(msg, message_realm=realm) + maybe_update_markdown_engines(realm.id, False) + converted = convert(msg, message_realm=realm) self.assertEqual(converted, '

Check out this file file:///Volumes/myserver/Users/Shared/pi.py

') def test_inline_bitcoin(self) -> None: @@ -601,7 +617,7 @@ class BugdownTest(ZulipTestCase): @override_settings(INLINE_IMAGE_PREVIEW=False) def test_image_preview_enabled(self) -> None: - ret = bugdown.image_preview_enabled() + ret = image_preview_enabled() self.assertEqual(ret, False) settings.INLINE_IMAGE_PREVIEW = True @@ -610,23 +626,23 @@ class BugdownTest(ZulipTestCase): message = Message(sender=sender_user_profile, sending_client=get_client("test")) realm = message.get_realm() - ret = bugdown.image_preview_enabled() + ret = image_preview_enabled() self.assertEqual(ret, True) - ret = bugdown.image_preview_enabled(no_previews=True) + ret = image_preview_enabled(no_previews=True) self.assertEqual(ret, False) - ret = bugdown.image_preview_enabled(message, realm) + ret = image_preview_enabled(message, realm) self.assertEqual(ret, True) - ret = bugdown.image_preview_enabled(message) + ret = image_preview_enabled(message) self.assertEqual(ret, True) - ret = bugdown.image_preview_enabled(message, realm, - no_previews=True) + ret = image_preview_enabled(message, realm, + no_previews=True) self.assertEqual(ret, False) - ret = bugdown.image_preview_enabled(message, no_previews=True) + ret = image_preview_enabled(message, no_previews=True) self.assertEqual(ret, False) @override_settings(INLINE_URL_EMBED_PREVIEW=False) @@ -637,23 +653,23 @@ class BugdownTest(ZulipTestCase): realm.inline_url_embed_preview = True # off by default realm.save(update_fields=['inline_url_embed_preview']) - ret = bugdown.url_embed_preview_enabled() + ret = url_embed_preview_enabled() self.assertEqual(ret, False) settings.INLINE_URL_EMBED_PREVIEW = True - ret = bugdown.url_embed_preview_enabled() + ret = url_embed_preview_enabled() self.assertEqual(ret, True) - ret = bugdown.image_preview_enabled(no_previews=True) + ret = image_preview_enabled(no_previews=True) self.assertEqual(ret, False) - ret = bugdown.url_embed_preview_enabled(message, realm) + ret = url_embed_preview_enabled(message, realm) self.assertEqual(ret, True) - ret = bugdown.url_embed_preview_enabled(message) + ret = url_embed_preview_enabled(message) self.assertEqual(ret, True) - ret = bugdown.url_embed_preview_enabled(message, no_previews=True) + ret = url_embed_preview_enabled(message, no_previews=True) self.assertEqual(ret, False) def test_inline_dropbox(self) -> None: @@ -708,13 +724,13 @@ class BugdownTest(ZulipTestCase): self.assertEqual(converted, '

Test: https://developer.github.com/assets/images/hero-circuit-bg.png

\n
') def test_twitter_id_extraction(self) -> None: - self.assertEqual(bugdown.get_tweet_id('http://twitter.com/#!/VizzQuotes/status/409030735191097344'), '409030735191097344') - self.assertEqual(bugdown.get_tweet_id('http://twitter.com/VizzQuotes/status/409030735191097344'), '409030735191097344') - self.assertEqual(bugdown.get_tweet_id('http://twitter.com/VizzQuotes/statuses/409030735191097344'), '409030735191097344') - self.assertEqual(bugdown.get_tweet_id('https://twitter.com/wdaher/status/1017581858'), '1017581858') - self.assertEqual(bugdown.get_tweet_id('https://twitter.com/wdaher/status/1017581858/'), '1017581858') - self.assertEqual(bugdown.get_tweet_id('https://twitter.com/windyoona/status/410766290349879296/photo/1'), '410766290349879296') - self.assertEqual(bugdown.get_tweet_id('https://twitter.com/windyoona/status/410766290349879296/'), '410766290349879296') + self.assertEqual(get_tweet_id('http://twitter.com/#!/VizzQuotes/status/409030735191097344'), '409030735191097344') + self.assertEqual(get_tweet_id('http://twitter.com/VizzQuotes/status/409030735191097344'), '409030735191097344') + self.assertEqual(get_tweet_id('http://twitter.com/VizzQuotes/statuses/409030735191097344'), '409030735191097344') + self.assertEqual(get_tweet_id('https://twitter.com/wdaher/status/1017581858'), '1017581858') + self.assertEqual(get_tweet_id('https://twitter.com/wdaher/status/1017581858/'), '1017581858') + self.assertEqual(get_tweet_id('https://twitter.com/windyoona/status/410766290349879296/photo/1'), '410766290349879296') + self.assertEqual(get_tweet_id('https://twitter.com/windyoona/status/410766290349879296/'), '410766290349879296') def test_inline_interesting_links(self) -> None: def make_link(url: str) -> str: @@ -852,17 +868,17 @@ class BugdownTest(ZulipTestCase): def test_fetch_tweet_data_settings_validation(self) -> None: with self.settings(TEST_SUITE=False, TWITTER_CONSUMER_KEY=None): - self.assertIs(None, bugdown.fetch_tweet_data('287977969287315459')) + self.assertIs(None, fetch_tweet_data('287977969287315459')) def test_content_has_emoji(self) -> None: - self.assertFalse(bugdown.content_has_emoji_syntax('boring')) - self.assertFalse(bugdown.content_has_emoji_syntax('hello: world')) - self.assertFalse(bugdown.content_has_emoji_syntax(':foobar')) - self.assertFalse(bugdown.content_has_emoji_syntax('::: hello :::')) + self.assertFalse(content_has_emoji_syntax('boring')) + self.assertFalse(content_has_emoji_syntax('hello: world')) + self.assertFalse(content_has_emoji_syntax(':foobar')) + self.assertFalse(content_has_emoji_syntax('::: hello :::')) - self.assertTrue(bugdown.content_has_emoji_syntax('foo :whatever:')) - self.assertTrue(bugdown.content_has_emoji_syntax('\n:whatever:')) - self.assertTrue(bugdown.content_has_emoji_syntax(':smile: ::::::')) + self.assertTrue(content_has_emoji_syntax('foo :whatever:')) + self.assertTrue(content_has_emoji_syntax('\n:whatever:')) + self.assertTrue(content_has_emoji_syntax(':smile: ::::::')) def test_realm_emoji(self) -> None: def emoji_img(name: str, file_name: str, realm_id: int) -> str: @@ -873,7 +889,7 @@ class BugdownTest(ZulipTestCase): # Needs to mock an actual message because that's how bugdown obtains the realm msg = Message(sender=self.example_user('hamlet')) - converted = bugdown.convert(":green_tick:", message_realm=realm, message=msg) + converted = convert(":green_tick:", message_realm=realm, message=msg) realm_emoji = RealmEmoji.objects.filter(realm=realm, name='green_tick', deactivated=False).get() @@ -881,7 +897,7 @@ class BugdownTest(ZulipTestCase): # Deactivate realm emoji. do_remove_realm_emoji(realm, 'green_tick') - converted = bugdown.convert(":green_tick:", message_realm=realm, message=msg) + converted = convert(":green_tick:", message_realm=realm, message=msg) self.assertEqual(converted, '

:green_tick:

') def test_deactivated_realm_emoji(self) -> None: @@ -890,7 +906,7 @@ class BugdownTest(ZulipTestCase): do_remove_realm_emoji(realm, 'green_tick') msg = Message(sender=self.example_user('hamlet')) - converted = bugdown.convert(":green_tick:", message_realm=realm, message=msg) + converted = convert(":green_tick:", message_realm=realm, message=msg) self.assertEqual(converted, '

:green_tick:

') def test_unicode_emoji(self) -> None: @@ -925,23 +941,23 @@ class BugdownTest(ZulipTestCase): msg = Message(sender=self.example_user('othello')) msg.set_topic_name("https://google.com/hello-world") - converted_topic = bugdown.topic_links(realm.id, msg.topic_name()) + converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual(converted_topic, ['https://google.com/hello-world']) msg.set_topic_name("http://google.com/hello-world") - converted_topic = bugdown.topic_links(realm.id, msg.topic_name()) + converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual(converted_topic, ['http://google.com/hello-world']) msg.set_topic_name("Without scheme google.com/hello-world") - converted_topic = bugdown.topic_links(realm.id, msg.topic_name()) + converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual(converted_topic, ['https://google.com/hello-world']) msg.set_topic_name("Without scheme random.words/hello-world") - converted_topic = bugdown.topic_links(realm.id, msg.topic_name()) + converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual(converted_topic, []) msg.set_topic_name("Try out http://ftp.debian.org, https://google.com/ and https://google.in/.") - converted_topic = bugdown.topic_links(realm.id, msg.topic_name()) + converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual(converted_topic, ['http://ftp.debian.org', 'https://google.com/', 'https://google.in/']) def test_realm_patterns(self) -> None: @@ -962,14 +978,14 @@ class BugdownTest(ZulipTestCase): flush_per_request_caches() content = "We should fix #224 and #115, but not issue#124 or #1124z or [trac #15](https://trac.example.com/ticket/16) today." - converted = bugdown.convert(content, message_realm=realm, message=msg) - converted_topic = bugdown.topic_links(realm.id, msg.topic_name()) + converted = convert(content, message_realm=realm, message=msg) + converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual(converted, '

We should fix #224 and #115, but not issue#124 or #1124z or trac #15 today.

') self.assertEqual(converted_topic, ['https://trac.example.com/ticket/444']) msg.set_topic_name("#444 https://google.com") - converted_topic = bugdown.topic_links(realm.id, msg.topic_name()) + converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual(converted_topic, ['https://trac.example.com/ticket/444', 'https://google.com']) RealmFilter(realm=realm, pattern=r'#(?P[a-zA-Z]+-[0-9]+)', @@ -977,14 +993,14 @@ class BugdownTest(ZulipTestCase): msg = Message(sender=self.example_user('hamlet')) content = '#ZUL-123 was fixed and code was deployed to production, also #zul-321 was deployed to staging' - converted = bugdown.convert(content, message_realm=realm, message=msg) + converted = convert(content, message_realm=realm, message=msg) self.assertEqual(converted, '

#ZUL-123 was fixed and code was deployed to production, also #zul-321 was deployed to staging

') - def assert_conversion(content: str, convert: bool=True) -> None: - converted = bugdown.convert(content, message_realm=realm, message=msg) - converted_topic = bugdown.topic_links(realm.id, content) - if convert: + def assert_conversion(content: str, should_have_converted: bool=True) -> None: + converted = convert(content, message_realm=realm, message=msg) + converted_topic = topic_links(realm.id, content) + if should_have_converted: self.assertTrue('https://trac.example.com' in converted) self.assertEqual(len(converted_topic), 1) self.assertTrue('https://trac.example.com' in converted_topic[0]) @@ -1012,7 +1028,7 @@ class BugdownTest(ZulipTestCase): # test nested realm patterns should avoid double matching RealmFilter(realm=realm, pattern=r'hello#(?P[0-9]+)', url_format_string=r'https://trac.example.com/hello/%(id)s').save() - converted_topic = bugdown.topic_links(realm.id, 'hello#123 #234') + converted_topic = topic_links(realm.id, 'hello#123 #234') self.assertEqual(converted_topic, ['https://trac.example.com/ticket/234', 'https://trac.example.com/hello/123']) def test_maybe_update_markdown_engines(self) -> None: @@ -1023,9 +1039,10 @@ class BugdownTest(ZulipTestCase): url_format_string=url_format_string) realm_filter.save() - bugdown.realm_filter_data = {} - bugdown.maybe_update_markdown_engines(None, False) - all_filters = bugdown.realm_filter_data + import zerver.lib.markdown + zerver.lib.markdown.realm_filter_data = {} + maybe_update_markdown_engines(None, False) + all_filters = zerver.lib.markdown.realm_filter_data zulip_filters = all_filters[realm.id] self.assertEqual(len(zulip_filters), 1) self.assertEqual(zulip_filters[0], @@ -1074,7 +1091,7 @@ class BugdownTest(ZulipTestCase): url_format_string=r"https://trac.example.com/ticket/%(id)s").save() boring_msg = Message(sender=self.example_user('othello')) boring_msg.set_topic_name("no match here") - converted_boring_topic = bugdown.topic_links(realm.id, boring_msg.topic_name()) + converted_boring_topic = topic_links(realm.id, boring_msg.topic_name()) self.assertEqual(converted_boring_topic, []) def test_is_status_message(self) -> None: @@ -1848,7 +1865,7 @@ class BugdownTest(ZulipTestCase): #**garçon** #**천국** @**Ignore Person** ''' self.assertEqual( - bugdown.possible_linked_stream_names(content), + possible_linked_stream_names(content), {'test here', 'Denmark', 'garçon', '천국'}, ) @@ -1925,19 +1942,19 @@ class BugdownTest(ZulipTestCase): client = get_client("zephyr_mirror") message = Message(sending_client=client, sender=self.mit_user("sipbtest")) - converted = bugdown.convert(msg, message_realm=realm, message=message) + converted = convert(msg, message_realm=realm, message=message) self.assertEqual( converted, "

**test**

", ) msg = "* test" - converted = bugdown.convert(msg, message_realm=realm, message=message) + converted = convert(msg, message_realm=realm, message=message) self.assertEqual( converted, "

* test

", ) msg = "https://lists.debian.org/debian-ctte/2014/02/msg00173.html" - converted = bugdown.convert(msg, message_realm=realm, message=message) + converted = convert(msg, message_realm=realm, message=message) self.assertEqual( converted, '

https://lists.debian.org/debian-ctte/2014/02/msg00173.html

', @@ -1945,7 +1962,7 @@ class BugdownTest(ZulipTestCase): def test_url_to_a(self) -> None: url = 'javascript://example.com/invalidURL' - converted = bugdown.url_to_a(db_data=None, url=url, text=url) + converted = url_to_a(db_data=None, url=url, text=url) self.assertEqual( converted, 'javascript://example.com/invalidURL', @@ -1961,8 +1978,8 @@ class BugdownTest(ZulipTestCase): self.assertEqual(converted, expected_output) realm = Realm.objects.create(string_id='code_block_processor_test') - bugdown.maybe_update_markdown_engines(realm.id, True) - converted = bugdown.convert(msg, message_realm=realm, email_gateway=True) + maybe_update_markdown_engines(realm.id, True) + converted = convert(msg, message_realm=realm, email_gateway=True) expected_output = '

Hello,

\n' + \ '

I am writing this message to test something. I am writing this message to test something.

' self.assertEqual(converted, expected_output) @@ -1974,7 +1991,7 @@ class BugdownTest(ZulipTestCase): msg = "http://example.com/#settings/" self.assertEqual( - bugdown.convert(msg, message_realm=realm, message=message), + convert(msg, message_realm=realm, message=message), '

http://example.com/#settings/

', ) @@ -1985,7 +2002,7 @@ class BugdownTest(ZulipTestCase): msg = "http://zulip.testserver/#narrow/stream/999-hello" self.assertEqual( - bugdown.convert(msg, message_realm=realm, message=message), + convert(msg, message_realm=realm, message=message), '

http://zulip.testserver/#narrow/stream/999-hello

', ) @@ -1996,7 +2013,7 @@ class BugdownTest(ZulipTestCase): msg = "http://zulip.testserver/#streams/all" self.assertEqual( - bugdown.convert(msg, message_realm=realm, message=message), + convert(msg, message_realm=realm, message=message), '

http://zulip.testserver/#streams/all

', ) @@ -2007,7 +2024,7 @@ class BugdownTest(ZulipTestCase): msg = "[hello](http://zulip.testserver/#narrow/stream/999-hello)" self.assertEqual( - bugdown.convert(msg, message_realm=realm, message=message), + convert(msg, message_realm=realm, message=message), '

hello

', ) @@ -2167,7 +2184,7 @@ class BugdownAvatarTestCase(ZulipTestCase): smushing!avatar(hamlet@example.org) is allowed ''' self.assertEqual( - bugdown.possible_avatar_emails(content), + possible_avatar_emails(content), {'foo@example.com', 'bar@yo.tv', 'hamlet@example.org'}, ) @@ -2177,7 +2194,7 @@ class BugdownAvatarTestCase(ZulipTestCase): user_profile = self.example_user('hamlet') msg = f'!avatar({user_profile.email})' - converted = bugdown.convert(msg, message=message) + converted = convert(msg, message=message) values = {'email': user_profile.email, 'id': user_profile.id} self.assertEqual( converted, @@ -2189,7 +2206,7 @@ class BugdownAvatarTestCase(ZulipTestCase): email = 'fakeuser@example.com' msg = f'!avatar({email})' - converted = bugdown.convert(msg, message=message) + converted = convert(msg, message=message) self.assertEqual( converted, '

{0}

'.format(email))