From 100c6889dc0725199bc49538216dee6a8c91e014 Mon Sep 17 00:00:00 2001
From: Mirei <42766108+mahdibland@users.noreply.github.com>
Date: Sat, 1 Oct 2022 21:03:10 +0330
Subject: [PATCH] Update list_merge.py

---
 utils/list_merge.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/utils/list_merge.py b/utils/list_merge.py
index 2e4c3a633..31d4f59e8 100644
--- a/utils/list_merge.py
+++ b/utils/list_merge.py
@@ -57,6 +57,12 @@ class sub_merge():
                 file.write('Url Subscription could not be parsed')
                 file.close()
                 print(f'Writing error of {remarks} to {ids:0>2d}.txt\n')
+                
+        print('Remove Duplicate Lines ...\n')
+        print('Before Count: %s' % str(content_list.__len__()))
+        content_list = list(set(content_list))
+        print('After Count: %s' % str(content_list.__len__()))
+        print('Remove Duplicate Lines Completed\n')
 
         print('Merging nodes...\n')
         content_raw = ''.join(content_list) # https://python3-cookbook.readthedocs.io/zh_CN/latest/c02/p14_combine_and_concatenate_strings.html