From 56cb5d0b5969cd13826164fc17c1f9950e472858 Mon Sep 17 00:00:00 2001
From: Haoming <hmstudy02@gmail.com>
Date: Tue, 9 Sep 2025 14:16:30 +0800
Subject: [PATCH] yeet

---
 backend/text_processing/classic_engine.py | 21 +--------------------
 backend/text_processing/t5_engine.py      | 22 ----------------------
 backend/text_processing/umt5_engine.py    | 22 ----------------------
 3 files changed, 1 insertion(+), 64 deletions(-)

diff --git a/backend/text_processing/classic_engine.py b/backend/text_processing/classic_engine.py
index 6d4ed27c..ad038a60 100644
--- a/backend/text_processing/classic_engine.py
+++ b/backend/text_processing/classic_engine.py
@@ -81,26 +81,7 @@ class ClassicTextProcessingEngine:
         model_embeddings.token_embedding = CLIPEmbeddingForTextualInversion(model_embeddings.token_embedding, self.embeddings, textual_inversion_key=embedding_key)
 
         vocab = self.tokenizer.get_vocab()
-
-        self.comma_token = vocab.get(",</w>", None)
-
-        self.token_mults = {}
-
-        tokens_with_parens = [(k, v) for k, v in vocab.items() if "(" in k or ")" in k or "[" in k or "]" in k]
-        for text, ident in tokens_with_parens:
-            mult = 1.0
-            for c in text:
-                if c == "[":
-                    mult /= 1.1
-                if c == "]":
-                    mult *= 1.1
-                if c == "(":
-                    mult *= 1.1
-                if c == ")":
-                    mult /= 1.1
-
-            if mult != 1.0:
-                self.token_mults[ident] = mult
+        self.comma_token = vocab[",</w>"]
 
     def empty_chunk(self):
         chunk = PromptChunk()
diff --git a/backend/text_processing/t5_engine.py b/backend/text_processing/t5_engine.py
index 294b2826..633e4c0c 100644
--- a/backend/text_processing/t5_engine.py
+++ b/backend/text_processing/t5_engine.py
@@ -27,28 +27,6 @@ class T5TextProcessingEngine:
         self.id_end = 1
         self.id_pad = 0
 
-        vocab = self.tokenizer.get_vocab()
-
-        self.comma_token = vocab.get(",</w>", None)
-
-        self.token_mults = {}
-
-        tokens_with_parens = [(k, v) for k, v in vocab.items() if "(" in k or ")" in k or "[" in k or "]" in k]
-        for text, ident in tokens_with_parens:
-            mult = 1.0
-            for c in text:
-                if c == "[":
-                    mult /= 1.1
-                if c == "]":
-                    mult *= 1.1
-                if c == "(":
-                    mult *= 1.1
-                if c == ")":
-                    mult /= 1.1
-
-            if mult != 1.0:
-                self.token_mults[ident] = mult
-
     def tokenize(self, texts):
         tokenized = self.tokenizer(texts, truncation=False, add_special_tokens=False)["input_ids"]
         return tokenized
diff --git a/backend/text_processing/umt5_engine.py b/backend/text_processing/umt5_engine.py
index b8c547dd..feae7b53 100644
--- a/backend/text_processing/umt5_engine.py
+++ b/backend/text_processing/umt5_engine.py
@@ -42,28 +42,6 @@ class UMT5TextProcessingEngine:
         self.pad_to_max_length = False
         self.min_padding = None
 
-        vocab = self.tokenizer.get_vocab()
-
-        self.comma_token = vocab.get(",</w>", None)
-
-        self.token_mults = {}
-
-        tokens_with_parens = [(k, v) for k, v in vocab.items() if "(" in k or ")" in k or "[" in k or "]" in k]
-        for text, ident in tokens_with_parens:
-            mult = 1.0
-            for c in text:
-                if c == "[":
-                    mult /= 1.1
-                if c == "]":
-                    mult *= 1.1
-                if c == "(":
-                    mult *= 1.1
-                if c == ")":
-                    mult /= 1.1
-
-            if mult != 1.0:
-                self.token_mults[ident] = mult
-
     def tokenize(self, texts):
         return self.tokenizer(texts)["input_ids"]