CodeCow/stable-diffusion-webui-forge
1
0
Fork 0
mirror of https://github.com/lllyasviel/stable-diffusion-webui-forge.git synced 2026-07-21 21:01:24 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

yeet

Browse Source
This commit is contained in:
Haoming 2025-09-09 14:16:30 +08:00
parent 61e99e07bc
commit 56cb5d0b59
3 changed files with 1 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
backend/text_processing/classic_engine.py
View File

@ -81,26 +81,7 @@ class ClassicTextProcessingEngine:
model_embeddings.token_embedding = CLIPEmbeddingForTextualInversion(model_embeddings.token_embedding, self.embeddings, textual_inversion_key=embedding_key)
vocab = self.tokenizer.get_vocab()
self.comma_token = vocab.get(",</w>", None)
self.token_mults = {}
tokens_with_parens = [(k, v) for k, v in vocab.items() if "(" in k or ")" in k or "[" in k or "]" in k]
for text, ident in tokens_with_parens:
mult = 1.0
for c in text:
if c == "[":
mult /= 1.1
if c == "]":
mult *= 1.1
if c == "(":
mult *= 1.1
if c == ")":
mult /= 1.1
if mult != 1.0:
self.token_mults[ident] = mult
self.comma_token = vocab[",</w>"]
def empty_chunk(self):
chunk = PromptChunk()

22
backend/text_processing/t5_engine.py
View File

@ -27,28 +27,6 @@ class T5TextProcessingEngine:
self.id_end = 1
self.id_pad = 0
vocab = self.tokenizer.get_vocab()
self.comma_token = vocab.get(",</w>", None)
self.token_mults = {}
tokens_with_parens = [(k, v) for k, v in vocab.items() if "(" in k or ")" in k or "[" in k or "]" in k]
for text, ident in tokens_with_parens:
mult = 1.0
for c in text:
if c == "[":
mult /= 1.1
if c == "]":
mult *= 1.1
if c == "(":
mult *= 1.1
if c == ")":
mult /= 1.1
if mult != 1.0:
self.token_mults[ident] = mult
def tokenize(self, texts):
tokenized = self.tokenizer(texts, truncation=False, add_special_tokens=False)["input_ids"]
return tokenized

22
backend/text_processing/umt5_engine.py
View File

@ -42,28 +42,6 @@ class UMT5TextProcessingEngine:
self.pad_to_max_length = False
self.min_padding = None
vocab = self.tokenizer.get_vocab()
self.comma_token = vocab.get(",</w>", None)
self.token_mults = {}
tokens_with_parens = [(k, v) for k, v in vocab.items() if "(" in k or ")" in k or "[" in k or "]" in k]
for text, ident in tokens_with_parens:
mult = 1.0
for c in text:
if c == "[":
mult /= 1.1
if c == "]":
mult *= 1.1
if c == "(":
mult *= 1.1
if c == ")":
mult /= 1.1
if mult != 1.0:
self.token_mults[ident] = mult
def tokenize(self, texts):
return self.tokenizer(texts)["input_ids"]