formatting

README.md

@@ -123,7 +123,6 @@ The documentation was moved from this README over to the project's [wiki](https:
 - LDSR - https://github.com/Hafiidz/latent-diffusion
 - Ideas for optimizations - https://github.com/basujindal/stable-diffusion
 - Doggettx - Cross Attention layer optimization - https://github.com/Doggettx/stable-diffusion, original idea for prompt editing.
-- InvokeAI, lstein - Cross Attention layer optimization - https://github.com/invoke-ai/InvokeAI (originally http://github.com/lstein/stable-diffusion)
 - Rinon Gal - Textual Inversion - https://github.com/rinongal/textual_inversion (we're not using his code, but we are using his ideas).
 - Idea for SD upscale - https://github.com/jquesnelle/txt2imghd
 - Noise generation for outpainting mk2 - https://github.com/parlance-zz/g-diffuser-bot

modules/hypernetworks/hypernetwork.py

@@ -42,7 +42,7 @@ class Hypernetwork:
     filename = None
     name = None
 
-    def __init__(self, name=None, enable_sizes=None):
+    def __init__(self, name=None):
         self.filename = None
         self.name = name
         self.layers = {}
@@ -50,7 +50,7 @@ class Hypernetwork:
         self.sd_checkpoint = None
         self.sd_checkpoint_name = None
 
-        for size in enable_sizes or []:
+        for size in [320, 640, 768, 1280]:
             self.layers[size] = (HypernetworkModule(size), HypernetworkModule(size))
 
     def weights(self):
@@ -175,7 +175,6 @@ def train_hypernetwork(hypernetwork_name, learn_rate, data_root, log_directory,
     filename = os.path.join(shared.cmd_opts.hypernetwork_dir, f'{hypernetwork_name}.pt')
 
     log_directory = os.path.join(log_directory, datetime.datetime.now().strftime("%Y-%m-%d"), hypernetwork_name)
-    unload = shared.opts.unload_models_when_training
 
     if save_hypernetwork_every > 0:
         hypernetwork_dir = os.path.join(log_directory, "hypernetworks")
@@ -189,13 +188,11 @@ def train_hypernetwork(hypernetwork_name, learn_rate, data_root, log_directory,
     else:
         images_dir = None
 
+    cond_model = shared.sd_model.cond_stage_model
+
     shared.state.textinfo = f"Preparing dataset from {html.escape(data_root)}..."
     with torch.autocast("cuda"):
-        ds = modules.textual_inversion.dataset.PersonalizedBase(data_root=data_root, width=512, height=512, repeats=1, placeholder_token=hypernetwork_name, model=shared.sd_model, device=devices.device, template_file=template_file, include_cond=True)
-
-    if unload:
-        shared.sd_model.cond_stage_model.to(devices.cpu)
-        shared.sd_model.first_stage_model.to(devices.cpu)
+        ds = modules.textual_inversion.dataset.PersonalizedBase(data_root=data_root, width=512, height=512, repeats=1, placeholder_token=hypernetwork_name, model=shared.sd_model, device=devices.device, template_file=template_file)
 
     hypernetwork = shared.loaded_hypernetwork
     weights = hypernetwork.weights()
@@ -214,7 +211,7 @@ def train_hypernetwork(hypernetwork_name, learn_rate, data_root, log_directory,
         return hypernetwork, filename
 
     pbar = tqdm.tqdm(enumerate(ds), total=steps - ititial_step)
-    for i, (x, text, cond) in pbar:
+    for i, (x, text) in pbar:
         hypernetwork.step = i + ititial_step
 
         if hypernetwork.step > steps:
@@ -224,11 +221,11 @@ def train_hypernetwork(hypernetwork_name, learn_rate, data_root, log_directory,
             break
 
         with torch.autocast("cuda"):
-            cond = cond.to(devices.device)
+            c = cond_model([text])
+
             x = x.to(devices.device)
-            loss = shared.sd_model(x.unsqueeze(0), cond)[0]
+            loss = shared.sd_model(x.unsqueeze(0), c)[0]
             del x
-            del cond
 
             losses[hypernetwork.step % losses.shape[0]] = loss.item()
 
@@ -247,10 +244,6 @@ def train_hypernetwork(hypernetwork_name, learn_rate, data_root, log_directory,
 
             preview_text = text if preview_image_prompt == "" else preview_image_prompt
 
-            optimizer.zero_grad()
-            shared.sd_model.cond_stage_model.to(devices.device)
-            shared.sd_model.first_stage_model.to(devices.device)
-
             p = processing.StableDiffusionProcessingTxt2Img(
                 sd_model=shared.sd_model,
                 prompt=preview_text,
@@ -262,10 +255,6 @@ def train_hypernetwork(hypernetwork_name, learn_rate, data_root, log_directory,
             processed = processing.process_images(p)
             image = processed.images[0]
 
-            if unload:
-                shared.sd_model.cond_stage_model.to(devices.cpu)
-                shared.sd_model.first_stage_model.to(devices.cpu)
-
             shared.state.current_image = image
             image.save(last_saved_image)
 

modules/hypernetworks/ui.py

@@ -5,15 +5,15 @@ import gradio as gr
 
 import modules.textual_inversion.textual_inversion
 import modules.textual_inversion.preprocess
-from modules import sd_hijack, shared, devices
+from modules import sd_hijack, shared
 from modules.hypernetworks import hypernetwork
 
 
-def create_hypernetwork(name, enable_sizes):
+def create_hypernetwork(name):
     fn = os.path.join(shared.cmd_opts.hypernetwork_dir, f"{name}.pt")
     assert not os.path.exists(fn), f"file {fn} already exists"
 
-    hypernet = modules.hypernetworks.hypernetwork.Hypernetwork(name=name, enable_sizes=[int(x) for x in enable_sizes])
+    hypernet = modules.hypernetworks.hypernetwork.Hypernetwork(name=name)
     hypernet.save(fn)
 
     shared.reload_hypernetworks()
@@ -25,8 +25,6 @@ def train_hypernetwork(*args):
 
     initial_hypernetwork = shared.loaded_hypernetwork
 
-    assert not shared.cmd_opts.lowvram and not shared.cmd_opts.medvram, 'Training models with lowvram or medvram is not possible'
-
     try:
         sd_hijack.undo_optimizations()
 
@@ -41,7 +39,5 @@ Hypernetwork saved to {html.escape(filename)}
         raise
     finally:
         shared.loaded_hypernetwork = initial_hypernetwork
-        shared.sd_model.cond_stage_model.to(devices.device)
-        shared.sd_model.first_stage_model.to(devices.device)
         sd_hijack.apply_optimizations()
 

modules/sd_hijack.py

@@ -10,7 +10,6 @@ from torch.nn.functional import silu
 import modules.textual_inversion.textual_inversion
 from modules import prompt_parser, devices, sd_hijack_optimizations, shared
 from modules.shared import opts, device, cmd_opts
-from modules.sd_hijack_optimizations import invokeAI_mps_available
 
 import ldm.modules.attention
 import ldm.modules.diffusionmodules.model
@@ -31,16 +30,8 @@ def apply_optimizations():
     elif cmd_opts.opt_split_attention_v1:
         print("Applying v1 cross attention optimization.")
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.split_cross_attention_forward_v1
-    elif not cmd_opts.disable_opt_split_attention and (cmd_opts.opt_split_attention_invokeai or not torch.cuda.is_available()):
-        if not invokeAI_mps_available and shared.device.type == 'mps':
-            print("The InvokeAI cross attention optimization for MPS requires the psutil package which is not installed.")
-            print("Applying v1 cross attention optimization.")
-            ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.split_cross_attention_forward_v1
-        else:
-            print("Applying cross attention optimization (InvokeAI).")
-            ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.split_cross_attention_forward_invokeAI
     elif not cmd_opts.disable_opt_split_attention and (cmd_opts.opt_split_attention or torch.cuda.is_available()):
-        print("Applying cross attention optimization (Doggettx).")
+        print("Applying cross attention optimization.")
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.split_cross_attention_forward
         ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.cross_attention_attnblock_forward
 

modules/sd_hijack_optimizations.py

@@ -1,7 +1,6 @@
 import math
 import sys
 import traceback
-import importlib
 
 import torch
 from torch import einsum
@@ -117,102 +116,6 @@ def split_cross_attention_forward(self, x, context=None, mask=None):
 
     return self.to_out(r2)
 
-
-def check_for_psutil():
-    try:
-        spec = importlib.util.find_spec('psutil')
-        return spec is not None
-    except ModuleNotFoundError:
-        return False
-
-invokeAI_mps_available = check_for_psutil()
-
-# -- Taken from https://github.com/invoke-ai/InvokeAI --
-if invokeAI_mps_available:
-    import psutil
-    mem_total_gb = psutil.virtual_memory().total // (1 << 30)
-
-def einsum_op_compvis(q, k, v):
-    s = einsum('b i d, b j d -> b i j', q, k)
-    s = s.softmax(dim=-1, dtype=s.dtype)
-    return einsum('b i j, b j d -> b i d', s, v)
-
-def einsum_op_slice_0(q, k, v, slice_size):
-    r = torch.zeros(q.shape[0], q.shape[1], v.shape[2], device=q.device, dtype=q.dtype)
-    for i in range(0, q.shape[0], slice_size):
-        end = i + slice_size
-        r[i:end] = einsum_op_compvis(q[i:end], k[i:end], v[i:end])
-    return r
-
-def einsum_op_slice_1(q, k, v, slice_size):
-    r = torch.zeros(q.shape[0], q.shape[1], v.shape[2], device=q.device, dtype=q.dtype)
-    for i in range(0, q.shape[1], slice_size):
-        end = i + slice_size
-        r[:, i:end] = einsum_op_compvis(q[:, i:end], k, v)
-    return r
-
-def einsum_op_mps_v1(q, k, v):
-    if q.shape[1] <= 4096: # (512x512) max q.shape[1]: 4096
-        return einsum_op_compvis(q, k, v)
-    else:
-        slice_size = math.floor(2**30 / (q.shape[0] * q.shape[1]))
-        return einsum_op_slice_1(q, k, v, slice_size)
-
-def einsum_op_mps_v2(q, k, v):
-    if mem_total_gb > 8 and q.shape[1] <= 4096:
-        return einsum_op_compvis(q, k, v)
-    else:
-        return einsum_op_slice_0(q, k, v, 1)
-
-def einsum_op_tensor_mem(q, k, v, max_tensor_mb):
-    size_mb = q.shape[0] * q.shape[1] * k.shape[1] * q.element_size() // (1 << 20)
-    if size_mb <= max_tensor_mb:
-        return einsum_op_compvis(q, k, v)
-    div = 1 << int((size_mb - 1) / max_tensor_mb).bit_length()
-    if div <= q.shape[0]:
-        return einsum_op_slice_0(q, k, v, q.shape[0] // div)
-    return einsum_op_slice_1(q, k, v, max(q.shape[1] // div, 1))
-
-def einsum_op_cuda(q, k, v):
-    stats = torch.cuda.memory_stats(q.device)
-    mem_active = stats['active_bytes.all.current']
-    mem_reserved = stats['reserved_bytes.all.current']
-    mem_free_cuda, _ = torch.cuda.mem_get_info(q.device)
-    mem_free_torch = mem_reserved - mem_active
-    mem_free_total = mem_free_cuda + mem_free_torch
-    # Divide factor of safety as there's copying and fragmentation
-    return self.einsum_op_tensor_mem(q, k, v, mem_free_total / 3.3 / (1 << 20))
-
-def einsum_op(q, k, v):
-    if q.device.type == 'cuda':
-        return einsum_op_cuda(q, k, v)
-
-    if q.device.type == 'mps':
-        if mem_total_gb >= 32:
-            return einsum_op_mps_v1(q, k, v)
-        return einsum_op_mps_v2(q, k, v)
-
-    # Smaller slices are faster due to L2/L3/SLC caches.
-    # Tested on i7 with 8MB L3 cache.
-    return einsum_op_tensor_mem(q, k, v, 32)
-
-def split_cross_attention_forward_invokeAI(self, x, context=None, mask=None):
-    h = self.heads
-
-    q = self.to_q(x)
-    context = default(context, x)
-
-    context_k, context_v = hypernetwork.apply_hypernetwork(shared.loaded_hypernetwork, context)
-    k = self.to_k(context_k) * self.scale
-    v = self.to_v(context_v)
-    del context, context_k, context_v, x
-
-    q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h=h), (q, k, v))
-    r = einsum_op(q, k, v)
-    return self.to_out(rearrange(r, '(b h) n d -> b n (h d)', h=h))
-
-# -- End of code from https://github.com/invoke-ai/InvokeAI --
-
 def xformers_attention_forward(self, x, context=None, mask=None):
     h = self.heads
     q_in = self.to_q(x)

modules/shared.py

@@ -50,10 +50,9 @@ parser.add_argument("--ldsr-models-path", type=str, help="Path to directory with
 parser.add_argument("--xformers", action='store_true', help="enable xformers for cross attention layers")
 parser.add_argument("--force-enable-xformers", action='store_true', help="enable xformers for cross attention layers regardless of whether the checking code thinks you can run it; do not make bug reports if this fails to work")
 parser.add_argument("--deepdanbooru", action='store_true', help="enable deepdanbooru interrogator")
-parser.add_argument("--opt-split-attention", action='store_true', help="force-enables Doggettx's cross-attention layer optimization. By default, it's on for torch cuda.")
-parser.add_argument("--opt-split-attention-invokeai", action='store_true', help="force-enables InvokeAI's cross-attention layer optimization. By default, it's on when cuda is unavailable.")
-parser.add_argument("--opt-split-attention-v1", action='store_true', help="enable older version of split attention optimization that does not consume all the VRAM it can find")
+parser.add_argument("--opt-split-attention", action='store_true', help="force-enables cross-attention layer optimization. By default, it's on for torch.cuda and off for other torch devices.")
 parser.add_argument("--disable-opt-split-attention", action='store_true', help="force-disables cross-attention layer optimization")
+parser.add_argument("--opt-split-attention-v1", action='store_true', help="enable older version of split attention optimization that does not consume all the VRAM it can find")
 parser.add_argument("--use-cpu", nargs='+',choices=['SD', 'GFPGAN', 'BSRGAN', 'ESRGAN', 'SCUNet', 'CodeFormer'], help="use CPU as torch device for specified modules", default=[])
 parser.add_argument("--listen", action='store_true', help="launch gradio with 0.0.0.0 as server name, allowing to respond to network requests")
 parser.add_argument("--port", type=int, help="launch gradio with given server port, you need root/admin rights for ports < 1024, defaults to 7860 if available", default=None)
@@ -228,10 +227,6 @@ options_templates.update(options_section(('system', "System"), {
     "multiple_tqdm": OptionInfo(True, "Add a second progress bar to the console that shows progress for an entire job."),
 }))
 
-options_templates.update(options_section(('training', "Training"), {
-    "unload_models_when_training": OptionInfo(False, "Unload VAE and CLIP form VRAM when training"),
-}))
-
 options_templates.update(options_section(('sd', "Stable Diffusion"), {
     "sd_model_checkpoint": OptionInfo(None, "Stable Diffusion checkpoint", gr.Dropdown, lambda: {"choices": modules.sd_models.checkpoint_tiles()}, show_on_main_page=True),
     "sd_hypernetwork": OptionInfo("None", "Stable Diffusion finetune hypernetwork", gr.Dropdown, lambda: {"choices": ["None"] + [x for x in hypernetworks.keys()]}),

modules/textual_inversion/dataset.py

@@ -8,14 +8,14 @@ from torchvision import transforms
 
 import random
 import tqdm
-from modules import devices, shared
+from modules import devices
 import re
 
 re_tag = re.compile(r"[a-zA-Z][_\w\d()]+")
 
 
 class PersonalizedBase(Dataset):
-    def __init__(self, data_root, width, height, repeats, flip_p=0.5, placeholder_token="*", model=None, device=None, template_file=None, include_cond=False):
+    def __init__(self, data_root, width, height, repeats, flip_p=0.5, placeholder_token="*", model=None, device=None, template_file=None):
 
         self.placeholder_token = placeholder_token
 
@@ -32,15 +32,12 @@ class PersonalizedBase(Dataset):
 
         assert data_root, 'dataset directory not specified'
 
-        cond_model = shared.sd_model.cond_stage_model
-
         self.image_paths = [os.path.join(data_root, file_path) for file_path in os.listdir(data_root)]
         print("Preparing dataset...")
         for path in tqdm.tqdm(self.image_paths):
-            try:
-                image = Image.open(path).convert('RGB').resize((self.width, self.height), PIL.Image.BICUBIC)
-            except Exception:
-                continue
+            image = Image.open(path)
+            image = image.convert('RGB')
+            image = image.resize((self.width, self.height), PIL.Image.BICUBIC)
 
             filename = os.path.basename(path)
             filename_tokens = os.path.splitext(filename)[0]
@@ -55,13 +52,7 @@ class PersonalizedBase(Dataset):
             init_latent = model.get_first_stage_encoding(model.encode_first_stage(torchdata.unsqueeze(dim=0))).squeeze()
             init_latent = init_latent.to(devices.cpu)
 
-            if include_cond:
-                text = self.create_text(filename_tokens)
-                cond = cond_model([text]).to(devices.cpu)
-            else:
-                cond = None
-
-            self.dataset.append((init_latent, filename_tokens, cond))
+            self.dataset.append((init_latent, filename_tokens))
 
         self.length = len(self.dataset) * repeats
 
@@ -72,12 +63,6 @@ class PersonalizedBase(Dataset):
     def shuffle(self):
         self.indexes = self.initial_indexes[torch.randperm(self.initial_indexes.shape[0])]
 
-    def create_text(self, filename_tokens):
-        text = random.choice(self.lines)
-        text = text.replace("[name]", self.placeholder_token)
-        text = text.replace("[filewords]", ' '.join(filename_tokens))
-        return text
-
     def __len__(self):
         return self.length
 
@@ -86,7 +71,10 @@ class PersonalizedBase(Dataset):
             self.shuffle()
 
         index = self.indexes[i % len(self.indexes)]
-        x, filename_tokens, cond = self.dataset[index]
+        x, filename_tokens = self.dataset[index]
 
-        text = self.create_text(filename_tokens)
-        return x, text, cond
+        text = random.choice(self.lines)
+        text = text.replace("[name]", self.placeholder_token)
+        text = text.replace("[filewords]", ' '.join(filename_tokens))
+
+        return x, text

modules/textual_inversion/image_embedding.py

@@ -0,0 +1,219 @@
+import base64
+import json
+import numpy as np
+import zlib
+from PIL import Image, PngImagePlugin, ImageDraw, ImageFont
+from fonts.ttf import Roboto
+import torch
+
+
+class EmbeddingEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, torch.Tensor):
+            return {'TORCHTENSOR': obj.cpu().detach().numpy().tolist()}
+        return json.JSONEncoder.default(self, obj)
+
+
+class EmbeddingDecoder(json.JSONDecoder):
+    def __init__(self, *args, **kwargs):
+        json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs)
+
+    def object_hook(self, d):
+        if 'TORCHTENSOR' in d:
+            return torch.from_numpy(np.array(d['TORCHTENSOR']))
+        return d
+
+
+def embedding_to_b64(data):
+    d = json.dumps(data, cls=EmbeddingEncoder)
+    return base64.b64encode(d.encode())
+
+
+def embedding_from_b64(data):
+    d = base64.b64decode(data)
+    return json.loads(d, cls=EmbeddingDecoder)
+
+
+def lcg(m=2**32, a=1664525, c=1013904223, seed=0):
+    while True:
+        seed = (a * seed + c) % m
+        yield seed % 255
+
+
+def xor_block(block):
+    g = lcg()
+    randblock = np.array([next(g) for _ in range(np.product(block.shape))]).astype(np.uint8).reshape(block.shape)
+    return np.bitwise_xor(block.astype(np.uint8), randblock & 0x0F)
+
+
+def style_block(block, sequence):
+    im = Image.new('RGB', (block.shape[1], block.shape[0]))
+    draw = ImageDraw.Draw(im)
+    i = 0
+    for x in range(-6, im.size[0], 8):
+        for yi, y in enumerate(range(-6, im.size[1], 8)):
+            offset = 0
+            if yi % 2 == 0:
+                offset = 4
+            shade = sequence[i % len(sequence)]
+            i += 1
+            draw.ellipse((x+offset, y, x+6+offset, y+6), fill=(shade, shade, shade))
+
+    fg = np.array(im).astype(np.uint8) & 0xF0
+
+    return block ^ fg
+
+
+def insert_image_data_embed(image, data):
+    d = 3
+    data_compressed = zlib.compress(json.dumps(data, cls=EmbeddingEncoder).encode(), level=9)
+    data_np_ = np.frombuffer(data_compressed, np.uint8).copy()
+    data_np_high = data_np_ >> 4
+    data_np_low = data_np_ & 0x0F
+
+    h = image.size[1]
+    next_size = data_np_low.shape[0] + (h-(data_np_low.shape[0] % h))
+    next_size = next_size + ((h*d)-(next_size % (h*d)))
+
+    data_np_low.resize(next_size)
+    data_np_low = data_np_low.reshape((h, -1, d))
+
+    data_np_high.resize(next_size)
+    data_np_high = data_np_high.reshape((h, -1, d))
+
+    edge_style = list(data['string_to_param'].values())[0].cpu().detach().numpy().tolist()[0][:1024]
+    edge_style = (np.abs(edge_style)/np.max(np.abs(edge_style))*255).astype(np.uint8)
+
+    data_np_low = style_block(data_np_low, sequence=edge_style)
+    data_np_low = xor_block(data_np_low)
+    data_np_high = style_block(data_np_high, sequence=edge_style[::-1])
+    data_np_high = xor_block(data_np_high)
+
+    im_low = Image.fromarray(data_np_low, mode='RGB')
+    im_high = Image.fromarray(data_np_high, mode='RGB')
+
+    background = Image.new('RGB', (image.size[0]+im_low.size[0]+im_high.size[0]+2, image.size[1]), (0, 0, 0))
+    background.paste(im_low, (0, 0))
+    background.paste(image, (im_low.size[0]+1, 0))
+    background.paste(im_high, (im_low.size[0]+1+image.size[0]+1, 0))
+
+    return background
+
+
+def crop_black(img, tol=0):
+    mask = (img > tol).all(2)
+    mask0, mask1 = mask.any(0), mask.any(1)
+    col_start, col_end = mask0.argmax(), mask.shape[1]-mask0[::-1].argmax()
+    row_start, row_end = mask1.argmax(), mask.shape[0]-mask1[::-1].argmax()
+    return img[row_start:row_end, col_start:col_end]
+
+
+def extract_image_data_embed(image):
+    d = 3
+    outarr = crop_black(np.array(image.convert('RGB').getdata()).reshape(image.size[1], image.size[0], d).astype(np.uint8)) & 0x0F
+    black_cols = np.where(np.sum(outarr, axis=(0, 2)) == 0)
+    if black_cols[0].shape[0] < 2:
+        print('No Image data blocks found.')
+        return None
+
+    data_block_lower = outarr[:, :black_cols[0].min(), :].astype(np.uint8)
+    data_block_upper = outarr[:, black_cols[0].max()+1:, :].astype(np.uint8)
+
+    data_block_lower = xor_block(data_block_lower)
+    data_block_upper = xor_block(data_block_upper)
+
+    data_block = (data_block_upper << 4) | (data_block_lower)
+    data_block = data_block.flatten().tobytes()
+
+    data = zlib.decompress(data_block)
+    return json.loads(data, cls=EmbeddingDecoder)
+
+
+def caption_image_overlay(srcimage, title, footerLeft, footerMid, footerRight, textfont=None):
+    from math import cos
+
+    image = srcimage.copy()
+
+    if textfont is None:
+        try:
+            textfont = ImageFont.truetype(opts.font or Roboto, fontsize)
+            textfont = opts.font or Roboto
+        except Exception:
+            textfont = Roboto
+
+    factor = 1.5
+    gradient = Image.new('RGBA', (1, image.size[1]), color=(0, 0, 0, 0))
+    for y in range(image.size[1]):
+        mag = 1-cos(y/image.size[1]*factor)
+        mag = max(mag, 1-cos((image.size[1]-y)/image.size[1]*factor*1.1))
+        gradient.putpixel((0, y), (0, 0, 0, int(mag*255)))
+    image = Image.alpha_composite(image.convert('RGBA'), gradient.resize(image.size))
+
+    draw = ImageDraw.Draw(image)
+    fontsize = 32
+    font = ImageFont.truetype(textfont, fontsize)
+    padding = 10
+
+    _, _, w, h = draw.textbbox((0, 0), title, font=font)
+    fontsize = min(int(fontsize * (((image.size[0]*0.75)-(padding*4))/w)), 72)
+    font = ImageFont.truetype(textfont, fontsize)
+    _, _, w, h = draw.textbbox((0, 0), title, font=font)
+    draw.text((padding, padding), title, anchor='lt', font=font, fill=(255, 255, 255, 230))
+
+    _, _, w, h = draw.textbbox((0, 0), footerLeft, font=font)
+    fontsize_left = min(int(fontsize * (((image.size[0]/3)-(padding))/w)), 72)
+    _, _, w, h = draw.textbbox((0, 0), footerMid, font=font)
+    fontsize_mid = min(int(fontsize * (((image.size[0]/3)-(padding))/w)), 72)
+    _, _, w, h = draw.textbbox((0, 0), footerRight, font=font)
+    fontsize_right = min(int(fontsize * (((image.size[0]/3)-(padding))/w)), 72)
+
+    font = ImageFont.truetype(textfont, min(fontsize_left, fontsize_mid, fontsize_right))
+
+    draw.text((padding, image.size[1]-padding),               footerLeft, anchor='ls', font=font, fill=(255, 255, 255, 230))
+    draw.text((image.size[0]/2, image.size[1]-padding),       footerMid, anchor='ms', font=font, fill=(255, 255, 255, 230))
+    draw.text((image.size[0]-padding, image.size[1]-padding), footerRight, anchor='rs', font=font, fill=(255, 255, 255, 230))
+
+    return image
+
+
+if __name__ == '__main__':
+
+    testEmbed = Image.open('test_embedding.png')
+    data = extract_image_data_embed(testEmbed)
+    assert data is not None
+
+    data = embedding_from_b64(testEmbed.text['sd-ti-embedding'])
+    assert data is not None
+
+    image = Image.new('RGBA', (512, 512), (255, 255, 200, 255))
+    cap_image = caption_image_overlay(image, 'title', 'footerLeft', 'footerMid', 'footerRight')
+
+    test_embed = {'string_to_param': {'*': torch.from_numpy(np.random.random((2, 4096)))}}
+
+    embedded_image = insert_image_data_embed(cap_image, test_embed)
+
+    retrived_embed = extract_image_data_embed(embedded_image)
+
+    assert str(retrived_embed) == str(test_embed)
+
+    embedded_image2 = insert_image_data_embed(cap_image, retrived_embed)
+
+    assert embedded_image == embedded_image2
+
+    g = lcg()
+    shared_random = np.array([next(g) for _ in range(100)]).astype(np.uint8).tolist()
+
+    reference_random = [253, 242, 127,  44, 157,  27, 239, 133,  38,  79, 167,   4, 177,
+                         95, 130,  79,  78,  14,  52, 215, 220, 194, 126,  28, 240, 179,
+                        160, 153, 149,  50, 105,  14,  21, 218, 199,  18,  54, 198, 193,
+                         38, 128,  19,  53, 195, 124,  75, 205,  12,   6, 145,   0,  28,
+                         30, 148,   8,  45, 218, 171,  55, 249,  97, 166,  12,  35,   0,
+                         41, 221, 122, 215, 170,  31, 113, 186,  97, 119,  31,  23, 185,
+                         66, 140,  30,  41,  37,  63, 137, 109, 216,  55, 159, 145,  82,
+                         204, 86,  73, 222,  44, 198, 118, 240,  97]
+
+    assert shared_random == reference_random
+
+    hunna_kay_random_sum = sum(np.array([next(g) for _ in range(100000)]).astype(np.uint8).tolist())
+
+    assert 12731374 == hunna_kay_random_sum

modules/textual_inversion/preprocess.py

@@ -46,10 +46,7 @@ def preprocess(process_src, process_dst, process_width, process_height, process_
     for index, imagefile in enumerate(tqdm.tqdm(files)):
         subindex = [0]
         filename = os.path.join(src, imagefile)
-        try:
-            img = Image.open(filename).convert("RGB")
-        except Exception:
-            continue
+        img = Image.open(filename).convert("RGB")
 
         if shared.state.interrupted:
             break

modules/textual_inversion/textual_inversion.py

@@ -7,10 +7,14 @@ import tqdm
 import html
 import datetime
 
+from PIL import Image, PngImagePlugin
 
 from modules import shared, devices, sd_hijack, processing, sd_models
 import modules.textual_inversion.dataset
 
+from modules.textual_inversion.image_embedding import (embedding_to_b64, embedding_from_b64,
+                                                       insert_image_data_embed, extract_image_data_embed,
+                                                       caption_image_overlay)
 
 class Embedding:
     def __init__(self, vec, name, step=None):
@@ -80,7 +84,18 @@ class EmbeddingDatabase:
         def process_file(path, filename):
             name = os.path.splitext(filename)[0]
 
-            data = torch.load(path, map_location="cpu")
+            data = []
+
+            if filename.upper().endswith('.PNG'):
+                embed_image = Image.open(path)
+                if 'sd-ti-embedding' in embed_image.text:
+                    data = embedding_from_b64(embed_image.text['sd-ti-embedding'])
+                    name = data.get('name', name)
+                else:
+                    data = extract_image_data_embed(embed_image)
+                    name = data.get('name', name)
+            else:
+                data = torch.load(path, map_location="cpu")
 
             # textual inversion embeddings
             if 'string_to_param' in data:
@@ -156,7 +171,8 @@ def create_embedding(name, num_vectors_per_token, init_text='*'):
     return fn
 
 
-def train_embedding(embedding_name, learn_rate, data_root, log_directory, training_width, training_height, steps, num_repeats, create_image_every, save_embedding_every, template_file, preview_image_prompt):
+
+def train_embedding(embedding_name, learn_rate, data_root, log_directory, training_width, training_height, steps, num_repeats, create_image_every, save_embedding_every, template_file, save_image_with_stored_embedding, preview_image_prompt):
     assert embedding_name, 'embedding not selected'
 
     shared.state.textinfo = "Initializing textual inversion training..."
@@ -178,6 +194,12 @@ def train_embedding(embedding_name, learn_rate, data_root, log_directory, traini
     else:
         images_dir = None
 
+    if create_image_every > 0 and save_image_with_stored_embedding:
+        images_embeds_dir = os.path.join(log_directory, "image_embeddings")
+        os.makedirs(images_embeds_dir, exist_ok=True)
+    else:
+        images_embeds_dir = None
+        
     cond_model = shared.sd_model.cond_stage_model
 
     shared.state.textinfo = f"Preparing dataset from {html.escape(data_root)}..."
@@ -189,6 +211,8 @@ def train_embedding(embedding_name, learn_rate, data_root, log_directory, traini
     embedding = hijack.embedding_db.word_embeddings[embedding_name]
     embedding.vec.requires_grad = True
 
+    optimizer = torch.optim.AdamW([embedding.vec], lr=learn_rate)
+
     losses = torch.zeros((32,))
 
     last_saved_file = "<none>"
@@ -201,24 +225,12 @@ def train_embedding(embedding_name, learn_rate, data_root, log_directory, traini
     tr_img_len = len([os.path.join(data_root, file_path) for file_path in os.listdir(data_root)])
     epoch_len = (tr_img_len * num_repeats) + tr_img_len
 
-    scheduleIter = iter(LearnSchedule(learn_rate, steps, ititial_step))
-    (learn_rate, end_step) = next(scheduleIter)
-    print(f'Training at rate of {learn_rate} until step {end_step}')
-
-    optimizer = torch.optim.AdamW([embedding.vec], lr=learn_rate)
-
     pbar = tqdm.tqdm(enumerate(ds), total=steps-ititial_step)
-    for i, (x, text, _) in pbar:
+    for i, (x, text) in pbar:
         embedding.step = i + ititial_step
 
-        if embedding.step > end_step:
-            try:
-                (learn_rate, end_step) = next(scheduleIter)
-            except:
-                break
-            tqdm.tqdm.write(f'Training at rate of {learn_rate} until step {end_step}')
-            for pg in optimizer.param_groups:
-                pg['lr'] = learn_rate
+        if embedding.step > steps:
+            break
 
         if shared.state.interrupted:
             break
@@ -236,10 +248,10 @@ def train_embedding(embedding_name, learn_rate, data_root, log_directory, traini
             loss.backward()
             optimizer.step()
 
-        epoch_num = embedding.step // len(ds)
-        epoch_step = embedding.step - (epoch_num * len(ds)) + 1
+        epoch_num = embedding.step // epoch_len
+        epoch_step = embedding.step - (epoch_num * epoch_len) + 1
 
-        pbar.set_description(f"[Epoch {epoch_num}: {epoch_step}/{len(ds)}]loss: {losses.mean():.7f}")
+        pbar.set_description(f"[Epoch {epoch_num}: {epoch_step}/{epoch_len}]loss: {losses.mean():.7f}")
 
         if embedding.step > 0 and embedding_dir is not None and embedding.step % save_embedding_every == 0:
             last_saved_file = os.path.join(embedding_dir, f'{embedding_name}-{embedding.step}.pt')
@@ -264,6 +276,26 @@ def train_embedding(embedding_name, learn_rate, data_root, log_directory, traini
             image = processed.images[0]
 
             shared.state.current_image = image
+
+            if save_image_with_stored_embedding and os.path.exists(last_saved_file):
+
+                last_saved_image_chunks = os.path.join(images_embeds_dir, f'{embedding_name}-{embedding.step}.png')
+
+                info = PngImagePlugin.PngInfo()
+                data = torch.load(last_saved_file)
+                info.add_text("sd-ti-embedding", embedding_to_b64(data))
+
+                title = "<{}>".format(data.get('name', '???'))
+                checkpoint = sd_models.select_checkpoint()
+                footer_left = checkpoint.model_name
+                footer_mid = '[{}]'.format(checkpoint.hash)
+                footer_right = '{}'.format(embedding.step)
+
+                captioned_image = caption_image_overlay(image, title, footer_left, footer_mid, footer_right)
+                captioned_image = insert_image_data_embed(captioned_image, data)
+
+                captioned_image.save(last_saved_image_chunks, "PNG", pnginfo=info)
+
             image.save(last_saved_image)
 
             last_saved_image += f", prompt: {preview_text}"
@@ -289,36 +321,3 @@ Last saved image: {html.escape(last_saved_image)}<br/>
 
     return embedding, filename
 
-class LearnSchedule:
-    def __init__(self, learn_rate, max_steps, cur_step=0):
-        pairs = learn_rate.split(',')
-        self.rates = []
-        self.it = 0
-        self.maxit = 0
-        for i, pair in enumerate(pairs):
-            tmp = pair.split(':')
-            if len(tmp) == 2:
-                step = int(tmp[1])
-                if step > cur_step:
-                    self.rates.append((float(tmp[0]), min(step, max_steps)))
-                    self.maxit += 1
-                    if step > max_steps:
-                        return
-                elif step == -1:
-                    self.rates.append((float(tmp[0]), max_steps))
-                    self.maxit += 1
-                    return
-            else:
-                self.rates.append((float(tmp[0]), max_steps))
-                self.maxit += 1
-                return
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        if self.it < self.maxit:
-            self.it += 1
-            return self.rates[self.it - 1]
-        else:
-            raise StopIteration

modules/textual_inversion/ui.py

@@ -22,9 +22,6 @@ def preprocess(*args):
 
 
 def train_embedding(*args):
-
-    assert not shared.cmd_opts.lowvram and not shared.cmd_opts.medvram, 'Training models with lowvram or medvram is not possible'
-
     try:
         sd_hijack.undo_optimizations()
 

modules/ui.py

@@ -1037,7 +1037,6 @@ def create_ui(wrap_gradio_gpu_call):
                     gr.HTML(value="<p style='margin-bottom: 0.7em'>Create a new hypernetwork</p>")
 
                     new_hypernetwork_name = gr.Textbox(label="Name")
-                    new_hypernetwork_sizes = gr.CheckboxGroup(label="Modules", value=["768", "320", "640", "1280"], choices=["768", "320", "640", "1280"])
 
                     with gr.Row():
                         with gr.Column(scale=3):
@@ -1070,7 +1069,7 @@ def create_ui(wrap_gradio_gpu_call):
                     gr.HTML(value="<p style='margin-bottom: 0.7em'>Train an embedding; must specify a directory with a set of 1:1 ratio images</p>")
                     train_embedding_name = gr.Dropdown(label='Embedding', choices=sorted(sd_hijack.model_hijack.embedding_db.word_embeddings.keys()))
                     train_hypernetwork_name = gr.Dropdown(label='Hypernetwork', choices=[x for x in shared.hypernetworks.keys()])
-                    learn_rate = gr.Textbox(label='Learning rate', placeholder="Learning rate", value = "5.0e-03")
+                    learn_rate = gr.Number(label='Learning rate', value=5.0e-03)
                     dataset_directory = gr.Textbox(label='Dataset directory', placeholder="Path to directory with input images")
                     log_directory = gr.Textbox(label='Log directory', placeholder="Path to directory where to write outputs", value="textual_inversion")
                     template_file = gr.Textbox(label='Prompt template file', value=os.path.join(script_path, "textual_inversion_templates", "style_filewords.txt"))
@@ -1080,6 +1079,7 @@ def create_ui(wrap_gradio_gpu_call):
                     num_repeats = gr.Number(label='Number of repeats for a single input image per epoch', value=100, precision=0)
                     create_image_every = gr.Number(label='Save an image to log directory every N steps, 0 to disable', value=500, precision=0)
                     save_embedding_every = gr.Number(label='Save a copy of embedding to log directory every N steps, 0 to disable', value=500, precision=0)
+                    save_image_with_stored_embedding = gr.Checkbox(label='Save images with embedding in PNG chunks', value=True)
                     preview_image_prompt = gr.Textbox(label='Preview prompt', value="")
 
                     with gr.Row():
@@ -1115,7 +1115,6 @@ def create_ui(wrap_gradio_gpu_call):
             fn=modules.hypernetworks.ui.create_hypernetwork,
             inputs=[
                 new_hypernetwork_name,
-                new_hypernetwork_sizes,
             ],
             outputs=[
                 train_hypernetwork_name,
@@ -1157,6 +1156,7 @@ def create_ui(wrap_gradio_gpu_call):
                 create_image_every,
                 save_embedding_every,
                 template_file,
+                save_image_with_stored_embedding,
                 preview_image_prompt,
             ],
             outputs=[
@@ -1345,7 +1345,6 @@ Requested path was: {f}
             shared.state.interrupt()
             settings_interface.gradio_ref.do_restart = True
 
-
         restart_gradio.click(
             fn=request_restart,
             inputs=[],

requirements.txt

@@ -4,7 +4,7 @@ fairscale==0.4.4
 fonts
 font-roboto
 gfpgan
-gradio==3.4.1
+gradio==3.4b3
 invisible-watermark
 numpy
 omegaconf

requirements_versions.txt

@@ -2,7 +2,7 @@ transformers==4.19.2
 diffusers==0.3.0
 basicsr==1.4.2
 gfpgan==1.3.8
-gradio==3.4.1
+gradio==3.4b3
 numpy==1.23.3
 Pillow==9.2.0
 realesrgan==0.3.0

scripts/xy_grid.py

@@ -197,7 +197,7 @@ class Script(scripts.Script):
             x_values = gr.Textbox(label="X values", visible=False, lines=1)
 
         with gr.Row():
-            y_type = gr.Dropdown(label="Y type", choices=[x.label for x in current_axis_options], value=current_axis_options[0].label, visible=False, type="index", elem_id="y_type")
+            y_type = gr.Dropdown(label="Y type", choices=[x.label for x in current_axis_options], value=current_axis_options[4].label, visible=False, type="index", elem_id="y_type")
             y_values = gr.Textbox(label="Y values", visible=False, lines=1)
         
         draw_legend = gr.Checkbox(label='Draw legend', value=True)

style.css

@@ -240,7 +240,6 @@ fieldset span.text-gray-500, .gr-block.gr-box span.text-gray-500,  label.block s
 #settings fieldset span.text-gray-500, #settings .gr-block.gr-box span.text-gray-500, #settings label.block span{
     position: relative;
     border: none;
-    margin-right: 8em;
 }
 
 .gr-panel div.flex-col div.justify-between label span{
@@ -496,13 +495,3 @@ canvas[key="mask"] {
     mix-blend-mode: multiply;
     pointer-events: none;
 }
-
-
-/* gradio 3.4.1 stuff for editable scrollbar values */
-.gr-box > div > div > input.gr-text-input{
-    position: absolute;
-    right: 0.5em;
-    top: -0.6em;
-    z-index: 200;
-    width: 8em;
-}

webui.py

@@ -124,8 +124,6 @@ def webui():
         modules.scripts.reload_scripts(os.path.join(script_path, "scripts"))
         print('Reloading modules: modules.ui')
         importlib.reload(modules.ui)
-        print('Refreshing Model List')
-        modules.sd_models.list_models()
         print('Restarting Gradio')