diff --git a/javascript/localization.js b/javascript/localization.js index 8f00c186..1abff708 100644 --- a/javascript/localization.js +++ b/javascript/localization.js @@ -2,13 +2,10 @@ // localization = {} -- the dict with translations is created by the backend var ignore_ids_for_localization = { - setting_sd_hypernetwork: 'OPTION', setting_sd_model_checkpoint: 'OPTION', modelmerger_primary_model_name: 'OPTION', modelmerger_secondary_model_name: 'OPTION', modelmerger_tertiary_model_name: 'OPTION', - train_embedding: 'OPTION', - train_hypernetwork: 'OPTION', txt2img_styles: 'OPTION', img2img_styles: 'OPTION', setting_random_artist_categories: 'OPTION', diff --git a/modules/api/api.py b/modules/api/api.py index a9aba03a..d1c0c76a 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -225,7 +225,6 @@ class Api: self.add_api_route("/sdapi/v1/latent-upscale-modes", self.get_latent_upscale_modes, methods=["GET"], response_model=list[models.LatentUpscalerModeItem]) self.add_api_route("/sdapi/v1/sd-models", self.get_sd_models, methods=["GET"], response_model=list[models.SDModelItem]) self.add_api_route("/sdapi/v1/sd-modules", self.get_sd_vaes_and_text_encoders, methods=["GET"], response_model=list[models.SDModuleItem]) - self.add_api_route("/sdapi/v1/hypernetworks", self.get_hypernetworks, methods=["GET"], response_model=list[models.HypernetworkItem]) self.add_api_route("/sdapi/v1/face-restorers", self.get_face_restorers, methods=["GET"], response_model=list[models.FaceRestorerItem]) self.add_api_route("/sdapi/v1/realesrgan-models", self.get_realesrgan_models, methods=["GET"], response_model=list[models.RealesrganItem]) self.add_api_route("/sdapi/v1/prompt-styles", self.get_prompt_styles, methods=["GET"], response_model=list[models.PromptStyleItem]) @@ -234,7 +233,6 @@ class Api: self.add_api_route("/sdapi/v1/refresh-checkpoints", self.refresh_checkpoints, methods=["POST"]) self.add_api_route("/sdapi/v1/refresh-vae", self.refresh_vae, methods=["POST"]) self.add_api_route("/sdapi/v1/create/embedding", self.create_embedding, methods=["POST"], response_model=models.CreateResponse) - self.add_api_route("/sdapi/v1/create/hypernetwork", self.create_hypernetwork, methods=["POST"], response_model=models.CreateResponse) self.add_api_route("/sdapi/v1/memory", self.get_memory, methods=["GET"], response_model=models.MemoryResponse) self.add_api_route("/sdapi/v1/unload-checkpoint", self.unloadapi, methods=["POST"]) self.add_api_route("/sdapi/v1/reload-checkpoint", self.reloadapi, methods=["POST"]) @@ -712,9 +710,6 @@ class Api: from modules_forge.main_entry import module_list return [{"model_name": x, "filename": module_list[x]} for x in module_list.keys()] - def get_hypernetworks(self): - return [{"name": name, "path": shared.hypernetworks[name]} for name in shared.hypernetworks] - def get_face_restorers(self): return [{"name":x.name(), "cmd_dir": getattr(x, "cmd_dir", None)} for x in shared.face_restorers] @@ -770,16 +765,6 @@ class Api: finally: shared.state.end() - def create_hypernetwork(self, args: dict): - try: - shared.state.begin(job="create_hypernetwork") - filename = create_hypernetwork(**args) # create empty embedding - return models.CreateResponse(info=f"create hypernetwork filename: {filename}") - except AssertionError as e: - return models.TrainResponse(info=f"create hypernetwork error: {e}") - finally: - shared.state.end() - def get_memory(self): try: import os diff --git a/modules/api/models.py b/modules/api/models.py index 816f8acb..e160c39f 100644 --- a/modules/api/models.py +++ b/modules/api/models.py @@ -191,12 +191,6 @@ class ProgressResponse(BaseModel): current_image: str | None = Field(default=None, title="Current image", description="The current image in base64 format. opts.show_progress_every_n_steps is required for this to work.") textinfo: str | None = Field(default=None, title="Info text", description="Info text used by WebUI.") -class TrainResponse(BaseModel): - info: str = Field(title="Train info", description="Response string from train embedding or hypernetwork task.") - -class CreateResponse(BaseModel): - info: str = Field(title="Create info", description="Response string from create embedding or hypernetwork task.") - fields = {} for key, metadata in opts.data_labels.items(): value = opts.data.get(key) @@ -264,10 +258,6 @@ class SDModuleItem(BaseModel): model_name: str = Field(title="Model Name") filename: str = Field(title="Filename") -class HypernetworkItem(BaseModel): - name: str = Field(title="Name") - path: Optional[str] = Field(title="Path") - class FaceRestorerItem(BaseModel): name: str = Field(title="Name") cmd_dir: Optional[str] = Field(title="Path") diff --git a/modules/cmd_args.py b/modules/cmd_args.py index a8fc29d6..70b06e08 100644 --- a/modules/cmd_args.py +++ b/modules/cmd_args.py @@ -35,7 +35,6 @@ parser.add_argument("--no-progressbar-hiding", action='store_true', help="do not parser.add_argument("--max-batch-count", type=int, default=16, help="does not do anything") parser.add_argument("--embeddings-dir", type=normalized_filepath, default=os.path.join(data_path, 'embeddings'), help="embeddings directory for textual inversion (default: embeddings)") parser.add_argument("--textual-inversion-templates-dir", type=normalized_filepath, default=os.path.join(script_path, 'textual_inversion_templates'), help="directory with textual inversion templates") -parser.add_argument("--hypernetwork-dir", type=normalized_filepath, default=os.path.join(models_path, 'hypernetworks'), help="hypernetwork directory") parser.add_argument("--localizations-dir", type=normalized_filepath, default=os.path.join(script_path, 'localizations'), help="localizations directory") parser.add_argument("--allow-code", action='store_true', help="allow custom script execution from webui") parser.add_argument("--medvram", action='store_true', help="enable stable diffusion model optimizations for sacrificing a little speed for low VRM usage") diff --git a/modules/extra_networks.py b/modules/extra_networks.py index ae8d42d9..1a41c5ae 100644 --- a/modules/extra_networks.py +++ b/modules/extra_networks.py @@ -23,11 +23,6 @@ def register_extra_network_alias(extra_network, alias): extra_network_aliases[alias] = extra_network -def register_default_extra_networks(): - from modules.extra_networks_hypernet import ExtraNetworkHypernet - register_extra_network(ExtraNetworkHypernet()) - - class ExtraNetworkParams: def __init__(self, items=None): self.items = items or [] @@ -64,18 +59,6 @@ class ExtraNetwork: in this case, all effects of this extra networks should be disabled. Can be called multiple times before deactivate() - each new call should override the previous call completely. - - For example, if this ExtraNetwork's name is 'hypernet' and user's prompt is: - - > "1girl, " - - params_list will be: - - [ - ExtraNetworkParams(items=["agm", "1.1"]), - ExtraNetworkParams(items=["ray"]) - ] - """ raise NotImplementedError @@ -88,20 +71,19 @@ class ExtraNetwork: def lookup_extra_networks(extra_network_data): - """returns a dict mapping ExtraNetwork objects to lists of arguments for those extra networks. + """ + returns a dict mapping ExtraNetwork objects to lists of arguments for those extra networks. Example input: { 'lora': [], 'lyco': [], - 'hypernet': [] } Example output: { : [, ], - : [] } """ diff --git a/modules/extra_networks_hypernet.py b/modules/extra_networks_hypernet.py deleted file mode 100644 index b6a6dc0e..00000000 --- a/modules/extra_networks_hypernet.py +++ /dev/null @@ -1,28 +0,0 @@ -from modules import extra_networks, shared -from modules.hypernetworks import hypernetwork - - -class ExtraNetworkHypernet(extra_networks.ExtraNetwork): - def __init__(self): - super().__init__('hypernet') - - def activate(self, p, params_list): - additional = shared.opts.sd_hypernetwork - - if additional != "None" and additional in shared.hypernetworks and not any(x for x in params_list if x.items[0] == additional): - hypernet_prompt_text = f"" - p.all_prompts = [f"{prompt}{hypernet_prompt_text}" for prompt in p.all_prompts] - params_list.append(extra_networks.ExtraNetworkParams(items=[additional, shared.opts.extra_networks_default_multiplier])) - - names = [] - multipliers = [] - for params in params_list: - assert params.items - - names.append(params.items[0]) - multipliers.append(float(params.items[1]) if len(params.items) > 1 else 1.0) - - hypernetwork.load_hypernetworks(names, multipliers) - - def deactivate(self, p): - pass diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py deleted file mode 100644 index 5674d906..00000000 --- a/modules/hypernetworks/hypernetwork.py +++ /dev/null @@ -1,435 +0,0 @@ -import datetime -import glob -import html -import os -import inspect -from contextlib import closing - -import torch -import tqdm -from einops import rearrange, repeat -from backend.nn.unet import default -from modules import devices, sd_models, shared, sd_samplers, hashes, errors -from modules.textual_inversion import textual_inversion -from torch import einsum -from torch.nn.init import normal_, xavier_normal_, xavier_uniform_, kaiming_normal_, kaiming_uniform_, zeros_ - -from collections import deque -from statistics import stdev, mean - - -optimizer_dict = {optim_name : cls_obj for optim_name, cls_obj in inspect.getmembers(torch.optim, inspect.isclass) if optim_name != "Optimizer"} - -class HypernetworkModule(torch.nn.Module): - activation_dict = { - "linear": torch.nn.Identity, - "relu": torch.nn.ReLU, - "leakyrelu": torch.nn.LeakyReLU, - "elu": torch.nn.ELU, - "swish": torch.nn.Hardswish, - "tanh": torch.nn.Tanh, - "sigmoid": torch.nn.Sigmoid, - } - activation_dict.update({cls_name.lower(): cls_obj for cls_name, cls_obj in inspect.getmembers(torch.nn.modules.activation) if inspect.isclass(cls_obj) and cls_obj.__module__ == 'torch.nn.modules.activation'}) - - def __init__(self, dim, state_dict=None, layer_structure=None, activation_func=None, weight_init='Normal', - add_layer_norm=False, activate_output=False, dropout_structure=None): - super().__init__() - - self.multiplier = 1.0 - - assert layer_structure is not None, "layer_structure must not be None" - assert layer_structure[0] == 1, "Multiplier Sequence should start with size 1!" - assert layer_structure[-1] == 1, "Multiplier Sequence should end with size 1!" - - linears = [] - for i in range(len(layer_structure) - 1): - - # Add a fully-connected layer - linears.append(torch.nn.Linear(int(dim * layer_structure[i]), int(dim * layer_structure[i+1]))) - - # Add an activation func except last layer - if activation_func == "linear" or activation_func is None or (i >= len(layer_structure) - 2 and not activate_output): - pass - elif activation_func in self.activation_dict: - linears.append(self.activation_dict[activation_func]()) - else: - raise RuntimeError(f'hypernetwork uses an unsupported activation function: {activation_func}') - - # Add layer normalization - if add_layer_norm: - linears.append(torch.nn.LayerNorm(int(dim * layer_structure[i+1]))) - - # Everything should be now parsed into dropout structure, and applied here. - # Since we only have dropouts after layers, dropout structure should start with 0 and end with 0. - if dropout_structure is not None and dropout_structure[i+1] > 0: - assert 0 < dropout_structure[i+1] < 1, "Dropout probability should be 0 or float between 0 and 1!" - linears.append(torch.nn.Dropout(p=dropout_structure[i+1])) - # Code explanation : [1, 2, 1] -> dropout is missing when last_layer_dropout is false. [1, 2, 2, 1] -> [0, 0.3, 0, 0], when its True, [0, 0.3, 0.3, 0]. - - self.linear = torch.nn.Sequential(*linears) - - if state_dict is not None: - self.fix_old_state_dict(state_dict) - self.load_state_dict(state_dict) - else: - for layer in self.linear: - if type(layer) == torch.nn.Linear or type(layer) == torch.nn.LayerNorm: - w, b = layer.weight.data, layer.bias.data - if weight_init == "Normal" or type(layer) == torch.nn.LayerNorm: - normal_(w, mean=0.0, std=0.01) - normal_(b, mean=0.0, std=0) - elif weight_init == 'XavierUniform': - xavier_uniform_(w) - zeros_(b) - elif weight_init == 'XavierNormal': - xavier_normal_(w) - zeros_(b) - elif weight_init == 'KaimingUniform': - kaiming_uniform_(w, nonlinearity='leaky_relu' if 'leakyrelu' == activation_func else 'relu') - zeros_(b) - elif weight_init == 'KaimingNormal': - kaiming_normal_(w, nonlinearity='leaky_relu' if 'leakyrelu' == activation_func else 'relu') - zeros_(b) - else: - raise KeyError(f"Key {weight_init} is not defined as initialization!") - devices.torch_npu_set_device() - self.to(devices.device) - - def fix_old_state_dict(self, state_dict): - changes = { - 'linear1.bias': 'linear.0.bias', - 'linear1.weight': 'linear.0.weight', - 'linear2.bias': 'linear.1.bias', - 'linear2.weight': 'linear.1.weight', - } - - for fr, to in changes.items(): - x = state_dict.get(fr, None) - if x is None: - continue - - del state_dict[fr] - state_dict[to] = x - - def forward(self, x): - return x + self.linear(x) * (self.multiplier if not self.training else 1) - - def trainables(self): - layer_structure = [] - for layer in self.linear: - if type(layer) == torch.nn.Linear or type(layer) == torch.nn.LayerNorm: - layer_structure += [layer.weight, layer.bias] - return layer_structure - - -#param layer_structure : sequence used for length, use_dropout : controlling boolean, last_layer_dropout : for compatibility check. -def parse_dropout_structure(layer_structure, use_dropout, last_layer_dropout): - if layer_structure is None: - layer_structure = [1, 2, 1] - if not use_dropout: - return [0] * len(layer_structure) - dropout_values = [0] - dropout_values.extend([0.3] * (len(layer_structure) - 3)) - if last_layer_dropout: - dropout_values.append(0.3) - else: - dropout_values.append(0) - dropout_values.append(0) - return dropout_values - - -class Hypernetwork: - filename = None - name = None - - def __init__(self, name=None, enable_sizes=None, layer_structure=None, activation_func=None, weight_init=None, add_layer_norm=False, use_dropout=False, activate_output=False, **kwargs): - self.filename = None - self.name = name - self.layers = {} - self.step = 0 - self.sd_checkpoint = None - self.sd_checkpoint_name = None - self.layer_structure = layer_structure - self.activation_func = activation_func - self.weight_init = weight_init - self.add_layer_norm = add_layer_norm - self.use_dropout = use_dropout - self.activate_output = activate_output - self.last_layer_dropout = kwargs.get('last_layer_dropout', True) - self.dropout_structure = kwargs.get('dropout_structure', None) - if self.dropout_structure is None: - self.dropout_structure = parse_dropout_structure(self.layer_structure, self.use_dropout, self.last_layer_dropout) - self.optimizer_name = None - self.optimizer_state_dict = None - self.optional_info = None - - for size in enable_sizes or []: - self.layers[size] = ( - HypernetworkModule(size, None, self.layer_structure, self.activation_func, self.weight_init, - self.add_layer_norm, self.activate_output, dropout_structure=self.dropout_structure), - HypernetworkModule(size, None, self.layer_structure, self.activation_func, self.weight_init, - self.add_layer_norm, self.activate_output, dropout_structure=self.dropout_structure), - ) - self.eval() - - def weights(self): - res = [] - for layers in self.layers.values(): - for layer in layers: - res += layer.parameters() - return res - - def train(self, mode=True): - for layers in self.layers.values(): - for layer in layers: - layer.train(mode=mode) - for param in layer.parameters(): - param.requires_grad = mode - - def to(self, device): - for layers in self.layers.values(): - for layer in layers: - layer.to(device) - - return self - - def set_multiplier(self, multiplier): - for layers in self.layers.values(): - for layer in layers: - layer.multiplier = multiplier - - return self - - def eval(self): - for layers in self.layers.values(): - for layer in layers: - layer.eval() - for param in layer.parameters(): - param.requires_grad = False - - def save(self, filename): - state_dict = {} - optimizer_saved_dict = {} - - for k, v in self.layers.items(): - state_dict[k] = (v[0].state_dict(), v[1].state_dict()) - - state_dict['step'] = self.step - state_dict['name'] = self.name - state_dict['layer_structure'] = self.layer_structure - state_dict['activation_func'] = self.activation_func - state_dict['is_layer_norm'] = self.add_layer_norm - state_dict['weight_initialization'] = self.weight_init - state_dict['sd_checkpoint'] = self.sd_checkpoint - state_dict['sd_checkpoint_name'] = self.sd_checkpoint_name - state_dict['activate_output'] = self.activate_output - state_dict['use_dropout'] = self.use_dropout - state_dict['dropout_structure'] = self.dropout_structure - state_dict['last_layer_dropout'] = (self.dropout_structure[-2] != 0) if self.dropout_structure is not None else self.last_layer_dropout - state_dict['optional_info'] = self.optional_info if self.optional_info else None - - if self.optimizer_name is not None: - optimizer_saved_dict['optimizer_name'] = self.optimizer_name - - torch.save(state_dict, filename) - if shared.opts.save_optimizer_state and self.optimizer_state_dict: - optimizer_saved_dict['hash'] = self.shorthash() - optimizer_saved_dict['optimizer_state_dict'] = self.optimizer_state_dict - torch.save(optimizer_saved_dict, filename + '.optim') - - def load(self, filename): - self.filename = filename - if self.name is None: - self.name = os.path.splitext(os.path.basename(filename))[0] - - state_dict = torch.load(filename, map_location='cpu') - - self.layer_structure = state_dict.get('layer_structure', [1, 2, 1]) - self.optional_info = state_dict.get('optional_info', None) - self.activation_func = state_dict.get('activation_func', None) - self.weight_init = state_dict.get('weight_initialization', 'Normal') - self.add_layer_norm = state_dict.get('is_layer_norm', False) - self.dropout_structure = state_dict.get('dropout_structure', None) - self.use_dropout = True if self.dropout_structure is not None and any(self.dropout_structure) else state_dict.get('use_dropout', False) - self.activate_output = state_dict.get('activate_output', True) - self.last_layer_dropout = state_dict.get('last_layer_dropout', False) - # Dropout structure should have same length as layer structure, Every digits should be in [0,1), and last digit must be 0. - if self.dropout_structure is None: - self.dropout_structure = parse_dropout_structure(self.layer_structure, self.use_dropout, self.last_layer_dropout) - - if shared.opts.print_hypernet_extra: - if self.optional_info is not None: - print(f" INFO:\n {self.optional_info}\n") - - print(f" Layer structure: {self.layer_structure}") - print(f" Activation function: {self.activation_func}") - print(f" Weight initialization: {self.weight_init}") - print(f" Layer norm: {self.add_layer_norm}") - print(f" Dropout usage: {self.use_dropout}" ) - print(f" Activate last layer: {self.activate_output}") - print(f" Dropout structure: {self.dropout_structure}") - - optimizer_saved_dict = torch.load(self.filename + '.optim', map_location='cpu') if os.path.exists(self.filename + '.optim') else {} - - if self.shorthash() == optimizer_saved_dict.get('hash', None): - self.optimizer_state_dict = optimizer_saved_dict.get('optimizer_state_dict', None) - else: - self.optimizer_state_dict = None - if self.optimizer_state_dict: - self.optimizer_name = optimizer_saved_dict.get('optimizer_name', 'AdamW') - if shared.opts.print_hypernet_extra: - print("Loaded existing optimizer from checkpoint") - print(f"Optimizer name is {self.optimizer_name}") - else: - self.optimizer_name = "AdamW" - if shared.opts.print_hypernet_extra: - print("No saved optimizer exists in checkpoint") - - for size, sd in state_dict.items(): - if type(size) == int: - self.layers[size] = ( - HypernetworkModule(size, sd[0], self.layer_structure, self.activation_func, self.weight_init, - self.add_layer_norm, self.activate_output, self.dropout_structure), - HypernetworkModule(size, sd[1], self.layer_structure, self.activation_func, self.weight_init, - self.add_layer_norm, self.activate_output, self.dropout_structure), - ) - - self.name = state_dict.get('name', self.name) - self.step = state_dict.get('step', 0) - self.sd_checkpoint = state_dict.get('sd_checkpoint', None) - self.sd_checkpoint_name = state_dict.get('sd_checkpoint_name', None) - self.eval() - - def shorthash(self): - sha256 = hashes.sha256(self.filename, f'hypernet/{self.name}') - - return sha256[0:10] if sha256 else None - - -def list_hypernetworks(path): - res = {} - for filename in sorted(glob.iglob(os.path.join(path, '**/*.pt'), recursive=True), key=str.lower): - name = os.path.splitext(os.path.basename(filename))[0] - # Prevent a hypothetical "None.pt" from being listed. - if name != "None": - res[name] = filename - return res - - -def load_hypernetwork(name): - path = shared.hypernetworks.get(name, None) - - if path is None: - return None - - try: - hypernetwork = Hypernetwork() - hypernetwork.load(path) - return hypernetwork - except Exception: - errors.report(f"Error loading hypernetwork {path}", exc_info=True) - return None - - -def load_hypernetworks(names, multipliers=None): - already_loaded = {} - - for hypernetwork in shared.loaded_hypernetworks: - if hypernetwork.name in names: - already_loaded[hypernetwork.name] = hypernetwork - - shared.loaded_hypernetworks.clear() - - for i, name in enumerate(names): - hypernetwork = already_loaded.get(name, None) - if hypernetwork is None: - hypernetwork = load_hypernetwork(name) - - if hypernetwork is None: - continue - - hypernetwork.set_multiplier(multipliers[i] if multipliers else 1.0) - shared.loaded_hypernetworks.append(hypernetwork) - - -def apply_single_hypernetwork(hypernetwork, context_k, context_v, layer=None): - hypernetwork_layers = (hypernetwork.layers if hypernetwork is not None else {}).get(context_k.shape[2], None) - - if hypernetwork_layers is None: - return context_k, context_v - - if layer is not None: - layer.hyper_k = hypernetwork_layers[0] - layer.hyper_v = hypernetwork_layers[1] - - context_k = devices.cond_cast_unet(hypernetwork_layers[0](devices.cond_cast_float(context_k))) - context_v = devices.cond_cast_unet(hypernetwork_layers[1](devices.cond_cast_float(context_v))) - return context_k, context_v - - -def apply_hypernetworks(hypernetworks, context, layer=None): - context_k = context - context_v = context - for hypernetwork in hypernetworks: - context_k, context_v = apply_single_hypernetwork(hypernetwork, context_k, context_v, layer) - - return context_k, context_v - - -def attention_CrossAttention_forward(self, x, context=None, mask=None, **kwargs): - h = self.heads - - q = self.to_q(x) - context = default(context, x) - - context_k, context_v = apply_hypernetworks(shared.loaded_hypernetworks, context, self) - k = self.to_k(context_k) - v = self.to_v(context_v) - - q, k, v = (rearrange(t, 'b n (h d) -> (b h) n d', h=h) for t in (q, k, v)) - - sim = einsum('b i d, b j d -> b i j', q, k) * self.scale - - if mask is not None: - mask = rearrange(mask, 'b ... -> b (...)') - max_neg_value = -torch.finfo(sim.dtype).max - mask = repeat(mask, 'b j -> (b h) () j', h=h) - sim.masked_fill_(~mask, max_neg_value) - - # attention, what we cannot get enough of - attn = sim.softmax(dim=-1) - - out = einsum('b i j, b j d -> b i d', attn, v) - out = rearrange(out, '(b h) n d -> b n (h d)', h=h) - return self.to_out(out) - - -def stack_conds(conds): - if len(conds) == 1: - return torch.stack(conds) - - # same as in reconstruct_multicond_batch - token_count = max([x.shape[0] for x in conds]) - for i in range(len(conds)): - if conds[i].shape[0] != token_count: - last_vector = conds[i][-1:] - last_vector_repeated = last_vector.repeat([token_count - conds[i].shape[0], 1]) - conds[i] = torch.vstack([conds[i], last_vector_repeated]) - - return torch.stack(conds) - - -def statistics(data): - if len(data) < 2: - std = 0 - else: - std = stdev(data) - total_information = f"loss:{mean(data):.3f}" + u"\u00B1" + f"({std/ (len(data) ** 0.5):.3f})" - recent_data = data[-32:] - if len(recent_data) < 2: - std = 0 - else: - std = stdev(recent_data) - recent_information = f"recent 32 loss:{mean(recent_data):.3f}" + u"\u00B1" + f"({std / (len(recent_data) ** 0.5):.3f})" - return total_information, recent_information diff --git a/modules/hypernetworks/ui.py b/modules/hypernetworks/ui.py deleted file mode 100644 index 8b6255e2..00000000 --- a/modules/hypernetworks/ui.py +++ /dev/null @@ -1,38 +0,0 @@ -import html - -import gradio as gr -import modules.hypernetworks.hypernetwork -from modules import devices, sd_hijack, shared - -not_available = ["hardswish", "multiheadattention"] -keys = [x for x in modules.hypernetworks.hypernetwork.HypernetworkModule.activation_dict if x not in not_available] - - -def create_hypernetwork(name, enable_sizes, overwrite_old, layer_structure=None, activation_func=None, weight_init=None, add_layer_norm=False, use_dropout=False, dropout_structure=None): - filename = modules.hypernetworks.hypernetwork.create_hypernetwork(name, enable_sizes, overwrite_old, layer_structure, activation_func, weight_init, add_layer_norm, use_dropout, dropout_structure) - - return gr.Dropdown.update(choices=sorted(shared.hypernetworks)), f"Created: {filename}", "" - - -def train_hypernetwork(*args): - shared.loaded_hypernetworks = [] - - assert not shared.cmd_opts.lowvram, 'Training models with lowvram is not possible' - - try: - sd_hijack.undo_optimizations() - - hypernetwork, filename = modules.hypernetworks.hypernetwork.train_hypernetwork(*args) - - res = f""" -Training {'interrupted' if shared.state.interrupted else 'finished'} at {hypernetwork.step} steps. -Hypernetwork saved to {html.escape(filename)} -""" - return res, "" - except Exception: - raise - finally: - shared.sd_model.cond_stage_model.to(devices.device) - shared.sd_model.first_stage_model.to(devices.device) - sd_hijack.apply_optimizations() - diff --git a/modules/infotext_utils.py b/modules/infotext_utils.py index cb703acd..5f12c262 100644 --- a/modules/infotext_utils.py +++ b/modules/infotext_utils.py @@ -18,7 +18,6 @@ sys.modules['modules.generation_parameters_copypaste'] = sys.modules[__name__] re_param_code = r'\s*(\w[\w \-/]+):\s*("(?:\\.|[^\\"])+"|[^,]*)(?:,|$)' re_param = re.compile(re_param_code) re_imagesize = re.compile(r"^(\d+)x(\d+)$") -re_hypernet_hash = re.compile("\(([0-9a-f]+)\)$") type_of_gr_update = type(gr.update()) @@ -341,10 +340,6 @@ Steps: 20, Sampler: Euler a, CFG scale: 7, Seed: 965400086, Size: 512x512, Model if "Clip skip" not in res: res["Clip skip"] = "1" - hypernet = res.get("Hypernet", None) - if hypernet is not None: - res["Prompt"] += f"""""" - if "Hires resize-1" not in res: res["Hires resize-1"] = 0 res["Hires resize-2"] = 0 diff --git a/modules/initialize.py b/modules/initialize.py index 1fa57910..c59dad65 100644 --- a/modules/initialize.py +++ b/modules/initialize.py @@ -122,17 +122,10 @@ def initialize_rest(*, reload_script_modules=False): sd_unet.list_unets() startup_timer.record("scripts list_unets") - from modules import shared_items - shared_items.reload_hypernetworks() - startup_timer.record("reload hypernetworks") - from modules import ui_extra_networks ui_extra_networks.initialize() ui_extra_networks.register_default_pages() from modules import extra_networks extra_networks.initialize() - extra_networks.register_default_extra_networks() startup_timer.record("initialize extra networks") - - return diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 483a2f9d..e8403e1b 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -376,7 +376,6 @@ def configure_forge_reference_checkout(a1111_home: Path): refs = [ ModelRef(arg_name="--ckpt-dir", relative_path="models/Stable-diffusion"), ModelRef(arg_name="--vae-dir", relative_path="models/VAE"), - ModelRef(arg_name="--hypernetwork-dir", relative_path="models/hypernetworks"), ModelRef(arg_name="--embeddings-dir", relative_path="embeddings"), ModelRef(arg_name="--lora-dir", relative_path="models/lora"), # Ref A1111 need to have sd-webui-controlnet installed. diff --git a/modules/shared.py b/modules/shared.py index 01cdd82e..750315db 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -29,10 +29,6 @@ weight_load_location: str = None xformers_available = memory_management.xformers_enabled() -hypernetworks = {} - -loaded_hypernetworks = [] - state: 'shared_state.State' = None prompt_styles: 'styles.StyleDatabase' = None @@ -88,6 +84,5 @@ reload_gradio_theme = shared_gradio_themes.reload_gradio_theme list_checkpoint_tiles = shared_items.list_checkpoint_tiles refresh_checkpoints = shared_items.refresh_checkpoints list_samplers = shared_items.list_samplers -reload_hypernetworks = shared_items.reload_hypernetworks hf_endpoint = os.getenv('HF_ENDPOINT', 'https://huggingface.co') diff --git a/modules/shared_init.py b/modules/shared_init.py index 38f1c324..b6823077 100644 --- a/modules/shared_init.py +++ b/modules/shared_init.py @@ -7,13 +7,11 @@ from modules.shared import cmd_opts def initialize(): - """Initializes fields inside the shared module in a controlled manner. - + """ + Initializes fields inside the shared module in a controlled manner. Should be called early because some other modules you can import mingt need these fields to be already set. """ - os.makedirs(cmd_opts.hypernetwork_dir, exist_ok=True) - from modules import options, shared_options shared.options_templates = shared_options.options_templates shared.opts = options.Options(shared_options.options_templates, shared_options.restricted_opts) @@ -39,4 +37,3 @@ def initialize(): from modules import memmon, devices shared.mem_mon = memmon.MemUsageMonitor("MemMon", devices.device, shared.opts) shared.mem_mon.start() - diff --git a/modules/shared_items.py b/modules/shared_items.py index 31730c14..c86dd2f6 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -65,13 +65,6 @@ def list_samplers(): return modules.sd_samplers.all_samplers -def reload_hypernetworks(): - from modules.hypernetworks import hypernetwork - from modules import shared - - shared.hypernetworks = hypernetwork.list_hypernetworks(cmd_opts.hypernetwork_dir) - - def get_infotext_names(): from modules import infotext_utils, shared res = {} diff --git a/modules/shared_options.py b/modules/shared_options.py index f287a8ec..ffadc67d 100644 --- a/modules/shared_options.py +++ b/modules/shared_options.py @@ -125,7 +125,6 @@ options_templates.update(options_section(('system', "System", "system"), { "samples_log_stdout": OptionInfo(False, "Always print all generation info to standard output"), "multiple_tqdm": OptionInfo(True, "Add a second progress bar to the console that shows progress for an entire job."), "enable_upscale_progressbar": OptionInfo(True, "Show a progress bar in the console for tiled upscaling."), - "print_hypernet_extra": OptionInfo(False, "Print extra hypernetwork information to console."), "list_hidden_files": OptionInfo(True, "Load models/files in hidden directories").info("directory is hidden if its name starts with \".\""), "disable_mmap_load_safetensors": OptionInfo(False, "Disable memmapping for loading .safetensors files.").info("fixes very slow loading speed in some cases"), "hide_ldm_prints": OptionInfo(True, "Prevent Stability-AI's ldm/sgm modules from printing noise to console."), @@ -281,7 +280,6 @@ options_templates.update(options_section(('extra_networks', "Extra Networks", "s "ui_extra_networks_tab_reorder": OptionInfo("", "Extra networks tab order").needs_reload_ui(), "textual_inversion_print_at_load": OptionInfo(False, "Print a list of Textual Inversion embeddings when loading model"), "textual_inversion_add_hashes_to_infotext": OptionInfo(True, "Add Textual Inversion hashes to infotext"), - "sd_hypernetwork": OptionInfo("None", "Add hypernetwork to prompt", gr.Dropdown, lambda: {"choices": ["None", *shared.hypernetworks]}, refresh=shared_items.reload_hypernetworks), })) options_templates.update(options_section(('ui_prompt_editing', "Prompt editing", "ui"), { diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index cd3ec346..7850efb1 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -708,10 +708,8 @@ def initialize(): def register_default_pages(): from modules.ui_extra_networks_textual_inversion import ExtraNetworksPageTextualInversion - # from modules.ui_extra_networks_hypernets import ExtraNetworksPageHypernetworks from modules.ui_extra_networks_checkpoints import ExtraNetworksPageCheckpoints register_page(ExtraNetworksPageTextualInversion()) - # register_page(ExtraNetworksPageHypernetworks()) register_page(ExtraNetworksPageCheckpoints()) diff --git a/modules/ui_extra_networks_hypernets.py b/modules/ui_extra_networks_hypernets.py deleted file mode 100644 index 2fb4bd19..00000000 --- a/modules/ui_extra_networks_hypernets.py +++ /dev/null @@ -1,48 +0,0 @@ -import os - -from modules import shared, ui_extra_networks -from modules.ui_extra_networks import quote_js -from modules.hashes import sha256_from_cache - - -class ExtraNetworksPageHypernetworks(ui_extra_networks.ExtraNetworksPage): - def __init__(self): - super().__init__('Hypernetworks') - - def refresh(self): - shared.reload_hypernetworks() - - def create_item(self, name, index=None, enable_filter=True): - full_path = shared.hypernetworks.get(name) - if full_path is None: - return - - path, ext = os.path.splitext(full_path) - sha256 = sha256_from_cache(full_path, f'hypernet/{name}') - shorthash = sha256[0:10] if sha256 else None - search_terms = [self.search_terms_from_path(path)] - if sha256: - search_terms.append(sha256) - return { - "name": name, - "filename": full_path, - "shorthash": shorthash, - "preview": self.find_preview(path), - "description": self.find_description(path), - "search_terms": search_terms, - "prompt": quote_js(f""), - "local_preview": f"{path}.preview.{shared.opts.samples_format}", - "sort_keys": {'default': index, **self.get_sort_keys(path + ext)}, - } - - def list_items(self): - # instantiate a list to protect against concurrent modification - names = list(shared.hypernetworks) - for index, name in enumerate(names): - item = self.create_item(name, index) - if item is not None: - yield item - - def allowed_directories_for_previews(self): - return [shared.cmd_opts.hypernetwork_dir] -