fast

backend/memory_management.py

@@ -147,6 +147,15 @@ except Exception:
 if directml_enabled:
     OOM_EXCEPTION = Exception
 
+if args.fast_fp16:
+    _ver = str(torch.version.__version__)
+    if int(_ver[0]) >= 2 and int(_ver[2]) >= 7:
+        torch.backends.cuda.allow_fp16_bf16_reduction_math_sdp(True)
+        torch.backends.cuda.matmul.allow_fp16_accumulation = True
+        print("allow_fp16_accumulation:", torch.backends.cuda.matmul.allow_fp16_accumulation)
+    else:
+        print("This version of pytorch does not support fp16_accumulation")
+
 XFORMERS_VERSION = ""
 XFORMERS_ENABLED_VAE = True
 if args.disable_xformers:

backend/operations.py

@@ -389,9 +389,9 @@ try:
                     with main_stream_worker(weight, bias, signal):
                         return functional_linear_4bits(x, weight, bias)
 
-    bnb_avaliable = True
+    bnb_available = True
 except:
-    bnb_avaliable = False
+    bnb_available = False
 
 
 from backend.operations_gguf import dequantize_tensor
@@ -452,7 +452,7 @@ def using_forge_operations(operations=None, device=None, dtype=None, manual_cast
     if operations is None:
         if bnb_dtype in ["gguf"]:
             operations = ForgeOperationsGGUF
-        elif bnb_avaliable and bnb_dtype in ["nf4", "fp4"]:
+        elif bnb_available and bnb_dtype in ["nf4", "fp4"]:
             operations = ForgeOperationsBNB4bits
         else:
             operations = ForgeOperations
@@ -515,55 +515,3 @@ def automatic_memory_management():
 
     print(f"Automatic Memory Management: {len(module_list)} Modules in {(end - start):.2f} seconds.")
     return
-
-
-class DynamicSwapInstaller:
-    @staticmethod
-    def _install_module(module: torch.nn.Module, target_device: torch.device):
-        original_class = module.__class__
-        module.__dict__["forge_backup_original_class"] = original_class
-
-        def hacked_get_attr(self, name: str):
-            if "_parameters" in self.__dict__:
-                _parameters = self.__dict__["_parameters"]
-                if name in _parameters:
-                    p = _parameters[name]
-                    if p is None:
-                        return None
-                    if p.__class__ == torch.nn.Parameter:
-                        return torch.nn.Parameter(p.to(target_device), requires_grad=p.requires_grad)
-                    else:
-                        return p.to(target_device)
-            if "_buffers" in self.__dict__:
-                _buffers = self.__dict__["_buffers"]
-                if name in _buffers:
-                    return _buffers[name].to(target_device)
-            return super(original_class, self).__getattr__(name)
-
-        module.__class__ = type(
-            "DynamicSwap_" + original_class.__name__,
-            (original_class,),
-            {
-                "__getattr__": hacked_get_attr,
-            },
-        )
-
-        return
-
-    @staticmethod
-    def _uninstall_module(module: torch.nn.Module):
-        if "forge_backup_original_class" in module.__dict__:
-            module.__class__ = module.__dict__.pop("forge_backup_original_class")
-        return
-
-    @staticmethod
-    def install_model(model: torch.nn.Module, target_device: torch.device):
-        for m in model.modules():
-            DynamicSwapInstaller._install_module(m, target_device)
-        return
-
-    @staticmethod
-    def uninstall_model(model: torch.nn.Module):
-        for m in model.modules():
-            DynamicSwapInstaller._uninstall_module(m)
-        return

backend/patcher/lora.py

@@ -383,7 +383,7 @@ class LoraLoader:
 
             bnb_layer = None
 
-            if hasattr(weight, "bnb_quantized") and operations.bnb_avaliable:
+            if hasattr(weight, "bnb_quantized") and operations.bnb_available:
                 bnb_layer = parent_layer
                 from backend.operations_bnb import functional_dequantize_4bit