diff --git a/backend/attention.py b/backend/attention.py index 7d652038..3fc79362 100644 --- a/backend/attention.py +++ b/backend/attention.py @@ -389,12 +389,15 @@ def slice_attention_vae(q, k, v): r1[:, :, i:end] = torch.bmm(v, s2) del s2 break - except memory_management.OOM_EXCEPTION as e: - memory_management.soft_empty_cache(True) - steps *= 2 + except Exception as e: + if not memory_management.is_oom(e): + raise e if steps > 128: raise e - logger.warning(f"Out of Memory Error; trying again... ({steps})") + + logger.warning("Out of Memory Error; retrying with higher steps...") + memory_management.soft_empty_cache() + steps *= 2 return r1 @@ -450,11 +453,14 @@ def pytorch_attention_vae(q, k, v): out = operations.scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=0.0, is_causal=False) out = out.transpose(2, 3).reshape(orig_shape) _fallback = False - except memory_management.OOM_EXCEPTION: + except Exception as e: + if not memory_management.is_oom(e): + raise e logger.warning("Out of Memory Error; retrying with Slice Attention") _fallback = True if _fallback: + memory_management.soft_empty_cache() out = slice_attention_vae(q.view(B, -1, C), k.view(B, -1, C).transpose(1, 2), v.view(B, -1, C).transpose(1, 2)).reshape(orig_shape) return out diff --git a/backend/memory_management.py b/backend/memory_management.py index 900731e5..ae81a226 100644 --- a/backend/memory_management.py +++ b/backend/memory_management.py @@ -196,6 +196,17 @@ except Exception: pass OOM_EXCEPTION = getattr(torch, "OutOfMemoryError", Exception) +ACCELERATOR_ERROR = getattr(torch, "AcceleratorError", RuntimeError) + + +def is_oom(e: Exception) -> bool: + if isinstance(e, OOM_EXCEPTION): + return True + if isinstance(e, ACCELERATOR_ERROR) or "out of memory" in str(e).lower(): + discard_cuda_async_error() + return True + return False + if args.disable_xformers: XFORMERS_IS_AVAILABLE = False diff --git a/backend/patcher/vae.py b/backend/patcher/vae.py index adbb83a7..db4af748 100644 --- a/backend/patcher/vae.py +++ b/backend/patcher/vae.py @@ -219,8 +219,10 @@ class VAE: if pixel_samples is None: pixel_samples = torch.empty((samples_in.shape[0],) + tuple(out.shape[1:]), device=self.output_device) pixel_samples[x : x + batch_number] = out - except memory_management.OOM_EXCEPTION: - print("Warning: Encountered Out of Memory during VAE decoding; Retrying with Tiled VAE Decoding...") + except Exception as e: + if not memory_management.is_oom(e): + raise e + memory_management.logger.warning("Encountered Out of Memory during VAE decoding; Retrying with Tiled VAE Decoding...") _tile = True if _tile: @@ -270,8 +272,10 @@ class VAE: samples = torch.empty((_samples.shape[0],) + tuple(out.shape[1:]), device=self.output_device) samples[x : x + batch_number] = out _tile = False - except memory_management.OOM_EXCEPTION: - print("Warning: Encountered Out of Memory during VAE Encoding; Retrying with Tiled VAE Encoding...") + except Exception as e: + if not memory_management.is_oom(e): + raise e + memory_management.logger.warning("Encountered Out of Memory during VAE Encoding; Retrying with Tiled VAE Encoding...") _tile = True if _tile: diff --git a/modules/call_queue.py b/modules/call_queue.py index f2a7ed00..07c5b39a 100644 --- a/modules/call_queue.py +++ b/modules/call_queue.py @@ -1,11 +1,11 @@ -import os.path -from functools import wraps import html +import os.path import time import traceback +from functools import wraps +from modules import devices, fifo_lock, profiling, progress, shared from modules_forge import main_thread -from modules import shared, progress, errors, devices, fifo_lock, profiling queue_lock = fifo_lock.FIFOLock() @@ -76,15 +76,14 @@ def wrap_gradio_call_no_job(func, extra_outputs=None, add_stats=False): res = list(func(*args, **kwargs)) except Exception as e: if main_thread.last_exception is not None: - e = main_thread.last_exception + error_message = main_thread.last_exception else: + error_message = f"{type(e).__name__}: {e}" traceback.print_exc() - print(e) if extra_outputs_array is None: - extra_outputs_array = [None, ''] + extra_outputs_array = [None, ""] - error_message = f'{type(e).__name__}: {e}' res = extra_outputs_array + [f"
{text_a},
[ Profile ]
" else: - profiling_html = '' + profiling_html = "" # last item is always HTML res[-1] += f"Time taken: