When using the model encoded with fp8, a CUDA error occurs. #28
Description
Although a normal model is created intermittently, it is very infrequent.
It works fine with FP16, but when you use FP8, an error always appears.
Exception in thread MemMon:
Traceback (most recent call last):
File "C:\Users\Diana\AppData\Local\Programs\Python\Python310\lib\threading.py", line 1016, in _bootstrap_inner
self.run()
File "W:\SynologyDrive\PonyXL\modules\memmon.py", line 53, in run
free, total = self.cuda_mem_get_info()
File "W:\SynologyDrive\PonyXL\modules\memmon.py", line 34, in cuda_mem_get_info
return torch.cuda.mem_get_info(index)
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\torch\cuda\memory.py", line 663, in mem_get_info
return torch.cuda.cudart().cudaMemGetInfo(device)
RuntimeError: CUDA error: device-side assert triggered
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
*** Error completing request
*** Arguments: ('task(xg4j7qzwrv8wv7p)', <gradio.routes.Request object at 0x000001CD8E63A710>, '__pony_p__, (__positive__),\n__001/hair_style_women__ hair, __001/color__ color hair, small_head,\nperfect __001/face__:0.5, __001/color__ color eyes, beautiful clear eyes, facial blush, glow parted lips, __001/expression__:0.5,\n__001/body_type__ body, __001/breast_size__, covered_nipples:0.1, cameltoe:0.3, small hands and feet,\n{hanbok|__001/fashion_all__|__001/clothes-latex__|__001/clothes-swimsuit__|__001/clothes-brezier__|__001/clothes-panty__|__001/clothes-lingerie__|__001/female_top__ and __001/female_bottom__|__001/female_see_through_clothes__|__001/female_undies__|__001/clothes-preppy_look__|__001/clothes-women_costume__|__001/clothes-women_suit__|__001/clothes-dress__|__001/shorts__|__001/skirt__|__001/wedding_dress__|__001/fashion_all__|__001/fashion_spring__|__001/fashion_summer__|__001/fashion_fall__|__001/fashion_winter__}, {__001/clothes-stockings__|__001/clothes-shoes__|bare foot}, __001/angle__, {__001/pose__|__001/female_poses__|dynamic pose},\n__001/Mood__, __001/weather__, __001/time__,\n{__001/background__|__001/best_cities__|__001/landmark__|__001/landscapes__|__001/landscape_composition__|__001/flower_garden__|__001/place__|__001/ocean__|__001/seaside_scenery__|__001/place_indoor__|__001/place_outdoor__|__001/travel_list_100__|__001/spring__|__001/summer__|__001/autumn__|__001/winter__|__001/global_destinations_500__|__001/travel_list_100__|__001/world_walks__|__001/world_small_towns__|__001/world_hikes__|__001/wonders_list__|__001/weirdest_places__},', '__pony_n__, __negative__, (nude), NSFW, BeyondSDXLv3,', [], 1, 1, 3, 960, 768, True, 0.23, 2, '4x-UltraMix_Balanced', 23, 0, 0, 'Use same checkpoint', 'Use same sampler', 'Use same scheduler', '', '', [], 0, 10, 'DPM++ 2S a', 'SGM Uniform', False, '', 0.8, -1, False, -1, 0, 0, 0, 0.03, True, 0.0, 2, 2048.0, 512, 512, True, 'ESRGAN_4x', 'None', 0, True, False, {'ad_model': 'deepfashion2_yolov8s-seg.pt', 'ad_model_classes': '', 'ad_tab_enable': True, 'ad_prompt': '__clothes_positive__, covered_nipples:0.1, cameltoe:0.3, ', 'ad_negative_prompt': '__clothes_negative__,', 'ad_confidence': 0.3, 'ad_mask_filter_method': 'Area', 'ad_mask_k': 0, 'ad_mask_min_ratio': 0, 'ad_mask_max_ratio': 1, 'ad_x_offset': 0, 'ad_y_offset': 0, 'ad_dilate_erode': 4, 'ad_mask_merge_invert': 'None', 'ad_mask_blur': 9, 'ad_denoising_strength': 0.17, 'ad_inpaint_only_masked': True, 'ad_inpaint_only_masked_padding': 32, 'ad_use_inpaint_width_height': False, 'ad_inpaint_width': 512, 'ad_inpaint_height': 512, 'ad_use_steps': False, 'ad_steps': 28, 'ad_use_cfg_scale': False, 'ad_cfg_scale': 3, 'ad_use_checkpoint': False, 'ad_checkpoint': 'Use same checkpoint', 'ad_use_vae': False, 'ad_vae': 'Use same VAE', 'ad_use_sampler': False, 'ad_sampler': 'DPM++ 2M', 'ad_scheduler': 'Use same scheduler', 'ad_use_noise_multiplier': False, 'ad_noise_multiplier': 0.5, 'ad_use_clip_skip': False, 'ad_clip_skip': 2, 'ad_restore_face': False, 'ad_controlnet_model': 'None', 'ad_controlnet_module': 'None', 'ad_controlnet_weight': 1, 'ad_controlnet_guidance_start': 0, 'ad_controlnet_guidance_end': 1, 'is_api': ()}, {'ad_model': 'hand_yolov9c.pt', 'ad_model_classes': '', 'ad_tab_enable': True, 'ad_prompt': '__hand_positive__,', 'ad_negative_prompt': '__hand_negative__,', 'ad_confidence': 0.3, 'ad_mask_filter_method': 'Area', 'ad_mask_k': 0, 'ad_mask_min_ratio': 0, 'ad_mask_max_ratio': 1, 'ad_x_offset': 0, 'ad_y_offset': 0, 'ad_dilate_erode': 4, 'ad_mask_merge_invert': 'None', 'ad_mask_blur': 9, 'ad_denoising_strength': 0.17, 'ad_inpaint_only_masked': True, 'ad_inpaint_only_masked_padding': 32, 'ad_use_inpaint_width_height': False, 'ad_inpaint_width': 512, 'ad_inpaint_height': 512, 'ad_use_steps': False, 'ad_steps': 28, 'ad_use_cfg_scale': False, 'ad_cfg_scale': 3, 'ad_use_checkpoint': False, 'ad_checkpoint': 'Use same checkpoint', 'ad_use_vae': False, 'ad_vae': 'Use same VAE', 'ad_use_sampler': False, 'ad_sampler': 'DPM++ 2M', 'ad_scheduler': 'Use same scheduler', 'ad_use_noise_multiplier': False, 'ad_noise_multiplier': 0.5, 'ad_use_clip_skip': False, 'ad_clip_skip': 2, 'ad_restore_face': False, 'ad_controlnet_model': 'None', 'ad_controlnet_module': 'None', 'ad_controlnet_weight': 0.55, 'ad_controlnet_guidance_start': 0, 'ad_controlnet_guidance_end': 1, 'is_api': ()}, {'ad_model': 'foot-yolov8x.pt', 'ad_model_classes': '', 'ad_tab_enable': True, 'ad_prompt': '__feet_positive__,', 'ad_negative_prompt': '__feet_negative__,', 'ad_confidence': 0.3, 'ad_mask_filter_method': 'Area', 'ad_mask_k': 0, 'ad_mask_min_ratio': 0, 'ad_mask_max_ratio': 1, 'ad_x_offset': 0, 'ad_y_offset': 0, 'ad_dilate_erode': 4, 'ad_mask_merge_invert': 'None', 'ad_mask_blur': 9, 'ad_denoising_strength': 0.17, 'ad_inpaint_only_masked': True, 'ad_inpaint_only_masked_padding': 32, 'ad_use_inpaint_width_height': False, 'ad_inpaint_width': 512, 'ad_inpaint_height': 512, 'ad_use_steps': False, 'ad_steps': 28, 'ad_use_cfg_scale': False, 'ad_cfg_scale': 3, 'ad_use_checkpoint': False, 'ad_checkpoint': 'Use same checkpoint', 'ad_use_vae': False, 'ad_vae': 'Use same VAE', 'ad_use_sampler': False, 'ad_sampler': 'DDIM', 'ad_scheduler': 'Beta', 'ad_use_noise_multiplier': False, 'ad_noise_multiplier': 0.5, 'ad_use_clip_skip': False, 'ad_clip_skip': 2, 'ad_restore_face': False, 'ad_controlnet_model': 'None', 'ad_controlnet_module': 'None', 'ad_controlnet_weight': 1, 'ad_controlnet_guidance_start': 0, 'ad_controlnet_guidance_end': 1, 'is_api': ()}, {'ad_model': 'face_yolov9c.pt', 'ad_model_classes': '', 'ad_tab_enable': True, 'ad_prompt': '__face_positive__, sexy korean girl,', 'ad_negative_prompt': '__face_negative__, western girl, american girl,', 'ad_confidence': 0.3, 'ad_mask_filter_method': 'Area', 'ad_mask_k': 0, 'ad_mask_min_ratio': 0, 'ad_mask_max_ratio': 1, 'ad_x_offset': 0, 'ad_y_offset': 0, 'ad_dilate_erode': 4, 'ad_mask_merge_invert': 'None', 'ad_mask_blur': 9, 'ad_denoising_strength': 0.17, 'ad_inpaint_only_masked': True, 'ad_inpaint_only_masked_padding': 32, 'ad_use_inpaint_width_height': False, 'ad_inpaint_width': 512, 'ad_inpaint_height': 512, 'ad_use_steps': False, 'ad_steps': 28, 'ad_use_cfg_scale': False, 'ad_cfg_scale': 7, 'ad_use_checkpoint': False, 'ad_checkpoint': 'Use same checkpoint', 'ad_use_vae': False, 'ad_vae': 'Use same VAE', 'ad_use_sampler': False, 'ad_sampler': 'DPM++ 2M', 'ad_scheduler': 'Use same scheduler', 'ad_use_noise_multiplier': False, 'ad_noise_multiplier': 1, 'ad_use_clip_skip': False, 'ad_clip_skip': 1, 'ad_restore_face': False, 'ad_controlnet_model': 'None', 'ad_controlnet_module': 'None', 'ad_controlnet_weight': 1, 'ad_controlnet_guidance_start': 0, 'ad_controlnet_guidance_end': 1, 'is_api': ()}, True, False, 1, False, False, False, 1.1, 1.5, 100, 0.7, False, False, True, False, False, 0, 'Gustavosta/MagicPrompt-Stable-Diffusion', '', False, 3, 99.3, 'Half Cosine Up', 0, 'Power Up', 3, 13.5, True, 'MEAN', 'AD', 0.97, True, 'Use same checkpoint', 'Use same vae', 1, 0, 'None', 'None', 'None', False, 0.15, 3, 0.4, 4, 'bicubic', 0.5, 2, True, False, True, False, False, False, 'Use same checkpoint', 'Use same vae', 'txt2img-1pass', 'None', '', '', 'Use same sampler', 'Use same scheduler', 'BMAB fast', 20, 7, 0.75, 0.5, 0, 1, False, False, 'Use same checkpoint', 'Use same vae', 'Select Model', 'None', '', '', 'Use same sampler', 'Use same scheduler', 20, 7, 0.75, 4, 0.35, False, 50, 200, 0.5, False, True, 'stretching', 'bottom', 'None', 0.85, 0.75, False, 'Use same checkpoint', 'Use same vae', True, '', '', 'Use same sampler', 'Use same scheduler', 'BMAB fast', 20, 7, 0.75, 1, 0, 0, 0.95, 1.1, 1.3, 1, 0, 0, 0, None, False, 1, False, '', False, False, False, True, True, 'Use same checkpoint', 'Use same vae', 'Use same sampler', 'Use same scheduler', 4, 2, 0.1, 1, 1, 0, 0.17, 3, 20, False, False, False, 'Score', 1, '', '', '', '', '', '', '', '', '', '', True, 1024, 1024, 3, 20, 64, 'Use same checkpoint', 'Use same vae', 'Use same sampler', 'Use same scheduler', 'Only masked', 32, 'BMAB Face(Normal)', 0.13, 5, 0.3, False, 0.26, False, True, False, 'subframe', '', '', 0.17, 3, True, 4, 0.3, 0.1, 'Only masked', 32, '', False, False, False, 0.4, 0.1, 0.9, 'Both', False, 0.3, 0, 1, False, 'Random', False, 0.3, 0, 1, 'Random', 'normal', False, False, 'normal', '', 'None', 0.5, False, False, 'Inpaint', 0.85, 0.4, 10, False, True, 'None', 1.5, 'None', True, 'AniMeMatrix-MB_EN.ttf', 'bottom-right', 'right', '0', '#000000', '#000000', 12, 100, 0, 0, 'W:\\SynologyDrive\\PonyXL\\AI_Mark_Pony.png', 'Defult', 'None', ControlNetUnit(is_ui=True, input_mode=<InputMode.SIMPLE: 'simple'>, batch_images='', output_dir='', loopback=False, enabled=False, module='depth_midas', model='None', weight=1.0, image=None, resize_mode=<ResizeMode.INNER_FIT: 'Crop and Resize'>, low_vram=False, processor_res=512, threshold_a=-1.0, threshold_b=-1.0, guidance_start=0.0, guidance_end=1.0, pixel_perfect=False, control_mode=<ControlMode.BALANCED: 'Balanced'>, inpaint_crop_input_image=False, hr_option=<HiResFixOption.BOTH: 'Both'>, save_detected_map=True, advanced_weighting=None, effective_region_mask=None, pulid_mode=<PuLIDMode.FIDELITY: 'Fidelity'>, union_control_type=<ControlNetUnionControlType.UNKNOWN: 'Unknown'>, ipadapter_input=None, mask=None, batch_mask_dir=None, animatediff_batch=False, batch_modifiers=[], batch_image_files=[], batch_keyframe_idx=None), ControlNetUnit(is_ui=True, input_mode=<InputMode.SIMPLE: 'simple'>, batch_images='', output_dir='', loopback=False, enabled=False, module='depth_midas', model='None', weight=1.0, image=None, resize_mode=<ResizeMode.INNER_FIT: 'Crop and Resize'>, low_vram=False, processor_res=512, threshold_a=-1.0, threshold_b=-1.0, guidance_start=0.0, guidance_end=1.0, pixel_perfect=False, control_mode=<ControlMode.BALANCED: 'Balanced'>, inpaint_crop_input_image=False, hr_option=<HiResFixOption.BOTH: 'Both'>, save_detected_map=True, advanced_weighting=None, effective_region_mask=None, pulid_mode=<PuLIDMode.FIDELITY: 'Fidelity'>, union_control_type=<ControlNetUnionControlType.UNKNOWN: 'Unknown'>, ipadapter_input=None, mask=None, batch_mask_dir=None, animatediff_batch=False, batch_modifiers=[], batch_image_files=[], batch_keyframe_idx=None), ControlNetUnit(is_ui=True, input_mode=<InputMode.SIMPLE: 'simple'>, batch_images='', output_dir='', loopback=False, enabled=False, module='depth_midas', model='None', weight=1.0, image=None, resize_mode=<ResizeMode.INNER_FIT: 'Crop and Resize'>, low_vram=False, processor_res=512, threshold_a=-1.0, threshold_b=-1.0, guidance_start=0.0, guidance_end=1.0, pixel_perfect=False, control_mode=<ControlMode.BALANCED: 'Balanced'>, inpaint_crop_input_image=False, hr_option=<HiResFixOption.BOTH: 'Both'>, save_detected_map=True, advanced_weighting=None, effective_region_mask=None, pulid_mode=<PuLIDMode.FIDELITY: 'Fidelity'>, union_control_type=<ControlNetUnionControlType.UNKNOWN: 'Unknown'>, ipadapter_input=None, mask=None, batch_mask_dir=None, animatediff_batch=False, batch_modifiers=[], batch_image_files=[], batch_keyframe_idx=None), True, 'both', 0.2, 0.8, 0.71, 0.25, 1, 0, -0.15, 0, True, False, 0, 1, 0, 'Version 2', 1.1, 0.6, 0, 0.5, 0, 1, 1.2, 0.4, 0, 0.5, 0, 1, 1, 1, 0, 0.5, 0, 1, False, 11, 0, 150, False, 1, 0.8, 3, 0, 0, 150, 4, False, 0, 0, 150, False, 'Constant', 0, 100, False, False, False, 2, 0.1, 0.5, 0, '', 0, 25, 1, False, False, False, 'BREAK', '-', 0.2, 10, None, False, '0', '0', 'inswapper_128.onnx', 'CodeFormer', 1, True, 'None', 1, 1, False, True, 1, 0, 0, False, 0.5, True, False, 'CUDA', False, 0, 'None', '', None, False, False, 0.5, 0, 'tab_single', False, False, 'positive', 'comma', 0, False, False, 'start', '', 0, '', [], 0, '', [], 0, '', [], False, False, False, True, False, False, False, 0, False, None, None, False, None, None, False, None, None, False, 50) {}
Traceback (most recent call last):
File "W:\SynologyDrive\PonyXL\modules\call_queue.py", line 74, in f
res = list(func(*args, **kwargs))
File "W:\SynologyDrive\PonyXL\modules\call_queue.py", line 53, in f
res = func(*args, **kwargs)
File "W:\SynologyDrive\PonyXL\modules\call_queue.py", line 37, in f
res = func(*args, **kwargs)
File "W:\SynologyDrive\PonyXL\modules\txt2img.py", line 109, in txt2img
processed = processing.process_images(p)
File "W:\SynologyDrive\PonyXL\modules\processing.py", line 847, in process_images
res = process_images_inner(p)
File "W:\SynologyDrive\PonyXL\extensions\sd-webui-controlnet\scripts\batch_hijack.py", line 59, in processing_process_images_hijack
return getattr(processing, '__controlnet_original_process_images_inner')(p, *args, **kwargs)
File "W:\SynologyDrive\PonyXL\modules\processing.py", line 966, in process_images_inner
p.setup_conds()
File "W:\SynologyDrive\PonyXL\modules\processing.py", line 1520, in setup_conds
super().setup_conds()
File "W:\SynologyDrive\PonyXL\modules\processing.py", line 502, in setup_conds
self.uc = self.get_conds_with_caching(prompt_parser.get_learned_conditioning, negative_prompts, total_steps, [self.cached_uc], self.extra_network_data)
File "W:\SynologyDrive\PonyXL\modules\processing.py", line 488, in get_conds_with_caching
cache[1] = function(shared.sd_model, required_prompts, steps, hires_steps, shared.opts.use_old_scheduling)
File "W:\SynologyDrive\PonyXL\modules\prompt_parser.py", line 188, in get_learned_conditioning
conds = model.get_learned_conditioning(texts)
File "W:\SynologyDrive\PonyXL\modules\sd_models_xl.py", line 32, in get_learned_conditioning
c = self.conditioner(sdxl_conds, force_zero_embeddings=['txt'] if force_zero_negative_prompt else [])
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\torch\nn\modules\module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\torch\nn\modules\module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "W:\SynologyDrive\PonyXL\repositories\generative-models\sgm\modules\encoders\modules.py", line 141, in forward
emb_out = embedder(batch[embedder.input_key])
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\torch\nn\modules\module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\torch\nn\modules\module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "W:\SynologyDrive\PonyXL\modules\sd_hijack_clip.py", line 313, in forward
return super().forward(texts)
File "W:\SynologyDrive\PonyXL\modules\sd_hijack_clip.py", line 227, in forward
z = self.process_tokens(tokens, multipliers)
File "W:\SynologyDrive\PonyXL\modules\sd_hijack_clip.py", line 269, in process_tokens
z = self.encode_with_transformers(tokens)
File "W:\SynologyDrive\PonyXL\modules\sd_hijack_clip.py", line 375, in encode_with_transformers
outputs = self.wrapped.transformer(input_ids=tokens, output_hidden_states=self.wrapped.layer == "hidden")
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\torch\nn\modules\module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\torch\nn\modules\module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\transformers\models\clip\modeling_clip.py", line 822, in forward
return self.text_model(
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\torch\nn\modules\module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\torch\nn\modules\module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\transformers\models\clip\modeling_clip.py", line 734, in forward
causal_attention_mask = _make_causal_mask(input_shape, hidden_states.dtype, device=hidden_states.device)
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\transformers\models\clip\modeling_clip.py", line 684, in _make_causal_mask
mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device)
RuntimeError: CUDA error: device-side assert triggered
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
---
Traceback (most recent call last):
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\gradio\routes.py", line 488, in run_predict
output = await app.get_blocks().process_api(
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\gradio\blocks.py", line 1431, in process_api
result = await self.call_function(
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\gradio\blocks.py", line 1103, in call_function
prediction = await anyio.to_thread.run_sync(
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\anyio\to_thread.py", line 33, in run_sync
return await get_asynclib().run_sync_in_worker_thread(
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\anyio\_backends\_asyncio.py", line 877, in run_sync_in_worker_thread
return await future
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\anyio\_backends\_asyncio.py", line 807, in run
result = context.run(func, *args)
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\gradio\utils.py", line 707, in wrapper
response = f(*args, **kwargs)
File "W:\SynologyDrive\PonyXL\modules\call_queue.py", line 104, in f
mem_stats = {k: -(v//-(1024*1024)) for k, v in shared.mem_mon.stop().items()}
File "W:\SynologyDrive\PonyXL\modules\memmon.py", line 92, in stop
return self.read()
File "W:\SynologyDrive\PonyXL\modules\memmon.py", line 77, in read
free, total = self.cuda_mem_get_info()
File "W:\SynologyDrive\PonyXL\modules\memmon.py", line 34, in cuda_mem_get_info
return torch.cuda.mem_get_info(index)
File "W:\SynologyDrive\PonyXL\venv\lib\site-packages\torch\cuda\memory.py", line 663, in mem_get_info
return torch.cuda.cudart().cudaMemGetInfo(device)
RuntimeError: CUDA error: device-side assert triggered
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.