freshly installed SDXL training problem #546
Description
This is for bugs only
Did you already ask in the discord?
No
You verified that this is a bug and not a feature request or question by asking in the discord?
No
Describe the bug
freshly installed sdxl training not working?
OS: CachyOS
GPU: Nvidia 4060 ti
Error Code:
`Running 1 job
Error running job: Failed to import diffusers.models.autoencoders.autoencoder_tiny because of the following error (look up to see its traceback):
/home/tobias/.triton/cache/QLAEYTJR4KV5WSBGJKRUAKVP475DE47NW7P4XMI2RFXBOIE5TZ4Q/cuda_utils.so: undefined symbol: cuModuleGetFunction
Error running on_error: cannot access local variable 'job' where it is not associated with a value
========================================
Result:
-
0 completed jobs
-
1 failure
========================================
Traceback (most recent call last):
Traceback (most recent call last):
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/utils/import_utils.py", line 1016, in _get_module
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/utils/import_utils.py", line 1016, in _get_module
return importlib.import_module("." + module_name, self.__name__)return importlib.import_module("." + module_name, self.__name__)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/importlib/init.py", line 90, in import_module
File "/usr/lib/python3.12/importlib/init.py", line 90, in import_module
return _bootstrap._gcd_import(name[level:], package, level)return _bootstrap._gcd_import(name[level:], package, level)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "", line 1387, in _gcd_import
File "", line 1387, in _gcd_import
File "", line 1360, in _find_and_load
File "", line 1360, in _find_and_load
File "", line 1310, in _find_and_load_unlocked
File "", line 1310, in _find_and_load_unlocked
File "", line 488, in _call_with_frames_removed
File "", line 488, in _call_with_frames_removed
File "", line 1387, in _gcd_import
File "", line 1387, in _gcd_import
File "", line 1360, in _find_and_load
File "", line 1360, in _find_and_load
File "", line 1331, in _find_and_load_unlocked
File "", line 1331, in _find_and_load_unlocked
File "", line 935, in _load_unlocked
File "", line 935, in _load_unlocked
File "", line 995, in exec_module
File "", line 995, in exec_module
File "", line 488, in _call_with_frames_removed
File "", line 488, in _call_with_frames_removed
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/models/autoencoders/init.py", line 1, in
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/models/autoencoders/init.py", line 1, in
from .autoencoder_asym_kl import AsymmetricAutoencoderKLfrom .autoencoder_asym_kl import AsymmetricAutoencoderKL
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/models/autoencoders/autoencoder_asym_kl.py", line 22, in
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/models/autoencoders/autoencoder_asym_kl.py", line 22, in
from ..modeling_utils import ModelMixinfrom ..modeling_utils import ModelMixin
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/models/modeling_utils.py", line 41, in
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/models/modeling_utils.py", line 41, in
from ..quantizers import DiffusersAutoQuantizer, DiffusersQuantizerfrom ..quantizers import DiffusersAutoQuantizer, DiffusersQuantizer
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/quantizers/init.py", line 16, in
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/quantizers/init.py", line 16, in
from .auto import DiffusersAutoQuantizerfrom .auto import DiffusersAutoQuantizer
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/quantizers/auto.py", line 22, in
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/quantizers/auto.py", line 22, in
from .bitsandbytes import BnB4BitDiffusersQuantizer, BnB8BitDiffusersQuantizerfrom .bitsandbytes import BnB4BitDiffusersQuantizer, BnB8BitDiffusersQuantizer
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/quantizers/bitsandbytes/init.py", line 2, in
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/quantizers/bitsandbytes/init.py", line 2, in
from .utils import dequantize_and_replace, dequantize_bnb_weight, replace_with_bnb_linearfrom .utils import dequantize_and_replace, dequantize_bnb_weight, replace_with_bnb_linear
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/quantizers/bitsandbytes/utils.py", line 32, in
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/quantizers/bitsandbytes/utils.py", line 32, in
import bitsandbytes as bnbimport bitsandbytes as bnb
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/bitsandbytes/init.py", line 20, in
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/bitsandbytes/init.py", line 20, in
from .nn import modulesfrom .nn import modules
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/bitsandbytes/nn/init.py", line 21, in
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/bitsandbytes/nn/init.py", line 21, in
from .triton_based_modules import (from .triton_based_modules import (
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/bitsandbytes/nn/triton_based_modules.py", line 6, in
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/bitsandbytes/nn/triton_based_modules.py", line 6, in
from bitsandbytes.triton.dequantize_rowwise import dequantize_rowwisefrom bitsandbytes.triton.dequantize_rowwise import dequantize_rowwise
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/bitsandbytes/triton/dequantize_rowwise.py", line 18, in
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/bitsandbytes/triton/dequantize_rowwise.py", line 18, in
@triton.autotune(@triton.autotune(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/triton/runtime/autotuner.py", line 378, in decorator
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/triton/runtime/autotuner.py", line 378, in decorator
return Autotuner(fn, fn.arg_names, configs, key, reset_to_zero, restore_value, pre_hook=pre_hook,return Autotuner(fn, fn.arg_names, configs, key, reset_to_zero, restore_value, pre_hook=pre_hook,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/triton/runtime/autotuner.py", line 130, in init
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/triton/runtime/autotuner.py", line 130, in init
self.do_bench = driver.active.get_benchmarker()self.do_bench = driver.active.get_benchmarker()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/triton/runtime/driver.py", line 23, in getattr
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/triton/runtime/driver.py", line 23, in getattr
self._initialize_obj()self._initialize_obj()
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/triton/runtime/driver.py", line 20, in _initialize_obj
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/triton/runtime/driver.py", line 20, in _initialize_obj
self._obj = self._init_fn()self._obj = self._init_fn()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/triton/runtime/driver.py", line 9, in _create_driver
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/triton/runtime/driver.py", line 9, in _create_driver
return actives[0]()return actives[0]()
^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/triton/backends/nvidia/driver.py", line 535, in init
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/triton/backends/nvidia/driver.py", line 535, in init
self.utils = CudaUtils() # TODO: make staticself.utils = CudaUtils() # TODO: make static
^^^^^^^^^^^^^^^^^^^^^^
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/triton/backends/nvidia/driver.py", line 89, in init
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/triton/backends/nvidia/driver.py", line 89, in init
mod = compile_module_from_src(Path(os.path.join(dirname, "driver.c")).read_text(), "cuda_utils")mod = compile_module_from_src(Path(os.path.join(dirname, "driver.c")).read_text(), "cuda_utils")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/triton/backends/nvidia/driver.py", line 71, in compile_module_from_src
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/triton/backends/nvidia/driver.py", line 71, in compile_module_from_src
mod = importlib.util.module_from_spec(spec)mod = importlib.util.module_from_spec(spec)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ImportErrorImportError: : /home/tobias/.triton/cache/QLAEYTJR4KV5WSBGJKRUAKVP475DE47NW7P4XMI2RFXBOIE5TZ4Q/cuda_utils.so: undefined symbol: cuModuleGetFunction/home/tobias/.triton/cache/QLAEYTJR4KV5WSBGJKRUAKVP475DE47NW7P4XMI2RFXBOIE5TZ4Q/cuda_utils.so: undefined symbol: cuModuleGetFunction
The above exception was the direct cause of the following exception:
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
Traceback (most recent call last):
File "/home/tobias/AI/ai-toolkit/run.py", line 120, in
File "/home/tobias/AI/ai-toolkit/run.py", line 120, in
main()main()
File "/home/tobias/AI/ai-toolkit/run.py", line 108, in main
File "/home/tobias/AI/ai-toolkit/run.py", line 108, in main
raise eraise e
File "/home/tobias/AI/ai-toolkit/run.py", line 95, in main
File "/home/tobias/AI/ai-toolkit/run.py", line 95, in main
job = get_job(config_file, args.name)job = get_job(config_file, args.name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/tobias/AI/ai-toolkit/toolkit/job.py", line 28, in get_job
File "/home/tobias/AI/ai-toolkit/toolkit/job.py", line 28, in get_job
from jobs import ExtensionJobfrom jobs import ExtensionJob
File "/home/tobias/AI/ai-toolkit/jobs/init.py", line 1, in
File "/home/tobias/AI/ai-toolkit/jobs/init.py", line 1, in
from .BaseJob import BaseJobfrom .BaseJob import BaseJob
File "/home/tobias/AI/ai-toolkit/jobs/BaseJob.py", line 5, in
File "/home/tobias/AI/ai-toolkit/jobs/BaseJob.py", line 5, in
from jobs.process import BaseProcessfrom jobs.process import BaseProcess
File "/home/tobias/AI/ai-toolkit/jobs/process/init.py", line 6, in
File "/home/tobias/AI/ai-toolkit/jobs/process/init.py", line 6, in
from .TrainVAEProcess import TrainVAEProcessfrom .TrainVAEProcess import TrainVAEProcess
File "/home/tobias/AI/ai-toolkit/jobs/process/TrainVAEProcess.py", line 18, in
File "/home/tobias/AI/ai-toolkit/jobs/process/TrainVAEProcess.py", line 18, in
from toolkit.image_utils import show_tensorsfrom toolkit.image_utils import show_tensors
File "/home/tobias/AI/ai-toolkit/toolkit/image_utils.py", line 14, in
File "/home/tobias/AI/ai-toolkit/toolkit/image_utils.py", line 14, in
from diffusers import AutoencoderTinyfrom diffusers import AutoencoderTiny
File "", line 1412, in _handle_fromlist
File "", line 1412, in _handle_fromlist
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/utils/import_utils.py", line 1007, in getattr
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/utils/import_utils.py", line 1007, in getattr
value = getattr(module, name)value = getattr(module, name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/utils/import_utils.py", line 1006, in getattr
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/utils/import_utils.py", line 1006, in getattr
module = self._get_module(self._class_to_module[name])module = self._get_module(self._class_to_module[name])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/utils/import_utils.py", line 1018, in _get_module
File "/home/tobias/AI/ai-toolkit/venv/lib/python3.12/site-packages/diffusers/utils/import_utils.py", line 1018, in _get_module
raise RuntimeError(raise RuntimeError(
RuntimeErrorRuntimeError: : Failed to import diffusers.models.autoencoders.autoencoder_tiny because of the following error (look up to see its traceback):
/home/tobias/.triton/cache/QLAEYTJR4KV5WSBGJKRUAKVP475DE47NW7P4XMI2RFXBOIE5TZ4Q/cuda_utils.so: undefined symbol: cuModuleGetFunctionFailed to import diffusers.models.autoencoders.autoencoder_tiny because of the following error (look up to see its traceback):
/home/tobias/.triton/cache/QLAEYTJR4KV5WSBGJKRUAKVP475DE47NW7P4XMI2RFXBOIE5TZ4Q/cuda_utils.so: undefined symbol: cuModuleGetFunction`