AttributeError: 'MatmulLtState' object has no attribute 'memory_efficient_backward'

[hissain]

System Info

ERROR:

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Cell In[4], line 10
      7 tokenizer.pad_token = tokenizer.eos_token
      9 peft_config = LoraConfig(task_type="CAUSAL_LM", r=2, lora_alpha=16, lora_dropout=0.01)
---> 10 model = get_peft_model(model, peft_config)
     11 model.print_trainable_parameters()

File ~/venvs/fine_tuning_llama/lib/python3.11/site-packages/peft/mapping.py:222, in get_peft_model(model, peft_config, adapter_name, mixed, autocast_adapter_dtype, revision, low_cpu_mem_usage)
    220 if peft_config.is_prompt_learning:
    221     peft_config = _prepare_prompt_learning_config(peft_config, model_config)
--> 222 return MODEL_TYPE_TO_PEFT_MODEL_MAPPING[peft_config.task_type](
    223     model, peft_config, adapter_name=adapter_name, autocast_adapter_dtype=autocast_adapter_dtype
    224 )

File ~/venvs/fine_tuning_llama/lib/python3.11/site-packages/peft/peft_model.py:1681, in PeftModelForCausalLM.__init__(self, model, peft_config, adapter_name, **kwargs)
   1678 def __init__(
   1679     self, model: torch.nn.Module, peft_config: PeftConfig, adapter_name: str = "default", **kwargs
   1680 ) -> None:
-> 1681     super().__init__(model, peft_config, adapter_name, **kwargs)
   1682     self.base_model_prepare_inputs_for_generation = self.base_model.prepare_inputs_for_generation

File ~/venvs/fine_tuning_llama/lib/python3.11/site-packages/peft/peft_model.py:176, in PeftModel.__init__(self, model, peft_config, adapter_name, autocast_adapter_dtype, low_cpu_mem_usage)
    174     ctx = init_empty_weights if low_cpu_mem_usage else nullcontext
    175     with ctx():
--> 176         self.base_model = cls(model, {adapter_name: peft_config}, adapter_name)
    177     self.set_additional_trainable_modules(peft_config, adapter_name)
    179 if hasattr(self.base_model, "_cast_adapter_dtype"):

File ~/venvs/fine_tuning_llama/lib/python3.11/site-packages/peft/tuners/lora/model.py:141, in LoraModel.__init__(self, model, config, adapter_name, low_cpu_mem_usage)
    140 def __init__(self, model, config, adapter_name, low_cpu_mem_usage: bool = False) -> None:
--> 141     super().__init__(model, config, adapter_name, low_cpu_mem_usage=low_cpu_mem_usage)

File ~/venvs/fine_tuning_llama/lib/python3.11/site-packages/peft/tuners/tuners_utils.py:184, in BaseTuner.__init__(self, model, peft_config, adapter_name, low_cpu_mem_usage)
    182 self._pre_injection_hook(self.model, self.peft_config[adapter_name], adapter_name)
    183 if peft_config != PeftType.XLORA or peft_config[adapter_name] != PeftType.XLORA:
--> 184     self.inject_adapter(self.model, adapter_name, low_cpu_mem_usage=low_cpu_mem_usage)
    186 # Copy the peft_config in the injected model.
    187 self.model.peft_config = self.peft_config

File ~/venvs/fine_tuning_llama/lib/python3.11/site-packages/peft/tuners/tuners_utils.py:501, in BaseTuner.inject_adapter(self, model, adapter_name, autocast_adapter_dtype, low_cpu_mem_usage)
    499         ctx = init_empty_weights if low_cpu_mem_usage else nullcontext
    500         with ctx():
--> 501             self._create_and_replace(peft_config, adapter_name, target, target_name, parent, current_key=key)
    503 if not self.targeted_module_names and not uses_dummy_target_modules:
    504     if excluded_modules and not unmatched_modules:
    505         # All targeted modules were excluded

File ~/venvs/fine_tuning_llama/lib/python3.11/site-packages/peft/tuners/lora/model.py:235, in LoraModel._create_and_replace(self, lora_config, adapter_name, target, target_name, parent, current_key)
    224     target.update_layer(
    225         adapter_name,
    226         r,
   (...)
    232         lora_bias=lora_config.lora_bias,
    233     )
    234 else:
--> 235     new_module = self._create_new_module(lora_config, adapter_name, target, **kwargs)
    236     if adapter_name not in self.active_adapters:
    237         # adding an additional adapter: it is not automatically trainable
    238         new_module.requires_grad_(False)

File ~/venvs/fine_tuning_llama/lib/python3.11/site-packages/peft/tuners/lora/model.py:354, in LoraModel._create_new_module(lora_config, adapter_name, target, **kwargs)
    352 new_module = None
    353 for dispatcher in dispatchers:
--> 354     new_module = dispatcher(target, adapter_name, lora_config=lora_config, **kwargs)
    355     if new_module is not None:  # first match wins
    356         break

File ~/venvs/fine_tuning_llama/lib/python3.11/site-packages/peft/tuners/lora/bnb.py:290, in dispatch_bnb_8bit(target, adapter_name, **kwargs)
    285 if loaded_in_8bit and isinstance(target_base_layer, bnb.nn.Linear8bitLt):
    286     eightbit_kwargs = kwargs.copy()
    287     eightbit_kwargs.update(
    288         {
    289             "has_fp16_weights": target.state.has_fp16_weights,
--> 290             "memory_efficient_backward": target.state.memory_efficient_backward,
    291             "threshold": target.state.threshold,
    292             "index": target.index,
    293         }
    294     )
    295     new_module = Linear8bitLt(target, adapter_name, **eightbit_kwargs)
    297 return new_module

AttributeError: 'MatmulLtState' object has no attribute 'memory_efficient_backward'

pip show bitsandbytes
Name: bitsandbytes
Version: 0.44.1

Reproduction

import torch
from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
from peft import LoraConfig, get_peft_model
from datasets import load_dataset

base_model_id = "/home/hissain/ml/models/Meta-Llama-3.1-8B-Instruct"
dataset_name = "scooterman/guanaco-llama3-1k"
new_model = "./models/NV-llama3-8b-SFT"

dataset = load_dataset(dataset_name, split="train")
model = AutoModelForCausalLM.from_pretrained(base_model_id, load_in_8bit=True, device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(base_model_id, add_eos_token=True, add_bos_token=True)
tokenizer.pad_token = tokenizer.eos_token

peft_config = LoraConfig(task_type="CAUSAL_LM", r=2, lora_alpha=16, lora_dropout=0.01)
model = get_peft_model(model, peft_config)
model.print_trainable_parameters()

Expected behavior

Should not get the error

[matthewdouglas]

Please update PEFT to the latest release: pip install -U peft. Does this solve the issue?

[hissain]

Please update PEFT to the latest release: pip install -U peft. Does this solve the issue?

It worked. Thanks.