fix monkey-patch for text_encoder

Author: takuma104

src/diffusers/loaders.py

@@ -946,14 +946,16 @@ def _modify_text_encoder(self, attn_processors: Dict[str, LoRAAttnProcessor]):
                 module = self.text_encoder.get_submodule(name)
                 # Construct a new function that performs the LoRA merging. We will monkey patch
                 # this forward pass.
-                lora_layer = getattr(attn_processors[name], self._get_lora_layer_attribute(name))
-                old_forward = module.forward
 
-                def new_forward(x):
-                    return old_forward(x) + lora_layer(x)
+                if name in attn_processors:
+                    module.lora_layer = getattr(attn_processors[name], self._get_lora_layer_attribute(name))
+                    module.old_forward = module.forward
 
-                # Monkey-patch.
-                module.forward = new_forward
+                    def new_forward(self, x):
+                        return self.old_forward(x) + self.lora_layer(x)
+
+                    # Monkey-patch.
+                    module.forward = new_forward.__get__(module)
 
     def _get_lora_layer_attribute(self, name: str) -> str:
         if "q_proj" in name: