|
| 1 | +from typing import Any, Dict, List, Optional, Tuple, Union |
| 2 | + |
| 3 | +import torch |
| 4 | +from torch import nn |
| 5 | + |
| 6 | +from ...models.controlnet import ControlNetModel, ControlNetOutput |
| 7 | +from ...models.modeling_utils import ModelMixin |
| 8 | + |
| 9 | + |
| 10 | +class MultiControlNetModel(ModelMixin): |
| 11 | + r""" |
| 12 | + Multiple `ControlNetModel` wrapper class for Multi-ControlNet |
| 13 | +
|
| 14 | + This module is a wrapper for multiple instances of the `ControlNetModel`. The `forward()` API is designed to be |
| 15 | + compatible with `ControlNetModel`. |
| 16 | +
|
| 17 | + Args: |
| 18 | + controlnets (`List[ControlNetModel]`): |
| 19 | + Provides additional conditioning to the unet during the denoising process. You must set multiple |
| 20 | + `ControlNetModel` as a list. |
| 21 | + """ |
| 22 | + |
| 23 | + def __init__(self, controlnets: Union[List[ControlNetModel], Tuple[ControlNetModel]]): |
| 24 | + super().__init__() |
| 25 | + self.nets = nn.ModuleList(controlnets) |
| 26 | + |
| 27 | + def forward( |
| 28 | + self, |
| 29 | + sample: torch.FloatTensor, |
| 30 | + timestep: Union[torch.Tensor, float, int], |
| 31 | + encoder_hidden_states: torch.Tensor, |
| 32 | + controlnet_cond: List[torch.tensor], |
| 33 | + conditioning_scale: List[float], |
| 34 | + class_labels: Optional[torch.Tensor] = None, |
| 35 | + timestep_cond: Optional[torch.Tensor] = None, |
| 36 | + attention_mask: Optional[torch.Tensor] = None, |
| 37 | + cross_attention_kwargs: Optional[Dict[str, Any]] = None, |
| 38 | + guess_mode: bool = False, |
| 39 | + return_dict: bool = True, |
| 40 | + ) -> Union[ControlNetOutput, Tuple]: |
| 41 | + for i, (image, scale, controlnet) in enumerate(zip(controlnet_cond, conditioning_scale, self.nets)): |
| 42 | + down_samples, mid_sample = controlnet( |
| 43 | + sample, |
| 44 | + timestep, |
| 45 | + encoder_hidden_states, |
| 46 | + image, |
| 47 | + scale, |
| 48 | + class_labels, |
| 49 | + timestep_cond, |
| 50 | + attention_mask, |
| 51 | + cross_attention_kwargs, |
| 52 | + guess_mode, |
| 53 | + return_dict, |
| 54 | + ) |
| 55 | + |
| 56 | + # merge samples |
| 57 | + if i == 0: |
| 58 | + down_block_res_samples, mid_block_res_sample = down_samples, mid_sample |
| 59 | + else: |
| 60 | + down_block_res_samples = [ |
| 61 | + samples_prev + samples_curr |
| 62 | + for samples_prev, samples_curr in zip(down_block_res_samples, down_samples) |
| 63 | + ] |
| 64 | + mid_block_res_sample += mid_sample |
| 65 | + |
| 66 | + return down_block_res_samples, mid_block_res_sample |
0 commit comments