Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions modules/dataLoader/ZImageBaseDataLoader.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
from modules.util.enum.ModelType import ModelType
from modules.util.TrainProgress import TrainProgress

from modules.dataLoader.zimage.PadTokens import PadTokens
from modules.dataLoader.zimage.StripPaddedTokens import StripPaddedTokens

from mgds.pipelineModules.DecodeTokens import DecodeTokens
from mgds.pipelineModules.DecodeVAE import DecodeVAE
from mgds.pipelineModules.EncodeQwenText import EncodeQwenText
Expand Down Expand Up @@ -45,7 +48,13 @@ def _preparation_modules(self, config: TrainConfig, model: ZImageModel):
if config.masked_training or config.model_type.has_mask_input():
modules.append(downscale_mask)

modules += [tokenize_prompt, encode_prompt]
strip_padded_tokens = StripPaddedTokens(
Comment thread
BitcrushedHeart marked this conversation as resolved.
Resolved
tokens_name='tokens',
tokens_mask_name='tokens_mask',
hidden_state_name='text_encoder_hidden_state',
)

modules += [tokenize_prompt, encode_prompt, strip_padded_tokens]
return modules

def _cache_modules(self, config: TrainConfig, model: ZImageModel, model_setup: BaseZImageSetup):
Expand Down Expand Up @@ -89,7 +98,14 @@ def _output_modules(self, config: TrainConfig, model: ZImageModel, model_setup:

output_names.append('text_encoder_hidden_state')

return self._output_modules_from_out_names(
pad_tokens = PadTokens(
tokens_name='tokens',
tokens_mask_name='tokens_mask',
hidden_state_name='text_encoder_hidden_state',
max_length=PROMPT_MAX_LENGTH,
)

return [pad_tokens] + self._output_modules_from_out_names(
model, model_setup,
output_names=output_names,
config=config,
Expand Down
58 changes: 58 additions & 0 deletions modules/dataLoader/zimage/PadTokens.py
Comment thread
BitcrushedHeart marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import torch
import torch.nn.functional as F
from mgds.PipelineModule import PipelineModule
from mgds.pipelineModuleTypes.RandomAccessPipelineModule import RandomAccessPipelineModule


class PadTokens(
PipelineModule,
RandomAccessPipelineModule,
):
def __init__(
self,
tokens_name: str,
tokens_mask_name: str,
hidden_state_name: str,
max_length: int,
):
super().__init__()
self.tokens_name = tokens_name
self.tokens_mask_name = tokens_mask_name
self.hidden_state_name = hidden_state_name
self.max_length = max_length

def length(self) -> int:
return self._get_previous_length(self.tokens_name)

def get_inputs(self) -> list[str]:
return [self.tokens_name, self.tokens_mask_name, self.hidden_state_name]

def get_outputs(self) -> list[str]:
return [self.tokens_name, self.tokens_mask_name, self.hidden_state_name]

def get_item(self, variation: int, index: int, requested_name: str = None) -> dict:
tokens = self._get_previous_item(variation, self.tokens_name, index)
tokens_mask = self._get_previous_item(variation, self.tokens_mask_name, index)
hidden_state = self._get_previous_item(variation, self.hidden_state_name, index)

current_length = tokens.shape[0]

if current_length >= self.max_length:
return {
self.tokens_name: tokens,
self.tokens_mask_name: tokens_mask,
self.hidden_state_name: hidden_state,
}

pad_length = self.max_length - current_length

tokens = F.pad(tokens, (0, pad_length), value=0)
tokens_mask = F.pad(tokens_mask, (0, pad_length), value=0)
# hidden_state shape: [N, hidden_dim] -> [max_length, hidden_dim]
hidden_state = F.pad(hidden_state, (0, 0, 0, pad_length), value=0.0)

return {
self.tokens_name: tokens,
self.tokens_mask_name: tokens_mask,
self.hidden_state_name: hidden_state,
}
45 changes: 45 additions & 0 deletions modules/dataLoader/zimage/StripPaddedTokens.py
Comment thread
BitcrushedHeart marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import torch
from mgds.PipelineModule import PipelineModule
from mgds.pipelineModuleTypes.RandomAccessPipelineModule import RandomAccessPipelineModule


class StripPaddedTokens(
Comment thread
BitcrushedHeart marked this conversation as resolved.
Resolved
PipelineModule,
RandomAccessPipelineModule,
):
def __init__(
self,
tokens_name: str,
tokens_mask_name: str,
hidden_state_name: str,
):
super().__init__()
self.tokens_name = tokens_name
self.tokens_mask_name = tokens_mask_name
self.hidden_state_name = hidden_state_name

def length(self) -> int:
return self._get_previous_length(self.tokens_name)

def get_inputs(self) -> list[str]:
return [self.tokens_name, self.tokens_mask_name, self.hidden_state_name]

def get_outputs(self) -> list[str]:
return [self.tokens_name, self.tokens_mask_name, self.hidden_state_name]

def get_item(self, variation: int, index: int, requested_name: str = None) -> dict:
tokens = self._get_previous_item(variation, self.tokens_name, index)
tokens_mask = self._get_previous_item(variation, self.tokens_mask_name, index)
hidden_state = self._get_previous_item(variation, self.hidden_state_name, index)

bool_mask = tokens_mask.bool()

tokens = tokens[bool_mask]
tokens_mask = tokens_mask[bool_mask]
hidden_state = hidden_state[bool_mask]

return {
self.tokens_name: tokens,
self.tokens_mask_name: tokens_mask,
self.hidden_state_name: hidden_state,
}