Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion modules/dataLoader/ChromaBaseDataLoader.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
from mgds.pipelineModules.EncodeT5Text import EncodeT5Text
from mgds.pipelineModules.EncodeVAE import EncodeVAE
from mgds.pipelineModules.MapData import MapData
from mgds.pipelineModules.PadMaskedTokens import PadMaskedTokens
from mgds.pipelineModules.PruneMaskedTokens import PruneMaskedTokens
from mgds.pipelineModules.RescaleImageChannels import RescaleImageChannels
from mgds.pipelineModules.SampleVAEDistribution import SampleVAEDistribution
from mgds.pipelineModules.SaveImage import SaveImage
Expand All @@ -38,6 +40,7 @@ def _preparation_modules(self, config: TrainConfig, model: ChromaModel):
encode_prompt = EncodeT5Text(tokens_in_name='tokens', tokens_attention_mask_in_name="tokens_mask", hidden_state_out_name='text_encoder_hidden_state', pooled_out_name=None, add_layer_norm=True,
text_encoder=model.text_encoder, hidden_state_output_index=-(1 + config.text_encoder_layer_skip), autocast_contexts=[model.autocast_context, model.text_encoder_autocast_context],
dtype=model.text_encoder_train_dtype.torch_dtype())
prune_masked_tokens = PruneMaskedTokens(tokens_name='tokens', tokens_mask_name='tokens_mask', hidden_state_name='text_encoder_hidden_state')

modules = [rescale_image, encode_image, image_sample]
if config.masked_training or config.model_type.has_mask_input():
Expand All @@ -47,6 +50,9 @@ def _preparation_modules(self, config: TrainConfig, model: ChromaModel):
if not config.train_text_encoder_or_embedding():
modules.append(encode_prompt)

if config.latent_caching and not config.train_text_encoder_or_embedding():
modules.append(prune_masked_tokens)

return modules

def _cache_modules(self, config: TrainConfig, model: ChromaModel, model_setup: BaseChromaSetup):
Expand Down Expand Up @@ -78,6 +84,8 @@ def _cache_modules(self, config: TrainConfig, model: ChromaModel, model_setup: B
)

def _output_modules(self, config: TrainConfig, model: ChromaModel, model_setup: BaseChromaSetup):
pad_masked_tokens = PadMaskedTokens(tokens_name='tokens', tokens_mask_name='tokens_mask', hidden_state_name='text_encoder_hidden_state', max_length=model.tokenizer.model_max_length)

output_names = [
'image_path', 'latent_image',
'prompt',
Expand All @@ -92,7 +100,7 @@ def _output_modules(self, config: TrainConfig, model: ChromaModel, model_setup:
if not config.train_text_encoder_or_embedding():
output_names.append('text_encoder_hidden_state')

return self._output_modules_from_out_names(
output_module_list = self._output_modules_from_out_names(
model, model_setup,
output_names=output_names,
config=config,
Expand All @@ -102,6 +110,11 @@ def _output_modules(self, config: TrainConfig, model: ChromaModel, model_setup:
train_dtype=model.train_dtype,
)

if config.latent_caching and not config.train_text_encoder_or_embedding():
output_module_list = [pad_masked_tokens] + output_module_list

return output_module_list

def _debug_modules(self, config: TrainConfig, model: ChromaModel): #TODO clean up
debug_dir = os.path.join(config.debug_dir, "dataloader")

Expand Down
15 changes: 14 additions & 1 deletion modules/dataLoader/QwenBaseDataLoader.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
from mgds.pipelineModules.DecodeVAE import DecodeVAE
from mgds.pipelineModules.EncodeQwenText import EncodeQwenText
from mgds.pipelineModules.EncodeVAE import EncodeVAE
from mgds.pipelineModules.PadMaskedTokens import PadMaskedTokens
from mgds.pipelineModules.PruneMaskedTokens import PruneMaskedTokens
from mgds.pipelineModules.RescaleImageChannels import RescaleImageChannels
from mgds.pipelineModules.SampleVAEDistribution import SampleVAEDistribution
from mgds.pipelineModules.SaveImage import SaveImage
Expand All @@ -41,6 +43,7 @@ def _preparation_modules(self, config: TrainConfig, model: QwenModel):
format_text=DEFAULT_PROMPT_TEMPLATE, additional_format_text_tokens=DEFAULT_PROMPT_TEMPLATE_CROP_START)
encode_prompt = EncodeQwenText(tokens_name='tokens', tokens_attention_mask_in_name='tokens_mask', hidden_state_out_name='text_encoder_hidden_state', tokens_attention_mask_out_name='tokens_mask',
text_encoder=model.text_encoder, hidden_state_output_index=-1, autocast_contexts=[model.autocast_context], dtype=model.train_dtype.torch_dtype(), crop_start=DEFAULT_PROMPT_TEMPLATE_CROP_START)
prune_masked_tokens = PruneMaskedTokens(tokens_name='tokens', tokens_mask_name='tokens_mask', hidden_state_name='text_encoder_hidden_state')

modules = [rescale_image, encode_image, image_sample]
if config.masked_training or config.model_type.has_mask_input():
Expand All @@ -51,6 +54,9 @@ def _preparation_modules(self, config: TrainConfig, model: QwenModel):
if not config.train_text_encoder_or_embedding():
modules.append(encode_prompt)

if config.latent_caching and not config.train_text_encoder_or_embedding():
modules.append(prune_masked_tokens)

return modules

def _cache_modules(self, config: TrainConfig, model: QwenModel, model_setup: BaseQwenSetup):
Expand Down Expand Up @@ -82,6 +88,8 @@ def _cache_modules(self, config: TrainConfig, model: QwenModel, model_setup: Bas
)

def _output_modules(self, config: TrainConfig, model: QwenModel, model_setup: BaseQwenSetup):
pad_masked_tokens = PadMaskedTokens(tokens_name='tokens', tokens_mask_name='tokens_mask', hidden_state_name='text_encoder_hidden_state', max_length=PROMPT_MAX_LENGTH)

output_names = [
'image_path', 'latent_image',
'prompt',
Expand All @@ -96,7 +104,7 @@ def _output_modules(self, config: TrainConfig, model: QwenModel, model_setup: Ba
if not config.train_text_encoder_or_embedding():
output_names.append('text_encoder_hidden_state')

return self._output_modules_from_out_names(
output_module_list = self._output_modules_from_out_names(
model, model_setup,
output_names=output_names,
config=config,
Expand All @@ -106,6 +114,11 @@ def _output_modules(self, config: TrainConfig, model: QwenModel, model_setup: Ba
train_dtype=model.train_dtype,
)

if config.latent_caching and not config.train_text_encoder_or_embedding():
output_module_list = [pad_masked_tokens] + output_module_list

return output_module_list

def _debug_modules(self, config: TrainConfig, model: QwenModel): #TODO clean up
debug_dir = os.path.join(config.debug_dir, "dataloader")

Expand Down
16 changes: 15 additions & 1 deletion modules/dataLoader/ZImageBaseDataLoader.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
from mgds.pipelineModules.DecodeVAE import DecodeVAE
from mgds.pipelineModules.EncodeQwenText import EncodeQwenText
from mgds.pipelineModules.EncodeVAE import EncodeVAE
from mgds.pipelineModules.PadMaskedTokens import PadMaskedTokens
from mgds.pipelineModules.PruneMaskedTokens import PruneMaskedTokens
from mgds.pipelineModules.RescaleImageChannels import RescaleImageChannels
from mgds.pipelineModules.SampleVAEDistribution import SampleVAEDistribution
from mgds.pipelineModules.SaveImage import SaveImage
Expand All @@ -39,13 +41,18 @@ def _preparation_modules(self, config: TrainConfig, model: ZImageModel):
raise NotImplementedError("Multiple data loader threads are not supported due to an issue with the transformers library: https://github.com/huggingface/transformers/issues/42673")
encode_prompt = EncodeQwenText(tokens_name='tokens', tokens_attention_mask_in_name='tokens_mask', hidden_state_out_name='text_encoder_hidden_state', tokens_attention_mask_out_name='tokens_mask',
text_encoder=model.text_encoder, hidden_state_output_index=-2, autocast_contexts=[model.autocast_context], dtype=model.train_dtype.torch_dtype())
prune_masked_tokens = PruneMaskedTokens(tokens_name='tokens', tokens_mask_name='tokens_mask', hidden_state_name='text_encoder_hidden_state')

modules = [rescale_image, encode_image, image_sample]

if config.masked_training or config.model_type.has_mask_input():
modules.append(downscale_mask)

modules += [tokenize_prompt, encode_prompt]

if config.latent_caching:
modules.append(prune_masked_tokens)

return modules

def _cache_modules(self, config: TrainConfig, model: ZImageModel, model_setup: BaseZImageSetup):
Expand Down Expand Up @@ -76,6 +83,8 @@ def _cache_modules(self, config: TrainConfig, model: ZImageModel, model_setup: B
)

def _output_modules(self, config: TrainConfig, model: ZImageModel, model_setup: BaseZImageSetup):
pad_masked_tokens = PadMaskedTokens(tokens_name='tokens', tokens_mask_name='tokens_mask', hidden_state_name='text_encoder_hidden_state', max_length=PROMPT_MAX_LENGTH)

output_names = [
'image_path', 'latent_image',
'prompt',
Expand All @@ -89,7 +98,7 @@ def _output_modules(self, config: TrainConfig, model: ZImageModel, model_setup:

output_names.append('text_encoder_hidden_state')

return self._output_modules_from_out_names(
output_module_list = self._output_modules_from_out_names(
model, model_setup,
output_names=output_names,
config=config,
Expand All @@ -99,6 +108,11 @@ def _output_modules(self, config: TrainConfig, model: ZImageModel, model_setup:
train_dtype=model.train_dtype,
)

if config.latent_caching:
output_module_list = [pad_masked_tokens] + output_module_list

return output_module_list

def _debug_modules(self, config: TrainConfig, model: ZImageModel):
debug_dir = os.path.join(config.debug_dir, "dataloader")

Expand Down