Repository navigation
Fix missing initializations for models created in 2022 #39772
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 8 commits
6c117b4
5c7c025
9f020a0
8a9dd35
d5c608a
1a89491
3ae1d23
62d6219
de3d656
38c9853
0f70e77
0a8b3bf
e72fac7
198942d
b1f0b47
8746788
e63912b
07a6443
c58c3c8
b954115
40c7240
da184dc
4f63715
c6f3b32
97e3b04
a586d99
b7d93f7
48b81fe
0220b9d
7dd274e
91fa7d2
c7e70f8
ab653bc
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -432,46 +432,46 @@ class CLIPSegPreTrainedModel(PreTrainedModel): | |
| base_model_prefix = "clip" | ||
| supports_gradient_checkpointing = True | ||
|
|
||
| def _init_weights(self, module): | ||
| def _init_weights(self, module: nn.Module): | ||
| """Initialize the weights""" | ||
| factor = self.config.initializer_factor | ||
| if isinstance(module, CLIPSegTextEmbeddings): | ||
| module.token_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02) | ||
| module.position_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02) | ||
| elif isinstance(module, CLIPSegVisionEmbeddings): | ||
| factor = self.config.initializer_factor | ||
| nn.init.normal_(module.class_embedding, mean=0.0, std=module.embed_dim**-0.5 * factor) | ||
| nn.init.normal_(module.class_embedding, std=module.embed_dim**-0.5 * factor) | ||
| nn.init.normal_(module.patch_embedding.weight, std=module.config.initializer_range * factor) | ||
| nn.init.normal_(module.position_embedding.weight, std=module.config.initializer_range * factor) | ||
| elif isinstance(module, CLIPSegAttention): | ||
| factor = self.config.initializer_factor | ||
| in_proj_std = (module.embed_dim**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor | ||
| out_proj_std = (module.embed_dim**-0.5) * factor | ||
| nn.init.normal_(module.q_proj.weight, std=in_proj_std) | ||
| nn.init.normal_(module.k_proj.weight, std=in_proj_std) | ||
| nn.init.normal_(module.v_proj.weight, std=in_proj_std) | ||
| nn.init.normal_(module.out_proj.weight, std=out_proj_std) | ||
| elif isinstance(module, CLIPSegMLP): | ||
| factor = self.config.initializer_factor | ||
| in_proj_std = (module.config.hidden_size**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor | ||
| fc_std = (2 * module.config.hidden_size) ** -0.5 * factor | ||
| nn.init.normal_(module.fc1.weight, std=fc_std) | ||
| nn.init.normal_(module.fc2.weight, std=in_proj_std) | ||
| elif isinstance(module, CLIPSegModel): | ||
| nn.init.normal_( | ||
| module.text_projection.weight, | ||
| std=module.text_embed_dim**-0.5 * self.config.initializer_factor, | ||
| std=module.text_embed_dim**-0.5 * factor, | ||
| ) | ||
| nn.init.normal_( | ||
| module.visual_projection.weight, | ||
| std=module.vision_embed_dim**-0.5 * self.config.initializer_factor, | ||
| std=module.vision_embed_dim**-0.5 * factor, | ||
| ) | ||
| module.logit_scale.data.fill_(self.config.logit_scale_init_value) | ||
|
|
||
| if isinstance(module, nn.LayerNorm): | ||
| module.bias.data.zero_() | ||
| module.weight.data.fill_(1.0) | ||
| if isinstance(module, nn.Linear) and module.bias is not None: | ||
| module.bias.data.zero_() | ||
| if isinstance(module, (nn.Linear, nn.ConvTranspose2d)): | ||
| module.weight.data.normal_(mean=0.0, std=factor * 0.02) | ||
| if module.bias is not None: | ||
| module.bias.data.zero_() | ||
|
Comment on lines
-473
to
+474
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This would overwrite most of the others, so need to make it coherent here
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I didn’t see the |
||
|
|
||
|
|
||
| # Copied from transformers.models.altclip.modeling_altclip.AltCLIPEncoder with AltCLIP->CLIPSeg | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1431,27 +1431,22 @@ def _init_weights(self, module: nn.Module): | |
| nn.init.xavier_uniform_(module.input_projection.weight, gain=xavier_std) | ||
| nn.init.constant_(module.input_projection.bias, 0) | ||
| # FPN | ||
| elif isinstance(module, MaskFormerFPNModel): | ||
| nn.init.xavier_uniform_(module.stem.get_submodule("0").weight, gain=xavier_std) | ||
|
|
||
| elif isinstance(module, MaskFormerFPNLayer): | ||
| nn.init.xavier_uniform_(module.proj[0].weight, gain=xavier_std) | ||
|
|
||
| elif isinstance(module, MaskFormerFPNConvLayer): | ||
| nn.init.xavier_uniform_(module.get_submodule("0").weight, gain=xavier_std) | ||
| # The MLP head | ||
| elif isinstance(module, MaskformerMLPPredictionHead): | ||
| elif isinstance(module, PredictionBlock): | ||
| # I was not able to find the correct initializer in the original implementation | ||
| # we'll use xavier | ||
| for submodule in module.modules(): | ||
| if isinstance(submodule, nn.Linear): | ||
| nn.init.xavier_uniform_(submodule.weight, gain=xavier_std) | ||
| nn.init.constant_(submodule.bias, 0) | ||
| elif isinstance(module, nn.LayerNorm): | ||
| nn.init.xavier_uniform_(module.get_submodule("0").weight, gain=xavier_std) | ||
| nn.init.constant_(module.get_submodule("0").bias, 0) | ||
|
Comment on lines
+1451
to
+1452
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let's initialize the correct module directly instead of that
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Does that mean keeping the original code? |
||
| elif isinstance(module, (nn.LayerNorm, nn.GroupNorm)): | ||
| module.bias.data.zero_() | ||
| module.weight.data.fill_(1.0) | ||
| # copied from DETR | ||
| if isinstance(module, (nn.Linear, nn.Conv2d, nn.BatchNorm2d)): | ||
| if isinstance(module, (nn.Linear, nn.Conv2d)): | ||
| # Slightly different from the TF version which uses truncated_normal for initialization | ||
| # cf https://github.com/pytorch/pytorch/pull/5617 | ||
| module.weight.data.normal_(mean=0.0, std=std) | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -596,7 +596,7 @@ def _init_weights(self, module: nn.Module): | |
| module.bias.data.zero_() | ||
| module.weight.data.fill_(1.0) | ||
| if isinstance(module, nn.Linear): | ||
| module.weight.data.normal_(mean=0.0, std=factor) | ||
| module.weight.data.normal_(mean=0.0, std=factor * 0.02) | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why?? |
||
| if module.bias is not None: | ||
| module.bias.data.zero_() | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -583,7 +583,7 @@ def _init_weights(self, module: nn.Module): | |
| module.bias.data.zero_() | ||
| module.weight.data.fill_(1.0) | ||
| if isinstance(module, nn.Linear): | ||
| module.weight.data.normal_(mean=0.0, std=factor) | ||
| module.weight.data.normal_(mean=0.0, std=factor * 0.02) | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. and here? |
||
| if module.bias is not None: | ||
| module.bias.data.zero_() | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -550,16 +550,20 @@ class ViltPreTrainedModel(PreTrainedModel): | |
| supports_gradient_checkpointing = True | ||
| _no_split_modules = ["ViltEmbeddings", "ViltSelfAttention"] | ||
|
|
||
| def _init_weights(self, module): | ||
| def _init_weights(self, module: nn.Module): | ||
| """Initialize the weights""" | ||
| if isinstance(module, (nn.Linear, nn.Conv2d)): | ||
| std = self.config.initializer_range | ||
| if isinstance(module, ViltEmbeddings): | ||
| nn.init.trunc_normal_(module.cls_token, std=std) | ||
| nn.init.trunc_normal_(module.position_embeddings, std=std) | ||
|
Comment on lines
+557
to
+558
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
| elif isinstance(module, (nn.Linear, nn.Conv2d)): | ||
| # Slightly different from the TF version which uses truncated_normal for initialization | ||
| # cf https://github.com/pytorch/pytorch/pull/5617 | ||
| module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) | ||
| module.weight.data.normal_(mean=0.0, std=std) | ||
| if module.bias is not None: | ||
| module.bias.data.zero_() | ||
| elif isinstance(module, nn.Embedding): | ||
| module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) | ||
| module.weight.data.normal_(mean=0.0, std=std) | ||
| if module.padding_idx is not None: | ||
| module.weight.data[module.padding_idx].zero_() | ||
| elif isinstance(module, nn.LayerNorm): | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Let's not change files only to rename a variable, which had a perfectly fine name before 🙂
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
By convention, it is better to use
factor = self.config.initializer_factor