From 00b364c707dac287e81ab6a952c386246a1025f3 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Sun, 30 Nov 2025 22:10:20 +0100 Subject: [PATCH 01/29] Up --- docs/source/en/_toctree.yml | 2 + docs/source/en/contributing.md | 515 ++++++++++++++++- docs/source/en/model_doc/ministral3.md | 114 ++++ docs/source/en/notebooks.md | 114 +++- src/transformers/integrations/mistral.py | 18 +- src/transformers/models/__init__.py | 1 + .../models/auto/configuration_auto.py | 2 + src/transformers/models/auto/modeling_auto.py | 5 + .../models/auto/tokenization_auto.py | 23 +- .../models/ministral3/__init__.py | 27 + .../ministral3/configuration_ministral3.py | 166 ++++++ .../convert_ministral3_weights_to_hf.py | 289 ++++++++++ .../models/ministral3/modeling_ministral3.py | 528 ++++++++++++++++++ .../models/ministral3/modular_ministral3.py | 124 ++++ tests/models/ministral3/__init__.py | 0 .../ministral3/test_modeling_ministral3.py | 239 ++++++++ 16 files changed, 2156 insertions(+), 11 deletions(-) mode change 120000 => 100644 docs/source/en/contributing.md create mode 100644 docs/source/en/model_doc/ministral3.md mode change 120000 => 100644 docs/source/en/notebooks.md create mode 100644 src/transformers/models/ministral3/__init__.py create mode 100644 src/transformers/models/ministral3/configuration_ministral3.py create mode 100644 src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py create mode 100644 src/transformers/models/ministral3/modeling_ministral3.py create mode 100644 src/transformers/models/ministral3/modular_ministral3.py create mode 100644 tests/models/ministral3/__init__.py create mode 100644 tests/models/ministral3/test_modeling_ministral3.py diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index f60dfc556c16..efb5738ecf35 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -598,6 +598,8 @@ title: MiniMax - local: model_doc/ministral title: Ministral + - local: model_doc/ministral3 + title: Ministral3 - local: model_doc/mistral title: Mistral - local: model_doc/mixtral diff --git a/docs/source/en/contributing.md b/docs/source/en/contributing.md deleted file mode 120000 index c97564d93a7f..000000000000 --- a/docs/source/en/contributing.md +++ /dev/null @@ -1 +0,0 @@ -../../../CONTRIBUTING.md \ No newline at end of file diff --git a/docs/source/en/contributing.md b/docs/source/en/contributing.md new file mode 100644 index 000000000000..d5b2b350a9a2 --- /dev/null +++ b/docs/source/en/contributing.md @@ -0,0 +1,514 @@ + + +# Contribute to πŸ€— Transformers + +Everyone is welcome to contribute, and we value everybody's contribution. Code +contributions are not the only way to help the community. Answering questions, helping +others, and improving the documentation are also immensely valuable. + +It also helps us if you spread the word! Reference the library in blog posts +about the awesome projects it made possible, shout out on Twitter every time it has +helped you, or simply ⭐️ the repository to say thank you. + +However you choose to contribute, please be mindful and respect our +[code of conduct](https://github.com/huggingface/transformers/blob/main/CODE_OF_CONDUCT.md). + +**This guide was heavily inspired by the awesome [scikit-learn guide to contributing](https://github.com/scikit-learn/scikit-learn/blob/main/CONTRIBUTING.md).** + +## Ways to contribute + +There are several ways you can contribute to πŸ€— Transformers: + +* Fix outstanding issues with the existing code. +* Submit issues related to bugs or desired new features. +* Implement new models. +* Contribute to the examples or to the documentation. + +If you don't know where to start, there is a special [Good First +Issue](https://github.com/huggingface/transformers/contribute) listing. It will give you a list of +open issues that are beginner-friendly and help you start contributing to open-source. The best way to do that is to open a Pull Request and link it to the issue that you'd like to work on. We try to give priority to opened PRs as we can easily track the progress of the fix, and if the contributor does not have time anymore, someone else can take the PR over. + +For something slightly more challenging, you can also take a look at the [Good Second Issue](https://github.com/huggingface/transformers/labels/Good%20Second%20Issue) list. In general though, if you feel like you know what you're doing, go for it and we'll help you get there! πŸš€ + +> All contributions are equally valuable to the community. πŸ₯° + +## Fixing outstanding issues + +If you notice an issue with the existing code and have a fix in mind, feel free to [start contributing](#create-a-pull-request) and open a Pull Request! + +## Submitting a bug-related issue or feature request + +Do your best to follow these guidelines when submitting a bug-related issue or a feature +request. It will make it easier for us to come back to you quickly and with good +feedback. + +### Did you find a bug? + +The πŸ€— Transformers library is robust and reliable thanks to users who report the problems they encounter. + +Before you report an issue, we would really appreciate it if you could **make sure the bug was not +already reported** (use the search bar on GitHub under Issues). Your issue should also be related to bugs in the library itself, and not your code. If you're unsure whether the bug is in your code or the library, please ask in the [forum](https://discuss.huggingface.co/) or on our [discord](https://discord.com/invite/hugging-face-879548962464493619) first. This helps us respond quicker to fixing issues related to the library versus general questions. + +> [!TIP] +> We have a [docs bot](https://huggingface.co/spaces/huggingchat/hf-docs-chat), and we highly encourage you to ask all your questions there. There is always a chance your bug can be fixed with a simple flag πŸ‘ΎπŸ”« + +Once you've confirmed the bug hasn't already been reported, please include the following information in your issue so we can quickly resolve it: + +* Your **OS type and version** and **Python**, and **PyTorch** versions when applicable. +* A short, self-contained, code snippet that allows us to reproduce the bug in + less than 30s. +* The *full* traceback if an exception is raised. +* Attach any other additional information, like screenshots, you think may help. + +To get the OS and software versions automatically, run the following command: + +```bash +transformers env +``` + +You can also run the same command from the root of the repository: + +```bash +python src/transformers/commands/transformers_cli.py env +``` + +### Do you want a new feature? + +If there is a new feature you'd like to see in πŸ€— Transformers, please open an issue and describe: + +1. What is the *motivation* behind this feature? Is it related to a problem or frustration with the library? Is it a feature related to something you need for a project? Is it something you worked on and think it could benefit the community? + + Whatever it is, we'd love to hear about it! + +2. Describe your requested feature in as much detail as possible. The more you can tell us about it, the better we'll be able to help you. +3. Provide a *code snippet* that demonstrates the features usage. +4. If the feature is related to a paper, please include a link. + +If your issue is well written we're already 80% of the way there by the time you create it. + +We have added [templates](https://github.com/huggingface/transformers/tree/main/templates) to help you get started with your issue. + +## Do you want to implement a new model? + +New models are constantly released and if you want to implement a new model, please provide the following information: + +* A short description of the model and a link to the paper. +* Link to the implementation if it is open-sourced. +* Link to the model weights if they are available. + +If you are willing to contribute the model yourself, let us know so we can help you add it to πŸ€— Transformers! + +We have a technical guide for [how to add a model to πŸ€— Transformers](https://huggingface.co/docs/transformers/modular_transformers). + +### Vision-Language Model Contribution Checklist + +If you're contributing a **vision-language model** (or any multimodal model that processes images/videos), please follow this checklist. Maintainers will use this to review your PR, and completing these steps will significantly increase the likelihood of your PR being merged quickly. + +**Required checklist for all vision-language model contributions:** + +☐ **1. Implement a modular file** + +All new models should use the modular architecture pattern. Create a `modular_.py` file using the modular model converter: + +- Use the CLI, [`transformers add-new-model-like`](https://github.com/huggingface/transformers/blob/main/src/transformers/cli/add_new_model_like.py) to generate a modular skeleton and get started +- All code should be in the modular file if possible. Modeling must be in it, it's better if configuration is in it as well. [Modular guide](./docs/source/en/modular_transformers.md#implementing-a-modular-file) shows a quick way to set up a modular file. +- Reuse existing patterns from similar models as much as possible +- You can make the model compatible with inference engines such as vLLM or SGLang, and enable zero-effort integration. See specific requirements for model implementation in ["Transformers modeling backend"](./docs/source/en/transformers_as_backend.md#multimodal-models) + +To verify your modular file is correct, run: + +```bash +python utils/modular_model_converter.py +``` + +This will generate the separate files (`modeling_*.py`, `configuration_*.py`, etc.) from your modular file. The CI will enforce that these generated files match your modular file. + +☐ **2. Add a fast image processor (for image models)** + +If your model processes images, implement a fast image processor that uses `torch` and `torchvision` instead of PIL/numpy for better inference performance: + +- See the detailed guide in [#36978](https://github.com/huggingface/transformers/issues/36978) +- Fast processors inherit from `BaseImageProcessorFast` +- Examples: `LlavaOnevisionImageProcessorFast`, `Idefics2ImageProcessorFast` + +☐ **3. Create a weight conversion script** + +Add a `convert__to_hf.py` script that converts the original model weights to the HuggingFace format: + +- Script should handle checkpoint loading, key mapping, and saving in HF format +- Include usage examples and documentation in the script +- Examples: [`convert_llava_onevision_weights_to_hf.py`](https://github.com/huggingface/transformers/blob/main/src/transformers/models/llava_onevision/convert_llava_onevision_weights_to_hf.py), [`convert_idefics2_weights_to_hf.py`](https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics2/convert_idefics2_weights_to_hf.py) + +☐ **4. Add integration tests with exact output matching** + +At minimum, add an `IntegrationTest` class that tests end-to-end generation (processing and modelling) with **exact** output matching: + +- For generative models: test that generated text matches expected output exactly +- For non-generative models: test that output logits match expected values +- Tests should use real checkpoints (load in 4-bit or half precision if the checkpoint is too big to fit in our CI runners) and real inputs +- Example pattern: + +```python +class MyModelIntegrationTest(unittest.TestCase): + @slow + def test_model_integration(self): + model = MyModelForConditionalGeneration.from_pretrained("org/model-name") + processor = AutoProcessor.from_pretrained("org/model-name") + + inputs = processor(images=image, text=prompt, return_tensors="pt") + output = model.generate(**inputs, max_new_tokens=20) + + EXPECTED_TEXT = "exact expected output" + self.assertEqual(processor.decode(output[0]), EXPECTED_TEXT) +``` + +See `tests/models/llava_onevision/test_modeling_llava_onevision.py` for complete examples. + +☐ **5. Update documentation** + +Add or update model documentation: + +- Create if the cli hasn't `docs/source/en/model_doc/.md` with usage examples +- Include model description, paper link, and basic usage with `Pipeline` and `AutoModel` +- Add the model to the appropriate TOC files + +☐ **6. Look for reusable patterns** + +The library has 400+ models with many established patterns: + +- Search for similar models (e.g., other vision-language models) +- Reuse attention mechanisms, layer implementations, and processing patterns +- Check models like LLaVA, Idefics2, Fuyu for vision-language patterns +- Use provided decorators like (`auto_docstring`, `can_return_tuple`, `check_model_inputs` and `_can_record_outputs`) where relevant. +- Don't reinvent the wheel + +☐ **7. Run quality checks and read the output** + +Before submitting your PR, install quality dependencies and run the full check suite: + +```bash +pip install -e ".[quality]" +make fixup +``` + +**Important**: Take time to read the output of `make fixup`. It will: +- Lint and format your code automatically +- Run consistency checks (imports, docstrings, etc.) +- Show any remaining issues that need manual fixes + +All checks must pass before your PR can be merged. + +**If this checklist is complete, your PR has a very high likelihood of being merged!** Following these steps makes the maintainers' work much easier and will reduce the number of review iterations, getting your important work out there faster. + +#### Copy-pastable checklist for maintainers + +Here's a condensed version maintainers can copy into PRs: + +```markdown +## Multimodal Model Addition Checklist + +Please ensure your PR completes all following items. See the [full checklist](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#vision-language-model-contribution-checklist) for details. + +- [ ] **Modular file**: `modular_.py` implemented and verified with `python utils/modular_model_converter.py ` +- [ ] **Fast image processor**: Implemented using `BaseImageProcessorFast` (see [#36978](https://github.com/huggingface/transformers/issues/36978)) +- [ ] **Conversion script**: `convert__to_hf.py` added with usage examples +- [ ] **Integration tests**: End-to-end tests with exact output matching (text or logits) +- [ ] **Documentation**: Model docs added/updated in `docs/source/en/model_doc/` +- [ ] **Pattern reuse**: Verified against similar models (LLaVA, Idefics2, etc.) +- [ ] **Quality checks**: `make fixup` passes with no errors + +``` + +## Do you want to add documentation? + +We're always looking for improvements to the documentation that make it more clear and accurate. Please let us know how the documentation can be improved such as typos and any content that is missing, unclear or inaccurate. We'll be happy to make the changes or help you make a contribution if you're interested! + +For more details about how to generate, build, and write the documentation, take a look at the documentation [README](https://github.com/huggingface/transformers/tree/main/docs). + +## Create a Pull Request + +Before writing any code, we strongly advise you to search through the existing PRs or +issues to make sure nobody is already working on the same thing. If you are +unsure, it is always a good idea to open an issue to get some feedback. + +You will need basic `git` proficiency to contribute to +πŸ€— Transformers. While `git` is not the easiest tool to use, it has the greatest +manual. Type `git --help` in a shell and enjoy! If you prefer books, [Pro +Git](https://git-scm.com/book/en/v2) is a very good reference. + +You'll need **[Python 3.9](https://github.com/huggingface/transformers/blob/main/setup.py#L449)** or above to contribute to πŸ€— Transformers. Follow the steps below to start contributing: + +1. Fork the [repository](https://github.com/huggingface/transformers) by + clicking on the **[Fork](https://github.com/huggingface/transformers/fork)** button on the repository's page. This creates a copy of the code + under your GitHub user account. + +2. Clone your fork to your local disk, and add the base repository as a remote: + + ```bash + git clone git@github.com:/transformers.git + cd transformers + git remote add upstream https://github.com/huggingface/transformers.git + ``` + +3. Create a new branch to hold your development changes: + + ```bash + git checkout -b a-descriptive-name-for-my-changes + ``` + + 🚨 **Do not** work on the `main` branch! + +4. Set up a development environment by running the following command in a virtual environment: + + ```bash + pip install -e ".[dev]" + ``` + + If πŸ€— Transformers was already installed in the virtual environment, remove + it with `pip uninstall transformers` before reinstalling it in editable + mode with the `-e` flag. + + Depending on your OS, and since the number of optional dependencies of Transformers is growing, you might get a + failure with this command. If that's the case make sure to install Pytorch then do: + + ```bash + pip install -e ".[quality]" + ``` + + which should be enough for most use cases. + +5. Develop the features in your branch. + + As you work on your code, you should make sure the test suite + passes. Run the tests impacted by your changes like this: + + ```bash + pytest tests/.py + ``` + + For more information about tests, check out the + [Testing](https://huggingface.co/docs/transformers/testing) guide. + + πŸ€— Transformers relies on `black` and `ruff` to format its source code + consistently. After you make changes, apply automatic style corrections and code verifications + that can't be automated in one go with: + + ```bash + make fixup + ``` + + This target is also optimized to only work with files modified by the PR you're working on. + + If you prefer to run the checks one after the other, the following command applies the + style corrections: + + ```bash + make style + ``` + + πŸ€— Transformers also uses `ruff` and a few custom scripts to check for coding mistakes. Quality + controls are run by the CI, but you can run the same checks with: + + ```bash + make quality + ``` + + Finally, we have a lot of scripts to make sure we don't forget to update + some files when adding a new model. You can run these scripts with: + + ```bash + make repo-consistency + ``` + + To learn more about those checks and how to fix any issues with them, check out the + [Checks on a Pull Request](https://huggingface.co/docs/transformers/pr_checks) guide. + + If you're modifying documents under the `docs/source` directory, make sure the documentation can still be built. This check will also run in the CI when you open a pull request. To run a local check + make sure you install the [documentation builder](https://github.com/huggingface/doc-builder). + + ```bash + pip install hf-doc-builder + ``` + + Run the following command from the root of the repository: + + ```bash + doc-builder build transformers docs/source/en --build_dir ~/tmp/test-build + ``` + + This will build the documentation in the `~/tmp/test-build` folder where you can inspect the generated + Markdown files with your favorite editor. You can also preview the docs on GitHub when you open a pull request. + + Once you're happy with your changes, add the changed files with `git add` and + record your changes locally with `git commit`: + + ```bash + git add modified_file.py + git commit + ``` + + Please remember to write [good commit + messages](https://chris.beams.io/posts/git-commit/) to clearly communicate the changes you made! + + To keep your copy of the code up to date with the original + repository, rebase your branch on `upstream/branch` *before* you open a pull request or if requested by a maintainer: + + ```bash + git fetch upstream + git rebase upstream/main + ``` + + Push your changes to your branch: + + ```bash + git push -u origin a-descriptive-name-for-my-changes + ``` + + If you've already opened a pull request, you'll need to force push with the `--force` flag. Otherwise, if the pull request hasn't been opened yet, you can just push your changes normally. + +6. Now you can go to your fork of the repository on GitHub and click on **Pull Request** to open a pull request. Make sure you tick off all the boxes on our [checklist](#pull-request-checklist) below. When you're ready, you can send your changes to the project maintainers for review. + +7. It's ok if maintainers request changes, it happens to our core contributors + too! So everyone can see the changes in the pull request, work in your local + branch and push the changes to your fork. They will automatically appear in + the pull request. + +### Pull request checklist + +☐ The pull request title should summarize your contribution.
+☐ If your pull request addresses an issue, please mention the issue number in the pull +request description to make sure they are linked (and people viewing the issue know you +are working on it).
+☐ To indicate a work in progress please prefix the title with `[WIP]`. These are +useful to avoid duplicated work, and to differentiate it from PRs ready to be merged.
+☐ Make sure existing tests pass.
+☐ If adding a new feature, also add tests for it.
+ +- If you are adding a new model, make sure you use + `ModelTester.all_model_classes = (MyModel, MyModelWithLMHead,...)` to trigger the common tests. +- If you are adding new `@slow` tests, make sure they pass using + `RUN_SLOW=1 python -m pytest tests/models/my_new_model/test_my_new_model.py`. +- If you are adding a new tokenizer, write tests and make sure + `RUN_SLOW=1 python -m pytest tests/models/{your_model_name}/test_tokenization_{your_model_name}.py` passes. +- CircleCI does not run the slow tests, but GitHub Actions does every night!
+ +☐ All public methods must have informative docstrings (see +[`modeling_bert.py`](https://github.com/huggingface/transformers/blob/main/src/transformers/models/bert/modeling_bert.py) +for an example).
+☐ Due to the rapidly growing repository, don't add any images, videos and other +non-text files that'll significantly weigh down the repository. Instead, use a Hub +repository such as [`hf-internal-testing`](https://huggingface.co/hf-internal-testing) +to host these files and reference them by URL. We recommend placing documentation +related images in the following repository: +[huggingface/documentation-images](https://huggingface.co/datasets/huggingface/documentation-images). +You can open a PR on this dataset repository and ask a Hugging Face member to merge it. + +For more information about the checks run on a pull request, take a look at our [Checks on a Pull Request](https://huggingface.co/docs/transformers/pr_checks) guide. + +### Tests + +An extensive test suite is included to test the library behavior and several examples. Library tests can be found in +the [tests](https://github.com/huggingface/transformers/tree/main/tests) folder and examples tests in the +[examples](https://github.com/huggingface/transformers/tree/main/examples) folder. + +We like `pytest` and `pytest-xdist` because it's faster. From the root of the +repository, specify a *path to a subfolder or a test file* to run the test: + +```bash +python -m pytest -n auto --dist=loadfile -s -v ./tests/models/my_new_model +``` + +Similarly, for the `examples` directory, specify a *path to a subfolder or test file* to run the test. For example, the following command tests the text classification subfolder in the PyTorch `examples` directory: + +```bash +pip install -r examples/xxx/requirements.txt # only needed the first time +python -m pytest -n auto --dist=loadfile -s -v ./examples/pytorch/text-classification +``` + +In fact, this is actually how our `make test` and `make test-examples` commands are implemented (not including the `pip install`)! + +You can also specify a smaller set of tests in order to test only the feature +you're working on. + +By default, slow tests are skipped but you can set the `RUN_SLOW` environment variable to +`yes` to run them. This will download many gigabytes of models so make sure you +have enough disk space, a good internet connection or a lot of patience! + + + +Remember to specify a *path to a subfolder or a test file* to run the test. Otherwise, you'll run all the tests in the `tests` or `examples` folder, which will take a very long time! + + + +```bash +RUN_SLOW=yes python -m pytest -n auto --dist=loadfile -s -v ./tests/models/my_new_model +RUN_SLOW=yes python -m pytest -n auto --dist=loadfile -s -v ./examples/pytorch/text-classification +``` + +Like the slow tests, there are other environment variables available which are not enabled by default during testing: + +- `RUN_CUSTOM_TOKENIZERS`: Enables tests for custom tokenizers. + +More environment variables and additional information can be found in the [testing_utils.py](https://github.com/huggingface/transformers/blob/main/src/transformers/testing_utils.py). + +πŸ€— Transformers uses `pytest` as a test runner only. It doesn't use any +`pytest`-specific features in the test suite itself. + +This means `unittest` is fully supported. Here's how to run tests with +`unittest`: + +```bash +python -m unittest discover -s tests -t . -v +python -m unittest discover -s examples -t examples -v +``` + +### Style guide + +For documentation strings, πŸ€— Transformers follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html). +Check our [documentation writing guide](https://github.com/huggingface/transformers/tree/main/docs#writing-documentation---specification) +for more information. + +### Develop on Windows + +On Windows (unless you're working in [Windows Subsystem for Linux](https://learn.microsoft.com/en-us/windows/wsl/) or WSL), you need to configure git to transform Windows `CRLF` line endings to Linux `LF` line endings: + +```bash +git config core.autocrlf input +``` + +One way to run the `make` command on Windows is with MSYS2: + +1. [Download MSYS2](https://www.msys2.org/), and we assume it's installed in `C:\msys64`. +2. Open the command line `C:\msys64\msys2.exe` (it should be available from the **Start** menu). +3. Run in the shell: `pacman -Syu` and install `make` with `pacman -S make`. +4. Add `C:\msys64\usr\bin` to your PATH environment variable. + +You can now use `make` from any terminal (PowerShell, cmd.exe, etc.)! πŸŽ‰ + +### Sync a forked repository with upstream main (the Hugging Face repository) + +When updating the main branch of a forked repository, please follow these steps to avoid pinging the upstream repository which adds reference notes to each upstream PR, and sends unnecessary notifications to the developers involved in these PRs. + +1. When possible, avoid syncing with the upstream using a branch and PR on the forked repository. Instead, merge directly into the forked main. +2. If a PR is absolutely necessary, use the following steps after checking out your branch: + + ```bash + git checkout -b your-branch-for-syncing + git pull --squash --no-commit upstream main + git commit -m '' + git push --set-upstream origin your-branch-for-syncing + ``` diff --git a/docs/source/en/model_doc/ministral3.md b/docs/source/en/model_doc/ministral3.md new file mode 100644 index 000000000000..49a8f7321343 --- /dev/null +++ b/docs/source/en/model_doc/ministral3.md @@ -0,0 +1,114 @@ + + + +# Ministral3 + +## Overview + +A balanced model in the Ministral 3 family, Ministral 3 8B is a powerful, efficient tiny language model with vision capabilities. + +This model is the instruct post-trained version, fine-tuned for instruction tasks, making it ideal for chat and instruction based use cases. + +The Ministral 3 family is designed for edge deployment, capable of running on a wide range of hardware. + +Key features: +- Vision: Enables the model to analyze images and provide insights based on visual content, in addition to text. +- Multilingual: Supports dozens of languages, including English, French, Spanish, German, Italian, Portuguese, Dutch, Chinese, Japanese, Korean, Arabic. +- System Prompt: Maintains strong adherence and support for system prompts. +- Agentic: Offers best-in-class agentic capabilities with native function calling and JSON outputting. +- Edge-Optimized: Delivers best-in-class performance at a small scale, deployable anywhere. +- Apache 2.0 License: Open-source license allowing usage and modification for both commercial and non-commercial purposes. +- Large Context Window: Supports a 256k context window. + +## Usage examples + +```py +from datetime import datetime, timedelta +import torch + +from mistral_common.protocol.instruct.request import ChatCompletionRequest +from mistral_common.tokens.tokenizers.mistral import MistralTokenizer +from huggingface_hub import hf_hub_download +from transformers import Mistral3ForConditionalGeneration + + +model_id = "mistralai/Ministral-3-3B-Instruct-2512" + +tokenizer = AutoTokenizer.from_pretrained(model_id) +model = Mistral3ForConditionalGeneration.from_pretrained( + model_id, torch_dtype=torch.bfloat16 +) + +image_url = "https://static.wikia.nocookie.net/essentialsdocs/images/7/70/Battle.png/revision/latest?cb=20220523172438" + +messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What action do you think I should take in this situation? List all the possible actions and explain why you think they are good or bad.", + }, + {"type": "image_url", "image_url": {"url": image_url}}, + ], + }, +] + +tokenized = tokenizer.apply_chat_template(messages, return_tensors="pt", return_dict=True) + +output = model.generate( + tokenized, + max_new_tokens=1000, +)[0] + +decoded_output = tokenizer.decode(output[len(tokenized.tokens) :]) +print(decoded_output) +``` + + +## Ministral3Config + +[[autodoc]] Ministral3Config + +## Ministral3PreTrainedModel + +[[autodoc]] Ministral3PreTrainedModel + - forward + +## Ministral3Model + +[[autodoc]] Ministral3Model + - forward + +## Ministral3ForCausalLM + +[[autodoc]] Ministral3ForCausalLM + +## Ministral3ForSequenceClassification + +[[autodoc]] Ministral3ForSequenceClassification + +## Ministral3ForTokenClassification + +[[autodoc]] Ministral3ForTokenClassification + +## Ministral3ForQuestionAnswering + +[[autodoc]] Ministral3ForQuestionAnswering diff --git a/docs/source/en/notebooks.md b/docs/source/en/notebooks.md deleted file mode 120000 index 10fb7a7b979a..000000000000 --- a/docs/source/en/notebooks.md +++ /dev/null @@ -1 +0,0 @@ -../../../notebooks/README.md \ No newline at end of file diff --git a/docs/source/en/notebooks.md b/docs/source/en/notebooks.md new file mode 100644 index 000000000000..faa777a6a313 --- /dev/null +++ b/docs/source/en/notebooks.md @@ -0,0 +1,113 @@ + + +# πŸ€— Transformers Notebooks + +You can find here a list of the official notebooks provided by Hugging Face. + +Also, we would like to list here interesting content created by the community. +If you wrote some notebook(s) leveraging πŸ€— Transformers and would like to be listed here, please open a +Pull Request so it can be included under the Community notebooks. + +## Hugging Face's notebooks πŸ€— + +### Documentation notebooks + +You can open any page of the documentation as a notebook in Colab (there is a button directly on said pages) but they are also listed here if you need them: + +| Notebook | Description | | | | +|:----------|:-------------|:-------------|:------------|------:| +| [Quicktour of the library](https://github.com/huggingface/notebooks/blob/main/transformers_doc/en/quicktour.ipynb) | A presentation of the various APIs in Transformers |[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/quicktour.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/en/transformers_doc/quicktour.ipynb)| | +| [Summary of the tasks](https://github.com/huggingface/notebooks/blob/main/transformers_doc/en/task_summary.ipynb) | How to run the models of the Transformers library task by task |[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/task_summary.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/task_summary.ipynb)| | +| [Preprocessing data](https://github.com/huggingface/notebooks/blob/main/transformers_doc/en/preprocessing.ipynb) | How to use a tokenizer to preprocess your data |[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/preprocessing.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/preprocessing.ipynb)|| +| [Fine-tuning a pretrained model](https://github.com/huggingface/notebooks/blob/main/transformers_doc/en/training.ipynb) | How to use the Trainer to fine-tune a pretrained model |[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/training.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/training.ipynb)|[![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/transformers_doc/en/training.ipynb)| +| [Summary of the tokenizers](https://github.com/huggingface/notebooks/blob/main/transformers_doc/en/tokenizer_summary.ipynb) | The differences between the tokenizers algorithm |[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/tokenizer_summary.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/tokenizer_summary.ipynb)|[![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/transformers_doc/en/tokenizer_summary.ipynb )| +| [Multilingual models](https://github.com/huggingface/notebooks/blob/main/transformers_doc/en/multilingual.ipynb) | How to use the multilingual models of the library |[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/multilingual.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/multilingual.ipynb)|[![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/transformers_doc/en/multilingual.ipynb)| + +### PyTorch Examples + +#### Natural Language Processing[[pytorch-nlp]] + +| Notebook | Description | | | | +|:----------|:-------------|:-------------|:-------------|------:| +| [Train your tokenizer](https://github.com/huggingface/notebooks/blob/main/examples/tokenizer_training.ipynb) | How to train and use your very own tokenizer |[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/tokenizer_training.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/tokenizer_training.ipynb)|[![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/examples/tokenizer_training.ipynb)| +| [Train your language model](https://github.com/huggingface/notebooks/blob/main/examples/language_modeling_from_scratch.ipynb) | How to easily start using transformers |[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling_from_scratch.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/language_modeling_from_scratch.ipynb)|[![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/examples/language_modeling_from_scratch.ipynb)| +| [How to fine-tune a model on text classification](https://github.com/huggingface/notebooks/blob/main/examples/text_classification.ipynb)| Show how to preprocess the data and fine-tune a pretrained model on any GLUE task. | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/text_classification.ipynb)| | +| [How to fine-tune a model on language modeling](https://github.com/huggingface/notebooks/blob/main/examples/language_modeling.ipynb)| Show how to preprocess the data and fine-tune a pretrained model on a causal or masked LM task. | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/language_modeling.ipynb)| [![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/examples/language_modeling.ipynb)| +| [How to fine-tune a model on token classification](https://github.com/huggingface/notebooks/blob/main/examples/token_classification.ipynb)| Show how to preprocess the data and fine-tune a pretrained model on a token classification task (NER, PoS). | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/token_classification.ipynb)|[![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/examples/token_classification.ipynb)| +| [How to fine-tune a model on question answering](https://github.com/huggingface/notebooks/blob/main/examples/question_answering.ipynb)| Show how to preprocess the data and fine-tune a pretrained model on SQUAD. | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/question_answering.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/question_answering.ipynb)|[![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/examples/question_answering.ipynb)| +| [How to fine-tune a model on multiple choice](https://github.com/huggingface/notebooks/blob/main/examples/multiple_choice.ipynb)| Show how to preprocess the data and fine-tune a pretrained model on SWAG. | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/multiple_choice.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/multiple_choice.ipynb)|[![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/examples/multiple_choice.ipynb)| +| [How to fine-tune a model on translation](https://github.com/huggingface/notebooks/blob/main/examples/translation.ipynb)| Show how to preprocess the data and fine-tune a pretrained model on WMT. | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/translation.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/translation.ipynb)|[![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/examples/translation.ipynb)| +| [How to fine-tune a model on summarization](https://github.com/huggingface/notebooks/blob/main/examples/summarization.ipynb)| Show how to preprocess the data and fine-tune a pretrained model on XSUM. | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/summarization.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/summarization.ipynb)| | +| [How to train a language model from scratch](https://github.com/huggingface/blog/blob/main/notebooks/01_how_to_train.ipynb)| Highlight all the steps to effectively train Transformer model on custom data | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/main/notebooks/01_how_to_train.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/blog/blob/main/notebooks/01_how_to_train.ipynb)| | +| [How to generate text](https://github.com/huggingface/blog/blob/main/notebooks/02_how_to_generate.ipynb)| How to use different decoding methods for language generation with transformers | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/main/notebooks/02_how_to_generate.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/blog/blob/main/notebooks/02_how_to_generate.ipynb)| | +| [Reformer](https://github.com/huggingface/blog/blob/main/notebooks/03_reformer.ipynb)| How Reformer pushes the limits of language modeling | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/blog/blob/main/notebooks/03_reformer.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/patrickvonplaten/blog/blob/main/notebooks/03_reformer.ipynb)| | + +#### Computer Vision[[pytorch-cv]] + +| Notebook | Description | | | | +|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------|------:| +| [How to fine-tune a model on image classification (Torchvision)](https://github.com/huggingface/notebooks/blob/main/examples/image_classification.ipynb) | Show how to preprocess the data using Torchvision and fine-tune any pretrained Vision model on Image Classification | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb)| [![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb)| +| [How to fine-tune a model on image classification (Albumentations)](https://github.com/huggingface/notebooks/blob/main/examples/image_classification_albumentations.ipynb) | Show how to preprocess the data using Albumentations and fine-tune any pretrained Vision model on Image Classification | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_albumentations.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/image_classification_albumentations.ipynb)| | +| [How to fine-tune a model on image classification (Kornia)](https://github.com/huggingface/notebooks/blob/main/examples/image_classification_kornia.ipynb) | Show how to preprocess the data using Kornia and fine-tune any pretrained Vision model on Image Classification | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_kornia.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/image_classification_kornia.ipynb)| | +| [How to perform zero-shot object detection with OWL-ViT](https://github.com/huggingface/notebooks/blob/main/examples/zeroshot_object_detection_with_owlvit.ipynb) | Show how to perform zero-shot object detection on images with text queries | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/zeroshot_object_detection_with_owlvit.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/zeroshot_object_detection_with_owlvit.ipynb)| | +| [How to fine-tune an image captioning model](https://github.com/huggingface/notebooks/blob/main/examples/image_captioning_blip.ipynb) | Show how to fine-tune BLIP for image captioning on a custom dataset | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_captioning_blip.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/image_captioning_blip.ipynb)| | +| [How to build an image similarity system with Transformers](https://github.com/huggingface/notebooks/blob/main/examples/image_similarity.ipynb) | Show how to build an image similarity system | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_similarity.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/image_similarity.ipynb)| | +| [How to fine-tune a SegFormer model on semantic segmentation](https://github.com/huggingface/notebooks/blob/main/examples/semantic_segmentation.ipynb) | Show how to preprocess the data and fine-tune a pretrained SegFormer model on Semantic Segmentation | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/semantic_segmentation.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/semantic_segmentation.ipynb)| | +| [How to fine-tune a VideoMAE model on video classification](https://github.com/huggingface/notebooks/blob/main/examples/video_classification.ipynb) | Show how to preprocess the data and fine-tune a pretrained VideoMAE model on Video Classification | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/video_classification.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/video_classification.ipynb)| | + +#### Audio[[pytorch-audio]] + +| Notebook | Description | | | +|:----------|:-------------|:-------------|------:| +| [How to fine-tune a speech recognition model in English](https://github.com/huggingface/notebooks/blob/main/examples/speech_recognition.ipynb)| Show how to preprocess the data and fine-tune a pretrained Speech model on TIMIT | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/speech_recognition.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/speech_recognition.ipynb)| +| [How to fine-tune a speech recognition model in any language](https://github.com/huggingface/notebooks/blob/main/examples/multi_lingual_speech_recognition.ipynb)| Show how to preprocess the data and fine-tune a multi-lingually pretrained speech model on Common Voice | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/multi_lingual_speech_recognition.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/multi_lingual_speech_recognition.ipynb)| +| [How to fine-tune a model on audio classification](https://github.com/huggingface/notebooks/blob/main/examples/audio_classification.ipynb)| Show how to preprocess the data and fine-tune a pretrained Speech model on Keyword Spotting | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/audio_classification.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/audio_classification.ipynb)| + +#### Biological Sequences[[pytorch-bio]] + +| Notebook | Description | | | +|:----------|:----------------------------------------------------------------------------------------|:-------------|------:| +| [How to fine-tune a pre-trained protein model](https://github.com/huggingface/notebooks/blob/main/examples/protein_language_modeling.ipynb) | See how to tokenize proteins and fine-tune a large pre-trained protein "language" model | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/protein_language_modeling.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/protein_language_modeling.ipynb) | +| [How to generate protein folds](https://github.com/huggingface/notebooks/blob/main/examples/protein_folding.ipynb) | See how to go from protein sequence to a full protein model and PDB file | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/protein_folding.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/protein_folding.ipynb) | +| [How to fine-tune a Nucleotide Transformer model](https://github.com/huggingface/notebooks/blob/main/examples/nucleotide_transformer_dna_sequence_modelling.ipynb) | See how to tokenize DNA and fine-tune a large pre-trained DNA "language" model | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/nucleotide_transformer_dna_sequence_modelling.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/nucleotide_transformer_dna_sequence_modelling.ipynb) | +| [Fine-tune a Nucleotide Transformer model with LoRA](https://github.com/huggingface/notebooks/blob/main/examples/nucleotide_transformer_dna_sequence_modelling_with_peft.ipynb) | Train even larger DNA models in a memory-efficient way | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/nucleotide_transformer_dna_sequence_modelling_with_peft.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/nucleotide_transformer_dna_sequence_modelling_with_peft.ipynb) | + +#### Other modalities[[pytorch-other]] + +| Notebook | Description | | | +|:----------|:----------------------------------------------------------------------------------------|:-------------|------:| +| [Probabilistic Time Series Forecasting](https://github.com/huggingface/notebooks/blob/main/examples/time-series-transformers.ipynb) | See how to train Time Series Transformer on a custom dataset | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/time-series-transformers.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/time-series-transformers.ipynb) | + +#### Utility notebooks[[pytorch-utility]] + +| Notebook | Description | | | +|:----------|:-------------|:-------------|------:| +| [How to export model to ONNX](https://github.com/huggingface/notebooks/blob/main/examples/onnx-export.ipynb)| Highlight how to export and run inference workloads through ONNX | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/onnx-export.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/onnx-export.ipynb)| + +### Optimum notebooks + +πŸ€— [Optimum](https://github.com/huggingface/optimum) is an extension of πŸ€— Transformers, providing a set of performance optimization tools enabling maximum efficiency to train and run models on targeted hardware. + +| Notebook | Description | | | +|:----------|:-------------|:-------------|------:| +| [How to quantize a model with ONNX Runtime for text classification](https://github.com/huggingface/notebooks/blob/main/examples/text_classification_quantization_ort.ipynb)| Show how to apply static and dynamic quantization on a model using [ONNX Runtime](https://github.com/microsoft/onnxruntime) for any GLUE task. | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification_quantization_ort.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/text_classification_quantization_ort.ipynb)| +| [How to fine-tune a model on text classification with ONNX Runtime](https://github.com/huggingface/notebooks/blob/main/examples/text_classification_ort.ipynb)| Show how to preprocess the data and fine-tune a model on any GLUE task using [ONNX Runtime](https://github.com/microsoft/onnxruntime). | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification_ort.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/text_classification_ort.ipynb)| +| [How to fine-tune a model on summarization with ONNX Runtime](https://github.com/huggingface/notebooks/blob/main/examples/summarization_ort.ipynb)| Show how to preprocess the data and fine-tune a model on XSUM using [ONNX Runtime](https://github.com/microsoft/onnxruntime). | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/summarization_ort.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/summarization_ort.ipynb)| + +## Community notebooks + +More notebooks developed by the community are available [here](https://hf.co/docs/transformers/community#community-notebooks). diff --git a/src/transformers/integrations/mistral.py b/src/transformers/integrations/mistral.py index 68807f20425b..c2fee6c63caa 100644 --- a/src/transformers/integrations/mistral.py +++ b/src/transformers/integrations/mistral.py @@ -84,19 +84,21 @@ def convert_tekken_tokenizer(tokenizer_file: str): # Extract vocab and special tokens vocab = mistral_tokenizer.instruct_tokenizer.tokenizer._tekken_token2id_nospecial - all_special = [ - token.get("token_str", str(token)) - if isinstance(token, dict) - else (token.value if hasattr(token, "value") else str(token)) - for token in mistral_tokenizer.instruct_tokenizer.tokenizer._all_special_tokens - ] - specials_tokens = {token: all_special.index(token) for token in all_special} + sorted_tokens = sorted(mistral_tokenizer.instruct_tokenizer.tokenizer._all_special_tokens, key=lambda x: x["rank"]) + all_special = [token["token_str"] for token in sorted_tokens] + + specials_tokens = {token: idx for idx, token in enumerate(all_special)} + specials_tokens.update(vocab) vocab = specials_tokens + # TODO(juliendenize): expose this in mistral-common to avoid accessing private attributes + # and improve maintainability + pattern = mistral_tokenizer.instruct_tokenizer.tokenizer._model._pat_str + # Convert tokenizer = PreTrainedTokenizerFast( - tokenizer_object=MistralConverter(vocab=vocab, additional_special_tokens=all_special).converted() + tokenizer_object=MistralConverter(vocab=vocab, additional_special_tokens=all_special, pattern=pattern).converted() ) # Post-process diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index f0d404718e7c..5dcdd6a66c7a 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -221,6 +221,7 @@ from .mimi import * from .minimax import * from .ministral import * + from .ministral3 import * from .mistral import * from .mistral3 import * from .mixtral import * diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 3c5dedcf292e..0f5ea70d2bf6 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -258,6 +258,7 @@ ("mimi", "MimiConfig"), ("minimax", "MiniMaxConfig"), ("ministral", "MinistralConfig"), + ("ministral3", "Ministral3Config"), ("mistral", "MistralConfig"), ("mistral3", "Mistral3Config"), ("mixtral", "MixtralConfig"), @@ -702,6 +703,7 @@ ("mimi", "Mimi"), ("minimax", "MiniMax"), ("ministral", "Ministral"), + ("ministral3", "Ministral3"), ("mistral", "Mistral"), ("mistral3", "Mistral3"), ("mixtral", "Mixtral"), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 64111d5e1b5f..0e8a75a9c6c5 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -258,6 +258,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("mimi", "MimiModel"), ("minimax", "MiniMaxModel"), ("ministral", "MinistralModel"), + ("ministral3", "Ministral3Model"), ("mistral", "MistralModel"), ("mistral3", "Mistral3Model"), ("mixtral", "MixtralModel"), @@ -697,6 +698,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("megatron-bert", "MegatronBertForCausalLM"), ("minimax", "MiniMaxForCausalLM"), ("ministral", "MinistralForCausalLM"), + ("ministral3", "Ministral3ForCausalLM"), ("mistral", "MistralForCausalLM"), ("mixtral", "MixtralForCausalLM"), ("mllama", "MllamaForCausalLM"), @@ -1249,6 +1251,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("megatron-bert", "MegatronBertForSequenceClassification"), ("minimax", "MiniMaxForSequenceClassification"), ("ministral", "MinistralForSequenceClassification"), + ("ministral3", "Ministral3ForSequenceClassification"), ("mistral", "MistralForSequenceClassification"), ("mixtral", "MixtralForSequenceClassification"), ("mobilebert", "MobileBertForSequenceClassification"), @@ -1343,6 +1346,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("megatron-bert", "MegatronBertForQuestionAnswering"), ("minimax", "MiniMaxForQuestionAnswering"), ("ministral", "MinistralForQuestionAnswering"), + ("ministral3", "Ministral3ForQuestionAnswering"), ("mistral", "MistralForQuestionAnswering"), ("mixtral", "MixtralForQuestionAnswering"), ("mobilebert", "MobileBertForQuestionAnswering"), @@ -1455,6 +1459,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("megatron-bert", "MegatronBertForTokenClassification"), ("minimax", "MiniMaxForTokenClassification"), ("ministral", "MinistralForTokenClassification"), + ("ministral3", "Ministral3ForTokenClassification"), ("mistral", "MistralForTokenClassification"), ("mixtral", "MixtralForTokenClassification"), ("mobilebert", "MobileBertForTokenClassification"), diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index deac13558652..e70be0ff3907 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -212,12 +212,30 @@ ("metaclip_2", "XLMRobertaTokenizerFast" if is_tokenizers_available() else None), ("mgp-str", "MgpstrTokenizer"), ("minimax", "GPT2Tokenizer" if is_tokenizers_available() else None), + ( + "ministral3", + ( + "MistralCommonTokenizer" + if is_mistral_common_available() + else ("LlamaTokenizer" if is_sentencepiece_available() else None), + "LlamaTokenizerFast" if is_tokenizers_available() and not is_mistral_common_available() else None, + ), + ), ( "mistral", "MistralCommonBackend" if is_mistral_common_available() else ("LlamaTokenizerFast" if is_tokenizers_available() else None), ), + ( + "mistral3", + ( + "MistralCommonBackend" + if is_mistral_common_available() + else ("LlamaTokenizer" if is_sentencepiece_available() else None), + "LlamaTokenizerFast" if is_tokenizers_available() and not is_mistral_common_available() else None, + ), + ), ( "mixtral", "MistralCommonBackend" @@ -384,7 +402,10 @@ def tokenizer_class_from_name(class_name: str) -> Union[type[Any], None]: for module_name, tokenizer_class in TOKENIZER_MAPPING_NAMES.items(): if tokenizer_class == class_name: module_name = model_type_to_module_name(module_name) - if module_name in ["mistral", "mixtral", "ministral"] and class_name == "MistralCommonBackend": + if ( + module_name in ["mistral", "mistral3", "mixtral", "ministral", "ministral3", "pixtral", "voxtral"] + and class_name == "MistralCommonTokenizer" + ): module = importlib.import_module(".tokenization_mistral_common", "transformers") else: module = importlib.import_module(f".{module_name}", "transformers.models") diff --git a/src/transformers/models/ministral3/__init__.py b/src/transformers/models/ministral3/__init__.py new file mode 100644 index 000000000000..59fdb90b750a --- /dev/null +++ b/src/transformers/models/ministral3/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2025 Mistral AI and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...utils import _LazyModule +from ...utils.import_utils import define_import_structure + + +if TYPE_CHECKING: + from .configuration_ministral3 import * + from .modeling_ministral3 import * +else: + import sys + + _file = globals()["__file__"] + sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) diff --git a/src/transformers/models/ministral3/configuration_ministral3.py b/src/transformers/models/ministral3/configuration_ministral3.py new file mode 100644 index 000000000000..5cd1144b58d4 --- /dev/null +++ b/src/transformers/models/ministral3/configuration_ministral3.py @@ -0,0 +1,166 @@ +# coding=utf-8 +# Copyright 2025 Mistral AI and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Ministral model configuration""" +from typing import Optional + +from ...configuration_utils import PreTrainedConfig +from ...modeling_rope_utils import RopeParameters +from ...utils import logging + + +logger = logging.get_logger(__name__) + + +class Ministral3Config(PreTrainedConfig): + r""" + This is the configuration class to store the configuration of a [`Ministral3Model`]. It is used to instantiate an + Mistral model according to the specified arguments, defining the model architecture. Instantiating a configuration + with the defaults will yield a similar configuration to that of the mistralai/Ministral-3-8B-Base-2512, mistralai/Ministral-3-8B-Instruct-2512 or mistralai/Ministral-3-8B-Reasoning-2512. + + [mistralai/Ministral-3-8B-Base-2512](https://huggingface.co/mistralai/Ministral-3-8B-Base-2512) + [mistralai/Ministral-3-8B-Instruct-2512](https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512) + [mistralai/Ministral-3-8B-Reasoning-2512](https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512) + + Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PreTrainedConfig`] for more information. + + Args: + vision_config (`Union[AutoConfig, dict]`, *optional*, defaults to `PixtralVisionConfig`): + The config object or dictionary of the vision backbone. + text_config (`Union[AutoConfig, dict]`, *optional*, defaults to `MistralConfig`): + The config object or dictionary of the text backbone. + image_token_index (`int`, *optional*, defaults to 10): + The image token index to encode the image prompt. + projector_hidden_act (`str`, *optional*, defaults to `"gelu"`): + The activation function used by the multimodal projector. + vision_feature_layer (`Union[int, list[int]]`, *optional*, defaults to -1): + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + multimodal_projector_bias (`bool`, *optional*, defaults to `False`): + Whether to use bias in the multimodal projector. + spatial_merge_size (`int`, *optional*, defaults to 2): + The downsampling factor for the spatial merge operation. + + Example: + + ```python + >>> from transformers import Ministral3ForConditionalGeneration, Ministral3Config, PixtralVisionConfig, MistralConfig + + >>> # Initializing a Pixtral-vision config + >>> vision_config = PixtralVisionConfig() + + >>> # Initializing a Mistral config + >>> text_config = Ministral3Config() + + >>> # Initializing a Mistral3 configuration + >>> configuration = Mistral3Config(vision_config, text_config) + + >>> # Initializing a model from the ministral configuration + >>> model = Ministral3ForConditionalGeneration(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "ministal3" + keys_to_ignore_at_inference = ["past_key_values"] + # Default tensor parallel plan for base model `MistralModel` + base_model_tp_plan = { + "layers.*.self_attn.q_proj": "colwise", + "layers.*.self_attn.k_proj": "colwise", + "layers.*.self_attn.v_proj": "colwise", + "layers.*.self_attn.o_proj": "rowwise", + "layers.*.mlp.gate_proj": "colwise", + "layers.*.mlp.up_proj": "colwise", + "layers.*.mlp.down_proj": "rowwise", + } + base_model_pp_plan = { + "embed_tokens": (["input_ids"], ["inputs_embeds"]), + "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), + "norm": (["hidden_states"], ["hidden_states"]), + } + + def __init__( + self, + vocab_size: Optional[int] = 131072, + hidden_size: Optional[int] = 4096, + intermediate_size: Optional[int] = 14336, + num_hidden_layers: Optional[int] = 34, + num_attention_heads: Optional[int] = 32, + num_key_value_heads: Optional[int] = 8, + head_dim: Optional[int] = 128, + hidden_act: Optional[str] = "silu", + max_position_embeddings: Optional[int] = 262144, + initializer_range: Optional[float] = 0.02, + rms_norm_eps: Optional[float] = 1e-5, + use_cache: Optional[bool] = True, + pad_token_id: Optional[int] = 11, + bos_token_id: Optional[int] = 1, + eos_token_id: Optional[int] = 2, + tie_word_embeddings: Optional[bool] = False, + rope_parameters: Optional[RopeParameters | dict[str, RopeParameters]] = { + "type": "yarn", + "rope_theta": 1000000.0, + "factor": 16.0, + "original_max_position_embeddings": 16384, + "beta_fast": 32.0, + "beta_slow": 1.0, + "mscale_all_dim": 1.0, + "mscale": 1.0, + "llama_4_scaling_beta": 0.1, + }, + sliding_window: Optional[int] = None, + attention_dropout: Optional[float] = 0.0, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.sliding_window = sliding_window + self.head_dim = head_dim if head_dim is not None else hidden_size // num_attention_heads + + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.attention_dropout = attention_dropout + + if "layer_types" in kwargs: + logger.warning_once( + "Detected Mistral model with layer_types. Consider using AutoModel or Ministral classes instead to enable alternating attention compatibility." + ) + + self.rope_parameters = rope_parameters + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + ignore_keys_at_rope_validation={"llama_4_scaling_beta"}, + **kwargs, + ) + + +__all__ = ["Ministral3Config"] diff --git a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py new file mode 100644 index 000000000000..77c2ca27118d --- /dev/null +++ b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py @@ -0,0 +1,289 @@ +# Copyright 2025 Mistral AI and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import json +import os +import re + +import torch +from safetensors.torch import load_file + +from transformers import ( + GenerationConfig, + Ministral3Config, + Ministral3ForCausalLM, + Mistral3Config, + Mistral3ForConditionalGeneration, + PixtralImageProcessorFast, + PixtralProcessor, + PixtralVisionConfig, +) +from transformers.integrations.mistral import convert_tekken_tokenizer + + +# fmt: off +STATE_DICT_MAPPING = { + # Text model keys + r"^output.weight": r"lm_head.weight", + r"^norm.weight": r"model.language_model.norm.weight", + r"^tok_embeddings.weight": r"model.language_model.embed_tokens.weight", + r"^layers.(\d+).attention_norm.weight": r"model.language_model.layers.\1.input_layernorm.weight", + r"^layers.(\d+).ffn_norm.weight": r"model.language_model.layers.\1.post_attention_layernorm.weight", + r"^layers.(\d+).attention.w(q|k|v|o).weight": r"model.language_model.layers.\1.self_attn.\2_proj.weight", + r"^layers.(\d+).feed_forward.w1.weight": r"model.language_model.layers.\1.mlp.gate_proj.weight", + r"^layers.(\d+).feed_forward.w2.weight": r"model.language_model.layers.\1.mlp.down_proj.weight", + r"^layers.(\d+).feed_forward.w3.weight": r"model.language_model.layers.\1.mlp.up_proj.weight", + + # Vision model keys + r"vision_encoder.transformer.layers.(\d+).attention_norm.weight": r"model.vision_tower.transformer.layers.\1.attention_norm.weight", + r"^vision_encoder.transformer.layers.(\d+).ffn_norm.weight": r"model.vision_tower.transformer.layers.\1.ffn_norm.weight", + r"^vision_encoder.transformer.layers.(\d+).attention.w(q|k|v|o).weight": r"model.vision_tower.transformer.layers.\1.attention.\2_proj.weight", + r"^vision_encoder.transformer.layers.(\d+).feed_forward.w1.weight": r"model.vision_tower.transformer.layers.\1.feed_forward.gate_proj.weight", + r"^vision_encoder.transformer.layers.(\d+).feed_forward.w2.weight": r"model.vision_tower.transformer.layers.\1.feed_forward.down_proj.weight", + r"^vision_encoder.transformer.layers.(\d+).feed_forward.w3.weight": r"model.vision_tower.transformer.layers.\1.feed_forward.up_proj.weight", + r"^vision_language_adapter.w_in": r"model.multi_modal_projector.linear_1", + r"^vision_language_adapter.w_out": r"model.multi_modal_projector.linear_2", + r"^vision_encoder.ln_pre.weight": r"model.vision_tower.ln_pre.weight", + r"^vision_encoder.patch_conv.weight": r"model.vision_tower.patch_conv.weight", + r"^patch_merger.merging_layer.weight": r"model.multi_modal_projector.patch_merger.merging_layer.weight", + r"^pre_mm_projector_norm.weight": r"model.multi_modal_projector.norm.weight", +} +# fmt: on + + +def map_old_key_to_new(old_key): + """Map of a key of the original state dict to the equivalent key in HF format""" + for pattern, replacement in STATE_DICT_MAPPING.items(): + new_key, n_replace = re.subn(pattern, replacement, old_key) + # Early exit of the loop + if n_replace > 0: + return new_key + + raise ValueError(f"Key: {old_key} could not be mapped (check the mapping).") + + +def read_json(path): + with open(path, "r") as f: + return json.load(f) + + +def permute_for_rope(tensor, n_heads, dim1, dim2): + """Permute the weights for the ROPE formulation.""" + tensor = tensor.view(n_heads, dim1 // n_heads // 2, 2, dim2) + tensor = tensor.transpose(1, 2) + tensor = tensor.reshape(dim1, dim2) + return tensor + + +def convert_state_dict(original_state_dict: dict, config: Mistral3Config): + """Convert a state dict file, when a single `nn.Module` is never sharded in different files (usual case).""" + new_dict = {} + + for old_key, tensor in original_state_dict.items(): + new_key = map_old_key_to_new(old_key) + + if "vision" in old_key: + num_attention_heads = config.vision_config.num_attention_heads + num_key_value_heads = num_attention_heads + hidden_size = config.vision_config.hidden_size + head_dim = config.vision_config.head_dim + key_value_dim = head_dim * num_attention_heads + query_dim = head_dim * num_attention_heads + else: + num_attention_heads = config.text_config.num_attention_heads + hidden_size = config.text_config.hidden_size + head_dim = config.text_config.head_dim + num_key_value_heads = config.text_config.num_key_value_heads + key_value_dim = head_dim * num_key_value_heads + query_dim = head_dim * num_attention_heads + + if "q_proj" in new_key: + tensor = permute_for_rope(tensor, num_attention_heads, query_dim, hidden_size) + elif "k_proj" in new_key: + tensor = permute_for_rope(tensor, num_key_value_heads, key_value_dim, hidden_size) + + new_dict[new_key] = tensor + return new_dict + + +def convert_config(original_config: dict, max_position_embeddings: int = 262144): + original_vision_config = original_config.pop("vision_encoder", None) + original_text_config = original_config + + # Text config + text_key_mapping = { + "hidden_size": "dim", + "num_hidden_layers": "n_layers", + "intermediate_size": "hidden_dim", + "num_attention_heads": "n_heads", + "num_key_value_heads": "n_kv_heads", + "rms_norm_eps": "norm_eps", + } + similar_text_keys_to_keep = [ + "head_dim", + "vocab_size", + ] + + new_text_config_kwargs = {k: original_text_config[v] for k, v in text_key_mapping.items()} + new_text_config_kwargs.update({k: v for k, v in original_text_config.items() if k in similar_text_keys_to_keep}) + tie_word_embeddings = original_text_config.get("tied_embeddings", False) + new_text_config_kwargs["tie_word_embeddings"] = tie_word_embeddings + new_text_config_kwargs["rope_parameters"] = { + "type": "yarn", + "rope_theta": original_config.get("rope_theta", 1000000.0), + "factor": float(original_config["yarn"]["factor"]), + "original_max_position_embeddings": original_config["yarn"]["original_max_position_embeddings"], + "beta_fast": float(original_config["yarn"]["beta"]), + "beta_slow": float(original_config["yarn"]["alpha"]), + "mscale_all_dim": 1.0, + "mscale": 1.0, + "llama_4_scaling_beta": original_config["llama_4_scaling"]["beta"], + } + + # These are not always defined depending on `params.json` + new_text_config_kwargs["sliding_window"] = original_text_config.get("sliding_window", None) + new_text_config_kwargs["max_position_embeddings"] = original_text_config.get( + "max_position_embeddings", original_text_config.get("max_seq_len", max_position_embeddings) + ) + # This may sometimes be a string in `params.json` + if new_text_config_kwargs["sliding_window"] is not None: + new_text_config_kwargs["sliding_window"] = int(new_text_config_kwargs["sliding_window"]) + + new_text_config = Ministral3Config(**new_text_config_kwargs) + + # No vision + if original_vision_config is None: + return new_text_config + + # Vision config + new_vision_config = original_vision_config + adapter_bias = new_vision_config.pop("adapter_bias", False) + _ = new_vision_config.pop("mm_projector_id", None) + _ = new_vision_config.pop("add_pre_mm_projector_layer_norm", None) + spatial_merge_size = new_vision_config.pop("spatial_merge_size") + image_token_id = new_vision_config.pop("image_token_id", 10) + _ = new_vision_config.pop("image_break_token_id", 12) + _ = new_vision_config.pop("image_end_token_id", 13) + _ = new_vision_config.pop("max_image_size") + new_vision_config = PixtralVisionConfig(hidden_act="silu", **new_vision_config) + + new_config = Mistral3Config( + vision_config=new_vision_config, + text_config=new_text_config, + multimodal_projector_bias=adapter_bias, + image_token_id=image_token_id, + spatial_merge_size=spatial_merge_size, + vision_feature_layer=-1, + ) + return new_config + + +def convert_and_write_model(input_dir: str, output_dir: str, max_position_embeddings: int): + """Convert the model and save it (this implicitly save the config as well).""" + params = read_json(os.path.join(input_dir, "params.json")) + config = convert_config(params, max_position_embeddings) + + full_state_dict = {} + # The model may be split between different files, but a single nn.Module is always fully present in a single file + shards = [file for file in os.listdir(input_dir) if file.endswith(".safetensors")] + for shard_file in shards: + original_state_dict = load_file(os.path.join(input_dir, shard_file)) + new_dict = convert_state_dict(original_state_dict, config) + full_state_dict.update(new_dict) + + if config.text_config.tie_word_embeddings: + full_state_dict["lm_head.weight"] = full_state_dict["model.language_model.embed_tokens.weight"] + + # Load weights into model and resave them + with torch.device("meta"): + if isinstance(config, Mistral3Config): + model = Mistral3ForConditionalGeneration(config) + elif isinstance(config, Ministral3Config): + model = Ministral3ForCausalLM(config) + else: + raise ValueError(f"Unknown config type {type(config)}.") + + model.load_state_dict(full_state_dict, strict=True, assign=True) + model.save_pretrained(output_dir) + return config + + +def convert_and_write_processor_and_tokenizer( + input_dir: str, output_dir: str, model_config: Mistral3Config | Ministral3ForCausalLM +): + """Convert the tokenizer and save it.""" + from mistral_common.tokens.tokenizers.tekken import Tekkenizer + + tokenizer_file = os.path.join(input_dir, "tekken.json") + tokenizer = convert_tekken_tokenizer(tokenizer_file) + tokenizer.add_special_tokens({"pad_token": ""}) + + # No vision + if isinstance(model_config, Ministral3Config): + tokenizer.save_pretrained(output_dir) + return + + tekkenizer = Tekkenizer.from_file(tokenizer_file) + config = read_json(os.path.join(input_dir, "params.json")) + patch_size = config["vision_encoder"]["patch_size"] + spatial_merge_size = config["vision_encoder"]["spatial_merge_size"] + max_image_size = config["vision_encoder"]["max_image_size"] + image_processor = PixtralImageProcessorFast(patch_size=patch_size, size={"longest_edge": max_image_size}) + + processor = PixtralProcessor( + tokenizer=tokenizer, + image_processor=image_processor, + image_token="[IMG]", + patch_size=patch_size, + spatial_merge_size=spatial_merge_size, + ) + + # Finally save it + processor.save_pretrained(output_dir) + + generation_config = GenerationConfig( + eos_token_id=tekkenizer.eos_id, + bos_token_id=tekkenizer.bos_id, + pad_token_id=tekkenizer.pad_id, + max_length=model_config.text_config.max_position_embeddings, + ) + + generation_config.save_pretrained(output_dir) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "input_dir", + help="Location of Mistral weights, which contains tokenizer.model and model folders", + ) + parser.add_argument( + "output_dir", + help="Location to write HF model and tokenizer", + ) + parser.add_argument( + "--max_position_embeddings", + type=int, + default=262144, + help="`max_position_embeddings` field in the config. This needs to be manually passed (not present anywhere otherwise).", + ) + + args = parser.parse_args() + + config = convert_and_write_model(args.input_dir, args.output_dir, args.max_position_embeddings) + convert_and_write_processor_and_tokenizer(args.input_dir, args.output_dir, config) + + +if __name__ == "__main__": + main() diff --git a/src/transformers/models/ministral3/modeling_ministral3.py b/src/transformers/models/ministral3/modeling_ministral3.py new file mode 100644 index 000000000000..a05966edb6c8 --- /dev/null +++ b/src/transformers/models/ministral3/modeling_ministral3.py @@ -0,0 +1,528 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/ministral3/modular_ministral3.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_ministral3.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +from collections.abc import Callable +from typing import Optional, Union + +import torch +from torch import nn + +from transformers.utils.generic import check_model_inputs + +from ...activations import ACT2FN +from ...cache_utils import Cache, DynamicCache +from ...generation import GenerationMixin +from ...integrations import use_kernel_forward_from_hub +from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask +from ...modeling_flash_attention_utils import FlashAttentionKwargs +from ...modeling_layers import GenericForQuestionAnswering, GenericForTokenClassification, GradientCheckpointingLayer +from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast +from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update +from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from .configuration_ministral3 import Ministral3Config + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + position_ids (`torch.Tensor`, *optional*): + Deprecated and unused. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +def eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + scaling: float, + dropout: float = 0.0, + **kwargs: Unpack[TransformersKwargs], +): + key_states = repeat_kv(key, module.num_key_value_groups) + value_states = repeat_kv(value, module.num_key_value_groups) + + attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling + if attention_mask is not None: + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) + attn_output = torch.matmul(attn_weights, value_states) + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attn_weights + + +def _get_llama_4_attn_scale(positions_ids: torch.Tensor, beta: float, max_position_embeddings: int) -> torch.Tensor: + scaling = 1 + beta * torch.log(1 + torch.floor(positions_ids / max_position_embeddings)) + return scaling.unsqueeze(-1) + + +class Ministral3Attention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: Ministral3Config, layer_idx: int): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.head_dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads + self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads + self.scaling = self.head_dim**-0.5 + self.attention_dropout = config.attention_dropout + self.is_causal = True + self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=False) + self.k_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=False) + self.v_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=False) + self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=False) + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], + attention_mask: Optional[torch.Tensor], + past_key_values: Optional[Cache] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, self.head_dim) + + query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) + key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) + value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) + + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + query_states = query_states * _get_llama_4_attn_scale( + cache_position, + self.config.rope_parameters.get("llama_4_scaling_beta"), + self.config.rope_parameters.get("original_max_position_embeddings"), + ).to(query_states.dtype) + + if past_key_values is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + sliding_window=getattr(self.config, "sliding_window", None), # main diff with Llama + **kwargs, + ) + + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + + +class Ministral3MLP(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, x): + down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + return down_proj + + +@use_kernel_forward_from_hub("RMSNorm") +class Ministral3RMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + Ministral3RMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" + + +class Ministral3DecoderLayer(GradientCheckpointingLayer): + def __init__(self, config: Ministral3Config, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = Ministral3Attention(config=config, layer_idx=layer_idx) + self.mlp = Ministral3MLP(config) + self.input_layernorm = Ministral3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = Ministral3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + use_cache: Optional[bool] = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + **kwargs: Unpack[TransformersKwargs], + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + # Self Attention + hidden_states, _ = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + return hidden_states + + +@auto_docstring +class Ministral3PreTrainedModel(PreTrainedModel): + config: Ministral3Config + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["Ministral3DecoderLayer"] + _skip_keys_device_placement = ["past_key_values"] + _supports_flash_attn = True + _supports_sdpa = True + _supports_flex_attn = True + + _can_compile_fullgraph = True + _supports_attention_backend = True + _can_record_outputs = { + "hidden_states": Ministral3DecoderLayer, + "attentions": Ministral3Attention, + } + + +class Ministral3RotaryEmbedding(nn.Module): + inv_freq: torch.Tensor # fix linting for `register_buffer` + + def __init__(self, config: Ministral3Config, device=None): + super().__init__() + self.max_seq_len_cached = config.max_position_embeddings + self.original_max_seq_len = config.max_position_embeddings + + self.config = config + + self.rope_type = self.config.rope_parameters["rope_type"] + rope_init_fn: Callable = self.compute_default_rope_parameters + if self.rope_type != "default": + rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] + inv_freq, self.attention_scaling = rope_init_fn(self.config, device) + + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.original_inv_freq = inv_freq + + @staticmethod + def compute_default_rope_parameters( + config: Optional[Ministral3Config] = None, + device: Optional["torch.device"] = None, + seq_len: Optional[int] = None, + ) -> tuple["torch.Tensor", float]: + """ + Computes the inverse frequencies according to the original RoPE implementation + Args: + config ([`~transformers.PreTrainedConfig`]): + The model configuration. + device (`torch.device`): + The device to use for initialization of the inverse frequencies. + seq_len (`int`, *optional*): + The current sequence length. Unused for this type of RoPE. + Returns: + Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the + post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). + """ + base = config.rope_parameters["rope_theta"] + dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads + + attention_factor = 1.0 # Unused in this type of RoPE + + # Compute the inverse frequencies + inv_freq = 1.0 / ( + base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim) + ) + return inv_freq, attention_factor + + @torch.no_grad() + @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope) + def forward(self, x, position_ids): + inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device) + position_ids_expanded = position_ids[:, None, :].float() + + device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): # Force float32 + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() * self.attention_scaling + sin = emb.sin() * self.attention_scaling + + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +@auto_docstring +class Ministral3Model(Ministral3PreTrainedModel): + def __init__(self, config: Ministral3Config): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [Ministral3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.norm = Ministral3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = Ministral3RotaryEmbedding(config=config) + self.gradient_checkpointing = False + + # Initialize weights and apply final processing + self.post_init() + + @check_model_inputs() + @auto_docstring + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs: Unpack[TransformersKwargs], + ) -> BaseModelOutputWithPast: + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + if use_cache and past_key_values is None: + past_key_values = DynamicCache(config=self.config) + + if cache_position is None: + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + cache_position = torch.arange( + past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device + ) + + if position_ids is None: + position_ids = cache_position.unsqueeze(0) + + mask_function = create_causal_mask if self.config.sliding_window is None else create_sliding_window_causal_mask + causal_mask = mask_function( + config=self.config, + input_embeds=inputs_embeds, + attention_mask=attention_mask, + cache_position=cache_position, + past_key_values=past_key_values, + position_ids=position_ids, + ) + + hidden_states = inputs_embeds + position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids) + + for decoder_layer in self.layers[: self.config.num_hidden_layers]: + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = self.norm(hidden_states) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values if use_cache else None, + ) + + +@auto_docstring +class Ministral3ForCausalLM(Ministral3PreTrainedModel, GenerationMixin): + _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} + _tp_plan = {"lm_head": "colwise_rep"} + _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} + + def __init__(self, config): + super().__init__(config) + self.model = Ministral3Model(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + logits_to_keep: Union[int, torch.Tensor] = 0, + **kwargs: Unpack[TransformersKwargs], + ) -> CausalLMOutputWithPast: + r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, Ministral3ForCausalLM + + >>> model = Ministral3ForCausalLM.from_pretrained("meta-ministral3/Ministral3-2-7b-hf") + >>> tokenizer = AutoTokenizer.from_pretrained("meta-ministral3/Ministral3-2-7b-hf") + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." + ```""" + outputs: BaseModelOutputWithPast = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + cache_position=cache_position, + **kwargs, + ) + + hidden_states = outputs.last_hidden_state + # Only compute necessary logits, and do not upcast them to float if we are not computing the loss + slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep + logits = self.lm_head(hidden_states[:, slice_indices, :]) + + loss = None + if labels is not None: + loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs) + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +class Ministral3ForTokenClassification(GenericForTokenClassification, Ministral3PreTrainedModel): + pass + + +@auto_docstring +class Ministral3ForSequenceClassification(PreTrainedModel): + config: Ministral3Config + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["Ministral3DecoderLayer"] + _skip_keys_device_placement = ["past_key_values"] + _supports_flash_attn = True + _supports_sdpa = True + _supports_flex_attn = True + + _can_compile_fullgraph = True + _supports_attention_backend = True + _can_record_outputs = { + "hidden_states": Ministral3DecoderLayer, + "attentions": Ministral3Attention, + } + + +class Ministral3ForQuestionAnswering(GenericForQuestionAnswering, Ministral3PreTrainedModel): + pass + + +__all__ = [ + "Ministral3ForCausalLM", + "Ministral3ForQuestionAnswering", + "Ministral3Model", + "Ministral3PreTrainedModel", + "Ministral3ForSequenceClassification", + "Ministral3ForTokenClassification", +] diff --git a/src/transformers/models/ministral3/modular_ministral3.py b/src/transformers/models/ministral3/modular_ministral3.py new file mode 100644 index 000000000000..b97b951be6e7 --- /dev/null +++ b/src/transformers/models/ministral3/modular_ministral3.py @@ -0,0 +1,124 @@ +from collections.abc import Callable +from typing import Optional + +import torch + +from ...cache_utils import Cache +from ...modeling_flash_attention_utils import FlashAttentionKwargs +from ...modeling_layers import ( + GenericForQuestionAnswering, + GenericForSequenceClassification, + GenericForTokenClassification, +) +from ...modeling_utils import ALL_ATTENTION_FUNCTIONS +from ...processing_utils import Unpack +from ...utils import auto_docstring, logging +from ..mistral.modeling_mistral import ( + MistralAttention, + MistralDecoderLayer, + MistralForCausalLM, + MistralModel, + MistralPreTrainedModel, + apply_rotary_pos_emb, + eager_attention_forward, +) + + +logger = logging.get_logger(__name__) + + +def _get_llama_4_attn_scale(positions_ids: torch.Tensor, beta: float, max_position_embeddings: int) -> torch.Tensor: + scaling = 1 + beta * torch.log(1 + torch.floor(positions_ids / max_position_embeddings)) + return scaling.unsqueeze(-1) + + +class Ministral3Attention(MistralAttention): + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], + attention_mask: Optional[torch.Tensor], + past_key_values: Optional[Cache] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, self.head_dim) + + query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) + key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) + value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) + + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + query_states = query_states * _get_llama_4_attn_scale( + cache_position, + self.config.rope_parameters.get("llama_4_scaling_beta"), + self.config.rope_parameters.get("original_max_position_embeddings"), + ).to(query_states.dtype) + + if past_key_values is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + sliding_window=getattr(self.config, "sliding_window", None), # main diff with Llama + **kwargs, + ) + + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + + +class Ministral3DecoderLayer(MistralDecoderLayer): + pass + + +@auto_docstring +class Ministral3PreTrainedModel(MistralPreTrainedModel): + pass + + +@auto_docstring +class Ministral3Model(MistralModel): + pass + + +@auto_docstring +class Ministral3ForCausalLM(MistralForCausalLM): + pass + + +class Ministral3ForTokenClassification(GenericForTokenClassification, Ministral3PreTrainedModel): + pass + + +class Ministral3ForSequenceClassification(GenericForSequenceClassification, MistralPreTrainedModel): + pass + + +class Ministral3ForQuestionAnswering(GenericForQuestionAnswering, Ministral3PreTrainedModel): + pass + + +__all__ = [ + "Ministral3ForCausalLM", + "Ministral3ForQuestionAnswering", + "Ministral3Model", + "Ministral3PreTrainedModel", + "Ministral3ForSequenceClassification", + "Ministral3ForTokenClassification", +] diff --git a/tests/models/ministral3/__init__.py b/tests/models/ministral3/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/models/ministral3/test_modeling_ministral3.py b/tests/models/ministral3/test_modeling_ministral3.py new file mode 100644 index 000000000000..5f88ea7b5835 --- /dev/null +++ b/tests/models/ministral3/test_modeling_ministral3.py @@ -0,0 +1,239 @@ +# coding=utf-8 +# Copyright 2025 the HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Testing suite for the PyTorch Ministral3 model.""" + +import gc +import logging +import unittest + +import pytest + +from transformers import AutoTokenizer, BitsAndBytesConfig, GenerationConfig, is_torch_available +from transformers.testing_utils import ( + backend_empty_cache, + cleanup, + require_bitsandbytes, + require_flash_attn, + require_torch, + require_torch_accelerator, + slow, + torch_device, +) + + +if is_torch_available(): + import torch + + from transformers import ( + AutoModelForCausalLM, + Ministral3ForCausalLM, + Ministral3Model, + ) + + +from ...causal_lm_tester import CausalLMModelTest, CausalLMModelTester + + +class Ministral3ModelTester(CausalLMModelTester): + if is_torch_available(): + base_model_class = Ministral3Model + + +@require_torch +class Ministral3ModelTest(CausalLMModelTest, unittest.TestCase): + model_tester_class = Ministral3ModelTester + + # TODO (ydshieh): Check this. See https://app.circleci.com/pipelines/github/huggingface/transformers/79245/workflows/9490ef58-79c2-410d-8f51-e3495156cf9c/jobs/1012146 + def is_pipeline_test_to_skip( + self, + pipeline_test_case_name, + config_class, + model_architecture, + tokenizer_name, + image_processor_name, + feature_extractor_name, + processor_name, + ): + return True + + @require_flash_attn + @require_torch_accelerator + @pytest.mark.flash_attn_test + @slow + def test_flash_attn_2_inference_equivalence_right_padding(self): + self.skipTest(reason="Ministral3 flash attention does not support right padding") + + +@require_torch +class Ministral3IntegrationTest(unittest.TestCase): + def tearDown(self): + cleanup(torch_device, gc_collect=True) + + @slow + def test_model_8b_logits(self): + input_ids = [1, 306, 4658, 278, 6593, 310, 2834, 338] + model = AutoModelForCausalLM.from_pretrained("mistralai/Ministral3-8B-Instruct-2410", device_map="auto") + assert isinstance(model, Ministral3ForCausalLM) + input_ids = torch.tensor([input_ids]).to(model.model.embed_tokens.weight.device) + with torch.no_grad(): + out = model(input_ids).logits.float().cpu() + # Expected mean on dim = -1 + EXPECTED_MEAN = torch.tensor([[-1.5029, -7.2815, 4.5190, 0.5930, -5.2526, 3.0765, -0.6314, 1.8068]]) + torch.testing.assert_close(out.mean(-1), EXPECTED_MEAN, rtol=1e-2, atol=1e-2) + # slicing logits[0, 0, 0:30] + EXPECTED_SLICE = torch.tensor([-3.9446, -3.9466, 0.6383, -3.9466, -3.9468, -3.9448, -3.9462, -3.9455, + -3.9451, -0.8244, -3.9472, -3.9458, -3.9460, -3.9406, -3.9462, -3.9462, + -3.9458, -3.9462, -3.9463, -3.9461, -3.9448, -3.9451, -3.9462, -3.9458, + -3.9455, -3.9452, -3.9458, -3.9469, -3.9460, -3.9464]) # fmt: skip + torch.testing.assert_close(out[0, 0, :30], EXPECTED_SLICE, rtol=1e-4, atol=1e-4) + + del model + backend_empty_cache(torch_device) + gc.collect() + + @slow + def test_model_8b_generation(self): + EXPECTED_TEXT_COMPLETION = "My favourite condiment is 100% natural, 100% organic, 100% free of" + prompt = "My favourite condiment is " + tokenizer = AutoTokenizer.from_pretrained("Mistralai/Ministral3-8B-Instruct-2410") + model = Ministral3ForCausalLM.from_pretrained("Mistralai/Ministral3-8B-Instruct-2410", device_map="auto") + input_ids = tokenizer.encode(prompt, return_tensors="pt").to(model.model.embed_tokens.weight.device) + + # greedy generation outputs + generated_ids = model.generate(input_ids, max_new_tokens=20, temperature=0) + text = tokenizer.decode(generated_ids[0], skip_special_tokens=True) + self.assertEqual(EXPECTED_TEXT_COMPLETION, text) + + del model + backend_empty_cache(torch_device) + gc.collect() + + @require_bitsandbytes + @slow + @require_flash_attn + @pytest.mark.flash_attn_test + def test_model_8b_long_prompt(self): + EXPECTED_OUTPUT_TOKEN_IDS = [36850, 4112] + # An input with 4097 tokens that is above the size of the sliding window + input_ids = [1] + [306, 338] * 2048 + model = Ministral3ForCausalLM.from_pretrained( + "Mistralai/Ministral3-8B-Instruct-2410", + device_map="auto", + dtype=torch.bfloat16, + attn_implementation="flash_attention_2", + ) + input_ids = torch.tensor([input_ids]).to(model.model.embed_tokens.weight.device) + generated_ids = model.generate(input_ids, max_new_tokens=4, temperature=0) + self.assertEqual(EXPECTED_OUTPUT_TOKEN_IDS, generated_ids[0][-2:].tolist()) + + # Assisted generation + assistant_model = model + assistant_model.generation_config.num_assistant_tokens = 2 + assistant_model.generation_config.num_assistant_tokens_schedule = "constant" + generated_ids = model.generate(input_ids, max_new_tokens=4, temperature=0) + self.assertEqual(EXPECTED_OUTPUT_TOKEN_IDS, generated_ids[0][-2:].tolist()) + + del assistant_model + del model + backend_empty_cache(torch_device) + gc.collect() + + @slow + @unittest.skip("not working with Ministral3") + @pytest.mark.torch_export_test + def test_export_text_with_hybrid_cache(self): + # TODO: Exportability is not working + from transformers.testing_utils import is_torch_greater_or_equal + + if not is_torch_greater_or_equal("2.6.0"): + self.skipTest(reason="This test requires torch >= 2.6 to run.") + + from transformers.integrations.executorch import TorchExportableModuleForDecoderOnlyLM + + model_id = "Mistralai/Ministral3-8B-Instruct-2410" + model = Ministral3ForCausalLM.from_pretrained( + model_id, + generation_config=GenerationConfig( + use_cache=True, + cache_implementation="static", + cache_config={ + "batch_size": 1, + "max_cache_len": 50, + }, + ), + ) + + # Export + HybridCache + model.eval() + exportable_module = TorchExportableModuleForDecoderOnlyLM(model) + exported_program = exportable_module.export( + input_ids=torch.tensor([[1]], dtype=torch.long, device=model.device), + cache_position=torch.tensor([0], dtype=torch.long, device=model.device), + ) + logging.info(f"\nExported program: {exported_program}") + + # Test generation with the exported model + prompt = "My favourite condiment is " + max_new_tokens_to_generate = 20 + # Generate text with the exported model + tokenizer = AutoTokenizer.from_pretrained(model_id) + export_generated_text = TorchExportableModuleForDecoderOnlyLM.generate( + exported_program, tokenizer, prompt, max_new_tokens=max_new_tokens_to_generate + ) + logging.info(f"\nExport generated texts: '{export_generated_text}'") + + input_text = tokenizer(prompt, return_tensors="pt") + with torch.no_grad(): + eager_outputs = model.generate( + **input_text, + max_new_tokens=max_new_tokens_to_generate, + do_sample=False, # Use greedy decoding to match the exported model + cache_implementation="static", + ) + + eager_generated_text = tokenizer.decode(eager_outputs[0], skip_special_tokens=True) + logging.info(f"\nEager generated texts: '{eager_generated_text}'") + + self.assertEqual(export_generated_text, eager_generated_text) + + @pytest.mark.flash_attn_test + @require_flash_attn + @slow + def test_past_sliding_window_generation(self): + try: + from datasets import load_dataset + except ImportError: + self.skipTest("datasets not found") + + model = Ministral3ForCausalLM.from_pretrained( + "mistralai/Ministral3-8B-Instruct-2410", + device_map="auto", + quantization_config=BitsAndBytesConfig(load_in_4bit=True), + ) + tokenizer = AutoTokenizer.from_pretrained("mistralai/Ministral3-8B-Instruct-2410", legacy=False) + + wiki = load_dataset("wikitext", "wikitext-103-raw-v1", split="validation") + chunks = [x["text"] for x in wiki.select(range(550)) if x["text"].strip()] + real_corpus = "\n".join(chunks) + prompt = f"[INST]{real_corpus} Question: Based on the text, at which depth of the continental shelf does H. Gammarus live?[/INST]" + + inputs = tokenizer(prompt, return_tensors="pt").to(model.device) + input_length = inputs.input_ids.shape[1] # around 33k tokens > 32k sliding window + outputs = model.generate(**inputs, max_new_tokens=100, do_sample=False) + output_text = tokenizer.decode(outputs[0][input_length:], skip_special_tokens=True) + self.assertEqual( + output_text, + " H. Gammarus lives on the continental shelf at depths of 0 – 150 metres ( 0 – 492 ft ) , although not normally deeper than 50 m ( 160 ft ) .", + ) From 05db6083e0fc8393c62d2c7f8d39f013780cf1fd Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Sun, 30 Nov 2025 22:12:14 +0100 Subject: [PATCH 02/29] WIP --- docs/source/en/contributing.md | 515 +-------------------------------- docs/source/en/notebooks.md | 114 +------- 2 files changed, 2 insertions(+), 627 deletions(-) mode change 100644 => 120000 docs/source/en/contributing.md mode change 100644 => 120000 docs/source/en/notebooks.md diff --git a/docs/source/en/contributing.md b/docs/source/en/contributing.md deleted file mode 100644 index d5b2b350a9a2..000000000000 --- a/docs/source/en/contributing.md +++ /dev/null @@ -1,514 +0,0 @@ - - -# Contribute to πŸ€— Transformers - -Everyone is welcome to contribute, and we value everybody's contribution. Code -contributions are not the only way to help the community. Answering questions, helping -others, and improving the documentation are also immensely valuable. - -It also helps us if you spread the word! Reference the library in blog posts -about the awesome projects it made possible, shout out on Twitter every time it has -helped you, or simply ⭐️ the repository to say thank you. - -However you choose to contribute, please be mindful and respect our -[code of conduct](https://github.com/huggingface/transformers/blob/main/CODE_OF_CONDUCT.md). - -**This guide was heavily inspired by the awesome [scikit-learn guide to contributing](https://github.com/scikit-learn/scikit-learn/blob/main/CONTRIBUTING.md).** - -## Ways to contribute - -There are several ways you can contribute to πŸ€— Transformers: - -* Fix outstanding issues with the existing code. -* Submit issues related to bugs or desired new features. -* Implement new models. -* Contribute to the examples or to the documentation. - -If you don't know where to start, there is a special [Good First -Issue](https://github.com/huggingface/transformers/contribute) listing. It will give you a list of -open issues that are beginner-friendly and help you start contributing to open-source. The best way to do that is to open a Pull Request and link it to the issue that you'd like to work on. We try to give priority to opened PRs as we can easily track the progress of the fix, and if the contributor does not have time anymore, someone else can take the PR over. - -For something slightly more challenging, you can also take a look at the [Good Second Issue](https://github.com/huggingface/transformers/labels/Good%20Second%20Issue) list. In general though, if you feel like you know what you're doing, go for it and we'll help you get there! πŸš€ - -> All contributions are equally valuable to the community. πŸ₯° - -## Fixing outstanding issues - -If you notice an issue with the existing code and have a fix in mind, feel free to [start contributing](#create-a-pull-request) and open a Pull Request! - -## Submitting a bug-related issue or feature request - -Do your best to follow these guidelines when submitting a bug-related issue or a feature -request. It will make it easier for us to come back to you quickly and with good -feedback. - -### Did you find a bug? - -The πŸ€— Transformers library is robust and reliable thanks to users who report the problems they encounter. - -Before you report an issue, we would really appreciate it if you could **make sure the bug was not -already reported** (use the search bar on GitHub under Issues). Your issue should also be related to bugs in the library itself, and not your code. If you're unsure whether the bug is in your code or the library, please ask in the [forum](https://discuss.huggingface.co/) or on our [discord](https://discord.com/invite/hugging-face-879548962464493619) first. This helps us respond quicker to fixing issues related to the library versus general questions. - -> [!TIP] -> We have a [docs bot](https://huggingface.co/spaces/huggingchat/hf-docs-chat), and we highly encourage you to ask all your questions there. There is always a chance your bug can be fixed with a simple flag πŸ‘ΎπŸ”« - -Once you've confirmed the bug hasn't already been reported, please include the following information in your issue so we can quickly resolve it: - -* Your **OS type and version** and **Python**, and **PyTorch** versions when applicable. -* A short, self-contained, code snippet that allows us to reproduce the bug in - less than 30s. -* The *full* traceback if an exception is raised. -* Attach any other additional information, like screenshots, you think may help. - -To get the OS and software versions automatically, run the following command: - -```bash -transformers env -``` - -You can also run the same command from the root of the repository: - -```bash -python src/transformers/commands/transformers_cli.py env -``` - -### Do you want a new feature? - -If there is a new feature you'd like to see in πŸ€— Transformers, please open an issue and describe: - -1. What is the *motivation* behind this feature? Is it related to a problem or frustration with the library? Is it a feature related to something you need for a project? Is it something you worked on and think it could benefit the community? - - Whatever it is, we'd love to hear about it! - -2. Describe your requested feature in as much detail as possible. The more you can tell us about it, the better we'll be able to help you. -3. Provide a *code snippet* that demonstrates the features usage. -4. If the feature is related to a paper, please include a link. - -If your issue is well written we're already 80% of the way there by the time you create it. - -We have added [templates](https://github.com/huggingface/transformers/tree/main/templates) to help you get started with your issue. - -## Do you want to implement a new model? - -New models are constantly released and if you want to implement a new model, please provide the following information: - -* A short description of the model and a link to the paper. -* Link to the implementation if it is open-sourced. -* Link to the model weights if they are available. - -If you are willing to contribute the model yourself, let us know so we can help you add it to πŸ€— Transformers! - -We have a technical guide for [how to add a model to πŸ€— Transformers](https://huggingface.co/docs/transformers/modular_transformers). - -### Vision-Language Model Contribution Checklist - -If you're contributing a **vision-language model** (or any multimodal model that processes images/videos), please follow this checklist. Maintainers will use this to review your PR, and completing these steps will significantly increase the likelihood of your PR being merged quickly. - -**Required checklist for all vision-language model contributions:** - -☐ **1. Implement a modular file** - -All new models should use the modular architecture pattern. Create a `modular_.py` file using the modular model converter: - -- Use the CLI, [`transformers add-new-model-like`](https://github.com/huggingface/transformers/blob/main/src/transformers/cli/add_new_model_like.py) to generate a modular skeleton and get started -- All code should be in the modular file if possible. Modeling must be in it, it's better if configuration is in it as well. [Modular guide](./docs/source/en/modular_transformers.md#implementing-a-modular-file) shows a quick way to set up a modular file. -- Reuse existing patterns from similar models as much as possible -- You can make the model compatible with inference engines such as vLLM or SGLang, and enable zero-effort integration. See specific requirements for model implementation in ["Transformers modeling backend"](./docs/source/en/transformers_as_backend.md#multimodal-models) - -To verify your modular file is correct, run: - -```bash -python utils/modular_model_converter.py -``` - -This will generate the separate files (`modeling_*.py`, `configuration_*.py`, etc.) from your modular file. The CI will enforce that these generated files match your modular file. - -☐ **2. Add a fast image processor (for image models)** - -If your model processes images, implement a fast image processor that uses `torch` and `torchvision` instead of PIL/numpy for better inference performance: - -- See the detailed guide in [#36978](https://github.com/huggingface/transformers/issues/36978) -- Fast processors inherit from `BaseImageProcessorFast` -- Examples: `LlavaOnevisionImageProcessorFast`, `Idefics2ImageProcessorFast` - -☐ **3. Create a weight conversion script** - -Add a `convert__to_hf.py` script that converts the original model weights to the HuggingFace format: - -- Script should handle checkpoint loading, key mapping, and saving in HF format -- Include usage examples and documentation in the script -- Examples: [`convert_llava_onevision_weights_to_hf.py`](https://github.com/huggingface/transformers/blob/main/src/transformers/models/llava_onevision/convert_llava_onevision_weights_to_hf.py), [`convert_idefics2_weights_to_hf.py`](https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics2/convert_idefics2_weights_to_hf.py) - -☐ **4. Add integration tests with exact output matching** - -At minimum, add an `IntegrationTest` class that tests end-to-end generation (processing and modelling) with **exact** output matching: - -- For generative models: test that generated text matches expected output exactly -- For non-generative models: test that output logits match expected values -- Tests should use real checkpoints (load in 4-bit or half precision if the checkpoint is too big to fit in our CI runners) and real inputs -- Example pattern: - -```python -class MyModelIntegrationTest(unittest.TestCase): - @slow - def test_model_integration(self): - model = MyModelForConditionalGeneration.from_pretrained("org/model-name") - processor = AutoProcessor.from_pretrained("org/model-name") - - inputs = processor(images=image, text=prompt, return_tensors="pt") - output = model.generate(**inputs, max_new_tokens=20) - - EXPECTED_TEXT = "exact expected output" - self.assertEqual(processor.decode(output[0]), EXPECTED_TEXT) -``` - -See `tests/models/llava_onevision/test_modeling_llava_onevision.py` for complete examples. - -☐ **5. Update documentation** - -Add or update model documentation: - -- Create if the cli hasn't `docs/source/en/model_doc/.md` with usage examples -- Include model description, paper link, and basic usage with `Pipeline` and `AutoModel` -- Add the model to the appropriate TOC files - -☐ **6. Look for reusable patterns** - -The library has 400+ models with many established patterns: - -- Search for similar models (e.g., other vision-language models) -- Reuse attention mechanisms, layer implementations, and processing patterns -- Check models like LLaVA, Idefics2, Fuyu for vision-language patterns -- Use provided decorators like (`auto_docstring`, `can_return_tuple`, `check_model_inputs` and `_can_record_outputs`) where relevant. -- Don't reinvent the wheel - -☐ **7. Run quality checks and read the output** - -Before submitting your PR, install quality dependencies and run the full check suite: - -```bash -pip install -e ".[quality]" -make fixup -``` - -**Important**: Take time to read the output of `make fixup`. It will: -- Lint and format your code automatically -- Run consistency checks (imports, docstrings, etc.) -- Show any remaining issues that need manual fixes - -All checks must pass before your PR can be merged. - -**If this checklist is complete, your PR has a very high likelihood of being merged!** Following these steps makes the maintainers' work much easier and will reduce the number of review iterations, getting your important work out there faster. - -#### Copy-pastable checklist for maintainers - -Here's a condensed version maintainers can copy into PRs: - -```markdown -## Multimodal Model Addition Checklist - -Please ensure your PR completes all following items. See the [full checklist](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#vision-language-model-contribution-checklist) for details. - -- [ ] **Modular file**: `modular_.py` implemented and verified with `python utils/modular_model_converter.py ` -- [ ] **Fast image processor**: Implemented using `BaseImageProcessorFast` (see [#36978](https://github.com/huggingface/transformers/issues/36978)) -- [ ] **Conversion script**: `convert__to_hf.py` added with usage examples -- [ ] **Integration tests**: End-to-end tests with exact output matching (text or logits) -- [ ] **Documentation**: Model docs added/updated in `docs/source/en/model_doc/` -- [ ] **Pattern reuse**: Verified against similar models (LLaVA, Idefics2, etc.) -- [ ] **Quality checks**: `make fixup` passes with no errors - -``` - -## Do you want to add documentation? - -We're always looking for improvements to the documentation that make it more clear and accurate. Please let us know how the documentation can be improved such as typos and any content that is missing, unclear or inaccurate. We'll be happy to make the changes or help you make a contribution if you're interested! - -For more details about how to generate, build, and write the documentation, take a look at the documentation [README](https://github.com/huggingface/transformers/tree/main/docs). - -## Create a Pull Request - -Before writing any code, we strongly advise you to search through the existing PRs or -issues to make sure nobody is already working on the same thing. If you are -unsure, it is always a good idea to open an issue to get some feedback. - -You will need basic `git` proficiency to contribute to -πŸ€— Transformers. While `git` is not the easiest tool to use, it has the greatest -manual. Type `git --help` in a shell and enjoy! If you prefer books, [Pro -Git](https://git-scm.com/book/en/v2) is a very good reference. - -You'll need **[Python 3.9](https://github.com/huggingface/transformers/blob/main/setup.py#L449)** or above to contribute to πŸ€— Transformers. Follow the steps below to start contributing: - -1. Fork the [repository](https://github.com/huggingface/transformers) by - clicking on the **[Fork](https://github.com/huggingface/transformers/fork)** button on the repository's page. This creates a copy of the code - under your GitHub user account. - -2. Clone your fork to your local disk, and add the base repository as a remote: - - ```bash - git clone git@github.com:/transformers.git - cd transformers - git remote add upstream https://github.com/huggingface/transformers.git - ``` - -3. Create a new branch to hold your development changes: - - ```bash - git checkout -b a-descriptive-name-for-my-changes - ``` - - 🚨 **Do not** work on the `main` branch! - -4. Set up a development environment by running the following command in a virtual environment: - - ```bash - pip install -e ".[dev]" - ``` - - If πŸ€— Transformers was already installed in the virtual environment, remove - it with `pip uninstall transformers` before reinstalling it in editable - mode with the `-e` flag. - - Depending on your OS, and since the number of optional dependencies of Transformers is growing, you might get a - failure with this command. If that's the case make sure to install Pytorch then do: - - ```bash - pip install -e ".[quality]" - ``` - - which should be enough for most use cases. - -5. Develop the features in your branch. - - As you work on your code, you should make sure the test suite - passes. Run the tests impacted by your changes like this: - - ```bash - pytest tests/.py - ``` - - For more information about tests, check out the - [Testing](https://huggingface.co/docs/transformers/testing) guide. - - πŸ€— Transformers relies on `black` and `ruff` to format its source code - consistently. After you make changes, apply automatic style corrections and code verifications - that can't be automated in one go with: - - ```bash - make fixup - ``` - - This target is also optimized to only work with files modified by the PR you're working on. - - If you prefer to run the checks one after the other, the following command applies the - style corrections: - - ```bash - make style - ``` - - πŸ€— Transformers also uses `ruff` and a few custom scripts to check for coding mistakes. Quality - controls are run by the CI, but you can run the same checks with: - - ```bash - make quality - ``` - - Finally, we have a lot of scripts to make sure we don't forget to update - some files when adding a new model. You can run these scripts with: - - ```bash - make repo-consistency - ``` - - To learn more about those checks and how to fix any issues with them, check out the - [Checks on a Pull Request](https://huggingface.co/docs/transformers/pr_checks) guide. - - If you're modifying documents under the `docs/source` directory, make sure the documentation can still be built. This check will also run in the CI when you open a pull request. To run a local check - make sure you install the [documentation builder](https://github.com/huggingface/doc-builder). - - ```bash - pip install hf-doc-builder - ``` - - Run the following command from the root of the repository: - - ```bash - doc-builder build transformers docs/source/en --build_dir ~/tmp/test-build - ``` - - This will build the documentation in the `~/tmp/test-build` folder where you can inspect the generated - Markdown files with your favorite editor. You can also preview the docs on GitHub when you open a pull request. - - Once you're happy with your changes, add the changed files with `git add` and - record your changes locally with `git commit`: - - ```bash - git add modified_file.py - git commit - ``` - - Please remember to write [good commit - messages](https://chris.beams.io/posts/git-commit/) to clearly communicate the changes you made! - - To keep your copy of the code up to date with the original - repository, rebase your branch on `upstream/branch` *before* you open a pull request or if requested by a maintainer: - - ```bash - git fetch upstream - git rebase upstream/main - ``` - - Push your changes to your branch: - - ```bash - git push -u origin a-descriptive-name-for-my-changes - ``` - - If you've already opened a pull request, you'll need to force push with the `--force` flag. Otherwise, if the pull request hasn't been opened yet, you can just push your changes normally. - -6. Now you can go to your fork of the repository on GitHub and click on **Pull Request** to open a pull request. Make sure you tick off all the boxes on our [checklist](#pull-request-checklist) below. When you're ready, you can send your changes to the project maintainers for review. - -7. It's ok if maintainers request changes, it happens to our core contributors - too! So everyone can see the changes in the pull request, work in your local - branch and push the changes to your fork. They will automatically appear in - the pull request. - -### Pull request checklist - -☐ The pull request title should summarize your contribution.
-☐ If your pull request addresses an issue, please mention the issue number in the pull -request description to make sure they are linked (and people viewing the issue know you -are working on it).
-☐ To indicate a work in progress please prefix the title with `[WIP]`. These are -useful to avoid duplicated work, and to differentiate it from PRs ready to be merged.
-☐ Make sure existing tests pass.
-☐ If adding a new feature, also add tests for it.
- -- If you are adding a new model, make sure you use - `ModelTester.all_model_classes = (MyModel, MyModelWithLMHead,...)` to trigger the common tests. -- If you are adding new `@slow` tests, make sure they pass using - `RUN_SLOW=1 python -m pytest tests/models/my_new_model/test_my_new_model.py`. -- If you are adding a new tokenizer, write tests and make sure - `RUN_SLOW=1 python -m pytest tests/models/{your_model_name}/test_tokenization_{your_model_name}.py` passes. -- CircleCI does not run the slow tests, but GitHub Actions does every night!
- -☐ All public methods must have informative docstrings (see -[`modeling_bert.py`](https://github.com/huggingface/transformers/blob/main/src/transformers/models/bert/modeling_bert.py) -for an example).
-☐ Due to the rapidly growing repository, don't add any images, videos and other -non-text files that'll significantly weigh down the repository. Instead, use a Hub -repository such as [`hf-internal-testing`](https://huggingface.co/hf-internal-testing) -to host these files and reference them by URL. We recommend placing documentation -related images in the following repository: -[huggingface/documentation-images](https://huggingface.co/datasets/huggingface/documentation-images). -You can open a PR on this dataset repository and ask a Hugging Face member to merge it. - -For more information about the checks run on a pull request, take a look at our [Checks on a Pull Request](https://huggingface.co/docs/transformers/pr_checks) guide. - -### Tests - -An extensive test suite is included to test the library behavior and several examples. Library tests can be found in -the [tests](https://github.com/huggingface/transformers/tree/main/tests) folder and examples tests in the -[examples](https://github.com/huggingface/transformers/tree/main/examples) folder. - -We like `pytest` and `pytest-xdist` because it's faster. From the root of the -repository, specify a *path to a subfolder or a test file* to run the test: - -```bash -python -m pytest -n auto --dist=loadfile -s -v ./tests/models/my_new_model -``` - -Similarly, for the `examples` directory, specify a *path to a subfolder or test file* to run the test. For example, the following command tests the text classification subfolder in the PyTorch `examples` directory: - -```bash -pip install -r examples/xxx/requirements.txt # only needed the first time -python -m pytest -n auto --dist=loadfile -s -v ./examples/pytorch/text-classification -``` - -In fact, this is actually how our `make test` and `make test-examples` commands are implemented (not including the `pip install`)! - -You can also specify a smaller set of tests in order to test only the feature -you're working on. - -By default, slow tests are skipped but you can set the `RUN_SLOW` environment variable to -`yes` to run them. This will download many gigabytes of models so make sure you -have enough disk space, a good internet connection or a lot of patience! - - - -Remember to specify a *path to a subfolder or a test file* to run the test. Otherwise, you'll run all the tests in the `tests` or `examples` folder, which will take a very long time! - - - -```bash -RUN_SLOW=yes python -m pytest -n auto --dist=loadfile -s -v ./tests/models/my_new_model -RUN_SLOW=yes python -m pytest -n auto --dist=loadfile -s -v ./examples/pytorch/text-classification -``` - -Like the slow tests, there are other environment variables available which are not enabled by default during testing: - -- `RUN_CUSTOM_TOKENIZERS`: Enables tests for custom tokenizers. - -More environment variables and additional information can be found in the [testing_utils.py](https://github.com/huggingface/transformers/blob/main/src/transformers/testing_utils.py). - -πŸ€— Transformers uses `pytest` as a test runner only. It doesn't use any -`pytest`-specific features in the test suite itself. - -This means `unittest` is fully supported. Here's how to run tests with -`unittest`: - -```bash -python -m unittest discover -s tests -t . -v -python -m unittest discover -s examples -t examples -v -``` - -### Style guide - -For documentation strings, πŸ€— Transformers follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html). -Check our [documentation writing guide](https://github.com/huggingface/transformers/tree/main/docs#writing-documentation---specification) -for more information. - -### Develop on Windows - -On Windows (unless you're working in [Windows Subsystem for Linux](https://learn.microsoft.com/en-us/windows/wsl/) or WSL), you need to configure git to transform Windows `CRLF` line endings to Linux `LF` line endings: - -```bash -git config core.autocrlf input -``` - -One way to run the `make` command on Windows is with MSYS2: - -1. [Download MSYS2](https://www.msys2.org/), and we assume it's installed in `C:\msys64`. -2. Open the command line `C:\msys64\msys2.exe` (it should be available from the **Start** menu). -3. Run in the shell: `pacman -Syu` and install `make` with `pacman -S make`. -4. Add `C:\msys64\usr\bin` to your PATH environment variable. - -You can now use `make` from any terminal (PowerShell, cmd.exe, etc.)! πŸŽ‰ - -### Sync a forked repository with upstream main (the Hugging Face repository) - -When updating the main branch of a forked repository, please follow these steps to avoid pinging the upstream repository which adds reference notes to each upstream PR, and sends unnecessary notifications to the developers involved in these PRs. - -1. When possible, avoid syncing with the upstream using a branch and PR on the forked repository. Instead, merge directly into the forked main. -2. If a PR is absolutely necessary, use the following steps after checking out your branch: - - ```bash - git checkout -b your-branch-for-syncing - git pull --squash --no-commit upstream main - git commit -m '' - git push --set-upstream origin your-branch-for-syncing - ``` diff --git a/docs/source/en/contributing.md b/docs/source/en/contributing.md new file mode 120000 index 000000000000..c97564d93a7f --- /dev/null +++ b/docs/source/en/contributing.md @@ -0,0 +1 @@ +../../../CONTRIBUTING.md \ No newline at end of file diff --git a/docs/source/en/notebooks.md b/docs/source/en/notebooks.md deleted file mode 100644 index faa777a6a313..000000000000 --- a/docs/source/en/notebooks.md +++ /dev/null @@ -1,113 +0,0 @@ - - -# πŸ€— Transformers Notebooks - -You can find here a list of the official notebooks provided by Hugging Face. - -Also, we would like to list here interesting content created by the community. -If you wrote some notebook(s) leveraging πŸ€— Transformers and would like to be listed here, please open a -Pull Request so it can be included under the Community notebooks. - -## Hugging Face's notebooks πŸ€— - -### Documentation notebooks - -You can open any page of the documentation as a notebook in Colab (there is a button directly on said pages) but they are also listed here if you need them: - -| Notebook | Description | | | | -|:----------|:-------------|:-------------|:------------|------:| -| [Quicktour of the library](https://github.com/huggingface/notebooks/blob/main/transformers_doc/en/quicktour.ipynb) | A presentation of the various APIs in Transformers |[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/quicktour.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/en/transformers_doc/quicktour.ipynb)| | -| [Summary of the tasks](https://github.com/huggingface/notebooks/blob/main/transformers_doc/en/task_summary.ipynb) | How to run the models of the Transformers library task by task |[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/task_summary.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/task_summary.ipynb)| | -| [Preprocessing data](https://github.com/huggingface/notebooks/blob/main/transformers_doc/en/preprocessing.ipynb) | How to use a tokenizer to preprocess your data |[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/preprocessing.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/preprocessing.ipynb)|| -| [Fine-tuning a pretrained model](https://github.com/huggingface/notebooks/blob/main/transformers_doc/en/training.ipynb) | How to use the Trainer to fine-tune a pretrained model |[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/training.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/training.ipynb)|[![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/transformers_doc/en/training.ipynb)| -| [Summary of the tokenizers](https://github.com/huggingface/notebooks/blob/main/transformers_doc/en/tokenizer_summary.ipynb) | The differences between the tokenizers algorithm |[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/tokenizer_summary.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/tokenizer_summary.ipynb)|[![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/transformers_doc/en/tokenizer_summary.ipynb )| -| [Multilingual models](https://github.com/huggingface/notebooks/blob/main/transformers_doc/en/multilingual.ipynb) | How to use the multilingual models of the library |[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/multilingual.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/multilingual.ipynb)|[![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/transformers_doc/en/multilingual.ipynb)| - -### PyTorch Examples - -#### Natural Language Processing[[pytorch-nlp]] - -| Notebook | Description | | | | -|:----------|:-------------|:-------------|:-------------|------:| -| [Train your tokenizer](https://github.com/huggingface/notebooks/blob/main/examples/tokenizer_training.ipynb) | How to train and use your very own tokenizer |[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/tokenizer_training.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/tokenizer_training.ipynb)|[![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/examples/tokenizer_training.ipynb)| -| [Train your language model](https://github.com/huggingface/notebooks/blob/main/examples/language_modeling_from_scratch.ipynb) | How to easily start using transformers |[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling_from_scratch.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/language_modeling_from_scratch.ipynb)|[![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/examples/language_modeling_from_scratch.ipynb)| -| [How to fine-tune a model on text classification](https://github.com/huggingface/notebooks/blob/main/examples/text_classification.ipynb)| Show how to preprocess the data and fine-tune a pretrained model on any GLUE task. | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/text_classification.ipynb)| | -| [How to fine-tune a model on language modeling](https://github.com/huggingface/notebooks/blob/main/examples/language_modeling.ipynb)| Show how to preprocess the data and fine-tune a pretrained model on a causal or masked LM task. | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/language_modeling.ipynb)| [![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/examples/language_modeling.ipynb)| -| [How to fine-tune a model on token classification](https://github.com/huggingface/notebooks/blob/main/examples/token_classification.ipynb)| Show how to preprocess the data and fine-tune a pretrained model on a token classification task (NER, PoS). | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/token_classification.ipynb)|[![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/examples/token_classification.ipynb)| -| [How to fine-tune a model on question answering](https://github.com/huggingface/notebooks/blob/main/examples/question_answering.ipynb)| Show how to preprocess the data and fine-tune a pretrained model on SQUAD. | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/question_answering.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/question_answering.ipynb)|[![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/examples/question_answering.ipynb)| -| [How to fine-tune a model on multiple choice](https://github.com/huggingface/notebooks/blob/main/examples/multiple_choice.ipynb)| Show how to preprocess the data and fine-tune a pretrained model on SWAG. | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/multiple_choice.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/multiple_choice.ipynb)|[![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/examples/multiple_choice.ipynb)| -| [How to fine-tune a model on translation](https://github.com/huggingface/notebooks/blob/main/examples/translation.ipynb)| Show how to preprocess the data and fine-tune a pretrained model on WMT. | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/translation.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/translation.ipynb)|[![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/examples/translation.ipynb)| -| [How to fine-tune a model on summarization](https://github.com/huggingface/notebooks/blob/main/examples/summarization.ipynb)| Show how to preprocess the data and fine-tune a pretrained model on XSUM. | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/summarization.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/summarization.ipynb)| | -| [How to train a language model from scratch](https://github.com/huggingface/blog/blob/main/notebooks/01_how_to_train.ipynb)| Highlight all the steps to effectively train Transformer model on custom data | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/main/notebooks/01_how_to_train.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/blog/blob/main/notebooks/01_how_to_train.ipynb)| | -| [How to generate text](https://github.com/huggingface/blog/blob/main/notebooks/02_how_to_generate.ipynb)| How to use different decoding methods for language generation with transformers | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/main/notebooks/02_how_to_generate.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/blog/blob/main/notebooks/02_how_to_generate.ipynb)| | -| [Reformer](https://github.com/huggingface/blog/blob/main/notebooks/03_reformer.ipynb)| How Reformer pushes the limits of language modeling | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/blog/blob/main/notebooks/03_reformer.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/patrickvonplaten/blog/blob/main/notebooks/03_reformer.ipynb)| | - -#### Computer Vision[[pytorch-cv]] - -| Notebook | Description | | | | -|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------|------:| -| [How to fine-tune a model on image classification (Torchvision)](https://github.com/huggingface/notebooks/blob/main/examples/image_classification.ipynb) | Show how to preprocess the data using Torchvision and fine-tune any pretrained Vision model on Image Classification | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb)| [![Open in AMD Dev Cloud](https://oneclickamd.ai/static/amd.svg?v=2)](https://oneclickamd.ai/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb)| -| [How to fine-tune a model on image classification (Albumentations)](https://github.com/huggingface/notebooks/blob/main/examples/image_classification_albumentations.ipynb) | Show how to preprocess the data using Albumentations and fine-tune any pretrained Vision model on Image Classification | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_albumentations.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/image_classification_albumentations.ipynb)| | -| [How to fine-tune a model on image classification (Kornia)](https://github.com/huggingface/notebooks/blob/main/examples/image_classification_kornia.ipynb) | Show how to preprocess the data using Kornia and fine-tune any pretrained Vision model on Image Classification | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_kornia.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/image_classification_kornia.ipynb)| | -| [How to perform zero-shot object detection with OWL-ViT](https://github.com/huggingface/notebooks/blob/main/examples/zeroshot_object_detection_with_owlvit.ipynb) | Show how to perform zero-shot object detection on images with text queries | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/zeroshot_object_detection_with_owlvit.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/zeroshot_object_detection_with_owlvit.ipynb)| | -| [How to fine-tune an image captioning model](https://github.com/huggingface/notebooks/blob/main/examples/image_captioning_blip.ipynb) | Show how to fine-tune BLIP for image captioning on a custom dataset | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_captioning_blip.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/image_captioning_blip.ipynb)| | -| [How to build an image similarity system with Transformers](https://github.com/huggingface/notebooks/blob/main/examples/image_similarity.ipynb) | Show how to build an image similarity system | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_similarity.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/image_similarity.ipynb)| | -| [How to fine-tune a SegFormer model on semantic segmentation](https://github.com/huggingface/notebooks/blob/main/examples/semantic_segmentation.ipynb) | Show how to preprocess the data and fine-tune a pretrained SegFormer model on Semantic Segmentation | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/semantic_segmentation.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/semantic_segmentation.ipynb)| | -| [How to fine-tune a VideoMAE model on video classification](https://github.com/huggingface/notebooks/blob/main/examples/video_classification.ipynb) | Show how to preprocess the data and fine-tune a pretrained VideoMAE model on Video Classification | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/video_classification.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/video_classification.ipynb)| | - -#### Audio[[pytorch-audio]] - -| Notebook | Description | | | -|:----------|:-------------|:-------------|------:| -| [How to fine-tune a speech recognition model in English](https://github.com/huggingface/notebooks/blob/main/examples/speech_recognition.ipynb)| Show how to preprocess the data and fine-tune a pretrained Speech model on TIMIT | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/speech_recognition.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/speech_recognition.ipynb)| -| [How to fine-tune a speech recognition model in any language](https://github.com/huggingface/notebooks/blob/main/examples/multi_lingual_speech_recognition.ipynb)| Show how to preprocess the data and fine-tune a multi-lingually pretrained speech model on Common Voice | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/multi_lingual_speech_recognition.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/multi_lingual_speech_recognition.ipynb)| -| [How to fine-tune a model on audio classification](https://github.com/huggingface/notebooks/blob/main/examples/audio_classification.ipynb)| Show how to preprocess the data and fine-tune a pretrained Speech model on Keyword Spotting | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/audio_classification.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/audio_classification.ipynb)| - -#### Biological Sequences[[pytorch-bio]] - -| Notebook | Description | | | -|:----------|:----------------------------------------------------------------------------------------|:-------------|------:| -| [How to fine-tune a pre-trained protein model](https://github.com/huggingface/notebooks/blob/main/examples/protein_language_modeling.ipynb) | See how to tokenize proteins and fine-tune a large pre-trained protein "language" model | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/protein_language_modeling.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/protein_language_modeling.ipynb) | -| [How to generate protein folds](https://github.com/huggingface/notebooks/blob/main/examples/protein_folding.ipynb) | See how to go from protein sequence to a full protein model and PDB file | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/protein_folding.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/protein_folding.ipynb) | -| [How to fine-tune a Nucleotide Transformer model](https://github.com/huggingface/notebooks/blob/main/examples/nucleotide_transformer_dna_sequence_modelling.ipynb) | See how to tokenize DNA and fine-tune a large pre-trained DNA "language" model | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/nucleotide_transformer_dna_sequence_modelling.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/nucleotide_transformer_dna_sequence_modelling.ipynb) | -| [Fine-tune a Nucleotide Transformer model with LoRA](https://github.com/huggingface/notebooks/blob/main/examples/nucleotide_transformer_dna_sequence_modelling_with_peft.ipynb) | Train even larger DNA models in a memory-efficient way | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/nucleotide_transformer_dna_sequence_modelling_with_peft.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/nucleotide_transformer_dna_sequence_modelling_with_peft.ipynb) | - -#### Other modalities[[pytorch-other]] - -| Notebook | Description | | | -|:----------|:----------------------------------------------------------------------------------------|:-------------|------:| -| [Probabilistic Time Series Forecasting](https://github.com/huggingface/notebooks/blob/main/examples/time-series-transformers.ipynb) | See how to train Time Series Transformer on a custom dataset | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/time-series-transformers.ipynb) | [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/time-series-transformers.ipynb) | - -#### Utility notebooks[[pytorch-utility]] - -| Notebook | Description | | | -|:----------|:-------------|:-------------|------:| -| [How to export model to ONNX](https://github.com/huggingface/notebooks/blob/main/examples/onnx-export.ipynb)| Highlight how to export and run inference workloads through ONNX | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/onnx-export.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/onnx-export.ipynb)| - -### Optimum notebooks - -πŸ€— [Optimum](https://github.com/huggingface/optimum) is an extension of πŸ€— Transformers, providing a set of performance optimization tools enabling maximum efficiency to train and run models on targeted hardware. - -| Notebook | Description | | | -|:----------|:-------------|:-------------|------:| -| [How to quantize a model with ONNX Runtime for text classification](https://github.com/huggingface/notebooks/blob/main/examples/text_classification_quantization_ort.ipynb)| Show how to apply static and dynamic quantization on a model using [ONNX Runtime](https://github.com/microsoft/onnxruntime) for any GLUE task. | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification_quantization_ort.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/text_classification_quantization_ort.ipynb)| -| [How to fine-tune a model on text classification with ONNX Runtime](https://github.com/huggingface/notebooks/blob/main/examples/text_classification_ort.ipynb)| Show how to preprocess the data and fine-tune a model on any GLUE task using [ONNX Runtime](https://github.com/microsoft/onnxruntime). | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification_ort.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/text_classification_ort.ipynb)| -| [How to fine-tune a model on summarization with ONNX Runtime](https://github.com/huggingface/notebooks/blob/main/examples/summarization_ort.ipynb)| Show how to preprocess the data and fine-tune a model on XSUM using [ONNX Runtime](https://github.com/microsoft/onnxruntime). | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/summarization_ort.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/examples/summarization_ort.ipynb)| - -## Community notebooks - -More notebooks developed by the community are available [here](https://hf.co/docs/transformers/community#community-notebooks). diff --git a/docs/source/en/notebooks.md b/docs/source/en/notebooks.md new file mode 120000 index 000000000000..10fb7a7b979a --- /dev/null +++ b/docs/source/en/notebooks.md @@ -0,0 +1 @@ +../../../notebooks/README.md \ No newline at end of file From b8526008e032e2dd0946e7ac045eb778509c8e2f Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Sun, 30 Nov 2025 22:01:56 +0000 Subject: [PATCH 03/29] WIP --- docs/source/en/model_doc/ministral3.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/source/en/model_doc/ministral3.md b/docs/source/en/model_doc/ministral3.md index 49a8f7321343..93eee195f191 100644 --- a/docs/source/en/model_doc/ministral3.md +++ b/docs/source/en/model_doc/ministral3.md @@ -40,20 +40,15 @@ Key features: ## Usage examples ```py -from datetime import datetime, timedelta import torch - -from mistral_common.protocol.instruct.request import ChatCompletionRequest -from mistral_common.tokens.tokenizers.mistral import MistralTokenizer -from huggingface_hub import hf_hub_download -from transformers import Mistral3ForConditionalGeneration +from transformers import Mistral3ForConditionalGeneration, MistralCommonBackend model_id = "mistralai/Ministral-3-3B-Instruct-2512" -tokenizer = AutoTokenizer.from_pretrained(model_id) +tokenizer = MistralCommonBackend.from_pretrained(model_id) model = Mistral3ForConditionalGeneration.from_pretrained( - model_id, torch_dtype=torch.bfloat16 + model_id, torch_dtype=torch.bfloat16, device_map="auto" ) image_url = "https://static.wikia.nocookie.net/essentialsdocs/images/7/70/Battle.png/revision/latest?cb=20220523172438" @@ -73,12 +68,17 @@ messages = [ tokenized = tokenizer.apply_chat_template(messages, return_tensors="pt", return_dict=True) +tokenized["input_ids"] = tokenized["input_ids"].to(device="cuda") +tokenized["pixel_values"] = tokenized["pixel_values"].to(dtype=torch.bfloat16, device="cuda") +image_sizes = [tokenized["pixel_values"].shape[-2:]] + output = model.generate( - tokenized, - max_new_tokens=1000, + **tokenized, + image_sizes=image_sizes, + max_new_tokens=512, )[0] -decoded_output = tokenizer.decode(output[len(tokenized.tokens) :]) +decoded_output = tokenizer.decode(output[len(tokenized["input_ids"][0]):]) print(decoded_output) ``` From 723db4352836796f77a000713c1ad8c60f097cfc Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Sun, 30 Nov 2025 22:20:01 +0000 Subject: [PATCH 04/29] WIP --- .../ministral3/test_modeling_ministral3.py | 146 ++---------------- 1 file changed, 10 insertions(+), 136 deletions(-) diff --git a/tests/models/ministral3/test_modeling_ministral3.py b/tests/models/ministral3/test_modeling_ministral3.py index 5f88ea7b5835..9093492519aa 100644 --- a/tests/models/ministral3/test_modeling_ministral3.py +++ b/tests/models/ministral3/test_modeling_ministral3.py @@ -20,7 +20,7 @@ import pytest -from transformers import AutoTokenizer, BitsAndBytesConfig, GenerationConfig, is_torch_available +from transformers import AutoTokenizer, Mistral3ForConditionalGeneration, is_torch_available from transformers.testing_utils import ( backend_empty_cache, cleanup, @@ -37,7 +37,6 @@ import torch from transformers import ( - AutoModelForCausalLM, Ministral3ForCausalLM, Ministral3Model, ) @@ -82,34 +81,27 @@ def tearDown(self): cleanup(torch_device, gc_collect=True) @slow - def test_model_8b_logits(self): + def test_model_3b_logits(self): input_ids = [1, 306, 4658, 278, 6593, 310, 2834, 338] - model = AutoModelForCausalLM.from_pretrained("mistralai/Ministral3-8B-Instruct-2410", device_map="auto") - assert isinstance(model, Ministral3ForCausalLM) - input_ids = torch.tensor([input_ids]).to(model.model.embed_tokens.weight.device) + model = Mistral3ForConditionalGeneration.from_pretrained("mistralai/Ministral-3-3B-Instruct-2512", device_map="auto") + input_ids = torch.tensor([input_ids]).to(model.device) with torch.no_grad(): out = model(input_ids).logits.float().cpu() # Expected mean on dim = -1 - EXPECTED_MEAN = torch.tensor([[-1.5029, -7.2815, 4.5190, 0.5930, -5.2526, 3.0765, -0.6314, 1.8068]]) + EXPECTED_MEAN = torch.tensor([[-1.1503, -1.9935, -0.4457, -1.0717, -1.9182, -1.1431, -0.9697, -1.7098]]) torch.testing.assert_close(out.mean(-1), EXPECTED_MEAN, rtol=1e-2, atol=1e-2) - # slicing logits[0, 0, 0:30] - EXPECTED_SLICE = torch.tensor([-3.9446, -3.9466, 0.6383, -3.9466, -3.9468, -3.9448, -3.9462, -3.9455, - -3.9451, -0.8244, -3.9472, -3.9458, -3.9460, -3.9406, -3.9462, -3.9462, - -3.9458, -3.9462, -3.9463, -3.9461, -3.9448, -3.9451, -3.9462, -3.9458, - -3.9455, -3.9452, -3.9458, -3.9469, -3.9460, -3.9464]) # fmt: skip - torch.testing.assert_close(out[0, 0, :30], EXPECTED_SLICE, rtol=1e-4, atol=1e-4) del model backend_empty_cache(torch_device) gc.collect() @slow - def test_model_8b_generation(self): - EXPECTED_TEXT_COMPLETION = "My favourite condiment is 100% natural, 100% organic, 100% free of" + def test_model_3b_generation(self): + EXPECTED_TEXT_COMPLETION = "My favourite condiment is 100% pure olive oil. It's a staple in my kitchen and I use it in" prompt = "My favourite condiment is " - tokenizer = AutoTokenizer.from_pretrained("Mistralai/Ministral3-8B-Instruct-2410") - model = Ministral3ForCausalLM.from_pretrained("Mistralai/Ministral3-8B-Instruct-2410", device_map="auto") - input_ids = tokenizer.encode(prompt, return_tensors="pt").to(model.model.embed_tokens.weight.device) + tokenizer = AutoTokenizer.from_pretrained("mistralai/Ministral-3-3B-Instruct-2512") + model = Mistral3ForConditionalGeneration.from_pretrained("mistralai/Ministral-3-3B-Instruct-2512", device_map="auto") + input_ids = tokenizer.encode(prompt, return_tensors="pt").to(model.device) # greedy generation outputs generated_ids = model.generate(input_ids, max_new_tokens=20, temperature=0) @@ -119,121 +111,3 @@ def test_model_8b_generation(self): del model backend_empty_cache(torch_device) gc.collect() - - @require_bitsandbytes - @slow - @require_flash_attn - @pytest.mark.flash_attn_test - def test_model_8b_long_prompt(self): - EXPECTED_OUTPUT_TOKEN_IDS = [36850, 4112] - # An input with 4097 tokens that is above the size of the sliding window - input_ids = [1] + [306, 338] * 2048 - model = Ministral3ForCausalLM.from_pretrained( - "Mistralai/Ministral3-8B-Instruct-2410", - device_map="auto", - dtype=torch.bfloat16, - attn_implementation="flash_attention_2", - ) - input_ids = torch.tensor([input_ids]).to(model.model.embed_tokens.weight.device) - generated_ids = model.generate(input_ids, max_new_tokens=4, temperature=0) - self.assertEqual(EXPECTED_OUTPUT_TOKEN_IDS, generated_ids[0][-2:].tolist()) - - # Assisted generation - assistant_model = model - assistant_model.generation_config.num_assistant_tokens = 2 - assistant_model.generation_config.num_assistant_tokens_schedule = "constant" - generated_ids = model.generate(input_ids, max_new_tokens=4, temperature=0) - self.assertEqual(EXPECTED_OUTPUT_TOKEN_IDS, generated_ids[0][-2:].tolist()) - - del assistant_model - del model - backend_empty_cache(torch_device) - gc.collect() - - @slow - @unittest.skip("not working with Ministral3") - @pytest.mark.torch_export_test - def test_export_text_with_hybrid_cache(self): - # TODO: Exportability is not working - from transformers.testing_utils import is_torch_greater_or_equal - - if not is_torch_greater_or_equal("2.6.0"): - self.skipTest(reason="This test requires torch >= 2.6 to run.") - - from transformers.integrations.executorch import TorchExportableModuleForDecoderOnlyLM - - model_id = "Mistralai/Ministral3-8B-Instruct-2410" - model = Ministral3ForCausalLM.from_pretrained( - model_id, - generation_config=GenerationConfig( - use_cache=True, - cache_implementation="static", - cache_config={ - "batch_size": 1, - "max_cache_len": 50, - }, - ), - ) - - # Export + HybridCache - model.eval() - exportable_module = TorchExportableModuleForDecoderOnlyLM(model) - exported_program = exportable_module.export( - input_ids=torch.tensor([[1]], dtype=torch.long, device=model.device), - cache_position=torch.tensor([0], dtype=torch.long, device=model.device), - ) - logging.info(f"\nExported program: {exported_program}") - - # Test generation with the exported model - prompt = "My favourite condiment is " - max_new_tokens_to_generate = 20 - # Generate text with the exported model - tokenizer = AutoTokenizer.from_pretrained(model_id) - export_generated_text = TorchExportableModuleForDecoderOnlyLM.generate( - exported_program, tokenizer, prompt, max_new_tokens=max_new_tokens_to_generate - ) - logging.info(f"\nExport generated texts: '{export_generated_text}'") - - input_text = tokenizer(prompt, return_tensors="pt") - with torch.no_grad(): - eager_outputs = model.generate( - **input_text, - max_new_tokens=max_new_tokens_to_generate, - do_sample=False, # Use greedy decoding to match the exported model - cache_implementation="static", - ) - - eager_generated_text = tokenizer.decode(eager_outputs[0], skip_special_tokens=True) - logging.info(f"\nEager generated texts: '{eager_generated_text}'") - - self.assertEqual(export_generated_text, eager_generated_text) - - @pytest.mark.flash_attn_test - @require_flash_attn - @slow - def test_past_sliding_window_generation(self): - try: - from datasets import load_dataset - except ImportError: - self.skipTest("datasets not found") - - model = Ministral3ForCausalLM.from_pretrained( - "mistralai/Ministral3-8B-Instruct-2410", - device_map="auto", - quantization_config=BitsAndBytesConfig(load_in_4bit=True), - ) - tokenizer = AutoTokenizer.from_pretrained("mistralai/Ministral3-8B-Instruct-2410", legacy=False) - - wiki = load_dataset("wikitext", "wikitext-103-raw-v1", split="validation") - chunks = [x["text"] for x in wiki.select(range(550)) if x["text"].strip()] - real_corpus = "\n".join(chunks) - prompt = f"[INST]{real_corpus} Question: Based on the text, at which depth of the continental shelf does H. Gammarus live?[/INST]" - - inputs = tokenizer(prompt, return_tensors="pt").to(model.device) - input_length = inputs.input_ids.shape[1] # around 33k tokens > 32k sliding window - outputs = model.generate(**inputs, max_new_tokens=100, do_sample=False) - output_text = tokenizer.decode(outputs[0][input_length:], skip_special_tokens=True) - self.assertEqual( - output_text, - " H. Gammarus lives on the continental shelf at depths of 0 – 150 metres ( 0 – 492 ft ) , although not normally deeper than 50 m ( 160 ft ) .", - ) From ce3015dc7cf33100c414f232f5e91c0a46e83d70 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 1 Dec 2025 09:46:55 +0100 Subject: [PATCH 05/29] Apply suggestions from code review Co-authored-by: Julien Denize <40604584+juliendenize@users.noreply.github.com> --- .../models/ministral3/configuration_ministral3.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/ministral3/configuration_ministral3.py b/src/transformers/models/ministral3/configuration_ministral3.py index 5cd1144b58d4..863b0920df10 100644 --- a/src/transformers/models/ministral3/configuration_ministral3.py +++ b/src/transformers/models/ministral3/configuration_ministral3.py @@ -57,19 +57,23 @@ class Ministral3Config(PreTrainedConfig): Example: ```python - >>> from transformers import Ministral3ForConditionalGeneration, Ministral3Config, PixtralVisionConfig, MistralConfig + >>> from transformers import Ministral3Config, Ministral3ForCausalLM, Mistral3Config, Mistral3ForConditionalGeneration, PixtralVisionConfig >>> # Initializing a Pixtral-vision config >>> vision_config = PixtralVisionConfig() - >>> # Initializing a Mistral config + >>> # Initializing a Ministral3 config >>> text_config = Ministral3Config() >>> # Initializing a Mistral3 configuration >>> configuration = Mistral3Config(vision_config, text_config) >>> # Initializing a model from the ministral configuration - >>> model = Ministral3ForConditionalGeneration(configuration) + >>> # Initializing a model from the Ministral3 configuration + >>> text_model = Ministral3ForCausalLM(text_config) + + >>> # Initializing a model from the Mistral3 configuration + >>> model = Mistral3ForConditionalGeneration(configuration) >>> # Accessing the model configuration >>> configuration = model.config From 8470dcad6362251c6b526709779a1cee1faf27d6 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 1 Dec 2025 10:11:29 +0100 Subject: [PATCH 06/29] Update src/transformers/models/ministral3/configuration_ministral3.py Co-authored-by: Julien Denize <40604584+juliendenize@users.noreply.github.com> --- src/transformers/models/ministral3/configuration_ministral3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/models/ministral3/configuration_ministral3.py b/src/transformers/models/ministral3/configuration_ministral3.py index 863b0920df10..a2a4ad7b2bfe 100644 --- a/src/transformers/models/ministral3/configuration_ministral3.py +++ b/src/transformers/models/ministral3/configuration_ministral3.py @@ -68,7 +68,6 @@ class Ministral3Config(PreTrainedConfig): >>> # Initializing a Mistral3 configuration >>> configuration = Mistral3Config(vision_config, text_config) - >>> # Initializing a model from the ministral configuration >>> # Initializing a model from the Ministral3 configuration >>> text_model = Ministral3ForCausalLM(text_config) From d551a4879286fd513bb315b884e48cdca4efacc7 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 1 Dec 2025 16:44:52 +0100 Subject: [PATCH 07/29] fix most tests --- .../models/ministral3/modeling_ministral3.py | 32 +++++++------------ .../models/ministral3/modular_ministral3.py | 2 +- .../ministral3/test_modeling_ministral3.py | 5 ++- 3 files changed, 15 insertions(+), 24 deletions(-) diff --git a/src/transformers/models/ministral3/modeling_ministral3.py b/src/transformers/models/ministral3/modeling_ministral3.py index a05966edb6c8..2d9a8de24283 100644 --- a/src/transformers/models/ministral3/modeling_ministral3.py +++ b/src/transformers/models/ministral3/modeling_ministral3.py @@ -15,10 +15,15 @@ from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin -from ...integrations import use_kernel_forward_from_hub +from ...integrations import use_kernel_forward_from_hub, use_kernel_func_from_hub from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_layers import GenericForQuestionAnswering, GenericForTokenClassification, GradientCheckpointingLayer +from ...modeling_layers import ( + GenericForQuestionAnswering, + GenericForSequenceClassification, + GenericForTokenClassification, + GradientCheckpointingLayer, +) from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel @@ -34,6 +39,7 @@ def rotate_half(x): return torch.cat((-x2, x1), dim=-1) +@use_kernel_func_from_hub("rotary_pos_emb") def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): """Applies Rotary Position Embedding to the query and key tensors. @@ -120,6 +126,7 @@ def __init__(self, config: Ministral3Config, layer_idx: int): self.k_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=False) self.v_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=False) self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=False) + self.rotary_fn = apply_rotary_pos_emb def forward( self, @@ -353,7 +360,7 @@ def __init__(self, config: Ministral3Config): # Initialize weights and apply final processing self.post_init() - @check_model_inputs() + @check_model_inputs @auto_docstring def forward( self, @@ -495,23 +502,8 @@ class Ministral3ForTokenClassification(GenericForTokenClassification, Ministral3 pass -@auto_docstring -class Ministral3ForSequenceClassification(PreTrainedModel): - config: Ministral3Config - base_model_prefix = "model" - supports_gradient_checkpointing = True - _no_split_modules = ["Ministral3DecoderLayer"] - _skip_keys_device_placement = ["past_key_values"] - _supports_flash_attn = True - _supports_sdpa = True - _supports_flex_attn = True - - _can_compile_fullgraph = True - _supports_attention_backend = True - _can_record_outputs = { - "hidden_states": Ministral3DecoderLayer, - "attentions": Ministral3Attention, - } +class Ministral3ForSequenceClassification(GenericForSequenceClassification, Ministral3PreTrainedModel): + pass class Ministral3ForQuestionAnswering(GenericForQuestionAnswering, Ministral3PreTrainedModel): diff --git a/src/transformers/models/ministral3/modular_ministral3.py b/src/transformers/models/ministral3/modular_ministral3.py index b97b951be6e7..fe35e378cd1c 100644 --- a/src/transformers/models/ministral3/modular_ministral3.py +++ b/src/transformers/models/ministral3/modular_ministral3.py @@ -106,7 +106,7 @@ class Ministral3ForTokenClassification(GenericForTokenClassification, Ministral3 pass -class Ministral3ForSequenceClassification(GenericForSequenceClassification, MistralPreTrainedModel): +class Ministral3ForSequenceClassification(GenericForSequenceClassification, Ministral3PreTrainedModel): pass diff --git a/tests/models/ministral3/test_modeling_ministral3.py b/tests/models/ministral3/test_modeling_ministral3.py index 9093492519aa..90910518fe50 100644 --- a/tests/models/ministral3/test_modeling_ministral3.py +++ b/tests/models/ministral3/test_modeling_ministral3.py @@ -15,7 +15,6 @@ """Testing suite for the PyTorch Ministral3 model.""" import gc -import logging import unittest import pytest @@ -24,7 +23,6 @@ from transformers.testing_utils import ( backend_empty_cache, cleanup, - require_bitsandbytes, require_flash_attn, require_torch, require_torch_accelerator, @@ -37,7 +35,6 @@ import torch from transformers import ( - Ministral3ForCausalLM, Ministral3Model, ) @@ -52,6 +49,8 @@ class Ministral3ModelTester(CausalLMModelTester): @require_torch class Ministral3ModelTest(CausalLMModelTest, unittest.TestCase): + _is_stateful = True + model_split_percents = [0.5, 0.6] model_tester_class = Ministral3ModelTester # TODO (ydshieh): Check this. See https://app.circleci.com/pipelines/github/huggingface/transformers/79245/workflows/9490ef58-79c2-410d-8f51-e3495156cf9c/jobs/1012146 From 4f2331f16dec521b92e70b9c09fee7c70acf4cf3 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 1 Dec 2025 16:48:11 +0100 Subject: [PATCH 08/29] update docsting --- .../ministral3/configuration_ministral3.py | 59 ++++++++++++++----- 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/src/transformers/models/ministral3/configuration_ministral3.py b/src/transformers/models/ministral3/configuration_ministral3.py index a2a4ad7b2bfe..d8e53bda58bf 100644 --- a/src/transformers/models/ministral3/configuration_ministral3.py +++ b/src/transformers/models/ministral3/configuration_ministral3.py @@ -37,22 +37,49 @@ class Ministral3Config(PreTrainedConfig): documentation from [`PreTrainedConfig`] for more information. Args: - vision_config (`Union[AutoConfig, dict]`, *optional*, defaults to `PixtralVisionConfig`): - The config object or dictionary of the vision backbone. - text_config (`Union[AutoConfig, dict]`, *optional*, defaults to `MistralConfig`): - The config object or dictionary of the text backbone. - image_token_index (`int`, *optional*, defaults to 10): - The image token index to encode the image prompt. - projector_hidden_act (`str`, *optional*, defaults to `"gelu"`): - The activation function used by the multimodal projector. - vision_feature_layer (`Union[int, list[int]]`, *optional*, defaults to -1): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - multimodal_projector_bias (`bool`, *optional*, defaults to `False`): - Whether to use bias in the multimodal projector. - spatial_merge_size (`int`, *optional*, defaults to 2): - The downsampling factor for the spatial merge operation. + vocab_size (`Optional`, *optional*, defaults to 131072): + Vocabulary size of the Ministral3 model. Defines the number of different tokens that can be represented by + the `inputs_ids` passed when calling [`Ministral3Model`]. + hidden_size (`Optional`, *optional*, defaults to 4096): + Dimensionality of the embeddings and hidden states. + intermediate_size (`Optional`, *optional*, defaults to 14336): + Dimensionality of the intermediate (feed-forward) layer. + num_hidden_layers (`Optional`, *optional*, defaults to 34): + Number of hidden layers in the Transformer decoder. + num_attention_heads (`Optional`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer decoder. + num_key_value_heads (`Optional`, *optional*, defaults to 8): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA); if + `num_key_value_heads=1`, the model will use Multi Query Attention (MQA); otherwise GQA is used. + head_dim (`Optional`, *optional*, defaults to 128): + The attention head dimension. If not specified, will default to `hidden_size // num_attention_heads`. + hidden_act (`Optional`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`Optional`, *optional*, defaults to 262144): + The maximum sequence length that this model might ever be used with. + initializer_range (`Optional`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`Optional`, *optional*, defaults to 1e-05): + The epsilon used by the rms normalization layers. + use_cache (`Optional`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + pad_token_id (`Optional`, *optional*, defaults to 11): + The id of the padding token. + bos_token_id (`Optional`, *optional*, defaults to 1): + The id of the "beginning-of-sequence" token. + eos_token_id (`Optional`, *optional*, defaults to 2): + The id of the "end-of-sequence" token. + tie_word_embeddings (`Optional`, *optional*, defaults to `False`): + Whether the model's input and output word embeddings should be tied. + rope_parameters (`Union`, *optional*, defaults to `{'type': 'yarn', 'rope_theta': 1000000.0, 'factor': 16.0, 'original_max_position_embeddings': 16384, 'beta_fast': 32.0, 'beta_slow': 1.0, 'mscale_all_dim': 1.0, 'mscale': 1.0, 'llama_4_scaling_beta': 0.1}`): + Dictionary containing the configuration parameters for the RoPE embeddings, including optional Yarn scaling + settings such as `factor`, `original_max_position_embeddings`, `mscale`, and `llama_4_scaling_beta`. + sliding_window (`Optional`, *optional*): + Sliding window attention window size. If `None`, full attention is used. + attention_dropout (`Optional`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. Example: From 4d550aea3602ae36ad85121234b704acf5b4d465 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 1 Dec 2025 16:48:32 +0100 Subject: [PATCH 09/29] fixup --- src/transformers/integrations/mistral.py | 4 +++- .../models/ministral3/configuration_ministral3.py | 3 ++- tests/models/ministral3/test_modeling_ministral3.py | 12 +++++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/transformers/integrations/mistral.py b/src/transformers/integrations/mistral.py index c2fee6c63caa..f10bf73f73a5 100644 --- a/src/transformers/integrations/mistral.py +++ b/src/transformers/integrations/mistral.py @@ -98,7 +98,9 @@ def convert_tekken_tokenizer(tokenizer_file: str): # Convert tokenizer = PreTrainedTokenizerFast( - tokenizer_object=MistralConverter(vocab=vocab, additional_special_tokens=all_special, pattern=pattern).converted() + tokenizer_object=MistralConverter( + vocab=vocab, additional_special_tokens=all_special, pattern=pattern + ).converted() ) # Post-process diff --git a/src/transformers/models/ministral3/configuration_ministral3.py b/src/transformers/models/ministral3/configuration_ministral3.py index d8e53bda58bf..e790170694a6 100644 --- a/src/transformers/models/ministral3/configuration_ministral3.py +++ b/src/transformers/models/ministral3/configuration_ministral3.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Ministral model configuration""" + from typing import Optional from ...configuration_utils import PreTrainedConfig @@ -97,7 +98,7 @@ class Ministral3Config(PreTrainedConfig): >>> # Initializing a model from the Ministral3 configuration >>> text_model = Ministral3ForCausalLM(text_config) - + >>> # Initializing a model from the Mistral3 configuration >>> model = Mistral3ForConditionalGeneration(configuration) diff --git a/tests/models/ministral3/test_modeling_ministral3.py b/tests/models/ministral3/test_modeling_ministral3.py index 90910518fe50..192f53f64d60 100644 --- a/tests/models/ministral3/test_modeling_ministral3.py +++ b/tests/models/ministral3/test_modeling_ministral3.py @@ -82,7 +82,9 @@ def tearDown(self): @slow def test_model_3b_logits(self): input_ids = [1, 306, 4658, 278, 6593, 310, 2834, 338] - model = Mistral3ForConditionalGeneration.from_pretrained("mistralai/Ministral-3-3B-Instruct-2512", device_map="auto") + model = Mistral3ForConditionalGeneration.from_pretrained( + "mistralai/Ministral-3-3B-Instruct-2512", device_map="auto" + ) input_ids = torch.tensor([input_ids]).to(model.device) with torch.no_grad(): out = model(input_ids).logits.float().cpu() @@ -96,10 +98,14 @@ def test_model_3b_logits(self): @slow def test_model_3b_generation(self): - EXPECTED_TEXT_COMPLETION = "My favourite condiment is 100% pure olive oil. It's a staple in my kitchen and I use it in" + EXPECTED_TEXT_COMPLETION = ( + "My favourite condiment is 100% pure olive oil. It's a staple in my kitchen and I use it in" + ) prompt = "My favourite condiment is " tokenizer = AutoTokenizer.from_pretrained("mistralai/Ministral-3-3B-Instruct-2512") - model = Mistral3ForConditionalGeneration.from_pretrained("mistralai/Ministral-3-3B-Instruct-2512", device_map="auto") + model = Mistral3ForConditionalGeneration.from_pretrained( + "mistralai/Ministral-3-3B-Instruct-2512", device_map="auto" + ) input_ids = tokenizer.encode(prompt, return_tensors="pt").to(model.device) # greedy generation outputs From d9d8aa603b2cd8a6d1f1d185d45388c6c256b2f9 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 1 Dec 2025 16:55:33 +0100 Subject: [PATCH 10/29] typo in the ocnfig --- .../ministral3/configuration_ministral3.py | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/transformers/models/ministral3/configuration_ministral3.py b/src/transformers/models/ministral3/configuration_ministral3.py index e790170694a6..985f0b524027 100644 --- a/src/transformers/models/ministral3/configuration_ministral3.py +++ b/src/transformers/models/ministral3/configuration_ministral3.py @@ -106,7 +106,7 @@ class Ministral3Config(PreTrainedConfig): >>> configuration = model.config ```""" - model_type = "ministal3" + model_type = "ministral3" keys_to_ignore_at_inference = ["past_key_values"] # Default tensor parallel plan for base model `MistralModel` base_model_tp_plan = { @@ -142,21 +142,24 @@ def __init__( bos_token_id: Optional[int] = 1, eos_token_id: Optional[int] = 2, tie_word_embeddings: Optional[bool] = False, - rope_parameters: Optional[RopeParameters | dict[str, RopeParameters]] = { - "type": "yarn", - "rope_theta": 1000000.0, - "factor": 16.0, - "original_max_position_embeddings": 16384, - "beta_fast": 32.0, - "beta_slow": 1.0, - "mscale_all_dim": 1.0, - "mscale": 1.0, - "llama_4_scaling_beta": 0.1, - }, + rope_parameters: Optional[RopeParameters | dict[str, RopeParameters]] = None, sliding_window: Optional[int] = None, attention_dropout: Optional[float] = 0.0, **kwargs, ): + if rope_parameters is None: + rope_parameters = { + "type": "yarn", + "rope_theta": 1000000.0, + "factor": 16.0, + "original_max_position_embeddings": 16384, + "beta_fast": 32.0, + "beta_slow": 1.0, + "mscale_all_dim": 1.0, + "mscale": 1.0, + "llama_4_scaling_beta": 0.1, + } + self.vocab_size = vocab_size self.max_position_embeddings = max_position_embeddings self.hidden_size = hidden_size From 580e2f1f8b3b119a9aa1f89415e3fa24677d520e Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 1 Dec 2025 17:06:58 +0100 Subject: [PATCH 11/29] make the last 3 tests pass --- .../models/ministral3/configuration_ministral3.py | 1 + tests/causal_lm_tester.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/ministral3/configuration_ministral3.py b/src/transformers/models/ministral3/configuration_ministral3.py index 985f0b524027..c1eb58dcd634 100644 --- a/src/transformers/models/ministral3/configuration_ministral3.py +++ b/src/transformers/models/ministral3/configuration_ministral3.py @@ -153,6 +153,7 @@ def __init__( "rope_theta": 1000000.0, "factor": 16.0, "original_max_position_embeddings": 16384, + "max_position_embeddings": max_position_embeddings, "beta_fast": 32.0, "beta_slow": 1.0, "mscale_all_dim": 1.0, diff --git a/tests/causal_lm_tester.py b/tests/causal_lm_tester.py index cc5095e69ce0..50c461ed4f44 100644 --- a/tests/causal_lm_tester.py +++ b/tests/causal_lm_tester.py @@ -439,7 +439,13 @@ def test_model_rope_scaling_from_config(self, scaling_type): set_seed(42) # Fixed seed at init time so the two models get the same random weights _set_config_rope_params( - config, {"rope_type": "default", "rope_theta": 10_000.0, "partial_rotary_factor": partial_rotary_factor} + config, + { + "rope_type": "default", + "rope_theta": 10_000.0, + "partial_rotary_factor": partial_rotary_factor, + "original_max_position_embeddings": 16384, + }, ) original_model = self.model_tester_class.base_model_class(config) original_model.to(torch_device) @@ -649,7 +655,9 @@ def _config_supports_rope_scaling(config: PreTrainedConfig) -> bool: def _set_config_rope_params(config: PreTrainedConfig, rope_params: dict) -> bool: """Recursively sets RoPE parameters on configs and subconfigs, by duplicating the same RoPE values.""" - config.rope_parameters = rope_params + config.rope_parameters = config.rope_parameters or {} + config.rope_parameters.update(rope_params) + if any(name in config.__class__.__name__.lower() for name in ["gemma3", "modernbert"]): config.rope_parameters = {layer_type: config.rope_parameters.copy() for layer_type in config.layer_types} From d2331f261358e55ea91c08f1caf274a0783296c3 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 1 Dec 2025 17:17:02 +0100 Subject: [PATCH 12/29] fix auto --- src/transformers/models/auto/tokenization_auto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index e70be0ff3907..31c6a783726b 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -215,7 +215,7 @@ ( "ministral3", ( - "MistralCommonTokenizer" + "MistralCommonBackend" if is_mistral_common_available() else ("LlamaTokenizer" if is_sentencepiece_available() else None), "LlamaTokenizerFast" if is_tokenizers_available() and not is_mistral_common_available() else None, From 44f1f301a6c9f7ddad2b657882637624ba96bad0 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 1 Dec 2025 17:18:28 +0100 Subject: [PATCH 13/29] nits --- tests/causal_lm_tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/causal_lm_tester.py b/tests/causal_lm_tester.py index 50c461ed4f44..4d6a450d7976 100644 --- a/tests/causal_lm_tester.py +++ b/tests/causal_lm_tester.py @@ -655,7 +655,7 @@ def _config_supports_rope_scaling(config: PreTrainedConfig) -> bool: def _set_config_rope_params(config: PreTrainedConfig, rope_params: dict) -> bool: """Recursively sets RoPE parameters on configs and subconfigs, by duplicating the same RoPE values.""" - config.rope_parameters = config.rope_parameters or {} + config.rope_parameters = getattr(config, "rope_parameters", {}) or {} config.rope_parameters.update(rope_params) if any(name in config.__class__.__name__.lower() for name in ["gemma3", "modernbert"]): From 19c2af371687f8e1d4ce841c40f645c7de915ca9 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 1 Dec 2025 16:32:30 +0000 Subject: [PATCH 14/29] WIP --- .../convert_ministral3_weights_to_hf.py | 36 +++++++++++++++++-- src/transformers/utils/quantization_config.py | 2 +- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py index 77c2ca27118d..dc445242c133 100644 --- a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py +++ b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py @@ -30,6 +30,7 @@ PixtralVisionConfig, ) from transformers.integrations.mistral import convert_tekken_tokenizer +from transformers.quantizers.auto import AutoHfQuantizer # fmt: off @@ -44,6 +45,14 @@ r"^layers.(\d+).feed_forward.w1.weight": r"model.language_model.layers.\1.mlp.gate_proj.weight", r"^layers.(\d+).feed_forward.w2.weight": r"model.language_model.layers.\1.mlp.down_proj.weight", r"^layers.(\d+).feed_forward.w3.weight": r"model.language_model.layers.\1.mlp.up_proj.weight", + r"^layers.(\d+).attention.w(q|k|v|o).qscale_act": r"model.language_model.layers.\1.self_attn.\2_proj.act_scale", + r"^layers.(\d+).feed_forward.w1.qscale_act": r"model.language_model.layers.\1.mlp.gate_proj.act_scale", + r"^layers.(\d+).feed_forward.w2.qscale_act": r"model.language_model.layers.\1.mlp.down_proj.act_scale", + r"^layers.(\d+).feed_forward.w3.qscale_act": r"model.language_model.layers.\1.mlp.up_proj.act_scale", + r"^layers.(\d+).attention.w(q|k|v|o).qscale_weight": r"model.language_model.layers.\1.self_attn.\2_proj.weight_scale_inv", + r"^layers.(\d+).feed_forward.w1.qscale_weight": r"model.language_model.layers.\1.mlp.gate_proj.weight_scale_inv", + r"^layers.(\d+).feed_forward.w2.qscale_weight": r"model.language_model.layers.\1.mlp.down_proj.weight_scale_inv", + r"^layers.(\d+).feed_forward.w3.qscale_weight": r"model.language_model.layers.\1.mlp.up_proj.weight_scale_inv", # Vision model keys r"vision_encoder.transformer.layers.(\d+).attention_norm.weight": r"model.vision_tower.transformer.layers.\1.attention_norm.weight", @@ -91,6 +100,9 @@ def convert_state_dict(original_state_dict: dict, config: Mistral3Config): new_dict = {} for old_key, tensor in original_state_dict.items(): + if "fake_quantizer" in old_key: + continue + new_key = map_old_key_to_new(old_key) if "vision" in old_key: @@ -108,9 +120,9 @@ def convert_state_dict(original_state_dict: dict, config: Mistral3Config): key_value_dim = head_dim * num_key_value_heads query_dim = head_dim * num_attention_heads - if "q_proj" in new_key: + if "q_proj" in new_key and new_key.endswith("weight"): tensor = permute_for_rope(tensor, num_attention_heads, query_dim, hidden_size) - elif "k_proj" in new_key: + elif "k_proj" in new_key and new_key.endswith("weight"): tensor = permute_for_rope(tensor, num_key_value_heads, key_value_dim, hidden_size) new_dict[new_key] = tensor @@ -178,6 +190,20 @@ def convert_config(original_config: dict, max_position_embeddings: int = 262144) _ = new_vision_config.pop("max_image_size") new_vision_config = PixtralVisionConfig(hidden_act="silu", **new_vision_config) + kwargs = {} + if original_config.get("quantization", {}).get("qformat_weight") == "fp8_e4m3": + assert original_config["quantization"]["qscheme_act"] == "TENSOR" + quantization_config = { + "activation_scheme": "static", + "modules_to_not_convert": None, + "quant_method": "fp8", + "weight_block_size": [ + 1, + 1, + ] + } + kwargs["quantization_config"] = quantization_config + new_config = Mistral3Config( vision_config=new_vision_config, text_config=new_text_config, @@ -185,6 +211,7 @@ def convert_config(original_config: dict, max_position_embeddings: int = 262144) image_token_id=image_token_id, spatial_merge_size=spatial_merge_size, vision_feature_layer=-1, + **kwargs, ) return new_config @@ -192,6 +219,7 @@ def convert_config(original_config: dict, max_position_embeddings: int = 262144) def convert_and_write_model(input_dir: str, output_dir: str, max_position_embeddings: int): """Convert the model and save it (this implicitly save the config as well).""" params = read_json(os.path.join(input_dir, "params.json")) + config = convert_config(params, max_position_embeddings) full_state_dict = {} @@ -214,6 +242,10 @@ def convert_and_write_model(input_dir: str, output_dir: str, max_position_embedd else: raise ValueError(f"Unknown config type {type(config)}.") + # let's swap nn.Linear to FP8 Linear before loading + hf_quantizer = AutoHfQuantizer.from_config(model.config.quantization_config) + + import ipdb; ipdb.set_trace() model.load_state_dict(full_state_dict, strict=True, assign=True) model.save_pretrained(output_dir) return config diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index 8ce450a60b9b..1413f0faf933 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -2003,7 +2003,7 @@ def post_init(self): Safety checker that arguments are correct """ self.activation_scheme = self.activation_scheme.lower() - if self.activation_scheme != "dynamic": + if self.activation_scheme not in ["dynamic", "static"]: raise ValueError(f"Activation scheme {self.activation_scheme} not supported") if len(self.weight_block_size) != 2: raise ValueError("weight_block_size must be a tuple of two integers") From 2cd4f1525a6da613e4f82caac4992ac802f0918c Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 1 Dec 2025 16:38:22 +0000 Subject: [PATCH 15/29] WIP --- .../models/ministral3/convert_ministral3_weights_to_hf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py index dc445242c133..b07b9cb4be55 100644 --- a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py +++ b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py @@ -242,10 +242,10 @@ def convert_and_write_model(input_dir: str, output_dir: str, max_position_embedd else: raise ValueError(f"Unknown config type {type(config)}.") - # let's swap nn.Linear to FP8 Linear before loading - hf_quantizer = AutoHfQuantizer.from_config(model.config.quantization_config) + # let's swap nn.Linear to FP8 Linear before loading + hf_quantizer = AutoHfQuantizer.from_config(model.config.quantization_config) + model = hf_quantizer.preprocess(model) - import ipdb; ipdb.set_trace() model.load_state_dict(full_state_dict, strict=True, assign=True) model.save_pretrained(output_dir) return config From 128d37c060fb6ac4b7eb7f5ea3cc3bc4d63c2cf6 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 1 Dec 2025 16:40:18 +0000 Subject: [PATCH 16/29] WIP --- .../models/ministral3/convert_ministral3_weights_to_hf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py index b07b9cb4be55..cebaa33786dc 100644 --- a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py +++ b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py @@ -244,7 +244,7 @@ def convert_and_write_model(input_dir: str, output_dir: str, max_position_embedd # let's swap nn.Linear to FP8 Linear before loading hf_quantizer = AutoHfQuantizer.from_config(model.config.quantization_config) - model = hf_quantizer.preprocess(model) + hf_quantizer.preprocess_model(model, model.config) model.load_state_dict(full_state_dict, strict=True, assign=True) model.save_pretrained(output_dir) From 0640a36eea7f91aee708f7d7ad8ade14ef018be7 Mon Sep 17 00:00:00 2001 From: medmekk Date: Mon, 1 Dec 2025 17:02:50 +0000 Subject: [PATCH 17/29] per tensor --- .../integrations/finegrained_fp8.py | 23 +++++++++++-------- .../convert_ministral3_weights_to_hf.py | 5 +--- src/transformers/utils/quantization_config.py | 4 ++-- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/transformers/integrations/finegrained_fp8.py b/src/transformers/integrations/finegrained_fp8.py index 16950dfe0d6f..a47a0cd0afd2 100644 --- a/src/transformers/integrations/finegrained_fp8.py +++ b/src/transformers/integrations/finegrained_fp8.py @@ -322,19 +322,24 @@ def __init__( self.in_features = in_features self.out_features = out_features + if block_size is not None: + self.block_size = block_size + else: + self.block_size = (out_features, in_features) + self.weight = torch.nn.Parameter(torch.empty(out_features, in_features, dtype=FP8Linear.dtype, device=device)) if self.weight.element_size() == 1: - scale_out_features = (out_features + block_size[0] - 1) // block_size[0] - scale_in_features = (in_features + block_size[1] - 1) // block_size[1] - self.weight_scale_inv = nn.Parameter( - torch.empty(scale_out_features, scale_in_features, dtype=torch.float32, device=device) - ) + scale_out_features = (out_features + self.block_size[0] - 1) // self.block_size[0] + scale_in_features = (in_features + self.block_size[1] - 1) // self.block_size[1] + if scale_out_features * scale_in_features == 1: + self.weight_scale_inv = nn.Parameter(torch.tensor(1.0, dtype=torch.float32, device=device)) + else: + self.weight_scale_inv = nn.Parameter( + torch.empty(scale_out_features, scale_in_features, dtype=torch.float32, device=device) + ) else: - self.register_parameter("weight_scale_inv", None) - - self.block_size = block_size - + self.register_parameter("weight_scale_inv", None) self.activation_scheme = activation_scheme if bias: diff --git a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py index cebaa33786dc..b9a5928ec28f 100644 --- a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py +++ b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py @@ -197,10 +197,7 @@ def convert_config(original_config: dict, max_position_embeddings: int = 262144) "activation_scheme": "static", "modules_to_not_convert": None, "quant_method": "fp8", - "weight_block_size": [ - 1, - 1, - ] + "weight_block_size": None } kwargs["quantization_config"] = quantization_config diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index 9359dea675f2..b63531a6c1d5 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -2009,9 +2009,9 @@ def post_init(self): self.activation_scheme = self.activation_scheme.lower() if self.activation_scheme not in ["dynamic", "static"]: raise ValueError(f"Activation scheme {self.activation_scheme} not supported") - if len(self.weight_block_size) != 2: + if self.weight_block_size is not None and len(self.weight_block_size) != 2: raise ValueError("weight_block_size must be a tuple of two integers") - if self.weight_block_size[0] <= 0 or self.weight_block_size[1] <= 0: + if self.weight_block_size is not None and (self.weight_block_size[0] <= 0 or self.weight_block_size[1] <= 0): raise ValueError("weight_block_size must be a tuple of two positive integers") def get_loading_attributes(self): From 4048042af473ff67de56cebbad16cbb4a1e85ef0 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 1 Dec 2025 17:04:38 +0000 Subject: [PATCH 18/29] WIP --- .../convert_ministral3_weights_to_hf.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py index cebaa33786dc..66bb8b8b03af 100644 --- a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py +++ b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py @@ -30,8 +30,8 @@ PixtralVisionConfig, ) from transformers.integrations.mistral import convert_tekken_tokenizer -from transformers.quantizers.auto import AutoHfQuantizer - +from transformers.quantizers.auto import AutoHfQuantizer, AutoQuantizationConfig +from transformers.integrations.finegrained_fp8 import replace_with_fp8_linear # fmt: off STATE_DICT_MAPPING = { @@ -195,14 +195,14 @@ def convert_config(original_config: dict, max_position_embeddings: int = 262144) assert original_config["quantization"]["qscheme_act"] == "TENSOR" quantization_config = { "activation_scheme": "static", - "modules_to_not_convert": None, + "modules_to_not_convert": ["model.vision_tower.*"], "quant_method": "fp8", "weight_block_size": [ 1, 1, ] } - kwargs["quantization_config"] = quantization_config + kwargs["quantization_config"] = AutoQuantizationConfig.from_dict(quantization_config) new_config = Mistral3Config( vision_config=new_vision_config, @@ -242,10 +242,9 @@ def convert_and_write_model(input_dir: str, output_dir: str, max_position_embedd else: raise ValueError(f"Unknown config type {type(config)}.") - # let's swap nn.Linear to FP8 Linear before loading - hf_quantizer = AutoHfQuantizer.from_config(model.config.quantization_config) - hf_quantizer.preprocess_model(model, model.config) - + # let's swap nn.Linear to FP8 Linear before loading + model = replace_with_fp8_linear(model, model.config.quantization_config.modules_to_not_convert, model.config.quantization_config) + import ipdb; ipdb.set_trace() model.load_state_dict(full_state_dict, strict=True, assign=True) model.save_pretrained(output_dir) return config From 67b161967b2d464e3b85e7e4751e3435b1f54ae8 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 1 Dec 2025 17:08:11 +0000 Subject: [PATCH 19/29] WIP --- .../models/ministral3/convert_ministral3_weights_to_hf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py index 11d82f5e07c5..89a682502c37 100644 --- a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py +++ b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py @@ -195,7 +195,7 @@ def convert_config(original_config: dict, max_position_embeddings: int = 262144) assert original_config["quantization"]["qscheme_act"] == "TENSOR" quantization_config = { "activation_scheme": "static", - "modules_to_not_convert": ["model.vision_tower.*"], + "modules_to_not_convert": ["model.vision_tower", "model.multi_modal_projector"], "quant_method": "fp8", "weight_block_size": None } @@ -241,7 +241,6 @@ def convert_and_write_model(input_dir: str, output_dir: str, max_position_embedd # let's swap nn.Linear to FP8 Linear before loading model = replace_with_fp8_linear(model, model.config.quantization_config.modules_to_not_convert, model.config.quantization_config) - import ipdb; ipdb.set_trace() model.load_state_dict(full_state_dict, strict=True, assign=True) model.save_pretrained(output_dir) return config From 142f794e4b8ccc962401b155bdc426c633fb9563 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 1 Dec 2025 17:12:22 +0000 Subject: [PATCH 20/29] WIP --- src/transformers/integrations/finegrained_fp8.py | 12 +++++++++++- .../ministral3/convert_ministral3_weights_to_hf.py | 8 ++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/transformers/integrations/finegrained_fp8.py b/src/transformers/integrations/finegrained_fp8.py index a47a0cd0afd2..6aa8ea8616b6 100644 --- a/src/transformers/integrations/finegrained_fp8.py +++ b/src/transformers/integrations/finegrained_fp8.py @@ -342,6 +342,9 @@ def __init__( self.register_parameter("weight_scale_inv", None) self.activation_scheme = activation_scheme + if self.activation_scheme == "static": + self.activation_scale = nn.Parameter(torch.tensor(1.0, dtype=torch.float32, device=device)) + if bias: self.bias = nn.Parameter(torch.empty(self.out_features)) else: @@ -361,7 +364,14 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: device_type = torch.accelerator.current_accelerator().type if is_torch_accelerator_available() else "cuda" torch_accelerator_module = getattr(torch, device_type, torch.cuda) with torch_accelerator_module.device(input.device): - qinput, scale = act_quant(input, self.block_size[1]) + if self.activation_scheme == "dynamic": + qinput, scale = act_quant(input, self.block_size[1]) + elif self.activation_scheme == "static": + scale = self.act_scale + qinput = (input / scale).to(torch.float8_e4m3fn) + else: + raise NotImplementedError("Not supported") + output = w8a8_block_fp8_matmul_triton( qinput, weight, diff --git a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py index 89a682502c37..c4b98c6cab91 100644 --- a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py +++ b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py @@ -45,10 +45,10 @@ r"^layers.(\d+).feed_forward.w1.weight": r"model.language_model.layers.\1.mlp.gate_proj.weight", r"^layers.(\d+).feed_forward.w2.weight": r"model.language_model.layers.\1.mlp.down_proj.weight", r"^layers.(\d+).feed_forward.w3.weight": r"model.language_model.layers.\1.mlp.up_proj.weight", - r"^layers.(\d+).attention.w(q|k|v|o).qscale_act": r"model.language_model.layers.\1.self_attn.\2_proj.act_scale", - r"^layers.(\d+).feed_forward.w1.qscale_act": r"model.language_model.layers.\1.mlp.gate_proj.act_scale", - r"^layers.(\d+).feed_forward.w2.qscale_act": r"model.language_model.layers.\1.mlp.down_proj.act_scale", - r"^layers.(\d+).feed_forward.w3.qscale_act": r"model.language_model.layers.\1.mlp.up_proj.act_scale", + r"^layers.(\d+).attention.w(q|k|v|o).qscale_act": r"model.language_model.layers.\1.self_attn.\2_proj.activation_scale", + r"^layers.(\d+).feed_forward.w1.qscale_act": r"model.language_model.layers.\1.mlp.gate_proj.activation_scale", + r"^layers.(\d+).feed_forward.w2.qscale_act": r"model.language_model.layers.\1.mlp.down_proj.activation_scale", + r"^layers.(\d+).feed_forward.w3.qscale_act": r"model.language_model.layers.\1.mlp.up_proj.activation_scale", r"^layers.(\d+).attention.w(q|k|v|o).qscale_weight": r"model.language_model.layers.\1.self_attn.\2_proj.weight_scale_inv", r"^layers.(\d+).feed_forward.w1.qscale_weight": r"model.language_model.layers.\1.mlp.gate_proj.weight_scale_inv", r"^layers.(\d+).feed_forward.w2.qscale_weight": r"model.language_model.layers.\1.mlp.down_proj.weight_scale_inv", From 70f89d0c5bec415400c774a02b304091e8ebd35c Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 1 Dec 2025 18:22:07 +0100 Subject: [PATCH 21/29] style --- .../models/ministral3/convert_ministral3_weights_to_hf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py index cebaa33786dc..089c8161a1c9 100644 --- a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py +++ b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py @@ -200,9 +200,9 @@ def convert_config(original_config: dict, max_position_embeddings: int = 262144) "weight_block_size": [ 1, 1, - ] + ], } - kwargs["quantization_config"] = quantization_config + kwargs["quantization_config"] = quantization_config new_config = Mistral3Config( vision_config=new_vision_config, From 547ac70493430c66b552ea51b4ab3777b183cf5a Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 1 Dec 2025 18:22:50 +0100 Subject: [PATCH 22/29] fixup --- src/transformers/integrations/finegrained_fp8.py | 2 +- .../ministral3/convert_ministral3_weights_to_hf.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/transformers/integrations/finegrained_fp8.py b/src/transformers/integrations/finegrained_fp8.py index 6aa8ea8616b6..8402c6da145e 100644 --- a/src/transformers/integrations/finegrained_fp8.py +++ b/src/transformers/integrations/finegrained_fp8.py @@ -339,7 +339,7 @@ def __init__( torch.empty(scale_out_features, scale_in_features, dtype=torch.float32, device=device) ) else: - self.register_parameter("weight_scale_inv", None) + self.register_parameter("weight_scale_inv", None) self.activation_scheme = activation_scheme if self.activation_scheme == "static": diff --git a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py index c4b98c6cab91..a812e8dcd023 100644 --- a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py +++ b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py @@ -29,9 +29,10 @@ PixtralProcessor, PixtralVisionConfig, ) -from transformers.integrations.mistral import convert_tekken_tokenizer -from transformers.quantizers.auto import AutoHfQuantizer, AutoQuantizationConfig from transformers.integrations.finegrained_fp8 import replace_with_fp8_linear +from transformers.integrations.mistral import convert_tekken_tokenizer +from transformers.quantizers.auto import AutoQuantizationConfig + # fmt: off STATE_DICT_MAPPING = { @@ -197,7 +198,7 @@ def convert_config(original_config: dict, max_position_embeddings: int = 262144) "activation_scheme": "static", "modules_to_not_convert": ["model.vision_tower", "model.multi_modal_projector"], "quant_method": "fp8", - "weight_block_size": None + "weight_block_size": None, } kwargs["quantization_config"] = AutoQuantizationConfig.from_dict(quantization_config) @@ -240,7 +241,9 @@ def convert_and_write_model(input_dir: str, output_dir: str, max_position_embedd raise ValueError(f"Unknown config type {type(config)}.") # let's swap nn.Linear to FP8 Linear before loading - model = replace_with_fp8_linear(model, model.config.quantization_config.modules_to_not_convert, model.config.quantization_config) + model = replace_with_fp8_linear( + model, model.config.quantization_config.modules_to_not_convert, model.config.quantization_config + ) model.load_state_dict(full_state_dict, strict=True, assign=True) model.save_pretrained(output_dir) return config From b212ea0995ea195157b0df2f79b031bac72f4e90 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 1 Dec 2025 17:25:50 +0000 Subject: [PATCH 23/29] WIP --- .../integrations/finegrained_fp8.py | 2 +- .../convert_ministral3_weights_to_hf.py | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/transformers/integrations/finegrained_fp8.py b/src/transformers/integrations/finegrained_fp8.py index 6aa8ea8616b6..16f89e2bd1e4 100644 --- a/src/transformers/integrations/finegrained_fp8.py +++ b/src/transformers/integrations/finegrained_fp8.py @@ -367,7 +367,7 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: if self.activation_scheme == "dynamic": qinput, scale = act_quant(input, self.block_size[1]) elif self.activation_scheme == "static": - scale = self.act_scale + scale = self.activation_scale qinput = (input / scale).to(torch.float8_e4m3fn) else: raise NotImplementedError("Not supported") diff --git a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py index c4b98c6cab91..59b1ef36a8ff 100644 --- a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py +++ b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py @@ -28,9 +28,10 @@ PixtralImageProcessorFast, PixtralProcessor, PixtralVisionConfig, + AutoTokenizer, ) from transformers.integrations.mistral import convert_tekken_tokenizer -from transformers.quantizers.auto import AutoHfQuantizer, AutoQuantizationConfig +from transformers.quantizers.auto import AutoQuantizationConfig from transformers.integrations.finegrained_fp8 import replace_with_fp8_linear # fmt: off @@ -239,9 +240,20 @@ def convert_and_write_model(input_dir: str, output_dir: str, max_position_embedd else: raise ValueError(f"Unknown config type {type(config)}.") - # let's swap nn.Linear to FP8 Linear before loading - model = replace_with_fp8_linear(model, model.config.quantization_config.modules_to_not_convert, model.config.quantization_config) + # let's swap nn.Linear to FP8 Linear before loading + model = replace_with_fp8_linear(model, model.config.quantization_config.modules_to_not_convert, model.config.quantization_config) + model.load_state_dict(full_state_dict, strict=True, assign=True) + # model.to("cuda") + + # tokenzier = AutoTokenizer.from_pretrained("mistralai/Ministral-3-3B-Instruct-2512") + # with torch.no_grad(): + # input_ids = tokenzier.encode("How many people live in Paris?", return_tensors="pt").to("cuda") + + # import ipdb; ipdb.set_trace() + # out = model.generate(input_ids, max_new_tokens=20) + + # assert False model.save_pretrained(output_dir) return config From b020d3b2139e2c07994060979552a168a446b6c4 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 1 Dec 2025 17:25:54 +0000 Subject: [PATCH 24/29] WIP --- .../ministral3/convert_ministral3_weights_to_hf.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py index 59b1ef36a8ff..507b2f3d269a 100644 --- a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py +++ b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py @@ -244,16 +244,6 @@ def convert_and_write_model(input_dir: str, output_dir: str, max_position_embedd model = replace_with_fp8_linear(model, model.config.quantization_config.modules_to_not_convert, model.config.quantization_config) model.load_state_dict(full_state_dict, strict=True, assign=True) - # model.to("cuda") - - # tokenzier = AutoTokenizer.from_pretrained("mistralai/Ministral-3-3B-Instruct-2512") - # with torch.no_grad(): - # input_ids = tokenzier.encode("How many people live in Paris?", return_tensors="pt").to("cuda") - - # import ipdb; ipdb.set_trace() - # out = model.generate(input_ids, max_new_tokens=20) - - # assert False model.save_pretrained(output_dir) return config From 2e4e5ae73feb173f74f4dd6e4541623d3eeb7159 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 1 Dec 2025 17:40:19 +0000 Subject: [PATCH 25/29] WIP --- src/transformers/integrations/finegrained_fp8.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/transformers/integrations/finegrained_fp8.py b/src/transformers/integrations/finegrained_fp8.py index 00db1bc5511f..209c2d22b1cf 100644 --- a/src/transformers/integrations/finegrained_fp8.py +++ b/src/transformers/integrations/finegrained_fp8.py @@ -184,15 +184,16 @@ def w8a8_block_fp8_matmul_triton( assert len(block_size) == 2 block_n, block_k = block_size[0], block_size[1] - assert A.shape[-1] == B.shape[-1] - assert A.shape[:-1] == As.shape[:-1] and A.is_contiguous() + assert A.shape[-1] == B.shape[-1], f"{A.shape}, {B.shape}" + assert A.shape[:-1] == As.shape[:-1], f"{A.shape}, {As.shape}" + assert A.is_contiguous() assert triton.cdiv(A.shape[-1], block_k) == As.shape[-1] M = A.numel() // A.shape[-1] assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2 N, K = B.shape - assert triton.cdiv(N, block_n) == Bs.shape[0] - assert triton.cdiv(K, block_k) == Bs.shape[1] + assert triton.cdiv(N, block_n) == Bs.shape[0], f"{N}, {block_n}, {Bs.shape}" + assert triton.cdiv(K, block_k) == Bs.shape[1], f"{K}, {block_k}, {Bs.shape}" C_shape = A.shape[:-1] + (N,) C = A.new_empty(C_shape, dtype=output_dtype) @@ -372,6 +373,9 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: else: raise NotImplementedError("Not supported") + scale = scale[None, None, None].repeat(qinput.shape[:2] + (1,)) + scale_inv = scale_inv[None, None] + output = w8a8_block_fp8_matmul_triton( qinput, weight, From 4aad125c2408471ab8b9c8e73c88e18a5c477ee8 Mon Sep 17 00:00:00 2001 From: medmekk Date: Mon, 1 Dec 2025 17:44:50 +0000 Subject: [PATCH 26/29] hack for now --- .../integrations/finegrained_fp8.py | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/transformers/integrations/finegrained_fp8.py b/src/transformers/integrations/finegrained_fp8.py index 209c2d22b1cf..820f8bc902a3 100644 --- a/src/transformers/integrations/finegrained_fp8.py +++ b/src/transformers/integrations/finegrained_fp8.py @@ -184,9 +184,9 @@ def w8a8_block_fp8_matmul_triton( assert len(block_size) == 2 block_n, block_k = block_size[0], block_size[1] - assert A.shape[-1] == B.shape[-1], f"{A.shape}, {B.shape}" - assert A.shape[:-1] == As.shape[:-1], f"{A.shape}, {As.shape}" - assert A.is_contiguous() + assert A.shape[-1] == B.shape[-1] + + assert A.shape[:-1] == As.shape[:-1] and A.is_contiguous() assert triton.cdiv(A.shape[-1], block_k) == As.shape[-1] M = A.numel() // A.shape[-1] @@ -372,18 +372,19 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: qinput = (input / scale).to(torch.float8_e4m3fn) else: raise NotImplementedError("Not supported") - - scale = scale[None, None, None].repeat(qinput.shape[:2] + (1,)) - scale_inv = scale_inv[None, None] - - output = w8a8_block_fp8_matmul_triton( - qinput, - weight, - scale, - scale_inv, - self.block_size, - output_dtype=input.dtype, - ) + if self.activation_scheme == "static": + output = F.linear(qinput.to(torch.bfloat16), weight.to(torch.bfloat16), None) * scale_inv * scale + output = output.to(input.dtype) + else: + output = w8a8_block_fp8_matmul_triton( + qinput, + weight, + scale, + scale_inv, + self.block_size, + output_dtype=input.dtype, + ) + # Blocks the CPU until all accelerator operations on the specified device are complete. It is used to ensure that the results of the # preceding operations are ready before proceeding torch_accelerator_module.synchronize() From 2b41984cc500c37e23dd26ed3e8facb67a3960bf Mon Sep 17 00:00:00 2001 From: medmekk Date: Mon, 1 Dec 2025 17:46:42 +0000 Subject: [PATCH 27/29] add todo --- src/transformers/integrations/finegrained_fp8.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/integrations/finegrained_fp8.py b/src/transformers/integrations/finegrained_fp8.py index 820f8bc902a3..d4f08a90d6d3 100644 --- a/src/transformers/integrations/finegrained_fp8.py +++ b/src/transformers/integrations/finegrained_fp8.py @@ -372,6 +372,7 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: qinput = (input / scale).to(torch.float8_e4m3fn) else: raise NotImplementedError("Not supported") + # TODO: fix this later to use the triton kernel if self.activation_scheme == "static": output = F.linear(qinput.to(torch.bfloat16), weight.to(torch.bfloat16), None) * scale_inv * scale output = output.to(input.dtype) From c1528a409faa5b38f7da7ddaeff47ac5447f491c Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 1 Dec 2025 18:49:44 +0100 Subject: [PATCH 28/29] fixup --- src/transformers/integrations/finegrained_fp8.py | 2 +- .../models/ministral3/convert_ministral3_weights_to_hf.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/transformers/integrations/finegrained_fp8.py b/src/transformers/integrations/finegrained_fp8.py index d4f08a90d6d3..f07edc4f4fa1 100644 --- a/src/transformers/integrations/finegrained_fp8.py +++ b/src/transformers/integrations/finegrained_fp8.py @@ -385,7 +385,7 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: self.block_size, output_dtype=input.dtype, ) - + # Blocks the CPU until all accelerator operations on the specified device are complete. It is used to ensure that the results of the # preceding operations are ready before proceeding torch_accelerator_module.synchronize() diff --git a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py index 14480e787487..9168a791f8e7 100644 --- a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py +++ b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py @@ -241,7 +241,9 @@ def convert_and_write_model(input_dir: str, output_dir: str, max_position_embedd raise ValueError(f"Unknown config type {type(config)}.") # let's swap nn.Linear to FP8 Linear before loading - model = replace_with_fp8_linear(model, model.config.quantization_config.modules_to_not_convert, model.config.quantization_config) + model = replace_with_fp8_linear( + model, model.config.quantization_config.modules_to_not_convert, model.config.quantization_config + ) model.load_state_dict(full_state_dict, strict=True, assign=True) model.save_pretrained(output_dir) From 730d489bc9d1e87e9e76192380be8e8f84f82ec5 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 1 Dec 2025 17:55:17 +0000 Subject: [PATCH 29/29] WIP --- .../models/ministral3/convert_ministral3_weights_to_hf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py index 14480e787487..f1a18403e5f1 100644 --- a/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py +++ b/src/transformers/models/ministral3/convert_ministral3_weights_to_hf.py @@ -241,7 +241,8 @@ def convert_and_write_model(input_dir: str, output_dir: str, max_position_embedd raise ValueError(f"Unknown config type {type(config)}.") # let's swap nn.Linear to FP8 Linear before loading - model = replace_with_fp8_linear(model, model.config.quantization_config.modules_to_not_convert, model.config.quantization_config) + if hasattr(model.config, "quantization_config"): + model = replace_with_fp8_linear(model, model.config.quantization_config.modules_to_not_convert, model.config.quantization_config) model.load_state_dict(full_state_dict, strict=True, assign=True) model.save_pretrained(output_dir)