remove deprecated (transformers v5) method use from examples, bump CI to use transformers v5

Author: speediedan

requirements/ci/requirements-oldest.txt

@@ -86,8 +86,10 @@ coverage==6.4
     #   finetuning-scheduler (pyproject.toml:dev)
     #   finetuning-scheduler (pyproject.toml:test)
     #   nbval
-cryptography==46.0.3 ; sys_platform == 'linux'
-    # via secretstorage
+cryptography==46.0.3
+    # via
+    #   google-auth
+    #   secretstorage
 cycler==0.12.1
     # via matplotlib
 databricks-sdk==0.80.0
@@ -152,7 +154,7 @@ gitdb==4.0.12
     # via gitpython
 gitpython==3.1.46
     # via mlflow-skinny
-google-auth==2.47.0
+google-auth==2.48.0
     # via databricks-sdk
 graphene==3.4.3
     # via mlflow
@@ -162,7 +164,7 @@ graphql-core==3.2.7
     #   graphql-relay
 graphql-relay==3.2.0
     # via graphene
-greenlet==3.3.0 ; platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'
+greenlet==3.3.1 ; platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'
     # via sqlalchemy
 gunicorn==23.0.0 ; sys_platform != 'win32'
     # via mlflow
@@ -300,7 +302,7 @@ more-itertools==10.8.0
     # via
     #   jaraco-classes
     #   jaraco-functools
-multidict==6.7.0
+multidict==6.7.1
     # via
     #   aiohttp
     #   yarl
@@ -565,7 +567,7 @@ send2trash==2.1.0
     # via notebook
 sentencepiece==0.2.0
     # via finetuning-scheduler (pyproject.toml)
-setuptools==80.10.1
+setuptools==80.10.2
     # via
     #   ipython
     #   lightning-utilities
@@ -695,7 +697,7 @@ virtualenv==20.36.1
     # via pre-commit
 waitress==3.0.2 ; sys_platform == 'win32'
     # via mlflow
-wcwidth==0.3.0
+wcwidth==0.4.0
     # via prompt-toolkit
 webencodings==0.5.1
     # via bleach

requirements/ci/requirements.txt

@@ -50,7 +50,7 @@ bleach[css]==6.3.0
     # via nbconvert
 blinker==1.9.0
     # via flask
-cachetools==6.2.4
+cachetools==6.2.5
     # via
     #   mlflow-skinny
     #   mlflow-tracing
@@ -72,6 +72,7 @@ click==8.3.1
     # via
     #   flask
     #   mlflow-skinny
+    #   typer-slim
     #   uvicorn
 cloudpickle==3.1.2
     # via mlflow-skinny
@@ -89,13 +90,14 @@ contourpy==1.3.2 ; python_full_version < '3.11'
     # via matplotlib
 contourpy==1.3.3 ; python_full_version >= '3.11'
     # via matplotlib
-coverage==7.13.1
+coverage==7.13.2
     # via
     #   finetuning-scheduler (pyproject.toml:dev)
     #   finetuning-scheduler (pyproject.toml:test)
     #   nbval
 cryptography==46.0.3
     # via
+    #   google-auth
     #   mlflow
     #   secretstorage
 cycler==0.12.1
@@ -173,7 +175,7 @@ gitdb==4.0.12
     # via gitpython
 gitpython==3.1.46
     # via mlflow-skinny
-google-auth==2.47.0
+google-auth==2.48.0
     # via databricks-sdk
 graphene==3.4.3
     # via mlflow
@@ -183,25 +185,26 @@ graphql-core==3.2.7
     #   graphql-relay
 graphql-relay==3.2.0
     # via graphene
-greenlet==3.3.0 ; platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'
+greenlet==3.3.1 ; platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'
     # via sqlalchemy
 gunicorn==23.0.0 ; sys_platform != 'win32'
     # via mlflow
 h11==0.16.0
     # via
     #   httpcore
     #   uvicorn
-hf-xet==1.2.0 ; platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'
+hf-xet==1.2.0 ; platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'
     # via huggingface-hub
 httpcore==1.0.9
     # via httpx
 httpx==0.28.1
     # via
     #   datasets
+    #   huggingface-hub
     #   jupyterlab
 huey==2.6.0
     # via mlflow
-huggingface-hub==0.36.0
+huggingface-hub==1.3.4
     # via
     #   datasets
     #   evaluate
@@ -317,7 +320,7 @@ jupyter-server==2.17.0
     #   notebook-shim
 jupyter-server-terminals==0.5.4
     # via jupyter-server
-jupyterlab==4.5.2
+jupyterlab==4.5.3
     # via notebook
 jupyterlab-pygments==0.3.0
     # via nbconvert
@@ -327,7 +330,7 @@ jupyterlab-server==2.28.0
     #   notebook
 jupyterlab-widgets==3.0.16
     # via ipywidgets
-jupytext==1.19.0
+jupytext==1.19.1
     # via finetuning-scheduler (pyproject.toml)
 keyring==25.7.0 ; platform_machine != 'ppc64le' and platform_machine != 's390x'
     # via twine
@@ -380,7 +383,7 @@ more-itertools==10.8.0 ; platform_machine != 'ppc64le' and platform_machine != '
     # via
     #   jaraco-classes
     #   jaraco-functools
-multidict==6.7.0
+multidict==6.7.1
     # via
     #   aiohttp
     #   yarl
@@ -414,7 +417,7 @@ nodeenv==1.10.0
     # via
     #   pre-commit
     #   pyright
-notebook==7.5.2
+notebook==7.5.3
     # via finetuning-scheduler (pyproject.toml)
 notebook-shim==0.2.4
     # via
@@ -644,12 +647,10 @@ requests==2.32.5
     #   datasets
     #   docker
     #   evaluate
-    #   huggingface-hub
     #   id
     #   jupyterlab-server
     #   mlflow-skinny
     #   requests-toolbelt
-    #   transformers
     #   twine
 requests-toolbelt==1.0.0
     # via twine
@@ -665,7 +666,7 @@ rfc3986-validator==0.1.1
     #   jupyter-events
 rfc3987-syntax==1.1.0
     # via jsonschema
-rich==14.2.0
+rich==14.3.1
     # via
     #   finetuning-scheduler (pyproject.toml)
     #   twine
@@ -699,11 +700,13 @@ send2trash==2.1.0
     # via jupyter-server
 sentencepiece==0.2.1
     # via finetuning-scheduler (pyproject.toml)
-setuptools==80.10.1
+setuptools==80.10.2
     # via
     #   jupyterlab
     #   lightning-utilities
     #   torch
+shellingham==1.5.4
+    # via huggingface-hub
 six==1.17.0
     # via
     #   python-dateutil
@@ -783,12 +786,16 @@ traitlets==5.14.3
     #   nbclient
     #   nbconvert
     #   nbformat
-transformers==4.57.6
+transformers==5.0.0
     # via finetuning-scheduler (pyproject.toml)
 twine==6.2.0
     # via
     #   finetuning-scheduler (pyproject.toml:dev)
     #   finetuning-scheduler (pyproject.toml:test)
+typer-slim==0.21.1
+    # via
+    #   huggingface-hub
+    #   transformers
 typeshed-client==2.8.2
     # via jsonargparse
 typing-extensions==4.15.0
@@ -820,6 +827,7 @@ typing-extensions==4.15.0
     #   sqlalchemy
     #   starlette
     #   torch
+    #   typer-slim
     #   typeshed-client
     #   typing-inspection
     #   uvicorn
@@ -843,7 +851,7 @@ virtualenv==20.36.1
     # via pre-commit
 waitress==3.0.2 ; sys_platform == 'win32'
     # via mlflow
-wcwidth==0.3.0
+wcwidth==0.4.0
     # via prompt-toolkit
 webcolors==25.10.0
     # via jsonschema

src/fts_examples/fts_superglue.py

@@ -173,7 +173,7 @@ def _convert_to_features(self, example_batch: LazyDict) -> BatchEncoding:
         """
         text_pairs = list(zip(example_batch[self.text_fields[0]], example_batch[self.text_fields[1]]))
         # Tokenize the text/text pairs
-        features = self.tokenizer.batch_encode_plus(
+        features = self.tokenizer(
             text_pairs, max_length=self.hparams.max_seq_length, padding="longest", truncation=True
         )
         # Rename label to labels to make it easier to pass to model forward

src/fts_examples/ipynb_src/fts_superglue_nb.py

@@ -303,7 +303,7 @@ def _convert_to_features(self, example_batch: datasets.arrow_dataset.LazyDict) -
         """
         text_pairs = list(zip(example_batch[self.text_fields[0]], example_batch[self.text_fields[1]]))
         # Tokenize the text/text pairs
-        features = self.tokenizer.batch_encode_plus(
+        features = self.tokenizer(
             text_pairs, max_length=self.hparams.max_seq_length, padding="longest", truncation=True
         )
         # Rename label to labels to make it easier to pass to model forward