Adapt and test huggingface_hub v1.0.0

setup.py

@@ -114,12 +114,12 @@
     "GitPython<3.1.19",
     "hf-doc-builder>=0.3.0",
     "hf_xet",
-    "huggingface-hub>=0.34.0,<1.0",
+    "huggingface-hub==1.0.0.rc1",
     "importlib_metadata",
     "ipadic>=1.0.0,<2.0",
     "jinja2>=3.1.0",
     "kenlm",
-    "kernels>=0.6.1,<=0.9",
+    "kernels>=0.10.2,<0.11",
     "librosa",
     "natten>=0.14.6,<0.15.0",
     "nltk<=3.8.1",

src/transformers/audio_utils.py

@@ -25,11 +25,8 @@
 from io import BytesIO
 from typing import TYPE_CHECKING, Any, Optional, Union
 
-
-if TYPE_CHECKING:
-    import torch
+import httpx
 import numpy as np
-import requests
 from packaging import version
 
 from .utils import (
@@ -42,6 +39,9 @@
 )
 
 
+if TYPE_CHECKING:
+    import torch
+
 if is_soundfile_available():
     import soundfile as sf
 
@@ -132,7 +132,9 @@ def load_audio_librosa(audio: Union[str, np.ndarray], sampling_rate=16000, timeo
 
     # Load audio from URL (e.g https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-Audio/audio/translate_to_chinese.wav)
     if audio.startswith("http://") or audio.startswith("https://"):
-        audio = librosa.load(BytesIO(requests.get(audio, timeout=timeout).content), sr=sampling_rate)[0]
+        audio = librosa.load(
+            BytesIO(httpx.get(audio, follow_redirects=True, timeout=timeout).content), sr=sampling_rate
+        )[0]
     elif os.path.isfile(audio):
         audio = librosa.load(audio, sr=sampling_rate)[0]
     return audio
@@ -174,7 +176,7 @@ def load_audio_as(
         # Load audio bytes from URL or file
         audio_bytes = None
         if audio.startswith(("http://", "https://")):
-            response = requests.get(audio, timeout=timeout)
+            response = httpx.get(audio, follow_redirects=True, timeout=timeout)
             response.raise_for_status()
             audio_bytes = response.content
         elif os.path.isfile(audio):

src/transformers/dependency_versions_table.py

@@ -23,12 +23,12 @@
     "GitPython": "GitPython<3.1.19",
     "hf-doc-builder": "hf-doc-builder>=0.3.0",
     "hf_xet": "hf_xet",
-    "huggingface-hub": "huggingface-hub>=0.34.0,<1.0",
+    "huggingface-hub": "huggingface-hub==1.0.0.rc1",
     "importlib_metadata": "importlib_metadata",
     "ipadic": "ipadic>=1.0.0,<2.0",
     "jinja2": "jinja2>=3.1.0",
     "kenlm": "kenlm",
-    "kernels": "kernels>=0.6.1,<=0.9",
+    "kernels": "kernels>=0.10.2,<0.11",
     "librosa": "librosa",
     "natten": "natten>=0.14.6,<0.15.0",
     "nltk": "nltk<=3.8.1",

src/transformers/file_utils.py

@@ -17,9 +17,6 @@
 This module should not be update anymore and is only left for backward compatibility.
 """
 
-from huggingface_hub import get_full_repo_name  # for backward compatibility
-from huggingface_hub.constants import HF_HUB_DISABLE_TELEMETRY as DISABLE_TELEMETRY  # for backward compatibility
-
 from . import __version__
 
 # Backward compatibility imports, to make sure all those objects can be found in file_utils

src/transformers/image_utils.py

@@ -19,8 +19,8 @@
 from io import BytesIO
 from typing import Optional, Union
 
+import httpx
 import numpy as np
-import requests
 
 from .utils import (
     ExplicitEnum,
@@ -462,7 +462,7 @@ def load_image(image: Union[str, "PIL.Image.Image"], timeout: Optional[float] =
         if image.startswith("http://") or image.startswith("https://"):
             # We need to actually check for a real protocol, otherwise it's impossible to use a local file
             # like http_huggingface_co.png
-            image = PIL.Image.open(BytesIO(requests.get(image, timeout=timeout).content))
+            image = PIL.Image.open(BytesIO(httpx.get(image, timeout=timeout, follow_redirects=True).content))
         elif os.path.isfile(image):
             image = PIL.Image.open(image)
         else:

src/transformers/modelcard.py

@@ -21,7 +21,7 @@
 from pathlib import Path
 from typing import Any, Optional, Union
 
-import requests
+import httpx
 import yaml
 from huggingface_hub import model_info
 from huggingface_hub.errors import OfflineModeIsEnabled
@@ -380,12 +380,7 @@ def __post_init__(self):
                 for tag in info.tags:
                     if tag.startswith("license:"):
                         self.license = tag[8:]
-            except (
-                requests.exceptions.HTTPError,
-                requests.exceptions.ConnectionError,
-                HFValidationError,
-                OfflineModeIsEnabled,
-            ):
+            except (httpx.HTTPError, HFValidationError, OfflineModeIsEnabled):
                 pass
 
     def create_model_index(self, metric_mapping):

src/transformers/pipelines/audio_classification.py

@@ -14,8 +14,8 @@
 import subprocess
 from typing import Any, Union
 
+import httpx
 import numpy as np
-import requests
 
 from ..utils import add_end_docstrings, is_torch_available, is_torchaudio_available, is_torchcodec_available, logging
 from .base import Pipeline, build_pipeline_init_args
@@ -168,7 +168,7 @@ def preprocess(self, inputs):
             if inputs.startswith("http://") or inputs.startswith("https://"):
                 # We need to actually check for a real protocol, otherwise it's impossible to use a local file
                 # like http_huggingface_co.png
-                inputs = requests.get(inputs).content
+                inputs = httpx.get(inputs, follow_redirects=True).content
             else:
                 with open(inputs, "rb") as f:
                     inputs = f.read()

src/transformers/pipelines/automatic_speech_recognition.py

@@ -14,8 +14,8 @@
 from collections import defaultdict
 from typing import TYPE_CHECKING, Any, Optional, Union
 
+import httpx
 import numpy as np
-import requests
 
 from ..generation import GenerationConfig
 from ..tokenization_utils import PreTrainedTokenizer
@@ -355,7 +355,7 @@ def preprocess(self, inputs, chunk_length_s=0, stride_length_s=None):
             if inputs.startswith("http://") or inputs.startswith("https://"):
                 # We need to actually check for a real protocol, otherwise it's impossible to use a local file
                 # like http_huggingface_co.png
-                inputs = requests.get(inputs).content
+                inputs = httpx.get(inputs, follow_redirects=True).content
             else:
                 with open(inputs, "rb") as f:
                     inputs = f.read()

src/transformers/pipelines/image_to_image.py

@@ -46,12 +46,13 @@ class ImageToImagePipeline(Pipeline):
 
     ```python
     >>> from PIL import Image
-    >>> import requests
+    >>> import httpx
+    >>> import io
 
     >>> from transformers import pipeline
 
     >>> upscaler = pipeline("image-to-image", model="caidas/swin2SR-classical-sr-x2-64")
-    >>> img = Image.open(requests.get("http://images.cocodataset.org/val2017/000000039769.jpg", stream=True).raw)
+    >>> img = Image.open(io.BytesIO(httpx.get("http://images.cocodataset.org/val2017/000000039769.jpg").content))
     >>> img = img.resize((64, 64))
     >>> upscaled_img = upscaler(img)
     >>> img.size

src/transformers/pipelines/video_classification.py

@@ -15,7 +15,7 @@
 from io import BytesIO
 from typing import Any, Optional, Union, overload
 
-import requests
+import httpx
 
 from ..utils import (
     add_end_docstrings,
@@ -142,7 +142,7 @@ def preprocess(self, video, num_frames=None, frame_sampling_rate=1):
             num_frames = self.model.config.num_frames
 
         if video.startswith("http://") or video.startswith("https://"):
-            video = BytesIO(requests.get(video).content)
+            video = BytesIO(httpx.get(video, follow_redirects=True).content)
 
         container = av.open(video)
 

src/transformers/pipelines/zero_shot_audio_classification.py

@@ -15,8 +15,8 @@
 from collections import UserDict
 from typing import Any, Union
 
+import httpx
 import numpy as np
-import requests
 
 from ..utils import (
     add_end_docstrings,
@@ -107,7 +107,7 @@ def preprocess(self, audio, candidate_labels=None, hypothesis_template="This is
             if audio.startswith("http://") or audio.startswith("https://"):
                 # We need to actually check for a real protocol, otherwise it's impossible to use a local file
                 # like http_huggingface_co.png
-                audio = requests.get(audio).content
+                audio = httpx.get(audio, follow_redirects=True).content
             else:
                 with open(audio, "rb") as f:
                     audio = f.read()

src/transformers/safetensors_conversion.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-import requests
+import httpx
 from huggingface_hub import Discussion, HfApi, get_repo_discussions
 
 from .utils import cached_file, http_user_agent, logging
@@ -44,10 +44,10 @@ def start(_sse_connection):
 
     data = {"data": [model_id, private, token]}
 
-    result = requests.post(sse_url, stream=True, json=data).json()
+    result = httpx.post(sse_url, follow_redirects=True, json=data).json()
     event_id = result["event_id"]
 
-    with requests.get(f"{sse_url}/{event_id}", stream=True) as sse_connection:
+    with httpx.stream("GET", f"{sse_url}/{event_id}") as sse_connection:
         try:
             logger.debug("Spawning safetensors automatic conversion.")
             start(sse_connection)

src/transformers/testing_utils.py

@@ -45,10 +45,9 @@
 from unittest import mock
 from unittest.mock import patch
 
-import huggingface_hub.utils
-import requests
+import httpx
 import urllib3
-from huggingface_hub import delete_repo
+from huggingface_hub import create_repo, delete_repo
 from packaging import version
 
 from transformers import Trainer
@@ -1848,7 +1847,7 @@ def __init__(self, namespace: Optional[str] = None, token: Optional[str] = None)
             repo_id = Path(tmp_dir).name
             if namespace is not None:
                 repo_id = f"{namespace}/{repo_id}"
-            self.repo_url = huggingface_hub.create_repo(repo_id, token=self.token)
+            self.repo_url = create_repo(repo_id, token=self.token)
 
     def __enter__(self):
         return self.repo_url
@@ -2660,13 +2659,14 @@ def wrapper(*args, **kwargs):
             while retry_count < max_attempts:
                 try:
                     return test_func_ref(*args, **kwargs)
-                # We catch all exceptions related to network issues from requests
+                # We catch all exceptions related to network issues from httpx
                 except (
-                    requests.exceptions.ConnectionError,
-                    requests.exceptions.Timeout,
-                    requests.exceptions.ReadTimeout,
-                    requests.exceptions.HTTPError,
-                    requests.exceptions.RequestException,
+                    httpx.HTTPError,
+                    httpx.RequestError,
+                    httpx.TimeoutException,
+                    httpx.ReadTimeout,
+                    httpx.ConnectError,
+                    httpx.NetworkError,
                 ) as err:
                     logger.error(
                         f"Test failed with {err} at try {retry_count}/{max_attempts} as it couldn't connect to the specified Hub repository."

src/transformers/trainer.py

@@ -49,7 +49,7 @@
 import numpy as np
 import torch
 import torch.distributed as dist
-from huggingface_hub import ModelCard, create_repo, upload_folder
+from huggingface_hub import CommitInfo, ModelCard, create_repo, upload_folder
 from packaging import version
 from torch import nn
 from torch.utils.data import DataLoader, Dataset, IterableDataset, RandomSampler, SequentialSampler
@@ -5117,7 +5117,7 @@ def push_to_hub(
         token: Optional[str] = None,
         revision: Optional[str] = None,
         **kwargs,
-    ) -> str:
+    ) -> CommitInfo:
         """
         Upload `self.model` and `self.processing_class` to the 🤗 model hub on the repo `self.args.hub_model_id`.
 

src/transformers/utils/__init__.py

@@ -16,8 +16,6 @@
 
 from functools import lru_cache
 
-from huggingface_hub import get_full_repo_name  # for backward compatibility
-from huggingface_hub.constants import HF_HUB_DISABLE_TELEMETRY as DISABLE_TELEMETRY  # for backward compatibility
 from packaging import version
 
 from .. import __version__

src/transformers/utils/attention_visualizer.py

@@ -11,9 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import io
 
-
-import requests
+import httpx
 from PIL import Image
 
 from ..masking_utils import create_causal_mask
@@ -180,7 +180,7 @@ def visualize_attention_mask(self, input_sentence: str, suffix=""):
         image_seq_length = None
         if self.config.model_type in PROCESSOR_MAPPING_NAMES:
             img = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/bee.jpg?download=true"
-            img = Image.open(requests.get(img, stream=True).raw)
+            img = Image.open(io.BytesIO(httpx.get(img, follow_redirects=True).content))
             image_seq_length = 5
             processor = AutoProcessor.from_pretrained(self.repo_id, image_seq_length=image_seq_length)
             if hasattr(processor, "image_token"):