Skip to content

Move internet tests in single file #3579

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Mar 17, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,9 @@ If you would like to run all tests:
pytest test -vvv
```

Tests that require internet access should be in
`test/test_internet.py`.

### Documentation

Torchvision uses [Google style](http://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html)
Expand Down
53 changes: 0 additions & 53 deletions test/test_datasets_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,18 +37,6 @@ def test_check_integrity(self):
self.assertTrue(utils.check_integrity(existing_fpath))
self.assertFalse(utils.check_integrity(nonexisting_fpath))

def test_get_redirect_url(self):
url = "http://www.vision.caltech.edu/visipedia-data/CUB-200-2011/CUB_200_2011.tgz"
expected = "https://drive.google.com/file/d/1hbzc_P1FuxMkcabkgn9ZKinBwW683j45/view"

actual = utils._get_redirect_url(url)
assert actual == expected

def test_get_redirect_url_max_hops_exceeded(self):
url = "http://www.vision.caltech.edu/visipedia-data/CUB-200-2011/CUB_200_2011.tgz"
with self.assertRaises(RecursionError):
utils._get_redirect_url(url, max_hops=0)

def test_get_google_drive_file_id(self):
url = "https://drive.google.com/file/d/1hbzc_P1FuxMkcabkgn9ZKinBwW683j45/view"
expected = "1hbzc_P1FuxMkcabkgn9ZKinBwW683j45"
Expand All @@ -61,47 +49,6 @@ def test_get_google_drive_file_id_invalid_url(self):

assert utils._get_google_drive_file_id(url) is None

def test_download_url(self):
with get_tmp_dir() as temp_dir:
url = "http://github.com/pytorch/vision/archive/master.zip"
try:
utils.download_url(url, temp_dir)
self.assertFalse(len(os.listdir(temp_dir)) == 0)
except URLError:
msg = "could not download test file '{}'".format(url)
warnings.warn(msg, RuntimeWarning)
raise unittest.SkipTest(msg)

def test_download_url_retry_http(self):
with get_tmp_dir() as temp_dir:
url = "https://github.com/pytorch/vision/archive/master.zip"
try:
utils.download_url(url, temp_dir)
self.assertFalse(len(os.listdir(temp_dir)) == 0)
except URLError:
msg = "could not download test file '{}'".format(url)
warnings.warn(msg, RuntimeWarning)
raise unittest.SkipTest(msg)

def test_download_url_dont_exist(self):
with get_tmp_dir() as temp_dir:
url = "http://github.com/pytorch/vision/archive/this_doesnt_exist.zip"
with self.assertRaises(URLError):
utils.download_url(url, temp_dir)

@unittest.mock.patch("torchvision.datasets.utils.download_file_from_google_drive")
def test_download_url_dispatch_download_from_google_drive(self, mock):
url = "https://drive.google.com/file/d/1hbzc_P1FuxMkcabkgn9ZKinBwW683j45/view"

id = "1hbzc_P1FuxMkcabkgn9ZKinBwW683j45"
filename = "filename"
md5 = "md5"

with get_tmp_dir() as root:
utils.download_url(url, root, filename, md5)

mock.assert_called_once_with(id, root, filename, md5)

def test_detect_file_type(self):
for file, expected in [
("foo.tar.xz", (".tar.xz", ".tar", ".xz")),
Expand Down
71 changes: 71 additions & 0 deletions test/test_internet.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
"""This file should contain all tests that need access to the internet (apart
from the ones in test_datasets_download.py)

We want to bundle all internet-related tests in one file, so the file can be
cleanly ignored in FB internal test infra.
"""

import os
import unittest
import unittest.mock
import warnings
from urllib.error import URLError

import torchvision.datasets.utils as utils
from common_utils import get_tmp_dir


class DatasetUtilsTester(unittest.TestCase):

def test_get_redirect_url(self):
url = "http://www.vision.caltech.edu/visipedia-data/CUB-200-2011/CUB_200_2011.tgz"
expected = "https://drive.google.com/file/d/1hbzc_P1FuxMkcabkgn9ZKinBwW683j45/view"

actual = utils._get_redirect_url(url)
assert actual == expected

def test_get_redirect_url_max_hops_exceeded(self):
url = "http://www.vision.caltech.edu/visipedia-data/CUB-200-2011/CUB_200_2011.tgz"
with self.assertRaises(RecursionError):
utils._get_redirect_url(url, max_hops=0)

def test_download_url(self):
with get_tmp_dir() as temp_dir:
url = "http://github.com/pytorch/vision/archive/master.zip"
try:
utils.download_url(url, temp_dir)
self.assertFalse(len(os.listdir(temp_dir)) == 0)
except URLError:
msg = "could not download test file '{}'".format(url)
warnings.warn(msg, RuntimeWarning)
raise unittest.SkipTest(msg)

def test_download_url_retry_http(self):
with get_tmp_dir() as temp_dir:
url = "https://github.com/pytorch/vision/archive/master.zip"
try:
utils.download_url(url, temp_dir)
self.assertFalse(len(os.listdir(temp_dir)) == 0)
except URLError:
msg = "could not download test file '{}'".format(url)
warnings.warn(msg, RuntimeWarning)
raise unittest.SkipTest(msg)

def test_download_url_dont_exist(self):
with get_tmp_dir() as temp_dir:
url = "http://github.com/pytorch/vision/archive/this_doesnt_exist.zip"
with self.assertRaises(URLError):
utils.download_url(url, temp_dir)

@unittest.mock.patch("torchvision.datasets.utils.download_file_from_google_drive")
def test_download_url_dispatch_download_from_google_drive(self, mock):
url = "https://drive.google.com/file/d/1hbzc_P1FuxMkcabkgn9ZKinBwW683j45/view"

id = "1hbzc_P1FuxMkcabkgn9ZKinBwW683j45"
filename = "filename"
md5 = "md5"

with get_tmp_dir() as root:
utils.download_url(url, root, filename, md5)

mock.assert_called_once_with(id, root, filename, md5)