From 41523a35d16c2e9d10f1d767f7bfb66ff9181ec1 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 23 May 2022 10:32:39 +0200 Subject: [PATCH 01/11] expand prototype test matrix to different Python versions --- .github/workflows/prototype-tests.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 518fa1cedc2..7e96537c17f 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -7,10 +7,18 @@ jobs: prototype: strategy: matrix: + python-version: + - "3.7" + - "3.8" + - "3.9" + - "3.10" os: - ubuntu-latest - - windows-latest - - macos-latest + include: + - python-version: "3.7" + os: windows-latest + - python-version: "3.7" + os: macos-latest fail-fast: false runs-on: ${{ matrix.os }} From 90babcd299b57b6b34915c8eeedb9a94a5d3021a Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 23 May 2022 14:54:38 +0200 Subject: [PATCH 02/11] fix python version --- .github/workflows/prototype-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 7e96537c17f..5ad512a4e63 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -27,7 +27,7 @@ jobs: - name: Set up python uses: actions/setup-python@v3 with: - python-version: 3.7 + python-version: ${{ matrix.python-version }} - name: Upgrade system packages run: python -m pip install --upgrade pip setuptools wheel From e0734cd9521d14c860c57a8b5829bb1cfc02b1f1 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 23 May 2022 15:20:42 +0200 Subject: [PATCH 03/11] try fix cub dataset --- torchvision/prototype/datasets/_builtin/cub200.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/cub200.py b/torchvision/prototype/datasets/_builtin/cub200.py index 0e5a80de825..4085c5bad32 100644 --- a/torchvision/prototype/datasets/_builtin/cub200.py +++ b/torchvision/prototype/datasets/_builtin/cub200.py @@ -107,10 +107,8 @@ def _2011_classify_archive(self, data: Tuple[str, Any]) -> Optional[int]: return 0 elif path.name == "train_test_split.txt": return 1 - elif path.name == "images.txt": - return 2 elif path.name == "bounding_boxes.txt": - return 3 + return 2 else: return None @@ -180,15 +178,17 @@ def _datapipe(self, resource_dps: List[IterDataPipe]) -> IterDataPipe[Dict[str, prepare_ann_fn: Callable if self._year == "2011": archive_dp, segmentations_dp = resource_dps - images_dp, split_dp, image_files_dp, bounding_boxes_dp = Demultiplexer( - archive_dp, 4, self._2011_classify_archive, drop_none=True, buffer_size=INFINITE_BUFFER_SIZE - ) + image_files_dp = Filter(archive_dp, path_comparator("name", "images.txt")) image_files_dp = CSVParser(image_files_dp, dialect="cub200") image_files_map = dict( (image_id, rel_posix_path.rsplit("/", maxsplit=1)[1]) for image_id, rel_posix_path in image_files_dp ) + images_dp, split_dp, bounding_boxes_dp = Demultiplexer( + archive_dp, 3, self._2011_classify_archive, drop_none=True, buffer_size=INFINITE_BUFFER_SIZE + ) + split_dp = CSVParser(split_dp, dialect="cub200") split_dp = Filter(split_dp, self._2011_filter_split) split_dp = Mapper(split_dp, getitem(0)) From cb8b43d13b53fff5f19c29be8ffab88a09b0fce1 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 23 May 2022 15:28:02 +0200 Subject: [PATCH 04/11] fix ImageNet --- test/test_prototype_builtin_datasets.py | 2 +- .../prototype/datasets/_builtin/imagenet.py | 28 ++++++------------- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/test/test_prototype_builtin_datasets.py b/test/test_prototype_builtin_datasets.py index 5a8c9e7eff8..5113e24516b 100644 --- a/test/test_prototype_builtin_datasets.py +++ b/test/test_prototype_builtin_datasets.py @@ -42,7 +42,7 @@ def test_coverage(): ) -@pytest.mark.filterwarnings("error") +# @pytest.mark.filterwarnings("error") class TestCommon: @pytest.mark.parametrize("name", datasets.list_datasets()) def test_info(self, name): diff --git a/torchvision/prototype/datasets/_builtin/imagenet.py b/torchvision/prototype/datasets/_builtin/imagenet.py index 1307757cef6..36f743c2f1c 100644 --- a/torchvision/prototype/datasets/_builtin/imagenet.py +++ b/torchvision/prototype/datasets/_builtin/imagenet.py @@ -1,4 +1,3 @@ -import enum import functools import pathlib import re @@ -10,7 +9,6 @@ IterKeyZipper, Mapper, Filter, - Demultiplexer, TarArchiveLoader, Enumerator, ) @@ -27,6 +25,7 @@ hint_shuffling, read_categories_file, path_accessor, + path_comparator, ) from torchvision.prototype.features import Label, EncodedImage @@ -46,9 +45,9 @@ def __init__(self, **kwargs: Any) -> None: super().__init__("Register on https://image-net.org/ and follow the instructions there.", **kwargs) -class ImageNetDemux(enum.IntEnum): - META = 0 - LABEL = 1 +# class ImageNetDemux(enum.IntEnum): +# META = 0 +# LABEL = 1 @register_dataset(NAME) @@ -108,12 +107,6 @@ def _prepare_train_data(self, data: Tuple[str, BinaryIO]) -> Tuple[Tuple[Label, def _prepare_test_data(self, data: Tuple[str, BinaryIO]) -> Tuple[None, Tuple[str, BinaryIO]]: return None, data - def _classifiy_devkit(self, data: Tuple[str, BinaryIO]) -> Optional[int]: - return { - "meta.mat": ImageNetDemux.META, - "ILSVRC2012_validation_ground_truth.txt": ImageNetDemux.LABEL, - }.get(pathlib.Path(data[0]).name) - # Although the WordNet IDs (wnids) are unique, the corresponding categories are not. For example, both n02012849 # and n03126707 are labeled 'crane' while the first means the bird and the latter means the construction equipment _WNID_MAP = { @@ -172,13 +165,11 @@ def _datapipe(self, resource_dps: List[IterDataPipe]) -> IterDataPipe[Dict[str, else: # config.split == "val": images_dp, devkit_dp = resource_dps - meta_dp, label_dp = Demultiplexer( - devkit_dp, 2, self._classifiy_devkit, drop_none=True, buffer_size=INFINITE_BUFFER_SIZE - ) - + meta_dp = Filter(devkit_dp, path_comparator("name", "meta.mat")) meta_dp = Mapper(meta_dp, self._extract_categories_and_wnids) - _, wnids = zip(*next(iter(meta_dp))) + _, wnids = zip(*list(meta_dp)[0]) + label_dp = Filter(devkit_dp, path_comparator("name", "ILSVRC2012_validation_ground_truth.txt")) label_dp = LineReader(label_dp, decode=True, return_path=False) # We cannot use self._wnids here, since we use a different order than the dataset label_dp = Mapper(label_dp, functools.partial(self._imagenet_label_to_wnid, wnids=wnids)) @@ -204,15 +195,12 @@ def __len__(self) -> int: "test": 100_000, }[self._split] - def _filter_meta(self, data: Tuple[str, Any]) -> bool: - return self._classifiy_devkit(data) == ImageNetDemux.META - def _generate_categories(self) -> List[Tuple[str, ...]]: self._split = "val" resources = self._resources() devkit_dp = resources[1].load(self._root) - meta_dp = Filter(devkit_dp, self._filter_meta) + meta_dp = Filter(devkit_dp, path_comparator("name", "meta.mat")) meta_dp = Mapper(meta_dp, self._extract_categories_and_wnids) categories_and_wnids = cast(List[Tuple[str, ...]], next(iter(meta_dp))) From 2882db3117344c946390508ae7de59d007e8133c Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 23 May 2022 16:19:27 +0200 Subject: [PATCH 05/11] reenable warnings to error --- test/test_prototype_builtin_datasets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_prototype_builtin_datasets.py b/test/test_prototype_builtin_datasets.py index 5113e24516b..5a8c9e7eff8 100644 --- a/test/test_prototype_builtin_datasets.py +++ b/test/test_prototype_builtin_datasets.py @@ -42,7 +42,7 @@ def test_coverage(): ) -# @pytest.mark.filterwarnings("error") +@pytest.mark.filterwarnings("error") class TestCommon: @pytest.mark.parametrize("name", datasets.list_datasets()) def test_info(self, name): From de35e284e8c9ee3dfdd7971d5b06d892cfc8f028 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 23 May 2022 15:20:42 +0200 Subject: [PATCH 06/11] Revert "try fix cub dataset" This reverts commit e0734cd9521d14c860c57a8b5829bb1cfc02b1f1. --- torchvision/prototype/datasets/_builtin/cub200.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/cub200.py b/torchvision/prototype/datasets/_builtin/cub200.py index 4085c5bad32..0e5a80de825 100644 --- a/torchvision/prototype/datasets/_builtin/cub200.py +++ b/torchvision/prototype/datasets/_builtin/cub200.py @@ -107,8 +107,10 @@ def _2011_classify_archive(self, data: Tuple[str, Any]) -> Optional[int]: return 0 elif path.name == "train_test_split.txt": return 1 - elif path.name == "bounding_boxes.txt": + elif path.name == "images.txt": return 2 + elif path.name == "bounding_boxes.txt": + return 3 else: return None @@ -178,17 +180,15 @@ def _datapipe(self, resource_dps: List[IterDataPipe]) -> IterDataPipe[Dict[str, prepare_ann_fn: Callable if self._year == "2011": archive_dp, segmentations_dp = resource_dps + images_dp, split_dp, image_files_dp, bounding_boxes_dp = Demultiplexer( + archive_dp, 4, self._2011_classify_archive, drop_none=True, buffer_size=INFINITE_BUFFER_SIZE + ) - image_files_dp = Filter(archive_dp, path_comparator("name", "images.txt")) image_files_dp = CSVParser(image_files_dp, dialect="cub200") image_files_map = dict( (image_id, rel_posix_path.rsplit("/", maxsplit=1)[1]) for image_id, rel_posix_path in image_files_dp ) - images_dp, split_dp, bounding_boxes_dp = Demultiplexer( - archive_dp, 3, self._2011_classify_archive, drop_none=True, buffer_size=INFINITE_BUFFER_SIZE - ) - split_dp = CSVParser(split_dp, dialect="cub200") split_dp = Filter(split_dp, self._2011_filter_split) split_dp = Mapper(split_dp, getitem(0)) From 6398629dc8a91bd36f2614a93a3a8104c1198373 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 23 May 2022 15:08:14 +0100 Subject: [PATCH 07/11] Revert "Indicate Celeba download parameter is deprecated and will be removed (#6059)" This reverts commit 49496c4f6201f05f2351788389a0863b087e78f4. From c5888e763b5fec792aad47a80d221324c3db6686 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 23 May 2022 15:28:02 +0200 Subject: [PATCH 08/11] Revert "fix ImageNet" This reverts commit cb8b43d13b53fff5f19c29be8ffab88a09b0fce1. --- .../prototype/datasets/_builtin/imagenet.py | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/imagenet.py b/torchvision/prototype/datasets/_builtin/imagenet.py index 36f743c2f1c..1307757cef6 100644 --- a/torchvision/prototype/datasets/_builtin/imagenet.py +++ b/torchvision/prototype/datasets/_builtin/imagenet.py @@ -1,3 +1,4 @@ +import enum import functools import pathlib import re @@ -9,6 +10,7 @@ IterKeyZipper, Mapper, Filter, + Demultiplexer, TarArchiveLoader, Enumerator, ) @@ -25,7 +27,6 @@ hint_shuffling, read_categories_file, path_accessor, - path_comparator, ) from torchvision.prototype.features import Label, EncodedImage @@ -45,9 +46,9 @@ def __init__(self, **kwargs: Any) -> None: super().__init__("Register on https://image-net.org/ and follow the instructions there.", **kwargs) -# class ImageNetDemux(enum.IntEnum): -# META = 0 -# LABEL = 1 +class ImageNetDemux(enum.IntEnum): + META = 0 + LABEL = 1 @register_dataset(NAME) @@ -107,6 +108,12 @@ def _prepare_train_data(self, data: Tuple[str, BinaryIO]) -> Tuple[Tuple[Label, def _prepare_test_data(self, data: Tuple[str, BinaryIO]) -> Tuple[None, Tuple[str, BinaryIO]]: return None, data + def _classifiy_devkit(self, data: Tuple[str, BinaryIO]) -> Optional[int]: + return { + "meta.mat": ImageNetDemux.META, + "ILSVRC2012_validation_ground_truth.txt": ImageNetDemux.LABEL, + }.get(pathlib.Path(data[0]).name) + # Although the WordNet IDs (wnids) are unique, the corresponding categories are not. For example, both n02012849 # and n03126707 are labeled 'crane' while the first means the bird and the latter means the construction equipment _WNID_MAP = { @@ -165,11 +172,13 @@ def _datapipe(self, resource_dps: List[IterDataPipe]) -> IterDataPipe[Dict[str, else: # config.split == "val": images_dp, devkit_dp = resource_dps - meta_dp = Filter(devkit_dp, path_comparator("name", "meta.mat")) + meta_dp, label_dp = Demultiplexer( + devkit_dp, 2, self._classifiy_devkit, drop_none=True, buffer_size=INFINITE_BUFFER_SIZE + ) + meta_dp = Mapper(meta_dp, self._extract_categories_and_wnids) - _, wnids = zip(*list(meta_dp)[0]) + _, wnids = zip(*next(iter(meta_dp))) - label_dp = Filter(devkit_dp, path_comparator("name", "ILSVRC2012_validation_ground_truth.txt")) label_dp = LineReader(label_dp, decode=True, return_path=False) # We cannot use self._wnids here, since we use a different order than the dataset label_dp = Mapper(label_dp, functools.partial(self._imagenet_label_to_wnid, wnids=wnids)) @@ -195,12 +204,15 @@ def __len__(self) -> int: "test": 100_000, }[self._split] + def _filter_meta(self, data: Tuple[str, Any]) -> bool: + return self._classifiy_devkit(data) == ImageNetDemux.META + def _generate_categories(self) -> List[Tuple[str, ...]]: self._split = "val" resources = self._resources() devkit_dp = resources[1].load(self._root) - meta_dp = Filter(devkit_dp, path_comparator("name", "meta.mat")) + meta_dp = Filter(devkit_dp, self._filter_meta) meta_dp = Mapper(meta_dp, self._extract_categories_and_wnids) categories_and_wnids = cast(List[Tuple[str, ...]], next(iter(meta_dp))) From 93777aeb281ed5f329b23fc6b579e5f6c9d600ac Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 30 Aug 2022 21:59:17 +0200 Subject: [PATCH 09/11] [SKIP CI] skip CircleCI From 7522df18aef99e4feb593ae9b077189c54d73172 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 30 Aug 2022 22:13:45 +0200 Subject: [PATCH 10/11] test on full matrix --- .github/workflows/prototype-tests.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 9b110d25c74..bc9cb5402c4 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -14,11 +14,8 @@ jobs: - "3.10" os: - ubuntu-latest - include: - - python-version: "3.7" - os: windows-latest - - python-version: "3.7" - os: macos-latest + - windows-latest + - macos-latest fail-fast: false runs-on: ${{ matrix.os }} From 4c770072eec7923467c0965966b448715d24361b Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 30 Aug 2022 22:13:52 +0200 Subject: [PATCH 11/11] [SKIP CI] skip CircleCI