AutoYAHP Part 1: Cleanup the Algorithms for AutoYAHP (#1056)

This PR refactors the algorithms and tests as will be required by AutoYAHP. It does not depend on AutoYAHP itself (a future PR will remove the underlying hparam classes).

- Refactored algorithm tests to not depend on hparams
- Reformatted the factorize and selective backprop docstrings so they would be correctly parsed by auto-yahp
- Refactor algorithm_settings.py to not depend on hparams and to return a list of pytest.param objects for a pytest.mark.parametrize. This change makes it more re-usable since it now includes information about markers required for each algorithm.
- Moved the TestTrainerAlgorithms into tests/algorithms/test_algorithms_train.py, since it tests the individual algorithms, not the trainer, and thus should live in tests/algorithms.
- Add helper methods for scanning a module to discover subclass implementations, check that the registry contains an entry, and test that a class is constructible from yaml

Author: ravi-mosaicml

.pre-commit-config.yaml

@@ -51,7 +51,6 @@ repos:
     hooks:
       - id: check-added-large-files
       - id: check-ast
-      - id: check-byte-order-marker
       - id: check-builtin-literals
         args:
           - --no-allow-dict-kwargs

composer/algorithms/factorize/factorize.py

@@ -117,9 +117,9 @@ class Factorize(Algorithm):
     See :func:`~composer.algorithms.factorize.factorize_matrix` and
     :func:`~composer.algorithms.factorize.factorize_conv2d` for more
     information about the factorization process. See
-    :class:`~composer.algorithms.factorize.FactorizedConv2d` and
-    :class:`~composer.algorithms.factorize.FactorizedLinear`
-    for more information about the factorized modules used to replace the
+    :class:`~composer.algorithms.factorize.FactorizedConv2d`
+    and :class:`~composer.algorithms.factorize.FactorizedLinear` for
+    more information about the factorized modules used to replace the
     original modules.
 
     Runs on :attr:`~composer.core.event.Event.INIT`.

composer/algorithms/selective_backprop/selective_backprop.py

@@ -179,18 +179,6 @@ class SelectiveBackprop(Algorithm):
 
      .. _Jiang et al, 2019: https://arxiv.org/abs/1910.00762
 
-     Args:
-         start (float, optional): SB interval start as fraction of training duration
-             Default: ``0.5``.
-         end (float, optional): SB interval end as fraction of training duration
-             Default: ``0.9``.
-         keep (float, optional): fraction of minibatch to select and keep for gradient computation
-             Default: ``0.5``.
-         scale_factor (float, optional): scale for downsampling input for selection forward pass
-             Default: ``1.``.
-         interrupt (int, optional): interrupt SB with a vanilla minibatch step every
-             ``interrupt`` batches. Default: ``2``.
-
     Example:
          .. testcode::
 
@@ -204,6 +192,18 @@ class SelectiveBackprop(Algorithm):
                  algorithms=[algorithm],
                  optimizers=[optimizer]
              )
+
+    Args:
+         start (float, optional): SB interval start as fraction of training duration
+             Default: ``0.5``.
+         end (float, optional): SB interval end as fraction of training duration
+             Default: ``0.9``.
+         keep (float, optional): fraction of minibatch to select and keep for gradient computation
+             Default: ``0.5``.
+         scale_factor (float, optional): scale for downsampling input for selection forward pass
+             Default: ``1.``.
+         interrupt (int, optional): interrupt SB with a vanilla minibatch step every
+             ``interrupt`` batches. Default: ``2``.
     """
 
     def __init__(self,

composer/core/algorithm.py

@@ -36,6 +36,10 @@ class Algorithm(Serializable, ABC):
       +----------------+-------------------------------------------------------------------------------+
     """
 
+    def __init__(self, *args, **kwargs):  # Stub signature for PyRight
+        del args, kwargs  # unused
+        pass
+
     @property
     def find_unused_parameters(self) -> bool:
         """Return True to indicate that the effect of this algorithm may cause some model parameters to be unused.

docs/source/templates/dataset_task_card.rst

@@ -1,19 +1,19 @@
 :orphan:
 
-##################
+#################
 Dataset/Task Name
-##################
+#################
 .. Based, in part, on [Datasheets for Datasets](https://arxiv.org/pdf/1803.09010.pdf) by Gebru et al.
 
 TLDR
-=======
+====
 ..
         TLDR about the dataset name, kind of data, tasks associated with it, and number of examples.
 
         *E.g., ImageNet is a dataset of natural images for 1000-way classification consisting of 1.2M training examples and 50K validation examples at about resolution 224x224.*
 
 Attribution
-============
+===========
 **Created By:** _______
 
 .. Who created the dataset (which team, research group) and on behalf of which entity (company, institution, organization)? Provide any links or citations as appropriate.
@@ -23,34 +23,34 @@ Attribution
 .. Under what license is the dataset available?*
 
 Using the Dataset
-==================
+=================
 
 Obtaining the Dataset
------------------------
+---------------------
 
 The dataset can be obtained from _________.
 
 .. URL, library, or other description of where to get it.
 
 Expected Format
-----------------
+---------------
 
 The Composer library expects this dataset to be stored as ________.
 
 .. What format does the Composer library expect this data to be stored in? E.g. JPEGs in folders for each class, TFRecords with a particular schema, etc.
 
 Steps to Obtain and Prepare Dataset
-------------------------------------
+-----------------------------------
 
 .. A list of step-by-step instructions necessary to obtain this dataset and place it in the right format or a link to such a guide that is available elsewhere.
 
 Performance Considerations:
-----------------------------
+---------------------------
 
 .. A list of performance considerations in order to use this dataset efficiently. E.g., the cost of loading the data for the first time, recommended storage medium, sensitivity of performance to memory size and disk throughput, costs associated with data augmentation, recommended batching strategies, etc.
 
 Technical Specifications
-=========================
+========================
 
 **Kind of Data: _____**
 

tests/algorithms/__init__.py

@@ -1,4 +1,2 @@
 # Copyright 2022 MosaicML Composer authors
 # SPDX-License-Identifier: Apache-2.0
-
-from tests.algorithms.algorithm_settings import get_settings as get_settings

tests/algorithms/algorithm_settings.py

@@ -8,8 +8,19 @@
 Each algorithm is keyed based on its name in the algorithm registry.
 """
 
-from composer.algorithms import algorithm_registry
+from typing import Any, Dict, Optional, Type
+
+import pytest
+from torch.utils.data import Dataset
+
+import composer
+from composer import Algorithm
+from composer.algorithms import (AGC, EMA, SAM, SWA, Alibi, AugMix, BlurPool, ChannelsLast, ColOut, CutMix, CutOut,
+                                 Factorize, GhostBatchNorm, LabelSmoothing, LayerFreezing, MixUp, NoOpModel,
+                                 ProgressiveResizing, RandAugment, SelectiveBackprop, SeqLengthWarmup, SqueezeExcite,
+                                 StochasticDepth)
 from composer.models import ComposerResNet
+from composer.models.base import ComposerModel
 from tests import common
 
 simple_vision_settings = {
@@ -23,7 +34,7 @@
     'dataset': (common.RandomImageDataset, {
         'is_PIL': True
     }),
-    'kwargs': {}
+    'kwargs': {},
 }
 
 simple_resnet_settings = {
@@ -34,32 +45,39 @@
     'dataset': (common.RandomImageDataset, {
         'shape': (3, 224, 224),
     }),
+    'kwargs': {},
 }
 
-_settings = {
-    'agc': simple_vision_settings,
-    'alibi': None,  # NLP settings needed
-    'augmix': None,  # requires PIL dataset to test
-    'blurpool': {
+_settings: Dict[Type[Algorithm], Optional[Dict[str, Any]]] = {
+    AGC: simple_vision_settings,
+    Alibi: None,  # NLP settings needed
+    AugMix: simple_vision_settings,
+    BlurPool: {
         'model': common.SimpleConvModel,
         'dataset': common.RandomImageDataset,
         'kwargs': {
             'min_channels': 0,
         },
     },
-    'channels_last': simple_vision_settings,
-    'colout': simple_vision_settings,
-    'cutmix': {
+    ChannelsLast: simple_vision_settings,
+    ColOut: simple_vision_settings,
+    CutMix: {
         'model': common.SimpleConvModel,
         'dataset': common.RandomImageDataset,
         'kwargs': {
             'num_classes': 2
         }
     },
-    'cutout': simple_vision_settings,
-    'ema': simple_vision_settings,
-    'factorize': simple_resnet_settings,
-    'ghost_batchnorm': {
+    CutOut: simple_vision_settings,
+    EMA: {
+        'model': common.SimpleConvModel,
+        'dataset': common.RandomImageDataset,
+        'kwargs': {
+            'half_life': "1ba",
+        },
+    },
+    Factorize: simple_resnet_settings,
+    GhostBatchNorm: {
         'model': (ComposerResNet, {
             'model_name': 'resnet18',
             'num_classes': 2
@@ -71,16 +89,17 @@
             'ghost_batch_size': 2,
         }
     },
-    'label_smoothing': simple_vision_settings,
-    'layer_freezing': simple_vision_settings,
-    'mixup': simple_vision_settings,
-    'progressive_resizing': simple_vision_settings,
-    'randaugment': None,  # requires PIL dataset to test
-    'sam': simple_vision_settings,
-    'selective_backprop': simple_vision_settings,
-    'seq_length_warmup': None,  # NLP settings needed
-    'squeeze_excite': simple_resnet_settings,
-    'stochastic_depth': {
+    LabelSmoothing: simple_vision_settings,
+    LayerFreezing: simple_vision_settings,
+    MixUp: simple_vision_settings,
+    ProgressiveResizing: simple_vision_settings,
+    RandAugment: simple_vision_settings,
+    NoOpModel: simple_vision_settings,
+    SAM: simple_vision_settings,
+    SelectiveBackprop: simple_vision_settings,
+    SeqLengthWarmup: None,  # NLP settings needed
+    SqueezeExcite: simple_resnet_settings,
+    StochasticDepth: {
         'model': (ComposerResNet, {
             'model_name': 'resnet50',
             'num_classes': 2
@@ -93,10 +112,11 @@
             'target_layer_name': 'ResNetBottleneck',
             'drop_rate': 0.2,
             'drop_distribution': 'linear',
-            'use_same_gpu_seed': False
+            'drop_warmup': "0.0dur",
+            'use_same_gpu_seed': False,
         }
     },
-    'swa': {
+    SWA: {
         'model': common.SimpleConvModel,
         'dataset': common.RandomImageDataset,
         'kwargs': {
@@ -105,37 +125,73 @@
             'update_interval': '1ep',
             'schedule_swa_lr': True,
         }
-    }
+    },
 }
 
 
-def get_settings(name: str):
-    """For a given algorithm name, creates the canonical setting
-    (algorithm, model, dataset) for testing.
+def _get_alg_settings(alg_cls: Type[Algorithm]):
+    if alg_cls not in _settings or _settings[alg_cls] is None:
+        raise ValueError(f"Algorithm {alg_cls.__name__} not in the settings dictionary.")
+    settings = _settings[alg_cls]
+    assert settings is not None
+    return settings
+
+
+def get_alg_kwargs(alg_cls: Type[Algorithm]) -> Dict[str, Any]:
+    """Return the kwargs for an algorithm."""
+    return _get_alg_settings(alg_cls)['kwargs']
 
-    Returns ``None`` if no settings provided.
+
+def get_alg_model(alg_cls: Type[Algorithm]) -> ComposerModel:
+    """Return an instance of the model for an algorithm."""
+    settings = _get_alg_settings(alg_cls)['model']
+    if isinstance(settings, tuple):
+        (cls, kwargs) = settings
+    else:
+        (cls, kwargs) = (settings, {})
+    return cls(**kwargs)
+
+
+def get_alg_dataset(alg_cls: Type[Algorithm]) -> Dataset:
+    """Return an instance of the dataset for an algorithm."""
+    settings = _get_alg_settings(alg_cls)['dataset']
+    if isinstance(settings, tuple):
+        (cls, kwargs) = settings
+    else:
+        (cls, kwargs) = (settings, {})
+    return cls(**kwargs)
+
+
+def get_algs_with_marks():
+    """Returns a list of algorithms appropriate markers for a subsequent call to pytest.mark.parameterize.
+    It applies markers as appropriate (e.g. XFAIL for algs missing config)
+    It reads from the algorithm registry
+
+    E.g. @pytest.mark.parametrize("alg_class", get_algs_with_marks())
     """
-    if name not in _settings:
-        raise ValueError(f'No settings for {name} found, please add.')
-
-    setting = _settings[name]
-    if setting is None:
-        return None
-
-    result = {}
-    for key in ('model', 'dataset'):
-        if isinstance(setting[key], tuple):
-            (obj, kwargs) = setting[key]
-        else:
-            (obj, kwargs) = (setting[key], {})
-
-        # create the object
-        result[key] = obj(**kwargs)
-
-    # create algorithm
-    kwargs = setting.get('kwargs', {})
-    hparams = algorithm_registry.get_algorithm_registry()[name]
-    result['algorithm'] = hparams(**kwargs).initialize_object()
-    result['algorithm_kwargs'] = kwargs
-
-    return result
+    ans = []
+    for alg_cls in common.get_module_subclasses(composer.algorithms, Algorithm):
+        marks = []
+        settings = _settings[alg_cls]
+
+        if alg_cls in (CutMix, MixUp, LabelSmoothing):
+            # see: https://github.com/mosaicml/composer/issues/362
+            pytest.importorskip("torch", minversion="1.10", reason="Pytorch 1.10 required.")
+
+        if alg_cls == SWA:
+            # TODO(matthew): Fix
+            marks.append(
+                pytest.mark.filterwarnings(
+                    r'ignore:Detected call of `lr_scheduler.step\(\)` before `optimizer.step\(\)`:UserWarning'))
+
+        if alg_cls == MixUp:
+            # TODO(Landen): Fix
+            marks.append(
+                pytest.mark.filterwarnings(r"ignore:Some targets have less than 1 total probability:UserWarning"))
+
+        if settings is None:
+            marks.append(pytest.mark.xfail(reason=f"Algorithm {alg_cls.__name__} is missing settings."))
+
+        ans.append(pytest.param(alg_cls, marks=marks, id=alg_cls.__name__))
+
+    return ans

tests/algorithms/test_algorithm_hparams.py

@@ -0,0 +1,28 @@
+# Copyright 2022 MosaicML Composer authors
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Type
+
+import pytest
+
+import composer.algorithms
+from composer.algorithms import AlgorithmHparams
+from composer.algorithms.algorithm_registry import registry as algorithm_registry
+from composer.core import Algorithm
+from tests.algorithms.algorithm_settings import get_alg_kwargs
+from tests.common import get_module_subclasses
+from tests.common.hparams import assert_in_registry, assert_yaml_loads
+
+
+@pytest.mark.parametrize("alg_hparams_cls", get_module_subclasses(composer.algorithms, AlgorithmHparams))
+def test_all_algs_in_registry(alg_hparams_cls: Type[AlgorithmHparams]):
+    assert_in_registry(alg_hparams_cls, algorithm_registry)
+
+
+@pytest.mark.xfail(reason="This test depends on AutoYAHP")
+@pytest.mark.parametrize("alg_cls", get_module_subclasses(composer.algorithms, Algorithm))
+def test_algs_load_from_yaml(alg_cls: Type[Algorithm]):
+    kwargs = get_alg_kwargs(alg_cls)
+    if kwargs is None:
+        pytest.xfail(f"Missing settings for algorithm {alg_cls.__name__}")
+    assert_yaml_loads(alg_cls, kwargs, expected=alg_cls)