super-gradients/tests/unit_tests/update_param_groups_unit_test.py

import unittest
from super_gradients.training import Trainer
from super_gradients.training.dataloaders.dataloaders import classification_test_dataloader
from super_gradients.training.metrics import Accuracy
from super_gradients.training.models import LeNet
from super_gradients.training.utils import HpmStruct, get_param
from super_gradients.training.utils.callbacks import TestLRCallback
import numpy as np


class TestNet(LeNet):
    """
    Toy test net with update_param_groups method that hard codes some lr.
    """
    def __init__(self):
        super(TestNet, self).__init__()

    def update_param_groups(
            self, param_groups: list, lr: float, epoch: int, iter: int, training_params: HpmStruct, total_batch: int
    ) -> list:
        initial_lr = get_param(training_params, "initial_lr")
        for param_group in param_groups:
            param_group["lr"] = initial_lr * (epoch + 1)
        return param_groups


class UpdateParamGroupsTest(unittest.TestCase):
    def test_lr_scheduling_with_update_param_groups(self):
        # Define Model
        net = TestNet()
        trainer = Trainer("lr_warmup_test", model_checkpoints_location='local')

        lrs = []
        phase_callbacks = [TestLRCallback(lr_placeholder=lrs)]

        train_params = {"max_epochs": 3,
                        "lr_mode": "step",
                        "lr_updates": [0, 1, 2],
                        "initial_lr": 0.1,
                        "lr_decay_factor": 1,
                        "loss": "cross_entropy", "optimizer": 'SGD',
                        "criterion_params": {}, "optimizer_params": {"weight_decay": 1e-4, "momentum": 0.9},
                        "train_metrics_list": [Accuracy()], "valid_metrics_list": [Accuracy()],
                        "loss_logging_items_names": ["Loss"], "metric_to_watch": "Accuracy",
                        "greater_metric_to_watch_is_better": True, "ema": False, "phase_callbacks": phase_callbacks,
                        }

        expected_lrs = np.array([0.1, 0.2, 0.3])
        trainer.train(model=net, training_params=train_params,
                      train_loader=classification_test_dataloader(),
                      valid_loader=classification_test_dataloader())

        self.assertTrue(np.allclose(np.array(lrs), expected_lrs, rtol=0.0000001))