super-gradients/tests/unit_tests/update_param_groups_unit_test.py

import unittest
from super_gradients.training import SgModel
from super_gradients.training.metrics import Accuracy
from super_gradients.training.datasets import ClassificationTestDatasetInterface
from super_gradients.training.models import LeNet
from super_gradients.training.utils import HpmStruct, get_param
from super_gradients.training.utils.callbacks import TestLRCallback
import numpy as np


class TestNet(LeNet):
    """
    Toy test net with update_param_groups method that hard codes some lr.
    """
    def __init__(self):
        super(TestNet, self).__init__()

    def update_param_groups(
            self, param_groups: list, lr: float, epoch: int, iter: int, training_params: HpmStruct, total_batch: int
    ) -> list:
        initial_lr = get_param(training_params, "initial_lr")
        for param_group in param_groups:
            param_group["lr"] = initial_lr * (epoch + 1)
        return param_groups


class UpdateParamGroupsTest(unittest.TestCase):
    def setUp(self) -> None:
        self.dataset_params = {"batch_size": 4}
        self.dataset = ClassificationTestDatasetInterface(dataset_params=self.dataset_params)
        self.arch_params = {'num_classes': 10}

    def test_lr_scheduling_with_update_param_groups(self):
        # Define Model
        net = TestNet()
        model = SgModel("lr_warmup_test", model_checkpoints_location='local')
        model.connect_dataset_interface(self.dataset)
        model.build_model(net, arch_params=self.arch_params)

        lrs = []
        phase_callbacks = [TestLRCallback(lr_placeholder=lrs)]

        train_params = {"max_epochs": 3,
                        "lr_mode": "step",
                        "lr_updates": [0, 1, 2],
                        "initial_lr": 0.1,
                        "lr_decay_factor": 1,
                        "loss": "cross_entropy", "optimizer": 'SGD',
                        "criterion_params": {}, "optimizer_params": {"weight_decay": 1e-4, "momentum": 0.9},
                        "train_metrics_list": [Accuracy()], "valid_metrics_list": [Accuracy()],
                        "loss_logging_items_names": ["Loss"], "metric_to_watch": "Accuracy",
                        "greater_metric_to_watch_is_better": True, "ema": False, "phase_callbacks": phase_callbacks,
                        }

        expected_lrs = np.array([0.1, 0.2, 0.3])
        model.train(train_params)

        self.assertTrue(np.allclose(np.array(lrs), expected_lrs, rtol=0.0000001))