super-gradients/.circleci/config.yml

version: 2.1

parameters:
  orb_version:
    type: string
    description: Deci ai ORB version https://circleci.com/developer/orbs/orb/deci-ai/circleci-common-orb
    default: "10.3.0"
#    default: "dev:alpha"

orbs:
  deci-common: deci-ai/circleci-common-orb@<< pipeline.parameters.orb_version >>

# This filter operates on SemVer2 tags only
release_tag_filter: &release_tag_filter
  filters:
    branches:
      ignore: /.*/
    tags:
      only: /^\d+\.\d+\.\d+$/

release_candidate_filter: &release_candidate_filter
  filters:
    branches:
      only: master


jobs:
  build:
    parameters:
      py_version:
        type: string
        default: latest
      package_name:
        type: string
    docker:
      - image: cimg/python:<< parameters.py_version >>
    resource_class: large
    steps:
      - deci-common/checkout_and_skip_build:
          check_version_file: true
      - deci-common/get_persisted_version_info
      - when:
          condition:
            and:
              - not:
                  equal: [ develop, << pipeline.git.branch >> ]
              - not:
                  equal: [ staging, << pipeline.git.branch >> ]
              - not:
                  equal: [ master, << pipeline.git.branch >> ]
          steps:
            - run:
                name: install Black Flake8 python linter
                command: |
                  pip install --user -r requirements.dev.txt
            - run:
                name: Lint all python files changed since develop branch
                command: |
                  flake8 --statistics --config scripts/flake8-config setup.py $(git diff --diff-filter ACM origin/master --name-only | grep 'py$' | grep -v 'experimental/' | grep -v 'experimental_models/')
            - run:
                name: Run Black on changed files against master branch
                command: |
                  black --check setup.py $(git diff --diff-filter ACM origin/master --name-only | grep 'py$' | grep -v 'experimental/' | grep -v 'experimental_models/')
      - run:
          name: add requirements.txt and requirements.pro.txt to source code
          command: |
            cp requirements.txt src/super_gradients/requirements.txt
            cp requirements.pro.txt src/super_gradients/requirements.pro.txt
      - run:
          name: install python dependencies
          command: |
            python3 -m venv venv
            . venv/bin/activate
            python3 -m pip install pip==22.0.4
            cat requirements.txt | cut -f1 -d"#" | xargs -n 1 -L 1 pip install --progress-bar off
      - run:
          name: edit package version
          command: |
            echo "${NEW_VERSION}" > version.txt
            cat version.txt
      - run:
          name: setup custom environment variables
          command: |
            echo 'export PYTHONPATH=/home/circleci/super_gradients' >> $BASH_ENV
            echo 'export UPLOAD_LOGS=FALSE' >> $BASH_ENV
      - run:
          name: install package
          no_output_timeout: 30m
          command: |
            . venv/bin/activate
            python3 -m pip install --extra-index-url https://pypi.ngc.nvidia.com .[pro]
      - run:
          name: run tests with coverage
          no_output_timeout: 30m
          command: |
            . venv/bin/activate
            coverage run --source=super_gradients -m unittest tests/deci_core_unit_test_suite_runner.py
            coverage report
            coverage html  # open htmlcov/index.html in a browser

      - store_artifacts:
          path: htmlcov

      - store_artifacts:
          path: ~/sg_logs

  release_candidate:
    parameters:
      py_version:
        type: string
    docker:
      - image: cimg/python:<< parameters.py_version >>
    steps:
      - deci-common/checkout_and_skip_build:
          check_version_file: true
      - deci-common/get_persisted_version_info
      - run:
          name: edit package version
          command: |
            echo $NEW_VERSION > version.txt
      - deci-common/pip_upload_package_codeartifact_dev:
          codeartifact_repository: "deci-packages"
      - deci-common/pip_test_package_installation_codeartifact_dev:
          package_name: "super-gradients"
          version: $NEW_VERSION
      - deci-common/git_config_automation_user
      - run:
          name: "commit version file"
          command: |
            git commit version.txt -m "Deci Services - Changed version to $NEW_VERSION"
      - deci-common/git_commit_and_tag:
          version: $NEW_VERSION

  release_version:
    parameters:
      py_version:
        type: string
    docker:
      - image: cimg/python:<< parameters.py_version >>
    steps:
      - deci-common/checkout_and_skip_build:
          check_version_file: true
      - run:
          name: verify __version__
          command: python3 tests/verify_version.py $CIRCLE_TAG
      - run:
          name: add requirements.txt and requirements.pro.txt to source code
          command: |
            cp requirements.txt src/super_gradients/requirements.txt
            cp requirements.pro.txt src/super_gradients/requirements.pro.txt
      - run:
          name: edit package version
          command: |
            echo $CIRCLE_TAG > version.txt

      - deci-common/pip_upload_package_codeartifact_all_accounts:
          codeartifact_repository: "deci-packages"

      - deci-common/pip_test_package_installation_codeartifact_dev:
          package_name: "super-gradients"
          version: $CIRCLE_TAG

      - deci-common/pip_test_package_installation_codeartifact_prod:
          package_name: "super-gradients"
          version: $CIRCLE_TAG

      - deci-common/pip_upload_package_codeartifact_prod:
          codeartifact_repository: "deci-toolkit"

      - deci-common/git_commit_and_tag:
          version: $CIRCLE_TAG
          delete_remote_tag_before_tagging: true

      - deci-common/tag_as:
          tag_name: "stable"
          delete_remote: true

      - deci-common/github_create_release:
          github_cli_token: $GITHUB_CLI_TOKEN
          directory_to_cd_into: "."
          tag: $CIRCLE_TAG
          notes: "This GitHub Release was done automatically by CircleCI"

  recipe_tests:
    machine: true
    resource_class: deci-ai/sg-gpu-on-premise
    parameters:
      sg_existing_env_path:
        type: string
        default: "/env/persistent_env"
      sg_new_env_name:
        type: string
        default: "${CIRCLE_BUILD_NUM}"
      sg_new_env_python_version:
        type: string
        default: "python3.8"
    steps:
      - checkout
      - run:
          name: install requirements and run recipe tests
          command: |
            << parameters.sg_new_env_python_version >> -m venv << parameters.sg_new_env_name >>
            source << parameters.sg_new_env_name >>/bin/activate
            python3.8 -m pip install --upgrade setuptools pip wheel
            python3.8 -m pip install -r requirements.txt
            python3.8 -m pip install git+https://github.com/Deci-AI/super-gradients.git@${CIRCLE_BRANCH}
            python3.8 -m pip install torch==1.12.0+cu116 torchvision==0.13.0+cu116 torchaudio==0.12.0 --extra-index-url https://download.pytorch.org/whl/cu116
            python3.8 src/super_gradients/examples/train_from_recipe_example/train_from_recipe.py --config-name=cifar10_resnet experiment_name=shortened_cifar10_resnet_accuracy_test training_hyperparams.max_epochs=100 training_hyperparams.average_best_models=False multi_gpu=DDP num_gpus=4
            python3.8 src/super_gradients/examples/convert_recipe_example/convert_recipe_example.py --config-name=cifar10_conversion_params experiment_name=shortened_cifar10_resnet_accuracy_test
            python3.8 src/super_gradients/examples/train_from_recipe_example/train_from_recipe.py --config-name=coco2017_yolox experiment_name=shortened_coco2017_yolox_n_map_test architecture=yolox_n training_hyperparams.loss=yolox_fast_loss training_hyperparams.max_epochs=10 training_hyperparams.average_best_models=False multi_gpu=DDP num_gpus=4
            python3.8 src/super_gradients/examples/train_from_recipe_example/train_from_recipe.py --config-name=cityscapes_regseg48 experiment_name=shortened_cityscapes_regseg48_iou_test training_hyperparams.max_epochs=10 training_hyperparams.average_best_models=False multi_gpu=DDP num_gpus=4
            coverage run --source=super_gradients -m unittest tests/deci_core_recipe_test_suite_runner.py

      - run:
          name: Remove new environment when failed
          command: "rm -r << parameters.sg_new_env_name >>"
          when: on_fail



workflows:
  release:
    jobs:
      - deci-common/persist_version_info:
          <<: *release_tag_filter
      - deci-common/codeartifact_login:
          name: "login_to_codeartifact_release"
          repo_name: "deci-packages"
          <<: *release_tag_filter
      - build:
          name: "build3.7"
          py_version: "3.7"
          package_name: "super-gradients"
          requires:
            - deci-common/persist_version_info
            - login_to_codeartifact_release
          <<: *release_tag_filter
      - recipe_tests:
          <<: *release_tag_filter
      - release_version:
          py_version: "3.7"
          requires:
            - "build3.7"
            - recipe_tests
          <<: *release_tag_filter
      - deci-common/pip_upload_package_from_codeartifact_to_global_pypi:
          package_name: "super-gradients"
          name: "upload_super_gradients_to_pypi"
          version: $CIRCLE_TAG
          requires:
            - "release_version"
          context:
            - pypi-public
          <<: *release_tag_filter

  build_and_deploy:
    jobs:
      - deci-common/persist_version_info
      - deci-common/codeartifact_login:
          repo_name: "deci-packages"

      - build:
          name: "build3.7"
          py_version: "3.7"
          package_name: "super-gradients"
          requires:
            - deci-common/persist_version_info
            - deci-common/codeartifact_login

      - release_candidate: # happens on merge
          py_version: "3.7"
          requires:
            - "build3.7"
          <<: *release_candidate_filter