thesis-doctorate/.gitlab-ci.yml

---
image: python:latest

variables:
  XDG_CACHE_HOME: $CI_PROJECT_DIR/.cache # Ensure tools use accessible cache directory.

stages:
  - validate
  - replicate

# Ensure files in repository adhere to convention standards.
validate:
  stage: validate
  cache:
    paths:
      - $XDG_CACHE_HOME/pre-commit
  variables:
    # TODO: Refactor notebooks and readd pre-commit checks.
    #       Notebooks aren't important at the moment, so disabling checks.
    SKIP: "nbqa-flake8,nbqa-mypy,nbqa-pydocstyle,nbqa-isort,nbqa-pyupgrade"
  script:
    - pip install pre-commit
    - pre-commit run --all-files

# Ensure results can be replicated on other machines.
replicate:
  stage: replicate
  cache:
    paths:
      - $XDG_CACHE_HOME/pypoetry
  variables:
    # Export AWS credentials.
    AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
    AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
    AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY

    # Define DVC remote cache location.
    URL_CACHE_DVC: s3://mraxilus-dvc-public/gitlab/thesis-doctorate
  script:
    # Install AWS.
    - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
    - unzip awscliv2.zip &> /dev/null
    - ./aws/install
    - aws --version
    - aws s3 ls $URL_CACHE_DVC

    # Install pyenv.
    - curl -L https://pyenv.run | bash
    - export PATH="~/.pyenv/bin:~/.pyenv/shims:$PATH"
    - pyenv install $(cat .python-version)
    - pyenv global $(cat .python-version)
    - python3 --version
    - pyenv --version

    # Install poetry.
    - curl -L https://install.python-poetry.org | python3 -
    - export PATH="~/.local/bin:$PATH"
    - poetry --version
    - poetry run which python

    # Install project dependencies.
    - poetry install --no-dev

    # Verify DVC configuration.
    - poetry run dvc config cache.type copy # Links fail for some reason.
    - poetry run dvc cache dir
    - poetry run dvc doctor
    - poetry run dvc status --cloud --remote amazon

    # Download cached data.
    - poetry run dvc fetch --remote amazon
    - poetry run dvc checkout --relink

    # Visualize the pipelines.
    - poetry run dvc dag --full clean@air
    - poetry run dvc dag --full clean@synthetic
    - poetry run dvc dag --full clean@water

    # Check that pipeline can be reproduced.
    - poetry run dvc repro --dry # Avoid running actual computations.