wepy.lysozyme_test/project.org_archive

#    -*- mode: org -*-


Archived entries from file /home/salotz/tree/lab/projects/wepy.lysozyme_test/project.org


* Running simulations
  :PROPERTIES:
  :ARCHIVE_TIME: 2019-11-14 Thu 21:19
  :ARCHIVE_FILE: ~/tree/lab/projects/wepy.lysozyme_test/project.org
  :ARCHIVE_CATEGORY: project
  :END:

** Standalone Runs

To validate that our system is okay we do a couple of standalone runs
on the system.

*** Script
Here is a script for running a basic kind of simulation:

#+begin_src python :tangle scripts/run.py
  import simtk.openmm as omm
  import simtk.openmm.app as omma

  import simtk.unit as unit

  import mdtraj as mdj

  from openmmtools import testsystems

  # from seh_prep.modules import StateDataFrameOMMReporter

  STEP_TIME = 0.002 * unit.picoseconds

  LANGEVIN_DEFAULTS = (
      300.0*unit.kelvin,
      1/unit.picosecond,
      STEP_TIME
  )

  PLATFORM = 'CPU'

  FRAME_INTERVAL_TIME = 10 * unit.picosecond

  FRAME_INTERVAL = int(round(FRAME_INTERVAL_TIME / STEP_TIME))

  def gen_sim(platform):

      integrator = omm.LangevinIntegrator(*LANGEVIN_DEFAULTS)

      test_sys = testsystems.LysozymeImplicit()

      # write a PDB file for reference

      top = mdj.Topology.from_openmm(test_sys.topology)
      traj = mdj.Trajectory(test_sys.positions.value_in_unit(unit.nanometer), topology=top)

      traj.save_pdb('ref.pdb')

      if platform == 'CUDA':
          platform_kwargs = {'Precision' : 'single'}
      elif platform == 'OpenCL':
          platform_kwargs = {"Precision" : 'single'}
      elif platform == 'CPU':
          platform_kwargs = {}
      elif platform == 'Reference':
          platform_kwargs = {}
      else:
          raise ValueError

      platform = omm.Platform.getPlatformByName(platform)

      simulation = omma.Simulation(test_sys.topology, test_sys.system, integrator,
                            platform=platform,
                            platformProperties=platform_kwargs)

      simulation.context.setPositions(test_sys.positions)


      # add some reporters
      simulation.reporters.append(omma.DCDReporter('traj.dcd',
                                                   reportInterval=FRAME_INTERVAL,
                                                   enforcePeriodicBox=False))

      # simulation.reporters.append(StateDataFrameOMMReporter(report_interval=10))

      simulation.reporters.append(omma.StateDataReporter('state.csv',
                                                         FRAME_INTERVAL,
                                                         step=True,
                                                         time=True,
                                                         potentialEnergy=True,
                                                         kineticEnergy=True,
                                                         totalEnergy=True,
                                                         temperature=True,
                                                         volume=True,
      ))

      return simulation

  if __name__ == "__main__":
      import sys
      import time

      # time in sampling to run
      sim_time = float(sys.argv[1]) #* unit.second
      platform = sys.argv[2]

      print("Running simulation for {}".format(sim_time))


      tranche_time = 10 * unit.picosecond
      tranche_steps = int(round(tranche_time / STEP_TIME))

      print(f"running tranches of {tranche_steps} steps")

      simulation = gen_sim(platform)

      print("Starting the simulation")
      curr_time = 0.
      step_counter = 0
      start_time = time.time()
      while curr_time < sim_time:
          print("starting from step {}".format(step_counter))
          print(type(tranche_steps))
          simulation.step(tranche_steps)
          curr_time = time.time() - start_time
          step_counter += tranche_steps
          print("Ran {}".format(step_counter))
          print("Time currently is: {}".format(curr_time))

      # add up the sampling time
      sampling_time = step_counter * STEP_TIME

      print(f"Ran a total of {step_counter} steps")
      print(f"Ran a total of {sampling_time}")

#+end_src


*** run settings

#+BEGIN_SRC toml :tangle hpcc/standalone/run_settings.toml
  # test run value
  walltime = "24:00:00"

  memory = "20gb"

  num_cpus = 2

  # test run value
  num_gpus = 1

  constraint = "[intel18|intel16]"

#+END_SRC


*** context settings

#+BEGIN_SRC toml :tangle hpcc/standalone/context_settings.toml
  [context]

  # its okay to leave out the exec dir, but you can specify this and the
  #run will get executed here and then be moved back
  # exec_dir = "$SCRATCH/exec"


  # epilog is something that can be run after the run, if it is empty
  # here that is okay, this is really only important for moving data
  # back after being executed somewhere else
  epilog = ""

  # this section is for the setup of the run in the script
  [context.setup]

  # this is the parent dir for the jobs, there should be a directory
  # here called 'jobs' and each job will will get it's own unique
  # directory
  goal_dir_path = "$SCRATCH/tree/lab/projects/wepy.lysozyme_test/hpcc/standalone"

  # will load these modules from lmod i.e. 'module load <modname>'
  lmod_modules = ["CUDA/10.0.130"]

  # everything in here is translated directly into environment
  # variables, e.g. "export ANACONDA_DIR=path/to/anaconda"
  [context.setup.env_vars]

  ANACONDA_DIR = "/mnt/home/lotzsamu/anaconda3"

#+END_SRC



*** Tasks

**** 1

#+begin_src bash :shebang "#!/bin/bash --login" :tangle hpcc/standalone/tasks/initial.sh

source $HOME/.bashrc

# use the ANACONDA_DIR which is set in the environment variables in
# the setup to setup this shell for anaconda
. ${ANACONDA_DIR}/etc/profile.d/conda.sh

# activate the proper virtualenv
conda activate lysozyme_test

# this will run for under 24 hours

script="$SCRATCH/tree/lab/projects/wepy.lysozyme_test/scripts/run.py"
${ANACONDA_DIR}/envs/lysozyme_test/bin/python $script 84000 CUDA
#+end_src


*** Retrieving results

#+begin_src bash
rsync -ravvhhiz \
  lotzsamu@rsync.hpcc.msu.edu:/mnt/gs18/scratch/users/lotzsamu/tree/lab/projects/wepy.lysozyme_test/hpcc/standalone/jobs/ \
  $HOME/tree/lab/projects/wepy.lysozyme_test/hpcc/standalone/jobs
#+end_src

** Wepy Runs

To validate that our system is okay we do a couple of standalone runs
on the system.

*** By Time
Here is a script for running a basic kind of simulation:

#+begin_src python :tangle scripts/wepy_run_by_time.py

  if __name__ == "__main__":

      import sys
      import logging
      from multiprocessing_logging import install_mp_handler

      from wepy_tools.sim_makers.openmm.lysozyme import LysozymeImplicitOpenMMSimMaker

      logging.getLogger().setLevel(logging.DEBUG)
      install_mp_handler()

      if sys.argv[1] == "-h" or sys.argv[1] == "--help":
          print("arguments: n_cycles, n_steps, n_walkers, n_workers, platform, resampler")
      else:
          walltime = int(sys.argv[1])
          #n_cycles = int(sys.argv[1])
          n_steps = int(sys.argv[2])
          n_walkers = int(sys.argv[3])
          n_workers = int(sys.argv[4])
          platform = sys.argv[5]
          resampler = sys.argv[6]

          print("Number of steps: {}".format(n_steps))
          print("Time to run: {}".format(walltime))
          # print("Number of cycles: {}".format(n_cycles))

      sim_maker = LysozymeImplicitOpenMMSimMaker()

      apparatus = sim_maker.make_apparatus(
          integrator='LangevinIntegrator',
          resampler=resampler,
          bc='UnbindingBC',
          platform=platform,
      )
      config = sim_maker.make_configuration(apparatus,
                                            work_mapper='TaskMapper',
                                            platform=platform)

      sim_manager = sim_maker.make_sim_manager(n_walkers, apparatus, config)

      sim_manager.run_simulation_by_time(walltime, n_steps, num_workers=n_workers)

#+end_src

*** By Num Cycles

Here is a script for running a basic kind of simulation:

#+begin_src python :tangle scripts/wepy_run_by_cycles.py

  if __name__ == "__main__":

      import sys
      import logging
      from multiprocessing_logging import install_mp_handler

      from wepy_tools.sim_makers.openmm.lysozyme import LysozymeImplicitOpenMMSimMaker

      logging.getLogger().setLevel(logging.DEBUG)
      install_mp_handler()

      if sys.argv[1] == "-h" or sys.argv[1] == "--help":
          print("arguments: n_cycles, n_steps, n_walkers, n_workers, platform, resampler")
          else:
          n_cycles = int(sys.argv[1])
          n_steps = int(sys.argv[2])
          n_walkers = int(sys.argv[3])
          n_workers = int(sys.argv[4])
          platform = sys.argv[5]
          resampler = sys.argv[6]

          print("Number of steps: {}".format(n_steps))
          print("Number of cycles: {}".format(n_cycles))

      sim_maker = LysozymeImplicitOpenMMSimMaker()

      apparatus = sim_maker.make_apparatus(
          integrator='LangevinIntegrator',
          resampler=resampler,
          bc='UnbindingBC',
          platform=platform,
      )
      config = sim_maker.make_configuration(apparatus,
                                            work_mapper='TaskMapper',
                                            platform=platform)

      sim_manager = sim_maker.make_sim_manager(n_walkers, apparatus, config)

      sim_manager.run_simulation_by_time(n_cycles, n_steps, num_workers=n_workers)

#+end_src


*** run settings

#+BEGIN_SRC toml :tangle hpcc/wepy/run_settings.toml
  # test run value
  walltime = "168:00:00"
  memory = "20gb"
  num_cpus = 4
  num_gpus = 8

  constraint = "[intel18|intel16]"

#+END_SRC


*** context settings

#+BEGIN_SRC toml :tangle hpcc/wepy/context_settings.toml
  [context]

  # its okay to leave out the exec dir, but you can specify this and the
  #run will get executed here and then be moved back
  # exec_dir = "$SCRATCH/exec"


  # epilog is something that can be run after the run, if it is empty
  # here that is okay, this is really only important for moving data
  # back after being executed somewhere else
  epilog = ""

  # this section is for the setup of the run in the script
  [context.setup]

  # this is the parent dir for the jobs, there should be a directory
  # here called 'jobs' and each job will will get it's own unique
  # directory
  goal_dir_path = "$SCRATCH/tree/lab/projects/wepy.lysozyme_test/hpcc/wepy"

  # will load these modules from lmod i.e. 'module load <modname>'
  lmod_modules = ["CUDA/10.0.130"]

  # everything in here is translated directly into environment
  # variables, e.g. "export ANACONDA_DIR=path/to/anaconda"
  [context.setup.env_vars]

  ANACONDA_DIR = "/mnt/home/lotzsamu/anaconda3"

#+END_SRC




*** Tasks

**** NoResampler test

#+begin_src bash :shebang "#!/bin/bash" :tangle hpcc/wepy/tasks/noresampler_test.sh
  # use the ANACONDA_DIR which is set in the environment variables in
  # the setup to setup this shell for anaconda
  . ${ANACONDA_DIR}/etc/profile.d/conda.sh

  # activate the proper virtualenv
  conda activate lysozyme_test

  ${ANACONDA_DIR}/envs/lysozyme_test/bin/python -m simtk.testInstallation

  WALLTIME=590400
  N_STEPS=10000 # 20 picoseconds
  N_WALKERS=48
  N_WORKERS=8
  PLATFORM="CUDA"
  RESAMPLER="NoResampler"

  script="$SCRATCH/tree/lab/projects/wepy.lysozyme_test/scripts/wepy_run_by_time.py"


  ${ANACONDA_DIR}/envs/lysozyme_test/bin/python $script \
                 $WALLTIME \
                 $N_STEPS \
                 $N_WALKERS \
                 $N_WORKERS \
                 $PLATFORM \
                 $RESAMPLER
#+end_src


**** WExplore test

#+begin_src bash :shebang "#!/bin/bash" :tangle hpcc/wepy/tasks/wexplore_test.sh
  # use the ANACONDA_DIR which is set in the environment variables in
  # the setup to setup this shell for anaconda
  . ${ANACONDA_DIR}/etc/profile.d/conda.sh

  # activate the proper virtualenv
  conda activate lysozyme_test

  ${ANACONDA_DIR}/envs/lysozyme_test/bin/python -m simtk.testInstallation

  WALLTIME=590400
  N_STEPS=10000 # 20 picoseconds
  N_WALKERS=48
  N_WORKERS=8
  PLATFORM="CUDA"
  RESAMPLER="WExploreResampler"

  script="$SCRATCH/tree/lab/projects/wepy.lysozyme_test/scripts/wepy_run_by_time.py"


  ${ANACONDA_DIR}/envs/lysozyme_test/bin/python $script \
                 $WALLTIME \
                 $N_STEPS \
                 $N_WALKERS \
                 $N_WORKERS \
                 $PLATFORM \
                 $RESAMPLER
#+end_src


**** REVO test

#+begin_src bash :shebang "#!/bin/bash" :tangle hpcc/wepy/tasks/revo_test.sh

  # use the ANACONDA_DIR which is set in the environment variables in
  # the setup to setup this shell for anaconda
  . ${ANACONDA_DIR}/etc/profile.d/conda.sh

  # activate the proper virtualenv
  conda activate lysozyme_test

  ${ANACONDA_DIR}/envs/lysozyme_test/bin/python -m simtk.testInstallation

  WALLTIME=590400
  N_STEPS=10000 # 20 picoseconds
  N_WALKERS=48
  N_WORKERS=8
  PLATFORM="CUDA"
  RESAMPLER="REVOResampler"

  script="$SCRATCH/tree/lab/projects/wepy.lysozyme_test/scripts/wepy_run_by_time.py"


  ${ANACONDA_DIR}/envs/lysozyme_test/bin/python $script \
                 $WALLTIME \
                 $N_STEPS \
                 $N_WALKERS \
                 $N_WORKERS \
                 $PLATFORM \
                 $RESAMPLER
#+end_src


*** Retrieving results

#+begin_src bash
rsync -ravvhhiz \
  lotzsamu@rsync.hpcc.msu.edu:/mnt/gs18/scratch/users/lotzsamu/tree/lab/projects/wepy.lysozyme_test/hpcc/wepy/jobs/ \
  $HOME/tree/lab/projects/wepy.lysozyme_test/hpcc/wepy/jobs
#+end_src






Archived entries from file /home/salotz/tree/lab/projects/wepy.lysozyme_test/project.org


* Analysis


We only consider the wepy runs here. The standalone was just a test.


** Tasks


*** Initialization

Domain specific stuff related to working with the project:
**** Header
#+BEGIN_SRC python :tangle lib/wepy_lysozyme/_tasks.py
  """Generated file from the analysis.org file. Do not edit directly."""
#+END_SRC


**** Imports

#+BEGIN_SRC python :tangle lib/seh_pathway_hopping/_tasks.py

  # standard library
  import os
  import os.path as osp
  import pickle

  # de facto standard library
  import numpy as np
  import pandas as pd
  import matplotlib.pyplot as plt
  import prefect

  # extra non-domain specific
  import joblib
  import sqlalchemy as sqla

  # 3rd party domain specific
  import mdtraj as mdj
  import simtk.unit as unit

  # auxiliary supporting repos
  import geomm
  import wepy

  # truly ad hoc

#+END_SRC

**** Paths

#+BEGIN_SRC python :tangle lib/seh_pathway_hopping/_tasks.py

  ## Paths

  # for localizing paths to very commonly used resources and resrouces
  # which may change schema. The directory structure for the rest is the
  # schema, so just use osp.join(project_path(), 'subpath/to/resource')
  # for the rest so a lot of work is reduced in specifying all of them

  def projects_path():

      user_home = osp.expanduser('~')
      studio = osp.join(user_home, 'tree/lab/studio')
      projects = osp.join(studio, 'projects/index')

      return projects


  def project_path():

      return osp.join(projects_path(), 'seh.pathway_hopping')

  def data_path():

      return osp.join(project_path(), 'data')

  def media_path():

      return osp.join(project_path(), 'media')

  def sqlite_path():

      return osp.join(project_path(), 'db/db.sqlite')

  def joblib_cache_path():

      return osp.join(project_path(), 'cache/joblib')
#+END_SRC

**** Setup

Set up caching of the tasks.

#+BEGIN_SRC python :tangle lib/seh_pathway_hopping/_tasks.py
  ## Setup

  # create the sqlite database

  # set up the joblib cache
  jlmem = joblib.Memory(joblib_cache_path())



  # set this when you want to do some recursion stuff with contigtrees
  def set_recursion_limit():
      recursion_limit = 5000
      import sys; sys.setrecursionlimit(recursion_limit)
      print("Setting recursion limit to {}".format(recursion_limit))

  # set the recursion depth since it is always needing to be increased
  set_recursion_limit()
#+END_SRC




*** Visualizing Trajectories

For each run we want to visualize trajectories of the simulations no
matter if we get exit points are not.

So we get traces for every end point.