first commit

Change-Id: Ica6a8c4c16b1bbba11fa6021dad7d1ac68fa32f1

Author: arnaudon

.gitignore

@@ -0,0 +1,139 @@
+SOURCES.txt
+*.svg
+*.png
+*.DS_Store
+
+*~
+*.pyc
+*.swo
+egg-info/
+*checkpoints
+*.run*
+*.pkl
+*.mvd*
+output/
+*profile_benchmark*
+*stdout
+*.swp
+.idea/
+*.c
+*.o
+*.so
+uild/*
+
+# jupyter notebook
+*jupyter-log*
+.ipynb_checkpoints/
+
+*.eqc
+
+*si_*
+*si_*
+*.log
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+*.c
+*.cpp
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# pytest
+.pytest_cache

.gitreview

@@ -0,0 +1,6 @@
+[gerrit]
+host=bbpcode.epfl.ch
+port=22
+project=cells/bluepyparallel
+defaultbranch=master
+defaultremote=origin

.pylintrc

@@ -0,0 +1,70 @@
+## look at http://docutils.sourceforge.net/sandbox/py-rest-doc/utils/pylintrc
+# for some of the options that are available
+
+[MESSAGES CONTROL]
+#C0103 -  Invalid name "%s" (should match %s) - matches too many things, like variables w/ single char names
+#R0904 - Too Many public methods
+#R0903 - Too Few public methods
+#W0511 - TODO in code
+#W0232 - Class has no __init__ method
+#R0922 - Abstract class is only referenced 1 times
+#R0801 - Similar lines in %d files
+#R0921 - Abstract class not referenced
+#W0141 - Used builtin function 'map'
+#R0401 - cyclic-import
+#I0013 - Ignore the 'Ignoring entire file' warning
+#W0142 - Used * or ** magic
+#W0622 - redefined-builti
+#C0325 - superfluous-parens
+#R0205 - useless-object-inheritance
+disable=C0103,R0904,R0903,W0511,W0232,R0922,R0801,R0921,W0141,R0401,I0013,W0142,W0622,C0325,R0205,C0330,C0302,E231,C0415,R0201,E1133,C0112,C0116,W0221
+
+[FORMAT]
+# Maximum number of characters on a single line.
+max-line-length=100
+
+[DESIGN]
+# Maximum number of arguments for function / method
+max-args=15
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore
+ignored-argument-names=_.*
+# Maximum number of locals for function / method body
+max-locals=25
+# Maximum number of return / yield for function / method body
+max-returns=6
+# Maximum number of branch for function / method body
+max-branches=20
+# Maximum number of statements in function / method body
+max-statements=60
+# Maximum number of parents for a class (see R0901).
+max-parents=10
+# Maximum number of attributes for a class (see R0902).
+max-attributes=40
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=60
+# checks for similarities and duplicated code. This computation may be
+# memory / CPU intensive, so you should disable it if you experiments some
+# problems.
+#
+
+[SIMILARITIES]
+# Minimum lines number of a similarity.
+min-similarity-lines=25
+# Ignore comments when computing similarities.
+ignore-comments=yes
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+[TYPECHECK]
+# List of classes names for which member attributes should not be checked
+# (useful for classes with attributes dynamically set).
+#ignored-classes=foo.bar
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis. It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=bluepyopt.deapext.*,sqlite3,seaborn,neurom,morphio,kgforge,dask,dask_mpi,neuron,bglibpy,voxcell,synthesis_workflow.synthesis,luigi_tools

README.rst

@@ -0,0 +1,74 @@
+# BluePyParallel: Bluebrain Python Embarassingly Parallel library
+
+
+Introduction
+============
+
+Provides an embarassingly parallel tool with sql backend.
+
+Running using Dask
+==================
+
+This is an example of a sbatch script that can be adapted to execute the script using multiple nodes and workers.
+
+Dask variables are not strictly required, but highly recommended, and they can be fine tuned.
+
+
+.. code:: bash
+
+    #!/bin/bash -l
+    #SBATCH --nodes=2             # Number of nodes
+    #SBATCH --time=24:00:00       # Time limit
+    #SBATCH --partition=prod      # Submit to the production 'partition'
+    #SBATCH --constraint=cpu      # Constraint the job to run on nodes with/without SSDs. If you want SSD, use only "nvme". If you want KNLs then "knl"
+    #SBATCH --exclusive           # only if you need to allocate whole node
+    #SBATCH --mem=0
+    #SBATCH --ntasks-per-node=72  # no of mpi ranks to use per node
+    #SBATCH --account=projXX      # your project number
+    #SBATCH --job-name=myscript
+    #SBATCH --output=myscript_out_%j
+    #SBATCH --error=myscript_err_%j
+    set -e
+    
+    module purge
+    module load unstable hpe-mpi
+    module unload unstable
+    
+    unset PMI_RANK  # for neuron
+    
+    # Dask configuration
+    export DASK_DISTRIBUTED__LOGGING__DISTRIBUTED="info"
+    export DASK_DISTRIBUTED__WORKER__USE_FILE_LOCKING=False
+    export DASK_DISTRIBUTED__WORKER__MEMORY__TARGET=False  # don't spill to disk
+    export DASK_DISTRIBUTED__WORKER__MEMORY__SPILL=False  # don't spill to disk
+    export DASK_DISTRIBUTED__WORKER__MEMORY__PAUSE=0.80  # pause execution at 80% memory use
+    export DASK_DISTRIBUTED__WORKER__MEMORY__TERMINATE=0.95  # restart the worker at 95% use
+    export DASK_DISTRIBUTED__WORKER__MULTIPROCESSING_METHOD=spawn
+    export DASK_DISTRIBUTED__WORKER__DAEMON=True
+    # Reduce dask profile memory usage/leak (see https://github.com/dask/distributed/issues/4091)
+    export DASK_DISTRIBUTED__WORKER__PROFILE__INTERVAL=10000ms  # Time between statistical profiling queries
+    export DASK_DISTRIBUTED__WORKER__PROFILE__CYCLE=1000000ms  # Time between starting new profile
+    
+    # Split tasks to avoid some dask errors (e.g. Event loop was unresponsive in Worker)
+    export PARALLEL_BATCH_SIZE=1000
+    
+    # Script parameters
+    OUTPUT="/path/to/mecombo_emodel.tsv"
+    CIRCUIT_CONFIG="/gpfs/bbp.cscs.ch/project/proj68/circuits/Isocortex/20190307/CircuitConfig"
+    MORPHOLOGY_PATH="/gpfs/bbp.cscs.ch/project/proj68/circuits/Isocortex/20190307/morphologies"
+    RELEASE_PATH="emodel_release"
+    N_CELLS=100
+    MTYPE="L5_TPC:A"
+    
+    # load the virtual env (alternatively, load the required modules)
+    source ~/venv/3.7.4-BluePyEModel/bin/activate
+    
+    srun -v \
+    BluePyEModel -v get_me_combos_parameters \
+    --circuit-config "$CIRCUIT_CONFIG" \
+    --morphology-path "$MORPHOLOGY_PATH" \
+    --release-path "$RELEASE_PATH" \
+    --output "$OUTPUT" \
+    --n-cells "$N_CELLS" \
+    --mtype "$MTYPE" \
+    --parallel-lib dask

bluepyparallel/__init__.py

@@ -0,0 +1,3 @@
+"""BluePyParallel functions."""
+from .evaluator import evaluate_combos
+from .parallel import init_parallel_factory