Merge branch 'master' into development

Author: pgleeson

.github/workflows/docker-image-quickrun.yml

@@ -0,0 +1,33 @@
+name: Docker Image Test - quick
+
+on:
+  push:
+    branches: [ master, dev* ]
+  pull_request:
+    branches: [ master, dev* ]
+
+jobs:
+
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build the Docker image
+      run: |
+        chmod a+w output/
+        ls -alt
+        ./build.sh
+    - name: Info on Docker images
+      run: |
+        docker -v
+        docker images
+    - name: Run the Docker image
+      run: |
+        ./run-quick.sh
+    - name: Info on generated files
+      run: |
+        ls -alt
+        ls -alt output
+        ls -alt output/*

.github/workflows/docker-image.yml

@@ -16,12 +16,18 @@ jobs:
     - uses: actions/checkout@v2
     - name: Build the Docker image
       run: |
+        chmod a+w output/
+        ls -alt
         ./build.sh
+    - name: Info on Docker images
+      run: |
+        docker -v
+        docker images
     - name: Run the Docker image
       run: |
         ./run.sh
     - name: Info on generated files
       run: |
+        ls -alt
         ls -alt output
-        ls -alt output/C2*
-        cat output/C2_*/report.json
+        ls -alt output/*

.gitignore

@@ -3,3 +3,4 @@ output/C2*
 /rebuild2.sh
 /output/siminfo.py
 /output/wor*
+/output/Sibernetic_*

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:22.04
 
 LABEL maintainer="David Lung ([email protected]); Padraig Gleeson ([email protected])"
 
@@ -36,7 +36,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends apt-utils \
   wget nano htop build-essential make git automake autoconf \
   g++ rpm libtool libncurses5-dev zlib1g-dev bison flex lsb-core \
   sudo xorg openbox x11-xserver-utils \
-  libxext-dev libncurses-dev python-dev mercurial \
+  libxext-dev libncurses-dev python3-dev mercurial \
   freeglut3-dev libglu1-mesa-dev libglew-dev python3-dev python3-pip python3-lxml  python3-scipy python3-tk \
   kmod dkms linux-source linux-headers-generic \
   maven openjdk-8-jdk \
@@ -50,7 +50,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends apt-utils \
 
 RUN sudo usermod -a -G video $USER
 
-USER $USER
+#USER $USER
 ENV HOME /home/$USER
 WORKDIR $HOME
 
@@ -62,29 +62,20 @@ RUN  sudo update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 10
 ################################################################################
 ########     Install NEURON simulator
 
-RUN sudo pip install neuron==7.8.1
+RUN pip3 install neuron==8.0.1
 
 
 ################################################################################
 ########     Install c302 for building neuronal network models
 
-
-# TODO remove this line after we have better dependency management.  The
-# current version of gitpython requires python >= 3.7, which is newer than the
-# python included in the base image. Therefore, we manually install an older
-# gitpython to be used with OpenWormData.
-# See https://github.com/openworm/OpenWorm/pull/316
-RUN sudo pip install 'gitpython==2.1.15' markupsafe
-
-RUN sudo pip install 'NeuroMLlite<0.5.0' # for python 3.6...
-
 RUN git clone https://github.com/openworm/c302.git && \
   cd c302 && \
-  git checkout ow-0.9.2 && \
+  git checkout ow-0.9.3 && \
   sudo pip install .
 
-# Note: owmeta, owmeta-core and pyNeuroML installed with the above library
+# Note: pyNeuroML installed with the above library
 
+RUN pip3 install owmeta-core==0.13.5
 RUN owm bundle remote --user add ow 'https://raw.githubusercontent.com/openworm/owmeta-bundles/master/index.json'
 
 
@@ -93,9 +84,7 @@ RUN owm bundle remote --user add ow 'https://raw.githubusercontent.com/openworm/
 
 RUN git clone https://github.com/openworm/sibernetic.git && \
   cd sibernetic && \
-  git checkout ow-0.9.1 # fixed to a specific branch
-
-RUN cp c302/pyopenworm.conf sibernetic/   # Temp step until PyOpenWorm can be run from any dir...
+  git checkout ow-0.9.3  # fixed to a specific branch
 
 
 ################################################################################
@@ -105,10 +94,6 @@ ENV C302_HOME=$HOME/c302/c302
 ENV SIBERNETIC_HOME=$HOME/sibernetic
 ENV PYTHONPATH=$PYTHONPATH:$HOME/c302:$SIBERNETIC_HOME
 
-# Not working with --chown=$USER:$USER
-COPY ./master_openworm.py $HOME/master_openworm.py
-RUN sudo chown $USER:$USER $HOME/master_openworm.py
-
 
 ################################################################################
 ########     Install Intel OpenCL libraries needed for Sibernetic
@@ -135,25 +120,16 @@ sudo apt install -y ocl-icd-opencl-dev vim
 ########     Build Sibernetic
 
 RUN cd sibernetic && \
-    sed -i -e "s/lpython2.7/lpython3.6m/g" makefile && \
-    sed -i -e "s/n2.7/n3.6/g" makefile && \
-    make clean && make all  # Use python 3 libs
-
-# intel i5, hd 5500, linux 4.15.0-39-generic
-# ./Release/Sibernetic -f worm -no_g device=CPU    190ms
-# ./Release/Sibernetic -f worm -no_g device=GPU    150ms (initialization takes some time)
-
-# Intel(R) Xeon(R) CPU E5-1650 v4 @ 3.60GHz, linux 4.4.0-139-generic
-# ./Release/Sibernetic -f worm -no_g device=CPU    60ms
-#
-# after installing the nvidia driver used in host:
-## wget http://us.download.nvidia.com/tesla/390.30/nvidia-diag-driver-local-repo-ubuntu1604-390.30_1.0-1_amd64.deb
-## sudo dpkg -i nvidia-diag-driver-local-repo-ubuntu1604-390.30_1.0-1_amd64.deb
-## sudo apt-key add /var/nvidia-diag-driver-local-repo-390.30/7fa2af80.pub
-## sudo apt-get update
-## sudo apt-get install -y cuda-drivers
-# ./Release/Sibernetic -f worm -no_g device=GPU    37ms
+    sed -i -e "s/n2.7/n3.10/g" makefile && \
+    make clean && make all && ldd ./Release/Sibernetic  # Use python 3 libs
+
+
+################################################################################
+########     Copy master python script
 
+# Not working with --chown=$USER:$USER
+COPY ./master_openworm.py $HOME/master_openworm.py
+RUN sudo chown $USER:$USER $HOME/master_openworm.py
 
 RUN echo '\n\nalias cd..="cd .."\nalias h=history\nalias ll="ls -alt"' >> ~/.bashrc
 

README.md

@@ -27,7 +27,7 @@ We have put together a [Docker container](https://hub.docker.com/r/openworm/open
 
 <img src="https://raw.githubusercontent.com/openworm/OpenWorm/master/img/muscle-activity.png" width="250"><img src="https://raw.githubusercontent.com/openworm/OpenWorm/master/img/neuron-activity.png" width="350">
 
-**NOTE**: Running the simulation for the full amount of time would produce content like the above.  However, in order to run in a reasonable amount of time, the default run time for the simulation is limited.  As such, you will see only a partial output, equivalent to about 5% of run time, compared to the examples above.  To extend the run time, use the `-d` argument as described below.
+**NOTE**: Running the simulation for the full amount of time would produce content like the above.  However, in order to run in a reasonable amount of time, **the default run time for the simulation is limited**.  As such, you will see only a partial output, equivalent to about 5% of run time, compared to the examples above.  To extend the run time, use the `-d` argument as described below.
 
 **Installation**
 

VERSION

@@ -0,0 +1 @@
+0.9.3

build.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
 
-docker build "$@" -t "openworm/openworm:0.9.3" .
+version=$(<VERSION) # Read version of Dockerfile from file VERSION
+docker build "$@" -t "openworm/openworm:$version" .

build2.sh

@@ -4,4 +4,5 @@
 # Note: Python 2 is no longer officially supported and this Docker image will
 # probably stop working eventually...
 
-docker build "$@" -t "openworm/openworm:0.9.3_py2"  -f Dockerfile2 .
+version=$(<VERSION) # Read version of Dockerfile from file VERSION
+docker build "$@" -t "openworm/openworm:${version}_py2"  -f Dockerfile2 .

master_openworm.py

@@ -2,7 +2,7 @@
 import matplotlib
 matplotlib.use('Agg')
 import shutil
-from subprocess import Popen, PIPE, check_output
+from subprocess import Popen, PIPE, check_output, STDOUT
 import os
 import pwd
 import shlex
@@ -76,22 +76,38 @@
 def execute_with_realtime_output(command, directory, env=None):
     p = None
     try:
-        p = Popen(shlex.split(command), stdout=PIPE, bufsize=1, cwd=directory, env=env)
+
+        print('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+        print('>> Executing command: %s'%command)
+        print('>> --------------------------------------------------------------')
+        p = Popen(shlex.split(command),
+                  stdout=PIPE,
+                  stderr=STDOUT,
+                  cwd=directory,
+                  env=env)
         with p.stdout:
             for line in iter(p.stdout.readline, b''):
-                 print(line.decode('utf-8'), end='')
+                 print('>>  %s'%line.decode('utf-8'), end='')
         p.wait() # wait for the subprocess to exit
     except KeyboardInterrupt as e:
         print("Caught CTRL+C")
         if p:
             p.kill()
         raise e
+    print('>> --------------------------------------------------------------')
+    print('>> Command exited with %i: %s'%(p.returncode,command))
+    print('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n')
+
+    if p.returncode!=0:
+        print('Exiting as the last command failed')
+        exit(p.returncode)
+
 
 
 sys.path.append(os.environ['C302_HOME'])
 
 try:
-    os.system('xhost +')
+    os.system('echo Granting permissions for xhost && xhost +')
 except:
     print("Unexpected error: %s" % sys.exc_info()[0])
 
@@ -110,23 +126,30 @@ def execute_with_realtime_output(command, directory, env=None):
 if 'DURATION' in os.environ:
     sim_duration = float(os.environ['DURATION'])
 
+noc302 = False
+if 'NOC302' in os.environ:
+    noc302 = bool(os.environ['NOC302'])
+
 DEFAULTS = {'duration': sim_duration,
             'dt': 0.005,
             'dtNrn': 0.05,
             'logstep': 100,
             'reference': 'FW',
             'c302params': 'C2',
             'verbose': False,
-            'device': 'GPU',
+            'device': 'CPU',
             'configuration': 'worm_crawl_half_resolution',
-            'noc302': False,
+            'noc302': noc302,
             'datareader': 'UpdatedSpreadsheetDataReader2',
             'outDir': OW_OUT_DIR}
 
 my_env = os.environ.copy()
-my_env["DISPLAY"] = ":44"
+DISPLAY = ':44'
+my_env["DISPLAY"] = DISPLAY
 
-os.system('Xvfb :44 -listen tcp -ac -screen 0 1920x1080x24+32 &') # TODO: terminate xvfb after recording
+# Xvfb or X virtual framebuffer is a display server implementing the X11 display server protocol.
+# In contrast to other display servers, Xvfb performs all graphical operations in virtual memory without showing any screen output.
+os.system('echo Starting xvfb && Xvfb %s -listen tcp -ac -screen 0 1920x1080x24+32 &'%DISPLAY) # TODO: terminate xvfb after recording
 
 try:
     command = """python sibernetic_c302.py
@@ -151,6 +174,9 @@ def execute_with_realtime_output(command, directory, env=None):
                 DEFAULTS['datareader'],
                 'simulations')
                 #DEFAULTS['outDir'])
+
+    if noc302: command += ' -noc302'
+
     execute_with_realtime_output(command, os.environ['SIBERNETIC_HOME'], env=my_env)
 except KeyboardInterrupt as e:
     pass
@@ -162,9 +188,13 @@ def execute_with_realtime_output(command, directory, env=None):
     for directory in dirnames:
         if directory.startswith('%s_%s' % (DEFAULTS['c302params'], DEFAULTS['reference'])):
             all_subdirs.append(os.path.join(dirpath, directory))
+        if directory.startswith('Sibernetic'):
+            all_subdirs.append(os.path.join(dirpath, directory))
 
 latest_subdir = max(all_subdirs, key=os.path.getmtime)
-
+print('\n========================================================================\n')
+print('Finished main simulation, data saved in: %s'%latest_subdir)
+execute_with_realtime_output('ls -alt %s'%latest_subdir, os.environ['SIBERNETIC_HOME'], env=my_env)
 
 try:
     os.mkdir('%s/output' % OW_OUT_DIR)
@@ -198,20 +228,30 @@ def execute_with_realtime_output(command, directory, env=None):
     print("Moving %s to %s"%(wcon, new_sim_out))
     shutil.move(wcon, new_sim_out)
 
+time.sleep(2)
 
 # Rerun and record simulation
-os.system('export DISPLAY=:44')
+execute_with_realtime_output('ls -alt /tmp/.X11-unix', os.environ['SIBERNETIC_HOME'], env=my_env)
+os.system('export DISPLAY=%s'%DISPLAY)
+execute_with_realtime_output('ls -alt /tmp/.X11-unix', os.environ['SIBERNETIC_HOME'], env=my_env)
 sibernetic_movie_name = '%s.mp4' % os.path.split(latest_subdir)[-1]
-os.system('tmux new-session -d -s SiberneticRecording "DISPLAY=:44 ffmpeg -r 30 -f x11grab -draw_mouse 0 -s 1920x1080 -i :44 -filter:v "crop=1200:800:100:100" -cpu-used 0 -b:v 384k -qmin 10 -qmax 42 -maxrate 384k -bufsize 1000k -an %s/%s"' % (new_sim_out, sibernetic_movie_name))
+command = 'tmux new-session -d -P -s SiberneticRecording "DISPLAY=%s ffmpeg -r 30 -f x11grab -draw_mouse 0 -s 1920x1080 -i %s -filter:v "crop=1200:800:100:100" -cpu-used 0 -b:v 384k -qmin 10 -qmax 42 -maxrate 384k -bufsize 1000k -an %s/%s"' % (DISPLAY, DISPLAY, new_sim_out, sibernetic_movie_name)
+execute_with_realtime_output(command, os.environ['SIBERNETIC_HOME'], env=my_env)
+
+time.sleep(3)
+
+execute_with_realtime_output('tmux list-sessions', os.environ['SIBERNETIC_HOME'], env=my_env)
 
 command = './Release/Sibernetic -f %s -l_from lpath=%s' % (DEFAULTS['configuration'], latest_subdir)
 execute_with_realtime_output(command, os.environ['SIBERNETIC_HOME'], env=my_env)
 
-os.system('tmux send-keys -t SiberneticRecording q')
-os.system('tmux send-keys -t SiberneticRecording "exit" C-m')
+execute_with_realtime_output('tmux send-keys -t SiberneticRecording q', os.environ['SIBERNETIC_HOME'], env=my_env)
+execute_with_realtime_output('tmux send-keys -t SiberneticRecording "exit" C-m', os.environ['SIBERNETIC_HOME'], env=my_env)
 
 time.sleep(3)
 
+execute_with_realtime_output('ls -alt %s'%latest_subdir, os.environ['SIBERNETIC_HOME'], env=my_env)
+
 # Remove black frames at the beginning of the recorded video
 command = "ffmpeg -i %s/%s -vf blackdetect=d=0:pic_th=0.70:pix_th=0.10 -an -f null - 2>&1 | grep blackdetect" % (new_sim_out, sibernetic_movie_name)
 outstr = str(check_output(command, shell=True).decode('utf-8'))
@@ -247,7 +287,7 @@ def execute_with_realtime_output(command, directory, env=None):
 os.system('ffmpeg -ss 1 -i %s/cut_%s -vf "select=gt(scene\,0.1)" -vsync vfr -vf fps=fps=1/1 %s' % (new_sim_out, sibernetic_movie_name, 'tmp/out%06d.jpg'))
 os.system('ffmpeg -r 100 -i %s -r 100 -vb 60M %s/speeded_%s' % ('tmp/out%06d.jpg', new_sim_out, sibernetic_movie_name))
 
-os.system('sudo rm -r tmp/*')
+os.system('rm -r tmp/*')
 
 
 

rebuild.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
 
-docker build "$@" -t "openworm/openworm:0.9.3"  --no-cache .
+version=$(<VERSION) # Read version of Dockerfile from file VERSION
+docker build "$@" -t "openworm/openworm:$version"  --no-cache .