Merge branch 'DEV'

# Conflicts:
#	README.md
#	pytaskmanager/__init__.py

Author: frankcorneliusmartin

.gitignore

@@ -1,14 +1,11 @@
 **/virt*
 **/__pycache__
 **/*.sqlite
-client/task*
-client2
-client3
-
 *.vpp
-client/logs
 *.sublime-project
 *.sublime-workspace
 master/logs
 pytaskmanager.egg-info
 .ipynb_checkpoints
+.idea/
+node_modules

README.md

@@ -16,7 +16,47 @@ Running a node/site requires a (virtual) machine that has:
 * Access to a local data store
 * Access to the internet and/or central server
 
-## Installation
-See the [wiki](https://github.com/IKNL/pytaskmanager/wiki) for detailed instructions on how to install the server and nodes. 
+Next to this registry, there is a node, which polls a central registry, to check for new tasks to execute. If a new task is available, the referenced container is pulled from the docker hub (e.g. `docker pull <container name>`) irrespective of whether it was pulled before. Afterwards, this container is executed (e.g. `docker run --rm -d <container name>`), with mounts for configuration files and output folders. Furthermore, the *internal* SPARQL endpoint URL is passed to the container as an environment variable (`$SPARQL_ENDPOINT=<url>`).
 
+This means the researcher is free to implement any application/script, embedded in a docker container. Sites running the node are able to limit the docker containers being able to run on their system. It is possible to limit the user/organization which developed the container (e.g. only allowing the regular expression `myConsortium/.*:.*` of container images), or even on the repository level (e.g. `myConsortium/myImage:.*`).
 
+# How to use it?
+
+## Prerequisites
+
+At the (hospital) site:
+
+* A Windows Server 2012R2 (or higher) machine, or a unix machine supporting Docker
+* Docker installed, and given rights to the user executing the node rights to perform docker commands
+* Python 2.7 or 3 (tested on both)
+
+At the central registry:
+
+* A Windows Server 2012R2 (or higher) machine, or a unix machine supporting Docker
+* Docker installed, and given rights to the user executing the node rights to perform docker commands
+* Python 2.7 or 3 (tested on both)
+
+## How to run?
+
+At the central registry:
+
+1. Please checkout this repository
+2. Run the python script master/TaskMaster.py (`python master/TaskMaster.py`). The registry will now run at port 5000, and the output is shown at the console.
+
+At the (hospital) sites:
+
+1. Checkout this repository
+2. Please adapt the config.json file to your site information, including the local URL to your internal SPARQL endpoint.
+3. Run the python script node/runScript.py (`python node/runScript.py`)
+4. **Optionally**: if you have a public IP address, you can also receive direct files (e.g. usefull if your site is a Trusted Third Party, and (encrypted) files are sent to you). To run this service, please execute the python script node/FileService.py (`python node/FileService.py`).
+
+## How to build and run an algorithm?
+
+The registry is based on REST commands. The docker containers are *only* needed for execution at the sites. As a researcher, this means you have to develop a docker container which can run on *every* site.
+
+To merge results from all sites, and to run the *centralised* part of your analysis, you can develop a script on your own computer. This computer can retrieve the results from the registry, perform its calculations, and (optionally, in an iterative algorithm) post a new request to run an image on the contributing sites. This can also be the same Docker image, using an updated configuration file.
+
+# How to contribute?
+If you have any requests, you can fork this repository, develop the addition/change, and send a pull request. If you have a request for a change, please add it to the issue tracker (see "Issues" in the left navigation bar).
+
+This readme and documentation still needs work, as the code for this infrastructure is still work in progress. If you have any question regarding use, please use the issue tracker as well. We might update the readme file accordingly, but also helps us to define where the need for help is.

pytaskmanager/VERSION

@@ -1 +1 @@
-0.1dev1
+0.1dev3

pytaskmanager/__init__.py

@@ -1,228 +1,8 @@
-import click
-
-import sys
 import os
-import shutil
-import yaml
-import logging
-
-# Define version and directories *before* importing submodules
-here = os.path.abspath(os.path.dirname(__file__))
 
 __version__ = ''
-
+here = os.path.abspath(os.path.dirname(__file__))
 with open(os.path.join(here, 'VERSION')) as fp:
     __version__ = fp.read()
 
-
-# default parameters for click.Path
-pparams = {
-    'exists': False,
-    'file_okay': False,
-    'dir_okay': True,
-}
-
-from . import server
-from .server import fixtures
-from .server import db
-from . import client
-from . import utest
-from . import util
-
 APPNAME = 'pytaskmanager'
-
-# ------------------------------------------------------------------------------
-# helper functions
-# ------------------------------------------------------------------------------
-def get_config_location(ctx, config, force_create):
-    """Ensure configuration file exists and return its location."""
-    if config is None:
-        # Get the location of config.yaml if not provided
-        filename = ctx.config_file
-    else:
-        # Use the config file provided as argument
-        filename = config
-
-    # Check that the config file exists and create it if necessary, but
-    # only if it was not explicitly provided!
-    if not os.path.exists(filename):
-        # We will always create a configuration file at the default location
-        # when necessary.
-        if config and not force_create:
-            click.echo("Configuration file '{}' does not exist and '--force-create' not specified!".format(filename))
-            click.echo("Aborting ...")
-            sys.exit(1)
-
-        # Make sure the directory exists
-        dirname = os.path.dirname(filename)
-
-        if dirname:
-            os.makedirs(dirname, exist_ok=True)
-
-        # Copy a default config file
-        if ctx.instance_type == 'server':
-            skeleton_file = 'server_config_skeleton.yaml'
-        elif ctx.instance_type == 'client':
-            skeleton_file = 'client_config_skeleton.yaml'
-        elif ctx.instance_type == 'unittest':
-            skeleton_file = 'unittest_config_skeleton.yaml'
-
-        src = os.path.join(here, '_data', skeleton_file)
-        dst = os.path.join(filename)
-        shutil.copy(src, dst)
-
-        if ctx.instance_type == 'server':
-            with open(dst, 'r') as fp:
-                cfg = yaml.load(fp)
-                print('-' * 80)
-                print(cfg)
-                print('-' * 80)
-
-            cfg['application']['logging']['file'] = ctx.instance_name + '.log'
-
-            with open(dst, 'w') as fp:
-                yaml.dump(cfg, fp)
-
-    return filename
-
-
-@click.group()
-def cli():
-    """Main entry point for CLI scripts."""
-    pass
-
-
-# ------------------------------------------------------------------------------
-# ptm test
-# ------------------------------------------------------------------------------
-@cli.command(name='test')
-@click.option('-c', '--config', default=None, type=click.Path(), help='location of the config file')
-def cli_test(config):
-    """Run unit tests."""
-    ctx = util.AppContext(APPNAME, 'unittest')
-    cfg_filename = get_config_location(ctx, config=None, force_create=False)
-    ctx.init(cfg_filename)
-    utest.run()
-
-
-# ------------------------------------------------------------------------------
-# ptm server
-# ------------------------------------------------------------------------------
-@cli.group(name='server')
-def cli_server():
-    """Subcommand `ptm server`."""
-    pass
-
-
-@cli_server.command(name='start')
-@click.option('-n', '--name', default='default', help='server instance to use')
-@click.option('-c', '--config', default=None, help='filename of config file; overrides --name if provided')
-@click.option('-e', '--environment', default='test', help='database environment to use')
-@click.option('--ip', default='0.0.0.0', help='ip address to listen on')
-@click.option('-p', '--port', default=5000, help='port to listen on')
-@click.option('--debug/--no-debug', default=True, help='run server in debug mode (auto-restart)')
-@click.option('--force-create', is_flag=True, help='Force creation of config file')
-def cli_server_start(name, config, environment, ip, port, debug, force_create):
-    """Start the server."""
-    click.echo("Starting server ...")
-    ctx = util.ServerContext(APPNAME, 'default')
-    # Load configuration and initialize logging system
-    cfg_filename = get_config_location(ctx, config, force_create)
-    ctx.init(cfg_filename, environment)
-
-    # Load the flask.Resources
-    server.init_resources(ctx)
-    # Run the server
-    server.run(ctx, ip, port, debug=debug)
-
-
-@cli_server.command(name='config_location')
-@click.option('-n', '--name', default='default', help='server instance to use')
-def cli_server_configlocation(name):
-    """Print the location of the default config file."""
-    # ctx = util.AppContext(APPNAME, 'server', name)
-    ctx = util.ServerContext(APPNAME, 'default')
-    cfg_filename = get_config_location(ctx, config=None, force_create=False)
-    click.echo('{}'.format(cfg_filename))
-
-
-@cli_server.command(name='passwd')
-@click.option('-n', '--name', default='default', help='server instance to use')
-@click.option('-c', '--config', default=None, help='filename of config file; overrides --name if provided')
-@click.option('-e', '--environment', default='test', help='database environment to use')
-@click.option('-p', '--password', prompt='Password', hide_input=True)
-def cli_server_passwd(name, config, environment, password):
-    """Set the root password."""
-    log = logging.getLogger('ptm')
-
-    ctx = util.AppContext(APPNAME, 'server', name)
-
-    # Load configuration and initialize logging system
-    cfg_filename = get_config_location(ctx, config)
-    ctx.init(cfg_filename, environment)
-
-    uri = ctx.get_database_location()
-    db.init(uri)
-
-    try:
-        root = db.User.getByUsername('root')
-    except Exception as e:
-        log.info("Creating user root")
-        root = db.User(username='root')
-
-    log.info("Setting password for root")
-    root.set_password(password)
-    root.save()
-
-    log.info("[DONE]")
-
-
-@cli_server.command(name='load_fixtures')
-@click.option('-n', '--name', default='default', help='server instance to use')
-@click.option('-e', '--environment', default='test', help='database environment to use')
-@click.option('-c', '--config', default=None, help='filename of config file; overrides --name if provided')
-def cli_server_load_fixtures(name, environment, config):
-    """Load fixtures for testing."""
-    click.echo("Loading fixtures.")
-    # ctx = util.AppContext(APPNAME, 'server', name)
-    ctx = util.ServerContext(APPNAME, 'default')
-
-    # Load configuration and initialize logging system
-    cfg_filename = get_config_location(ctx, config, force_create=False)
-    ctx.init(cfg_filename, environment)
-
-    fixtures.init(ctx)
-    fixtures.create()
-
-
-# ------------------------------------------------------------------------------
-# ptm client
-# ------------------------------------------------------------------------------
-@cli.group(name='client')
-def cli_client():
-    """Subcommand `ptm client`."""
-    pass
-
-
-@cli_client.command(name='config_location')
-@click.option('-n', '--name', default='default', help='client instance to use')
-def cli_server_configlocation(name):
-    """Print the location of the default config file."""
-    ctx = util.AppContext(APPNAME, 'client', name)
-    cfg_filename = get_config_location(ctx, config=None, force_create=False)
-    click.echo('{}'.format(cfg_filename))
-
-
-@cli_client.command(name='start')
-@click.option('-n', '--name', default='default', help='client instance to use')
-@click.option('-c', '--config', default=None, help='filename of config file; overrides --name if provided')
-def cli_client_start(name, config):
-    """Start the client."""
-    ctx = util.AppContext(APPNAME, 'client', name)
-
-    # Load configuration and initialize logging system
-    cfg_filename = get_config_location(ctx, config, force_create=False)
-    ctx.init(cfg_filename)
-
-    # Run the client
-    client.run(ctx)

pytaskmanager/_data/client_config_skeleton.yaml

@@ -0,0 +1,44 @@
+application:
+  api_key: 
+  server_url: https://api.distributedlearning.ai
+  port:
+  api_path: ''
+  server_entry_point: /token
+  delay: 30 # seconds
+  task_dir: tasks
+  docker_host: 127.0.0.1
+  
+  database_uri: 
+  databases:
+    default: 
+
+  logging:
+      level:        DEBUG                  # Can be on of 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'
+      file:         node.log               # Filename of logfile
+      use_console:  True                   # Log output to the console?
+      backup_count: 5                      # Number of logs to keep
+      max_size:     1024                   # Specified in kB (i.e. 1024 means a maximum file size of 1MB)
+      format:       "%(asctime)s - %(name)-14s - %(levelname)-8s - %(message)s"
+      datefmt:      "%H:%M:%S"
+
+# environments:
+#   test:
+#   prod:
+#     api_key: 
+#     server_url: https://api.distributedlearning.ai
+#     api_path: ''
+#     server_entry_point: /token
+#     delay: 30 # seconds
+#     task_dir: tasks
+#     docker_host: 127.0.0.1
+#     database_uri: 
+
+#     logging:
+#         level:        DEBUG                  # Can be on of 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'
+#         file:         node.log       # Filename of logfile
+#         use_console:  True                   # Log output to the console?
+#         backup_count: 5                      # Number of logs to keep
+#         max_size:     1024                   # Specified in kB (i.e. 1024 means a maximum file size of 1MB)
+#         format:       "%(asctime)s - %(name)-14s - %(levelname)-8s - %(message)s"
+#         # datefmt:      "%Y-%m-%d %H:%M:%S"
+#         datefmt:      "%H:%M:%S"