Merge pull request #5 from arturgontijo/_new_daemon

New daemon

Author: arturgontijo

Dockerfile

@@ -1,5 +1,7 @@
 FROM ubuntu:18.04
 
+ARG snetd_version="v0.1.7"
+
 ENV SINGNET_REPOS=/opt/singnet
 
 RUN mkdir -p ${SINGNET_REPOS}
@@ -15,9 +17,9 @@ RUN apt-get install -y python3 python3-pip
 RUN cd ${SINGNET_REPOS} && \
     mkdir snet-daemon && \
     cd snet-daemon && \
-    wget https://github.com/singnet/snet-daemon/releases/download/v0.1.6/snet-daemon-v0.1.6-linux-amd64.tar.gz && \
-    tar -xvf snet-daemon-v0.1.6-linux-amd64.tar.gz && \
-    mv snet-daemon-v0.1.6-linux-amd64/snetd /usr/bin/snetd
+    wget https://github.com/singnet/snet-daemon/releases/download/${snetd_version}/snet-daemon-${snetd_version}-linux-amd64.tar.gz && \
+    tar -xvf snet-daemon-${snetd_version}-linux-amd64.tar.gz && \
+    mv snet-daemon-${snetd_version}-linux-amd64/snetd /usr/bin/snetd
 
 RUN cd ${SINGNET_REPOS} && \
     git clone https://github.com/singnet/example-service.git && \

README.md

@@ -13,15 +13,15 @@ Simple arithmetic service compatible with SingularityNET
 * Clone the git repository:
 
 ```
-$ git clone [email protected]:singnet/example-service.git
-$ cd example-service
+git clone [email protected]:singnet/example-service.git
+cd example-service
 ```
 
 * Install the dependencies and compile the protobuf file:
 
 ```
-$ pip3 install -r requirements.txt
-$ sh buildproto.sh
+pip3 install -r requirements.txt
+sh buildproto.sh
 ```
 
 ### Running
@@ -31,13 +31,13 @@ $ sh buildproto.sh
 * Run the example service directly (without `SNET Daemon`):
 
 ```
-$ python3 run_example_service.py --no-daemon
+python3 run_example_service.py --no-daemon
 ```
 
 * To test it run the script:
 
 ```
-$ python3 test_example_service.py
+python3 test_example_service.py
 ```
 
 #### With SingularityNET Daemon
@@ -59,6 +59,9 @@ Create the `SNET Daemon`'s config JSON file (`snetd.config.json`).
    "PASSTHROUGH_ENDPOINT": "http://SERVICE_GRPC_HOST:SERVICE_GRPC_PORT",
    "ORGANIZATION_ID": "ORGANIZATION_ID",
    "SERVICE_ID": "SERVICE_ID",
+   "PAYMENT_CHANNEL_STORAGE_SERVER": {
+       "DATA_DIR": "/opt/singnet/etcd/"
+   },
    "LOG": {
        "LEVEL": "debug",
        "OUTPUT": {
@@ -68,20 +71,20 @@ Create the `SNET Daemon`'s config JSON file (`snetd.config.json`).
 }
 ```
 
-For example, using the Kovan testnet, replace tags with:
+For example, using the Ropsten testnet, replace tags with:
 
 - `DAEMON_HOST:DAEMON_PORT`: localhost:7000
-- `https://JSON_RPC_ENDPOINT`: https://kovan.infura.io
-- `REGISTRY_ADDRESS`: 0xe331bf20044a5b24c1a744abc90c1fd711d2c08d
+- `https://JSON_RPC_ENDPOINT`: https://ropsten.infura.io
+- `REGISTRY_ADDRESS`: 0x5156fde2ca71da4398f8c76763c41bc9633875e4
 - `http://SERVICE_GRPC_HOST:SERVICE_GRPC_PORT`: http://localhost:7003
 - `ORGANIZATION_ID`: example-organization
 - `SERVICE_ID`: example-service
 
-For example, using the Ropsten testnet, replace tags with:
+For example, using the Kovan testnet, replace tags with:
 
 - `DAEMON_HOST:DAEMON_PORT`: localhost:7000
-- `https://JSON_RPC_ENDPOINT`: https://ropsten.infura.io
-- `REGISTRY_ADDRESS`: 0x5156fde2ca71da4398f8c76763c41bc9633875e4
+- `https://JSON_RPC_ENDPOINT`: https://kovan.infura.io
+- `REGISTRY_ADDRESS`: 0xe331bf20044a5b24c1a744abc90c1fd711d2c08d
 - `http://SERVICE_GRPC_HOST:SERVICE_GRPC_PORT`: http://localhost:7003
 - `ORGANIZATION_ID`: example-organization
 - `SERVICE_ID`: example-service
@@ -90,24 +93,33 @@ See [SingularityNet daemon configuration](https://github.com/singnet/snet-daemon
 
 ##### Running Service + Daemon on Host
 
-* Run the script without flag to launch both `SNET Daemon` and the service
+* Run the script without flag to launch both `SNET Daemon` and the service. But first,
+download the latest `SNET Daemon` [release here](https://github.com/singnet/snet-daemon/releases).
 
 ```
-$ python3 run_example_service.py
+python3 run_example_service.py
 ```
 
 ##### Running Service + Daemon in Docker Container
 
-* Build the docker image and run a Container from it:
+* Build the docker image (with proper `SNET Daemon` version) and run a Container from it:
 
 ```
-$ docker build -t snet_example_service https://github.com/singnet/example-service.git#master
-$ export ETCD_HOST_FOLDER=$HOME/singnet/etcd/example-service/
-$ export ETCD_CONTAINER_FOLDER=/opt/singnet/example-service/storage-data-dir-1.etcd/
-$ docker run -p 7000:7000 -v $ETCD_HOST_FOLDER:$ETCD_CONTAINER_FOLDER -ti snet_example_service bash
+SNETD_VERSION="v0.1.7"
+docker build \
+    --build-arg snetd_version=$SNETD_VERSION \
+    -t snet_example_service \
+    https://github.com/singnet/example-service.git#master
+
+export ETCD_HOST=$HOME/.snet/etcd/example-service/
+export ETCD_CONTAINER=/opt/singnet/etcd/
+docker run \
+    -p 7000:7000 \
+    -v $ETCD_HOST:$ETCD_CONTAINER \
+    -ti snet_example_service bash
 ```
 
-Note that the `$ETCD_(HOST|CONTAINER)_FOLDER` are useful to keep your service's etcd folder outside the container.
+Note that the `$ETCD_(HOST|CONTAINER)` are useful to keep your service's etcd folder outside the container.
 
 From this point we follow the tutorial in the Docker Container's prompt.
 
@@ -117,21 +129,24 @@ After this, run the service (with `SNET Daemon`), make sure you have the `snetd.
 # cat snetd.config.json
 {
    "DAEMON_END_POINT": "localhost:7000",
-   "ETHEREUM_JSON_RPC_ENDPOINT": "https://kovan.infura.io",
+   "ETHEREUM_JSON_RPC_ENDPOINT": "https://ropsten.infura.io",
    "IPFS_END_POINT": "http://ipfs.singularitynet.io:80",
-   "REGISTRY_ADDRESS_KEY": "0xe331bf20044a5b24c1a744abc90c1fd711d2c08d",
+   "REGISTRY_ADDRESS_KEY": "0x5156fde2ca71da4398f8c76763c41bc9633875e4",
    "PASSTHROUGH_ENABLED": true,
    "PASSTHROUGH_ENDPOINT": "http://localhost:7003",
-   "ORGANIZATION_ID": "example-organization",
-   "SERVICE_ID": "example-service",
+   "ORGANIZATION_ID": "my-organization",
+   "SERVICE_ID": "my-service",
+   "PAYMENT_CHANNEL_STORAGE_SERVER": {
+       "DATA_DIR": "/opt/singnet/etcd/"
+   },
    "LOG": {
        "LEVEL": "debug",
        "OUTPUT": {
             "TYPE": "stdout"
            }
    }
 }
-# python3 run_example_service.py &
+# python3 run_example_service.py --daemon-config snetd.config.json &
 ```
 
 ### Testing

run_example_service.py

@@ -1,7 +1,12 @@
 import sys
+import os
+import signal
+import time
 import subprocess
 import logging
 import pathlib
+import glob
+import json
 import argparse
 
 from service import registry
@@ -13,25 +18,35 @@
 def main():
     parser = argparse.ArgumentParser(description="Run services")
     parser.add_argument("--no-daemon", action="store_false", dest="run_daemon", help="do not start the daemon")
+    parser.add_argument("--daemon-config",
+                        dest="daemon_config",
+                        help="Path of daemon configuration file, without config it won't be started",
+                        required=False
+                        )
+    parser.add_argument("--ssl", action="store_true", dest="run_ssl", help="start the daemon with SSL")
     args = parser.parse_args()
     root_path = pathlib.Path(__file__).absolute().parent
-
+    
     # All services modules go here
     service_modules = ["service.example_service"]
-
+    
     # Call for all the services listed in service_modules
-    all_p = start_all_services(root_path, service_modules, args.run_daemon)
-
-    # Wait for all subprocesses
+    all_p = start_all_services(root_path, service_modules, args.run_daemon, args.daemon_config, args.run_ssl)
+    
+    # Continuous checking all subprocess
     try:
-        for p in all_p:
-            p.wait()
+        while True:
+            for p in all_p:
+                p.poll()
+                if p.returncode and p.returncode != 0:
+                    kill_and_exit(all_p)
+            time.sleep(1)
     except Exception as e:
         log.error(e)
         raise
 
 
-def start_all_services(cwd, service_modules, run_daemon):
+def start_all_services(cwd, service_modules, run_daemon, daemon_config, run_ssl):
     """
     Loop through all service_modules and start them.
     For each one, an instance of Daemon "snetd" is created.
@@ -41,32 +56,59 @@ def start_all_services(cwd, service_modules, run_daemon):
     for i, service_module in enumerate(service_modules):
         service_name = service_module.split(".")[-1]
         log.info("Launching {} on port {}".format(str(registry[service_name]), service_module))
-        all_p += start_service(cwd, service_module, run_daemon)
+        all_p += start_service(cwd, service_module, run_daemon, daemon_config, run_ssl)
     return all_p
 
 
-def start_service(cwd, service_module, run_daemon):
+def start_service(cwd, service_module, run_daemon, daemon_config, run_ssl):
     """
     Starts SNET Daemon ("snetd") and the python module of the service
     at the passed gRPC port.
     """
+    
+    def add_ssl_configs(conf):
+        """Add SSL keys to snetd.config.json"""
+        with open(conf, "r") as f:
+            snetd_configs = json.load(f)
+            snetd_configs["ssl_cert"] = "/opt/singnet/.certs/fullchain.pem"
+            snetd_configs["ssl_key"] = "/opt/singnet/.certs/privkey.pem"
+        with open(conf, "w") as f:
+            json.dump(snetd_configs, f, sort_keys=True, indent=4)
+    
     all_p = []
     if run_daemon:
-        all_p.append(start_snetd(str(cwd)))
+        if daemon_config:
+            all_p.append(start_snetd(str(cwd), daemon_config))
+        else:
+            for idx, config_file in enumerate(glob.glob("./snetd_configs/*.json")):
+                if run_ssl:
+                    add_ssl_configs(config_file)
+                all_p.append(start_snetd(str(cwd), config_file))
     service_name = service_module.split(".")[-1]
     grpc_port = registry[service_name]["grpc"]
     p = subprocess.Popen([sys.executable, "-m", service_module, "--grpc-port", str(grpc_port)], cwd=str(cwd))
     all_p.append(p)
     return all_p
 
 
-def start_snetd(cwd):
+def start_snetd(cwd, config_file=None):
     """
     Starts the Daemon "snetd":
     """
     cmd = ["snetd", "serve"]
+    if config_file:
+        cmd = ["snetd", "serve", "--config", config_file]
     return subprocess.Popen(cmd, cwd=str(cwd))
 
 
+def kill_and_exit(all_p):
+    for p in all_p:
+        try:
+            os.kill(p.pid, signal.SIGTERM)
+        except Exception as e:
+            log.error(e)
+    exit(1)
+
+
 if __name__ == "__main__":
     main()

snetd_configs/snetd.mainnet.json

@@ -0,0 +1,38 @@
+{
+  "daemon_end_point": "0.0.0.0:7002",
+  "ethereum_json_rpc_endpoint": "https://mainnet.infura.io",
+  "ipfs_end_point": "http://ipfs.singularitynet.io:80",
+  "registry_address_key": "0xb3180a92e210b45e3447976a412ac0df859febaf",
+  "passthrough_enabled": true,
+  "passthrough_endpoint": "http://localhost:7003",
+  "organization_id": "snet",
+  "service_id": "example-service",
+
+  "payment_channel_storage_server": {
+    "id": "storage-mainnet",
+    "host": "127.0.0.1",
+    "client_port": 2379,
+    "peer_port": 2380,
+    "token": "unique-token",
+    "cluster": "storage-mainnet=http://127.0.0.1:2380",
+    "data_dir": "/opt/singnet/etcd/mainnet",
+    "enabled": true
+  },
+
+  "payment_channel_storage_client": {
+    "connection_timeout": "5s",
+    "request_timeout": "3s",
+    "endpoints": ["http://127.0.0.1:2379"]
+  },
+
+  "log": {
+    "level": "debug",
+    "output": {
+      "current_link": "./snetd-mainnet.log",
+      "file_pattern": "./snetd-mainnet.%Y%m%d.log",
+      "rotation_count": 0,
+      "rotation_time_in_sec": 86400,
+      "type": "file"
+    }
+  }
+}

snetd_configs/snetd.ropsten.json

@@ -0,0 +1,38 @@
+{
+  "daemon_end_point": "0.0.0.0:7052",
+  "ethereum_json_rpc_endpoint": "https://ropsten.infura.io",
+  "ipfs_end_point": "http://ipfs.singularitynet.io:80",
+  "registry_address_key": "0x5156fde2ca71da4398f8c76763c41bc9633875e4",
+  "passthrough_enabled": true,
+  "passthrough_endpoint": "http://localhost:7003",
+  "organization_id": "snet",
+  "service_id": "example-service",
+
+  "payment_channel_storage_server": {
+    "id": "storage-ropsten",
+    "host": "127.0.0.1",
+    "client_port": 2381,
+    "peer_port": 2382,
+    "token": "unique-token",
+    "cluster": "storage-ropsten=http://127.0.0.1:2382",
+    "data_dir": "/opt/singnet/etcd/ropsten",
+    "enabled": true
+  },
+
+  "payment_channel_storage_client": {
+    "connection_timeout": "5s",
+    "request_timeout": "3s",
+    "endpoints": ["http://127.0.0.1:2381"]
+  },
+
+  "log": {
+    "level": "debug",
+    "output": {
+      "current_link": "./snetd-ropsten.log",
+      "file_pattern": "./snetd-ropsten.%Y%m%d.log",
+      "rotation_count": 0,
+      "rotation_time_in_sec": 86400,
+      "type": "file"
+    }
+  }
+}