Merge pull request #52 from RADAR-base/release-0.2.0

Release 0.2.0

Author: blootsvoets

.dockerignore

@@ -1,3 +1,7 @@
-.idea
-.gradle*
-build
+/.idea/
+/.gradle*
+/.git*
+/out/
+/config/
+/build/
+/libs/

.gitignore

@@ -24,3 +24,4 @@ backend.log
 smtp.env
 
 /out/
+/libs/

.travis.yml

@@ -8,7 +8,6 @@ services:
 
 env:
   DOCKER_COMPOSE_VERSION: 1.16.1
-  TERM: dumb
 
 cache:
   directories:
@@ -26,8 +25,10 @@ before_install:
   - chmod +x "$HOME/bin/docker-compose";
 script:
   - ./gradlew check
+  - cd src/integrationTest/docker
   - sudo docker-compose up -d zookeeper-1 kafka-1 schema-registry-1 && sleep 30 && sudo docker-compose run --rm integration-test
   - sudo docker-compose down
+  - cd ../../..
 after_script:
   - ./gradlew sendCoverageToCodacy
 

CODEOWNERS

@@ -1,4 +1,4 @@
 # These owners will be the default owners for everything in the repo.
 # Unless a later match takes precedence, they will be requested for review when someone
 # opens a pull request.
-*       @blootsvoets @nivemaham @fnobilia
+*       @blootsvoets @nivemaham @yatharthranjan

Dockerfile

@@ -1,13 +1,50 @@
-FROM confluentinc/cp-base:3.3.0
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
-ENV TERM=dumb
+FROM openjdk:8-alpine AS builder
 
 RUN mkdir /code
 WORKDIR /code
 
-COPY gradle /code/gradle
-COPY gradle.properties build.gradle settings.gradle  gradlew /code/
-RUN ./gradlew downloadDependencies
-COPY src /code/src
+ENV GRADLE_OPTS -Dorg.gradle.daemon=false -Dorg.gradle.project.profile=prod
 
-ENTRYPOINT ["./gradlew"]
+COPY ./gradle/wrapper /code/gradle/wrapper
+COPY ./gradlew /code/
+RUN ./gradlew --version
+
+COPY ./gradle/profile.prod.gradle /code/gradle/
+COPY ./build.gradle ./gradle.properties ./settings.gradle /code/
+
+RUN ./gradlew downloadRuntimeDependencies
+
+COPY ./src/ /code/src
+
+RUN ./gradlew distTar && \
+  tar xf build/distributions/*.tar && \
+  rm build/distributions/*.tar
+
+FROM confluentinc/cp-base:3.3.1
+
+MAINTAINER Nivethika M <nivethika@thehyve.nl> , Joris Borgdorff <joris@thehyve.nl>
+
+LABEL description="RADAR-CNS Backend streams and monitor"
+
+ENV KAFKA_REST_PROXY http://rest-proxy:8082
+ENV KAFKA_SCHEMA_REGISTRY http://schema-registry:8081
+
+COPY --from=builder /code/radar-backend-*/bin/* /usr/bin/
+COPY --from=builder /code/radar-backend-*/lib/* /usr/lib/
+
+# Load topics validator
+COPY ./src/main/docker/radar-backend-init /usr/bin
+
+CMD ["radar-backend-init"]

README.md

@@ -12,16 +12,16 @@ RADAR-Backend provides an abstract layer to monitor and analyze streams of weara
 The following are the prerequisites to run RADAR-Backend on your machine:
 
 - Java 8
-- [Confluent Platform 3.1.2](http://docs.confluent.io/3.1.2/installation.html) ( Running instances of Zookeeper, Kafka-broker(s), Schema-Registry and Kafka-REST-Proxy services ).
+- [Confluent Platform 3.3.1](http://docs.confluent.io/3.3.1/installation.html) ( Running instances of Zookeeper, Kafka-broker(s), Schema-Registry and Kafka-REST-Proxy services ).
 - SMTP server to send notifications from the monitors.
 
 ## Installation
 
 1. Install the dependencies mentioned above.
-2. Clone RADAR-Backend repository.
+2. Clone radar-backend repository.
 
     ```shell
-    git clone https://github.com/RADAR-CNS/RADAR-Backend.git
+    git clone https://github.com/RADAR-CNS/radar-backend.git
     ```
 3. Build the project from project directory
 
@@ -33,10 +33,14 @@ The following are the prerequisites to run RADAR-Backend on your machine:
     ./gradlew clean
     
     # Build
-    ./gradlew build
+    ./gradlew distTar
+    
+    # Unpack the binaries
+    sudo mkdir -p /usr/local && \
+    sudo tar --strip-components 1 -C /usr/local xzf build/distributions/*.tar.gz
     ```
-   The build process creates separate jar files for each component. Built jars are located under `build/libs` folder.
-
+   
+   Now the backend is available as the `/usr/local/bin/radar-backend` script.
 
 ## Usage
 
@@ -63,13 +67,12 @@ The RADAR command-line has three subcommands: `stream`, `monitor` and `mock`. Th
   - android_empatica_e4_temperature_output
   - android_phone_usage_event
   - android_phone_usage_event_output
-3. Run `radarbackend.jar` with configured `radar.yml` and `stream` argument    
+3. Run `radar-backend` with configured `radar.yml` and `stream` argument    
 
     ```shell
-    java -jar radarbackend-1.0.jar -c path/to/radar.yml stream
+    radar-backend -c path/to/radar.yml stream
     ```
-       
-       
+
 The phone usage event stream uses an internal cache of 1 million elements, which may take about 50 MB of memory. Adjust `org.radarcns.stream.phone.PhoneUsageStream.MAX_CACHE_SIZE` to change it. 
 
 ### RADAR-backend monitors
@@ -111,11 +114,45 @@ To get email notifications for Empatica E4 battery status, an email server witho
       topics:
         - android_empatica_e4_temperature
       ```
+
+3. For Source Statistics monitors, configure what source topics to monitor to output some basic output statistics (like last time seen)
+    
+    ```yaml
+    statistics_monitors:
+      # Human readable monitor name
+      - name: Empatica E4
+        # topics to aggregate. This can take any number of topics that may
+	# lead to slightly different statistics
+        topics:
+          - android_empatica_e4_blood_volume_pulse_1min
+        # Topic to write results to. This should follow the convention
+	# source_statistics_[provider]_[model] with produer and model as
+	# defined in RADAR-Schemas
+        output_topic: source_statistics_empatica_e4
+	# Maximum batch size to aggregate before sending results.
+	# Defaults to 1000.
+        max_batch_size: 500
+	# Flush timeout in milliseconds. If the batch size is not larger than
+	# max_batch_size for this amount of time, the current batch is
+	# forcefully flushed to the output topic.
+	# Defaults to 60000 = 1 minute.
+	flush_timeout: 15000
+      - name: Biovotion VSM1
+        topics:
+          - android_biovotion_vsm1_acceleration_1min
+        output_topic: source_statistics_biovotion_vsm1
+      - name: RADAR pRMT
+        topics:
+          - android_phone_acceleration_1min
+          - android_phone_bluetooth_devices
+          - android_phone_sms
+        output_topic: source_statistics_radar_prmt
+    ```
 
-3. Run `radarbackend.jar` with configured `radar.yml` and `monitor` argument
+3. Run `radar-backend` with configured `radar.yml` and `monitor` argument
 
     ```shell
-    java -jar radarbackend-1.0.jar -c path/to/radar.yml monitor
+    radar-backend -c path/to/radar.yml monitor
     ```
 
 ### Send mock data to the backend
@@ -150,21 +187,33 @@ To get email notifications for Empatica E4 battery status, an email server witho
 3. To generate data on some `backend_mock_empatica_e4_<>` topic with a number of devices, run (substitute `<num-devices>` with the needed number of devices):
 
     ```shell
-    java -jar radarbackend-1.0.jar -c path/to/radar.yml mock --devices <num-devices>
+    radar-backend -c path/to/radar.yml mock --devices <num-devices>
     ```
 
     Press `Ctrl-C` to stop.
 
 4. To generate the file data configured in point 2, run
 
     ```shell
-    java -jar radarbackend-1.0.jar -c path/to/radar.yml mock --file mock_data.yml
+    radar-backend -c path/to/radar.yml mock --file mock_data.yml
     ```
 
     The data sending will automatically be stopped.
 
+### Docker image
+
+The backend is [published to Docker Hub](https://hub.docker.com/r/radarcns/radar-backend-kafka). Mount a `/etc/radar.yml` file to configure either the streams or the monitor.
+
+This image requires the following environment variable:
+
+- `KAFKA_REST_PROXY`: a valid Rest-Proxy instance
+- `KAFKA_SCHEMA_REGISTRY`: a valid Confluent Schema Registry.
+- `KAFKA_BROKERS`: number of brokers expected (default: 3).
+
+For a complete use case scenario, check the RADAR-CNS `docker-compose` file available [here](https://github.com/RADAR-CNS/RADAR-Docker/blob/backend-integration/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml)
 
 ## Contributing
+
 Code should be formatted using the [Google Java Code Style Guide](https://google.github.io/styleguide/javaguide.html).
 If you want to contribute a feature or fix browse our [issues](https://github.com/RADAR-CNS/RADAR-Backend/issues), and please make a pull request.