From a8d02a2f7bc1acf42429db8a875b7482d31f8f16 Mon Sep 17 00:00:00 2001 From: Willem Dekker Date: Thu, 2 Jan 2020 21:32:48 +0100 Subject: [PATCH 01/69] Added Portainer Agent Added Portainer Agent this allows you to connect your stack to a other Portainer instance/ server --- .templates/portainer_agent/service.yml | 9 +++++++++ menu.sh | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 .templates/portainer_agent/service.yml diff --git a/.templates/portainer_agent/service.yml b/.templates/portainer_agent/service.yml new file mode 100644 index 00000000..d03b849a --- /dev/null +++ b/.templates/portainer_agent/service.yml @@ -0,0 +1,9 @@ + portainer_agent: + image: portainer/agent + container_name: agent + ports: + - 9002:9001 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /var/lib/docker/volumes:/var/lib/docker/volumes + restart: always diff --git a/menu.sh b/menu.sh index 6c11d62f..0102246a 100755 --- a/menu.sh +++ b/menu.sh @@ -5,6 +5,7 @@ pushd ~/IOTstack declare -A cont_array=( [portainer]="Portainer" + [portainer_agent]="Portainer agent" [nodered]="Node-RED" [influxdb]="InfluxDB" [telegraf]="Telegraf (Requires InfluxDB and Mosquitto)" @@ -32,7 +33,7 @@ declare -A cont_array=( ) declare -a armhf_keys=("portainer" "nodered" "influxdb" "grafana" "mosquitto" "telegraf" "mariadb" "postgres" "adminer" "openhab" "zigbee2mqtt" "pihole" "plex" "tasmoadmin" "rtl_433" "espruinohub" - "motioneye" "webthings_gateway" "blynk_server" "nextcloud" "diyhue" "homebridge" "python") + "motioneye" "webthings_gateway" "blynk_server" "nextcloud" "diyhue" "homebridge" "python" "portainer_agent") sys_arch=$(uname -m) From 48b67d93284c3b16cd09b6cc511f653d6d336751 Mon Sep 17 00:00:00 2001 From: kauffman-repo <58049841+kauffman-repo@users.noreply.github.com> Date: Thu, 9 Jan 2020 20:04:51 +0100 Subject: [PATCH 02/69] Update directoryfix.sh fix for username not equal pi --- .templates/python/directoryfix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.templates/python/directoryfix.sh b/.templates/python/directoryfix.sh index 437d4b45..422a9885 100755 --- a/.templates/python/directoryfix.sh +++ b/.templates/python/directoryfix.sh @@ -4,7 +4,7 @@ if [ ! -d ./volumes/python/app ]; then sudo mkdir -p ./volumes/python/app - sudo chown -R pi:pi ./volumes/python + sudo chown -R $(whoami):$(whoami) ./volumes/python echo 'print("hello world")' >./volumes/python/app/app.py fi From 09351d6525c21c57b34b1b9df7c1ad9a3f479fab Mon Sep 17 00:00:00 2001 From: kauffman-repo <58049841+kauffman-repo@users.noreply.github.com> Date: Thu, 9 Jan 2020 20:33:36 +0100 Subject: [PATCH 03/69] Update directoryfix.sh --- .templates/python/directoryfix.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.templates/python/directoryfix.sh b/.templates/python/directoryfix.sh index 422a9885..93725145 100755 --- a/.templates/python/directoryfix.sh +++ b/.templates/python/directoryfix.sh @@ -2,9 +2,12 @@ # Directoryfix for python +#current user +u=$(whoami) + if [ ! -d ./volumes/python/app ]; then sudo mkdir -p ./volumes/python/app - sudo chown -R $(whoami):$(whoami) ./volumes/python + sudo chown -R $u:$u ./volumes/python echo 'print("hello world")' >./volumes/python/app/app.py fi From a39d01c5ad32409e5a950d4580a2a47ce88ff7ac Mon Sep 17 00:00:00 2001 From: Phill Kelley Date: Sun, 12 Jan 2020 17:03:47 +1100 Subject: [PATCH 04/69] Adds a Gitea container to IOTstack Gitea is a lightweight implementation of a Git server. Makers who are already running a local IOTstack server may prefer to commit their work to a local repository running on their Raspberry Pi, rather than GitHub/GitLab/etc. Implementing Gitea under the IOTStack umbrella inherits the existing IOTstack backup mechanisms for free. Candidate page for the wiki at https://www.dropbox.com/s/onqndgyamo2h74i/Gitea.md.zip?dl=1 --- .templates/gitea/gitea.env | 1 + .templates/gitea/service.yml | 14 ++++++++++++++ menu.sh | 3 ++- 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 .templates/gitea/gitea.env create mode 100644 .templates/gitea/service.yml diff --git a/.templates/gitea/gitea.env b/.templates/gitea/gitea.env new file mode 100644 index 00000000..27f76ca3 --- /dev/null +++ b/.templates/gitea/gitea.env @@ -0,0 +1 @@ +# initially empty diff --git a/.templates/gitea/service.yml b/.templates/gitea/service.yml new file mode 100644 index 00000000..fb7685a6 --- /dev/null +++ b/.templates/gitea/service.yml @@ -0,0 +1,14 @@ + gitea: + container_name: gitea + image: kapdap/gitea-rpi + restart: unless-stopped + user: "0" + ports: + - "7920:3000/tcp" + - "2222:22/tcp" + env_file: + - ./services/gitea/gitea.env + volumes: + - ./volumes/gitea/data:/data + - /etc/timezone:/etc/timezone:ro + - /etc/localtime:/etc/localtime:ro diff --git a/menu.sh b/menu.sh index 6c11d62f..ce27097b 100755 --- a/menu.sh +++ b/menu.sh @@ -28,11 +28,12 @@ declare -A cont_array=( [diyhue]="diyHue" [homebridge]="Homebridge" [python]="Python 3" + [gitea]="Gitea" ) declare -a armhf_keys=("portainer" "nodered" "influxdb" "grafana" "mosquitto" "telegraf" "mariadb" "postgres" "adminer" "openhab" "zigbee2mqtt" "pihole" "plex" "tasmoadmin" "rtl_433" "espruinohub" - "motioneye" "webthings_gateway" "blynk_server" "nextcloud" "diyhue" "homebridge" "python") + "motioneye" "webthings_gateway" "blynk_server" "nextcloud" "diyhue" "homebridge" "python" "gitea") sys_arch=$(uname -m) From eb9b982804ece6fc49d96107729988462e1dab16 Mon Sep 17 00:00:00 2001 From: Tomasmerca <32470264+Tomasmerca@users.noreply.github.com> Date: Wed, 15 Jan 2020 00:24:48 -0300 Subject: [PATCH 05/69] Added filter.acl file to mosquitto/config Route filter.acl to add the option in mosquitto/mosquitto.conf of filtering wich topics can be read or writen by defined users. --- .templates/mosquitto/service.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.templates/mosquitto/service.yml b/.templates/mosquitto/service.yml index 8a62d776..a7d80921 100644 --- a/.templates/mosquitto/service.yml +++ b/.templates/mosquitto/service.yml @@ -10,4 +10,5 @@ - ./volumes/mosquitto/data:/mosquitto/data - ./volumes/mosquitto/log:/mosquitto/log - ./services/mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf + - ./services/mosquitto/filter.acl:/mosquitto/config/filter.acl From c4adb556e757eef31bc53ffee04ff0b46bc396c9 Mon Sep 17 00:00:00 2001 From: Tomasmerca <32470264+Tomasmerca@users.noreply.github.com> Date: Wed, 15 Jan 2020 00:33:08 -0300 Subject: [PATCH 06/69] Created empty file whit example filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added the file where the user creates filters. The example filters don´t affect usage in any way. --- .templates/mosquitto/filter.acl | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .templates/mosquitto/filter.acl diff --git a/.templates/mosquitto/filter.acl b/.templates/mosquitto/filter.acl new file mode 100644 index 00000000..e1682311 --- /dev/null +++ b/.templates/mosquitto/filter.acl @@ -0,0 +1,6 @@ +user admin +topic read # +topic write # + +pattern read # +pattern write # From 89f46478138dddcc177876c6d93ec9c0de4cb0e1 Mon Sep 17 00:00:00 2001 From: Tomasmerca <32470264+Tomasmerca@users.noreply.github.com> Date: Wed, 15 Jan 2020 00:36:48 -0300 Subject: [PATCH 07/69] Added options for password and filter config --- .templates/mosquitto/mosquitto.conf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.templates/mosquitto/mosquitto.conf b/.templates/mosquitto/mosquitto.conf index 9f63ac19..5827737d 100644 --- a/.templates/mosquitto/mosquitto.conf +++ b/.templates/mosquitto/mosquitto.conf @@ -1,4 +1,10 @@ persistence true persistence_location /mosquitto/data/ log_dest file /mosquitto/log/mosquitto.log + +#Uncomment to enable passwords #password_file /mosquitto/config/pwfile +#allow_anonymous false + +#Uncomment to enable filters +#acl_file /mosquitto/config/filter.acl From 72e6345b9bdf2fb8850fa9838df0cd4091b9d027 Mon Sep 17 00:00:00 2001 From: RavHilton <49377971+RavHilton@users.noreply.github.com> Date: Thu, 6 Feb 2020 16:34:48 +1000 Subject: [PATCH 08/69] Update menu.sh Version "2" of the docker-compose.yml cannot use "Healthcheck" instructions. Only 2.1+ can do this. I propose we upgrade the version to at least 3 so we can incorporate HealthCheck commands and any other new features as appropriate. We're running the latest version of docker - the latest version of the docker-compose.yml is 3.7 - unless there's a really, really good reason to use an older format of the docker-compose.yml, can we please update it to at least 3.0, preferably 3.7? --- menu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/menu.sh b/menu.sh index 6c11d62f..f9a0c4c9 100755 --- a/menu.sh +++ b/menu.sh @@ -231,7 +231,7 @@ case $mainmenu_selection in #if no container is selected then dont overwrite the docker-compose.yml file if [ -n "$container_selection" ]; then touch docker-compose.yml - echo "version: '2'" >docker-compose.yml + echo "version: '3.7'" >docker-compose.yml echo "services:" >>docker-compose.yml #set the ACL for the stack From 818063fb8136c82d22fdcef9e383a5539e0cab75 Mon Sep 17 00:00:00 2001 From: PiRune Date: Fri, 28 Feb 2020 20:50:40 +0100 Subject: [PATCH 09/69] adding deconz docker compose setup --- .templates/deconz/service.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .templates/deconz/service.yml diff --git a/.templates/deconz/service.yml b/.templates/deconz/service.yml new file mode 100644 index 00000000..8c57832b --- /dev/null +++ b/.templates/deconz/service.yml @@ -0,0 +1,29 @@ +deconz: + image: marthoc/deconz + container_name: deconz + restart: unless-stopped + network_mode: bridge + ports: + - '80:80' + - '443:443' + - '5901:5901' + volumes: + - ./volumes/deconz/:/root/.local/share/dresden-elektronik/deCONZ + devices: + #ConBee II: + - /dev/ttyACM0 + #ConBee: + # - /dev/ttyUSB0 + #RaspBee: + # - /dev/ttyAMA0 or /dev/ttyS0 + environment: + - DECONZ_WEB_PORT=80 + - DECONZ_WS_PORT=443 + - DECONZ_VNC_MODE=1 + - DECONZ_VNC_PORT=5901 + - DECONZ_VNC_PASSWORD=changeme + - DEBUG_INFO=1 + - DEBUG_APS=0 + - DEBUG_ZCL=0 + - DEBUG_ZDP=0 + - DEBUG_OTAU=0 From 6461ed1c1db3401f327be65c1729f53531d629ee Mon Sep 17 00:00:00 2001 From: PiRune Date: Fri, 28 Feb 2020 20:59:44 +0100 Subject: [PATCH 10/69] deconz added to menu --- menu.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/menu.sh b/menu.sh index 6c11d62f..05c4bdbd 100755 --- a/menu.sh +++ b/menu.sh @@ -15,6 +15,7 @@ declare -A cont_array=( [adminer]="Adminer" [openhab]="openHAB" [zigbee2mqtt]="zigbee2mqtt" + [deconz]="deCONZ" [pihole]="Pi-Hole" [plex]="Plex media server" [tasmoadmin]="TasmoAdmin" @@ -31,7 +32,7 @@ declare -A cont_array=( ) declare -a armhf_keys=("portainer" "nodered" "influxdb" "grafana" "mosquitto" "telegraf" "mariadb" "postgres" - "adminer" "openhab" "zigbee2mqtt" "pihole" "plex" "tasmoadmin" "rtl_433" "espruinohub" + "adminer" "openhab" "zigbee2mqtt" "deconz" "pihole" "plex" "tasmoadmin" "rtl_433" "espruinohub" "motioneye" "webthings_gateway" "blynk_server" "nextcloud" "diyhue" "homebridge" "python") sys_arch=$(uname -m) From 413c0ae322356fbf62bde85bbca590b66d19191e Mon Sep 17 00:00:00 2001 From: PiRune Date: Mon, 2 Mar 2020 21:15:59 +0100 Subject: [PATCH 11/69] Update service.yml --- .templates/deconz/service.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.templates/deconz/service.yml b/.templates/deconz/service.yml index 8c57832b..401246e1 100644 --- a/.templates/deconz/service.yml +++ b/.templates/deconz/service.yml @@ -1,4 +1,4 @@ -deconz: + deconz: image: marthoc/deconz container_name: deconz restart: unless-stopped From 050fdb157d1a0b778529176d0845a8b528dfbf5d Mon Sep 17 00:00:00 2001 From: PiRune Date: Mon, 2 Mar 2020 21:41:05 +0100 Subject: [PATCH 12/69] deconz added --- .templates/deconz/service.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.templates/deconz/service.yml b/.templates/deconz/service.yml index 401246e1..41339439 100644 --- a/.templates/deconz/service.yml +++ b/.templates/deconz/service.yml @@ -6,7 +6,7 @@ ports: - '80:80' - '443:443' - - '5901:5901' + - '5901:5900' volumes: - ./volumes/deconz/:/root/.local/share/dresden-elektronik/deCONZ devices: @@ -17,10 +17,7 @@ #RaspBee: # - /dev/ttyAMA0 or /dev/ttyS0 environment: - - DECONZ_WEB_PORT=80 - - DECONZ_WS_PORT=443 - DECONZ_VNC_MODE=1 - - DECONZ_VNC_PORT=5901 - DECONZ_VNC_PASSWORD=changeme - DEBUG_INFO=1 - DEBUG_APS=0 From 614e56e06030797dd7776a7665c1a3975c385f08 Mon Sep 17 00:00:00 2001 From: PiRune Date: Mon, 2 Mar 2020 22:11:33 +0100 Subject: [PATCH 13/69] added node-red-contrib-deconz --- .templates/nodered/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.templates/nodered/build.sh b/.templates/nodered/build.sh index ba5ab374..81685931 100755 --- a/.templates/nodered/build.sh +++ b/.templates/nodered/build.sh @@ -35,6 +35,7 @@ node_selection=$(whiptail --title "Node-RED nodes" --checklist --separate-output "node-red-contrib-owntracks" " " "OFF" \ "node-red-contrib-alexa-local" " " "OFF" \ "node-red-contrib-heater-controller" " " "OFF" \ + "node-red-contrib-deconz" " " "OFF" \ 3>&1 1>&2 2>&3) ##echo "$check_selection" From cb3e1b0d1a8b41f2363c838a5be831879a232d37 Mon Sep 17 00:00:00 2001 From: Gabriele Pongelli Date: Sun, 22 Mar 2020 16:57:56 +0100 Subject: [PATCH 14/69] added transmission container --- .templates/transmission/service.yml | 16 ++++++++++++++++ menu.sh | 3 ++- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 .templates/transmission/service.yml diff --git a/.templates/transmission/service.yml b/.templates/transmission/service.yml new file mode 100644 index 00000000..f6e46e8d --- /dev/null +++ b/.templates/transmission/service.yml @@ -0,0 +1,16 @@ + transmission: + image: linuxserver/transmission + container_name: transmission + environment: + - PUID=1000 + - PGID=1000 + - TZ=Europe/Rome + volumes: + - transm_config_volume:/config + - transm_download_volume:/downloads + - transm_watch_volume:/watch + ports: + - 9091:9091 + - 51413:51413 + - 51413:51413/udp + restart: unless-stopped diff --git a/menu.sh b/menu.sh index 6c11d62f..5682538d 100755 --- a/menu.sh +++ b/menu.sh @@ -8,6 +8,7 @@ declare -A cont_array=( [nodered]="Node-RED" [influxdb]="InfluxDB" [telegraf]="Telegraf (Requires InfluxDB and Mosquitto)" + [transmission]="transmission" [grafana]="Grafana" [mosquitto]="Eclipse-Mosquitto" [postgres]="Postgres" @@ -30,7 +31,7 @@ declare -A cont_array=( [python]="Python 3" ) -declare -a armhf_keys=("portainer" "nodered" "influxdb" "grafana" "mosquitto" "telegraf" "mariadb" "postgres" +declare -a armhf_keys=("portainer" "nodered" "influxdb" "grafana" "mosquitto" "telegraf" "mariadb" "postgres" "transmission" "adminer" "openhab" "zigbee2mqtt" "pihole" "plex" "tasmoadmin" "rtl_433" "espruinohub" "motioneye" "webthings_gateway" "blynk_server" "nextcloud" "diyhue" "homebridge" "python") From 7751ed0b2615ecfd115817c28644568e069d098d Mon Sep 17 00:00:00 2001 From: Gabriele Pongelli Date: Sun, 22 Mar 2020 16:58:52 +0100 Subject: [PATCH 15/69] add volumes management from template --- .templates/transmission/volumes.yml | 21 +++++++++++++++++++++ menu.sh | 11 +++++++++++ 2 files changed, 32 insertions(+) create mode 100644 .templates/transmission/volumes.yml diff --git a/.templates/transmission/volumes.yml b/.templates/transmission/volumes.yml new file mode 100644 index 00000000..597c494b --- /dev/null +++ b/.templates/transmission/volumes.yml @@ -0,0 +1,21 @@ + + transm_download_volume: + driver: local + driver_opts: + o: bind + type: none + device: + + transm_watch_volume: + driver: local + driver_opts: + o: bind + type: none + device: + + transm_config_volume: + driver: local + driver_opts: + o: bind + type: none + device: diff --git a/menu.sh b/menu.sh index 5682538d..581bab20 100755 --- a/menu.sh +++ b/menu.sh @@ -117,6 +117,9 @@ function yml_builder() { #if an env file exists check for timezone [ -f "./services/$1/$1.env" ] && timezones ./services/$1/$1.env + # if a volumes.yml exists, append to overall volumes.yml file + [ -f "./services/$1/volumes.yml" ] && cat "./services/$1/volumes.yml" >> docker-volumes.yml + #add new line then append service echo "" >>docker-compose.yml cat $service >>docker-compose.yml @@ -260,6 +263,14 @@ case $mainmenu_selection in done fi fi + + # if a container needs volume, put it at the end of docker-compose + if [ -f docker-volumes.yml ]; then + echo "" >> docker-compose.yml + echo "volumes:" >> docker-compose.yml + cat docker-volumes.yml >> docker-compose.yml + rm docker-volumes.yml + fi echo "docker-compose successfully created" echo "run 'docker-compose up -d' to start the stack" From da5a5496d46f02da321ff40416efc51b528fefc9 Mon Sep 17 00:00:00 2001 From: peyanski Date: Sat, 4 Apr 2020 08:32:06 +0300 Subject: [PATCH 16/69] update menu.sh The url for the supervised HASSIO is changed, because the old one is resulting with error 404 and you cannot finish the installation --- menu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/menu.sh b/menu.sh index 6c11d62f..f651383f 100755 --- a/menu.sh +++ b/menu.sh @@ -406,7 +406,7 @@ case $mainmenu_selection in "tinker" " " \ 3>&1 1>&2 2>&3) if [ -n "$hassio_machine" ]; then - curl -sL https://raw.githubusercontent.com/home-assistant/hassio-installer/master/hassio_install.sh | sudo bash -s -- -m $hassio_machine + curl -sL https://raw.githubusercontent.com/home-assistant/supervised-installer/master/installer.sh | sudo bash -s -- -m $hassio_machine else echo "no selection" exit From d5b991b5dd50ac255549d670294000b82316bca9 Mon Sep 17 00:00:00 2001 From: Willem Dekker Date: Sun, 19 Apr 2020 12:15:40 +0200 Subject: [PATCH 17/69] small update to container name --- .templates/portainer_agent/service.yml | 2 +- README.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.templates/portainer_agent/service.yml b/.templates/portainer_agent/service.yml index d03b849a..b7c12309 100644 --- a/.templates/portainer_agent/service.yml +++ b/.templates/portainer_agent/service.yml @@ -1,6 +1,6 @@ portainer_agent: image: portainer/agent - container_name: agent + container_name: portainer-agent ports: - 9002:9001 volumes: diff --git a/README.md b/README.md index b59ba57e..6f361324 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ This Docker stack consists of: * Postgres * Mosquitto mqtt * Portainer +* Portainer agent * Adminer * openHAB * Home Assistant (HASSIO) From b0681140fbd7c7b226f1b71495d0696bd701aa9b Mon Sep 17 00:00:00 2001 From: Willem Dekker Date: Sun, 19 Apr 2020 12:57:37 +0200 Subject: [PATCH 18/69] first import --- .github/workflows/main.yml | 18 +++ ...Accessing-your-Device-from-the-internet.md | 43 +++++++ docs/Adminer.md | 11 ++ docs/Backups.md | 112 +++++++++++++++++ docs/Blynk_server.md | 81 ++++++++++++ docs/Custom.md | 13 ++ docs/Docker-commands.md | 14 +++ docs/EspruinoHub.md | 11 ++ docs/Getting-Started.md | 82 ++++++++++++ docs/Grafana.md | 26 ++++ docs/Home-Assistant.md | 51 ++++++++ docs/Home.md | 71 +++++++++++ docs/How-the-script-works.md | 5 + docs/InfluxDB.md | 8 ++ docs/MariaDB.md | 23 ++++ docs/Misc.md | 9 ++ docs/Mosquitto.md | 14 +++ docs/MotionEye.md | 43 +++++++ docs/Native-RTL_433.md | 3 + docs/Networking.md | 27 ++++ docs/NextCloud.md | 62 ++++++++++ docs/Node-RED.md | 117 ++++++++++++++++++ docs/Pi-hole.md | 7 ++ docs/Plex.md | 11 ++ docs/Portainer-agent.md | 18 +++ docs/Portainer.md | 25 ++++ docs/PostgreSQL.md | 6 + docs/Python.md | 50 ++++++++ docs/RPIEasy_native.md | 13 ++ docs/RTL_433-docker.md | 28 +++++ docs/TasmoAdmin.md | 9 ++ docs/Understanding-Containers.md | 1 + docs/Updating-the-Project.md | 20 +++ docs/Zigbee2MQTT.md | 34 +++++ docs/diyHue.md | 19 +++ docs/openHAB.md | 5 + docs/x2go.md | 23 ++++ mkdocs.yml | 4 + 38 files changed, 1117 insertions(+) create mode 100644 .github/workflows/main.yml create mode 100644 docs/Accessing-your-Device-from-the-internet.md create mode 100644 docs/Adminer.md create mode 100644 docs/Backups.md create mode 100644 docs/Blynk_server.md create mode 100644 docs/Custom.md create mode 100644 docs/Docker-commands.md create mode 100644 docs/EspruinoHub.md create mode 100644 docs/Getting-Started.md create mode 100644 docs/Grafana.md create mode 100644 docs/Home-Assistant.md create mode 100644 docs/Home.md create mode 100644 docs/How-the-script-works.md create mode 100644 docs/InfluxDB.md create mode 100644 docs/MariaDB.md create mode 100644 docs/Misc.md create mode 100644 docs/Mosquitto.md create mode 100644 docs/MotionEye.md create mode 100644 docs/Native-RTL_433.md create mode 100644 docs/Networking.md create mode 100644 docs/NextCloud.md create mode 100644 docs/Node-RED.md create mode 100644 docs/Pi-hole.md create mode 100644 docs/Plex.md create mode 100644 docs/Portainer-agent.md create mode 100644 docs/Portainer.md create mode 100644 docs/PostgreSQL.md create mode 100644 docs/Python.md create mode 100644 docs/RPIEasy_native.md create mode 100644 docs/RTL_433-docker.md create mode 100644 docs/TasmoAdmin.md create mode 100644 docs/Understanding-Containers.md create mode 100644 docs/Updating-the-Project.md create mode 100644 docs/Zigbee2MQTT.md create mode 100644 docs/diyHue.md create mode 100644 docs/openHAB.md create mode 100644 docs/x2go.md create mode 100644 mkdocs.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..d8887048 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,18 @@ +name: Publish docs via GitHub Pages +on: + push: + branches: + - master + +jobs: + build: + name: Deploy docs + runs-on: ubuntu-latest + steps: + - name: Checkout master + uses: actions/checkout@v1 + + - name: Deploy docs + uses: mhausenblas/mkdocs-deploy-gh-pages@master + env: + PERSONAL_TOKEN: ${{ secrets.PERSONAL_TOKEN }} diff --git a/docs/Accessing-your-Device-from-the-internet.md b/docs/Accessing-your-Device-from-the-internet.md new file mode 100644 index 00000000..908a9143 --- /dev/null +++ b/docs/Accessing-your-Device-from-the-internet.md @@ -0,0 +1,43 @@ +The challenge most of us face with remotely accessing your home network is that you don't have a static IP. From time to time the IP that your ISP assigns to you changes and it's difficult to keep up. Fortunately, there is a solution, a DynamicDNS. The section below shows you how to set up an easy to remember address that follows your public IP no matter when it changes. + +Secondly, how do you get into your home network? Your router has a firewall that is designed to keep the rest of the internet out of your network to protect you. Here we install a VPN and configure the firewall to only allow very secure VPN traffic in. + +# DuckDNS +If you want to have a dynamic DNS point to your Public IP I added a helper script. +Register with duckdns.org and create a subdomain name. Then edit the `nano ~/IOTstack/duck/duck.sh` file and add your + +```bash +DOMAINS="YOUR_DOMAINS" +DUCKDNS_TOKEN="YOUR_DUCKDNS_TOKEN" +``` + +first test the script to make sure it works `sudo ~/IOTstack/duck/duck.sh` then `cat /var/log/duck.log`. If you get KO then something has gone wrong and you should check out your settings in the script. If you get an OK then you can do the next step. + +Create a cron job by running the following command `crontab -e` + +You will be asked to use an editor option 1 for nano should be fine +paste the following in the editor `*/5 * * * * sudo ~/IOTstack/duck/duck.sh >/dev/null 2>&1` then ctrl+s and ctrl+x to save + +Your Public IP should be updated every five minutes + +# PiVPN +pimylifeup.com has an excellent tutorial on how to install [PiVPN](https://pimylifeup.com/raspberry-pi-vpn-server/) + +In point 17 and 18 they mention using noip for their dynamic DNS. Here you can use the DuckDNS address if you created one. + +Don't forget you need to open the port 1194 on your firewall. Most people won't be able to VPN from inside their network so download OpenVPN client for your mobile phone and try to connect over mobile data. ([More info.](https://en.wikipedia.org/wiki/Hairpinning)) + +Once you activate your VPN (from your phone/laptop/work computer) you will effectively be on your home network and you can access your devices as if you were on the wifi at home. + +I personally use the VPN any time I'm on public wifi, all your traffic is secure. + +# Zerotier +https://www.zerotier.com/ + +Zerotier is an alternative to PiVPN that doesn't require port forwarding on your router. It does however require registering for their free tier service [here](https://my.zerotier.com/login). + +Kevin Zhang has written a how to guide [here](https://iamkelv.in/blog/2017/06/zerotier.html). Just note that the install link is outdated and should be: +``` +curl -s 'https://raw.githubusercontent.com/zerotier/ZeroTierOne/master/doc/contact%40zerotier.com.gpg' | gpg --import && \ +if z=$(curl -s 'https://install.zerotier.com/' | gpg); then echo "$z" | sudo bash; fi +``` diff --git a/docs/Adminer.md b/docs/Adminer.md new file mode 100644 index 00000000..e469e927 --- /dev/null +++ b/docs/Adminer.md @@ -0,0 +1,11 @@ +# References +- [Docker](https://hub.docker.com/_/adminer) +- [Website](https://www.adminer.org/) + +# About + +This is a nice tool for managing databases. Web interface has moved to port 9080. There was an issue where openHAB and Adminer were using the same ports. If you have an port conflict edit the docker-compose.yml and under the adminer service change the line to read: +``` + ports: + - 9080:8080 +``` diff --git a/docs/Backups.md b/docs/Backups.md new file mode 100644 index 00000000..aa47fdcf --- /dev/null +++ b/docs/Backups.md @@ -0,0 +1,112 @@ +Because containers can easily be rebuilt from docker hub we only have to back up the data in the "volumes" directory. + +# Cloud Backups +## Dropbox-Uploader +This a great utility to easily upload data from your Pi to the cloud. https://magpi.raspberrypi.org/articles/dropbox-raspberry-pi. It can be installed from the Menu under Backups. +## rclone (Google Drive) +This is a service to upload to Google Drive. The config is described [here]( https://medium.com/@artur.klauser/mounting-google-drive-on-raspberry-pi-f5002c7095c2). Install it from the menu then follow the link for these sections: +* Getting a Google Drive Client ID +* Setting up the Rclone Configuration + +When naming the service in `rclone config` ensure to call it "gdrive" + +**The Auto-mounting instructions for the drive in the link don't work on Rasbian**. Auto-mounting of the drive isn't necessary for the backup script. + +If you want your Google Drive to mount on every boot then follow the instructions at the bottom of the wiki page + + +# Influxdb +`~/IOTstack/scripts/backup_influxdb.sh` does a database snapshot and stores it in ~/IOTstack/backups/influxdb/db . This can be restored with the help a script (that I still need to write) + +# Docker backups +The script `~/IOTstack/scripts/docker_backup.sh` performs the master backup for the stack. + +This script can be placed in a cron job to backup on a schedule. +Edit the crontab with`crontab -e` +Then add `0 23 * * * ~/IOTstack/scripts/docker_backup.sh >/dev/null 2>&1` to have a backup every night at 23:00. + +This script cheats by copying the volume folder live. The correct way would be to stop the stack first then copy the volumes and restart. The cheating method shouldn't be a problem unless you have fast changing data like in influxdb. This is why the script makes a database export of influxdb and ignores its volume. + +## Cloud integration +The docker_backup.sh script now no longer requires modification to enable cloud backups. It now tests for the presence of and enable file in the backups folder +### Drobox-Uploader +The backup tests for a file called `~/IOTstack/backups/dropbox`, if it is present it will upload to dropbox. To disable dropbox upload delete the file. To enable run `sudo touch ~/IOTstack/backups/dropbox` +### rclone +The backup tests for a file called `~/IOTstack/backups/rclone`, if it is present it will upload to google drive. To disable rclone upload delete the file. To enable run `sudo touch ~/IOTstack/backups/rclone` + +### Pruning online backups +@877dev has added functionality to prune both local and cloud backups. For dropbox make sure you dont have any files that contain spaces in your backup directory as the script cannot handle it at this time. + +## Restoring a backup +The "volumes" directory contains all the persistent data necessary to recreate the container. The docker-compose.yml and the environment files are optional as they can be regenerated with the menu. Simply copy the volumes directory into the IOTstack directory, Rebuild the stack and start. + +# Added your Dropbox token incorrectly or aborted the install at the token screen + +Make sure you are running the latest version of the project [link](https://github.com/gcgarner/IOTstack/wiki/Updating-the-Project). + +Run `~/Dropbox-Uploader/dropbox_uploader.sh unlink` and if you have added it key then it will prompt you to confirm its removal. If no key was found it will ask you for a new key. + +Confirm by running `~/Dropbox-Uploader/dropbox_uploader.sh` it should ask you for your key if you removed it or show you the following prompt if it has the key: + +``` + $ ~/Dropbox-Uploader/dropbox_uploader.sh +Dropbox Uploader v1.0 +Andrea Fabrizi - andrea.fabrizi@gmail.com + +Usage: /home/pi/Dropbox-Uploader/dropbox_uploader.sh [PARAMETERS] COMMAND... + +Commands: + upload + download [LOCAL_FILE/DIR] + delete + move + copy + mkdir +.... + +``` + +Ensure you **are not** running as sudo as this will store your api in the /root directory as `/root/.dropbox_uploader` + +If you ran the command with sudo the remove the old token file if it exists with either `sudo rm /root/.dropbox_uploader` or `sudo ~/Dropbox-Uploader/dropbox_uploader.sh unlink` + +# Auto-mount Gdrive with rclone + +To enable rclone to mount on boot you will need to make a user service. Run the following commands + +```bash +mkdir -p ~/.config/systemd/user +nano ~/.config/systemd/user/gdrive.service +``` +Copy the following code into the editor, save and exit + +``` +[Unit] +Description=rclone: Remote FUSE filesystem for cloud storage +Documentation=man:rclone(1) + +[Service] +Type=notify +ExecStartPre=/bin/mkdir -p %h/mnt/gdrive +ExecStart= \ + /usr/bin/rclone mount \ + --fast-list \ + --vfs-cache-mode writes \ + gdrive: %h/mnt/gdrive + +[Install] +WantedBy=default.target +``` +enable it to start on boot with: (no sudo) +```bash +systemctl --user enable gdrive.service +``` +start with +```bash +systemctl --user start gdrive.service +``` +if you no longer want it to start on boot then type: +```bash +systemctl --user disable gdrive.service +``` + diff --git a/docs/Blynk_server.md b/docs/Blynk_server.md new file mode 100644 index 00000000..7b3d7759 --- /dev/null +++ b/docs/Blynk_server.md @@ -0,0 +1,81 @@ +This is a custom implementation of Blynk Server + +```yml + blynk_server: + build: ./services/blynk_server/. + container_name: blynk_server + restart: unless-stopped + ports: + - 8180:8080 + - 8441:8441 + - 9443:9443 + volumes: + - ./volumes/blynk_server/data:/data +``` + +To connect to the admin interface navigate to `:9443/admin` + +I don't know anything about this service so you will need to read though the setup on the [Project Homepage](https://github.com/blynkkk/blynk-server) + +When setting up the application on your mobile be sure to select custom setup [here](https://github.com/blynkkk/blynk-server#app-and-sketch-changes) + +Writeup From @877dev + +**Getting started:** +Log into admin panel at https://youripaddress:9443/admin +(Use your Pi's IP address, and ignore Chrome warning). + +Default credentials: +user:admin@blynk.cc +pass:admin + +**Change username and password:** +Click on Users > "email address" and edit email, name and password. +Save changes +Restarting the container using Portainer may be required to take effect. + +**Setup gmail:** +Optional step, useful for getting the auth token emailed to you. +(To be added once confirmed working....) + +**iOS/Android app setup:** +Login the app as per the photos [HERE](https://github.com/blynkkk/blynk-server#app-and-sketch-changes) +Press "New Project" +Give it a name, choose device "Raspberry Pi 3 B" so you have plenty of [virtual pins](http://help.blynk.cc/en/articles/512061-what-is-virtual-pins) available, and lastly select WiFi. +Create project and the [auth token](https://docs.blynk.cc/#getting-started-getting-started-with-the-blynk-app-4-auth-token) will be emailed to you (if emails configured). You can also find the token in app under the phone app settings, or in the admin web interface by clicking Users>"email address" and scroll down to token. + +**Quick usage guide for app:** +Press on the empty page, the widgets will appear from the right. +Select your widget, let's say a button. +It appears on the page, press on it to configure. +Give it a name and colour if you want. +Press on PIN, and select virtual. Choose any pin i.e. V0 +Press ok. +To start the project running, press top right Play button. +You will get an offline message, because no devices are connected to your project via the token. +Enter node red..... + +**Node red:** +Install node-red-contrib-blynk-ws from pallette manager +Drag a "write event" node into your flow, and connect to a debug node +Configure the Blynk node for the first time: +```URL: wss://youripaddress:9443/websockets``` more info [HERE](https://github.com/gablau/node-red-contrib-blynk-ws/blob/master/README.md#how-to-use) +Enter your [auth token](https://docs.blynk.cc/#getting-started-getting-started-with-the-blynk-app-4-auth-token) from before and save/exit. +When you deploy the flow, notice the app shows connected message, as does the Blynk node. +Press the button on the app, you will notice the payload is sent to the debug node. + +**What next?** +Further information and advanced setup: +https://github.com/blynkkk/blynk-server + +Check the documentation: +https://docs.blynk.cc/ + +Visit the community forum pages: +https://community.blynk.cc/ + +Interesting post by Peter Knight on MQTT/Node Red flows: +https://community.blynk.cc/t/my-home-automation-projects-built-with-mqtt-and-node-red/29045 + +Some Blynk flow examples: +https://github.com/877dev/Node-Red-flow-examples \ No newline at end of file diff --git a/docs/Custom.md b/docs/Custom.md new file mode 100644 index 00000000..b86caf44 --- /dev/null +++ b/docs/Custom.md @@ -0,0 +1,13 @@ +If you have a container that you want to stop and start with the stack you can now use the custom container option. This you can use for testing or in prep for a Pull Request. + +You will need to create a directory for your container call `IOTstack/services/` + +Inside that container create a `service.yml` containing your service and configurations. Have a look at one of the other services for inspiration. + +Create a file called `IOTstack/services/custom.txt` and and enter your container names, one per line + +Run the menu.sh and build the stack. After you have made the selection you will be asked if you want to add the custom containers. + +Now your container will be part of the docker-compose.yml file and will respond to the docker-compose up -d commands. + +Docker creates volumes under root and your container may require different ownership on the volume directory. You can see an example of the workaround for this in the python template in `IOTstack/.templates/python/directoryfix.sh` \ No newline at end of file diff --git a/docs/Docker-commands.md b/docs/Docker-commands.md new file mode 100644 index 00000000..b0989987 --- /dev/null +++ b/docs/Docker-commands.md @@ -0,0 +1,14 @@ +# Aliases + +I've added bash aliases for stopping and starting the stack. They can be installed in the docker commands menu. These commands no longer need to be executed from the IOTstack directory and can be executed in any directory + +```bash +alias iotstack_up="docker-compose -f ~/IOTstack/docker-compose.yml up -d" +alias iotstack_down="docker-compose -f ~/IOTstack/docker-compose.yml down" +alias iotstack_start="docker-compose -f ~/IOTstack/docker-compose.yml start" +alias iotstack_stop="docker-compose -f ~/IOTstack/docker-compose.yml stop" +alias iotstack_update="docker-compose -f ~/IOTstack/docker-compose.yml pull" +alias iotstack_build="docker-compose -f ~/IOTstack/docker-compose.yml build" +``` + +You can now type `iotstack_up`, they even accept additional parameters `iotstack_stop portainer` diff --git a/docs/EspruinoHub.md b/docs/EspruinoHub.md new file mode 100644 index 00000000..1db2ca53 --- /dev/null +++ b/docs/EspruinoHub.md @@ -0,0 +1,11 @@ +This is a testing container + +I tried it however the container keeps restarting `docker logs espruinohub` I get "BLE Broken?" but could just be i dont have any BLE devices nearby + +web interface is on "{your_Pis_IP}:1888" + +see https://github.com/espruino/EspruinoHub#status--websocket-mqtt--espruino-web-ide for other details + +there were no recommendations for persistent data volumes. so `docker-compose down` may destroy all you configurations so use `docker-compose stop` in stead + +please let me know about your success or issues [here](https://github.com/gcgarner/IOTstack/issues/84) \ No newline at end of file diff --git a/docs/Getting-Started.md b/docs/Getting-Started.md new file mode 100644 index 00000000..da733f54 --- /dev/null +++ b/docs/Getting-Started.md @@ -0,0 +1,82 @@ +# Download the project + +On the lite image you will need to install git first +``` +sudo apt-get install git +``` +Then download with +``` +git clone https://github.com/gcgarner/IOTstack.git ~/IOTstack +``` +Due to some script restraints, this project needs to be stored in ~/IOTstack + +To enter the directory run: +``` +cd ~/IOTstack +``` +# The Menu +I've added a menu to make things easier. It is good however to familiarise yourself with how things are installed. +The menu can be used to install docker and then build the docker-compose.yml file necessary for starting the stack and it runs a few common commands. I do recommend you start to learn the docker and docker-compose commands if you plan using docker in the long run. I've added several helper scripts, have a look inside. + +Navigate to the project folder and run `./menu.sh` + +## Installing from the menu +Select the first option and follow the prompts + +## Build the docker-compose file +docker-compose uses the `docker-compose.yml` file to configure all the services. Run through the menu to select the options you want to install. + +## Docker commands +This menu executes shell scripts in the root of the project. It is not necessary to run them from the menu. Open up the shell script files to see what is inside and what they do + +## Miscellaneous commands +Some helpful commands have been added like disabling swap + +# Running Docker commands +From this point on make sure you are executing the commands from inside the project folder. Docker-compose commands need to be run from the folder where the docker-compose.yml is located. If you want to move the folder make sure you move the whole project folder. + +## Starting and Stopping containers +to start the stack navigate to the project folder containing the docker-compose.yml file + +To start the stack run: +`docker-compose up -d` or `./scripts/start.sh` + +To stop: +`docker-compose stop` stops without removing containers + +To remove the stack: +`docker-compose down` stops containers, deletes them and removes the network + +The first time you run 'start' the stack docker will download all the images for the web. Depending on how many containers you selected and your internet speed this can take a long while. + +## Persistent data +Docker allows you to map folders inside your containers to folders on the disk. This is done with the "volume" key. There are two types of volumes. Any modification to the container reflects in the volume. + +### Sharing files between the Pi and containers +Have a look a the wiki on how to share files between Node-RED and the Pi. [Wiki](https://github.com/gcgarner/IOTstack/wiki/Node-RED#sharing-files-between-node-red-and-the-host) + +## Updating the images +If a new version of a container image is available on docker hub it can be updated by a pull command. + +Use the `docker-compose stop` command to stop the stack + +Pull the latest version from docker hub with one of the following command + +`docker-compose pull` or the script `./scripts/update.sh` + +Start the new stack based on the updated images + +`docker-compose up -d` + +## Node-RED error after modifications to setup files +The Node-RED image differs from the rest of the images in this project. It uses the "build" key. It uses a dockerfile for the setup to inject the nodes for pre-installation. If you get an error for Node-RED run `docker-compose build` then `docker-compose up -d` + +## Deleting containers, volumes and images + +`./prune-images.sh` will remove all images not associated with a container. If you run it while the stack is up it will ignore any in-use images. If you run this while you stack is down it will delete all images and you will have to redownload all images from scratch. This command can be helpful to reclaim disk space after updating your images, just make sure to run it while your stack is running as not to delete the images in use. (your data will still be safe in your volume mapping) + +## Deleting folder volumes +If you want to delete the influxdb data folder run the following command `sudo rm -r volumes/influxdb/`. Only the data folder is deleted leaving the env file intact. review the docker-compose.yml file to see where the file volumes are stored. + +You can use git to delete all files and folders to return your folder to the freshly cloned state, AS IN YOU WILL LOSE ALL YOUR DATA. +`sudo git clean -d -x -f` will return the working tree to its clean state. USE WITH CAUTION! \ No newline at end of file diff --git a/docs/Grafana.md b/docs/Grafana.md new file mode 100644 index 00000000..198ef4ac --- /dev/null +++ b/docs/Grafana.md @@ -0,0 +1,26 @@ +# References +- [Docker](https://hub.docker.com/r/grafana/grafana) +- [Website](https://grafana.com/) + +# Security +Grafana's default credentials are username "admin" password "admin" it will ask you to choose a new password on boot. Go to `:3000` in your web browser. + +# Overwriting grafana.ini settings + +A list of the settings available in grafana.ini are listed [here](https://grafana.com/docs/installation/configuration/) + +To overwrite a setting edit the IOTstack/services/grafana/grafana.env file. The format is `GF__` + +An example would be: +``` +GF_PATHS_DATA=/var/lib/grafana +GF_PATHS_LOGS=/var/log/grafana +# [SERVER] +GF_SERVER_ROOT_URL=http://localhost:3000/grafana +GF_SERVER_SERVE_FROM_SUB_PATH=true +# [SECURITY] +GF_SECURITY_ADMIN_USER=admin +GF_SECURITY_ADMIN_PASSWORD=admin +``` + +After the alterations run `docker-compose up -d` to pull them in diff --git a/docs/Home-Assistant.md b/docs/Home-Assistant.md new file mode 100644 index 00000000..4634c969 --- /dev/null +++ b/docs/Home-Assistant.md @@ -0,0 +1,51 @@ +# References +- [Docker](https://hub.docker.com/r/homeassistant/home-assistant/) +- [Webpage](https://www.home-assistant.io/) + +Hass.io is a home automation platform running on Python 3. It is able to track and control all devices at home and offer a platform for automating control. Port binding is `8123`. +Hass.io is exposed to your hosts' network in order to discover devices on your LAN. That means that it does not sit inside docker's network. + +# Menu installation +Hass.io now has a seperate installation in the menu. The old version was incorrect and should be removed. Be sure to update you project and install the correct version. + +You will be asked to select you device type during the installation. Hass.io is no longer dependant on the IOTstack, it has its own service for maintaining its uptime. + +# Installation +The installation of Hass.io takes up to 20 minutes (depending on your internet connection). Refrain from restarting your Pi until it had come online and you are able to create a user account + +# Removal + +To remove Hass.io you first need to stop the service that controls it. Run the following in the terminal: + +```bash +sudo systemctl stop hassio-supervisor.service +sudo systemctl disable hassio-supervisor.service +``` + +This should stop the main service however there are two additional container that still need to be address + +This will stop the service and disable it from starting on the next boot + +Next you need to stop the hassio_dns and hassio_supervisor + +```bash +docker stop hassio_supervisor +docker stop hassio_dns +docker stop homeassistant +``` + +If you want to remove the containers + +```bash +docker rm hassio_supervisor +docker rm hassio_dns +docker stop homeassistant +``` + +After rebooting you should be able to reinstall + +The stored file are located in `/usr/share/hassio` which can be removed if you need to + +Double check with `docker ps` to see if there are other hassio containers running. They can stopped and removed in the same fashion for the dns and supervisor + +You can use Portainer to view what is running and clean up the unused images. \ No newline at end of file diff --git a/docs/Home.md b/docs/Home.md new file mode 100644 index 00000000..dc8fdafc --- /dev/null +++ b/docs/Home.md @@ -0,0 +1,71 @@ +# Wiki + +The README is moving to the Wiki, It's easier to add content and example to the Wiki vs the README.md + +* [Getting Started](https://github.com/SensorsIot/IOTstack/wiki/Getting-Started) +* [Updating the project](https://github.com/SensorsIot/IOTstack/wiki/Updating-the-Project) +* [How the script works](https://github.com/SensorsIot/IOTstack/wiki/How-the-script-works) +* [Understanding Containers](https://github.com/SensorsIot/IOTstack/wiki/Understanding-Containers) + +*** + +# Docker + +* [Commands](https://github.com/SensorsIot/IOTstack/wiki/Docker-commands) +* [Docker Networks](https://github.com/SensorsIot/IOTstack/wiki/Networking) + +*** + +# Containers +* [Portainer](https://github.com/SensorsIot/IOTstack/wiki/Portainer) +* [Portainer Agent](https://github.com/SensorsIot/IOTstack/wiki/Portainer-agent) +* [Node-RED](https://github.com/SensorsIot/IOTstack/wiki/Node-RED) +* [Grafana](https://github.com/SensorsIot/IOTstack/wiki/Grafana) +* [Mosquitto](https://github.com/SensorsIot/IOTstack/wiki/Mosquitto) +* [PostgreSQL](https://github.com/SensorsIot/IOTstack/wiki/PostgreSQL) +* [Adminer](https://github.com/SensorsIot/IOTstack/wiki/Adminer) +* [openHAB](https://github.com/SensorsIot/IOTstack/wiki/openHAB) +* [Home Assistant](https://github.com/SensorsIot/IOTstack/wiki/Home-Assistant) +* [Pi-Hole](https://github.com/SensorsIot/IOTstack/wiki/Pi-hole) +* [zigbee2MQTT](https://github.com/SensorsIot/IOTstack/wiki/Zigbee2MQTT) +* [Plex](https://github.com/SensorsIot/IOTstack/wiki/Plex) +* [TasmoAdmin](https://github.com/SensorsIot/IOTstack/wiki/TasmoAdmin) +* [RTL_433](https://github.com/SensorsIot/IOTstack/wiki/RTL_433-docker) +* [EspruinoHub (testing)](https://github.com/SensorsIot/IOTstack/wiki/EspruinoHub) +* [Next-Cloud](https://github.com/SensorsIot/IOTstack/wiki/NextCloud) +* [MariaDB](https://github.com/SensorsIot/IOTstack/wiki/MariaDB) +* [MotionEye](https://github.com/SensorsIot/IOTstack/wiki/MotionEye) +* [Blynk Server](https://github.com/SensorsIot/IOTstack/wiki/Blynk_server) +* [diyHue](https://github.com/SensorsIot/IOTstack/wiki/diyHue) +* [Python](https://github.com/SensorsIot/IOTstack/wiki/Python) +* [Custom containers](https://github.com/SensorsIot/IOTstack/wiki/Custom) + +*** + +# Native installs + +* [RTL_433](https://github.com/SensorsIot/IOTstack/wiki/Native-RTL_433) +* [RPIEasy](https://github.com/SensorsIot/IOTstack/wiki/RPIEasy_native) + +*** + +# Backups + +* [Docker backups](https://github.com/SensorsIot/IOTstack/wiki/Backups) +* Recovery (coming soon) +*** + +# Remote Access + +* [VPN and Dynamic DNS](https://github.com/SensorsIot/IOTstack/wiki/Accessing-your-Device-from-the-internet) +* [x2go](https://github.com/SensorsIot/IOTstack/wiki/x2go) + +*** + +# Miscellaneous + +* [log2ram](https://github.com/SensorsIot/IOTstack/wiki/Misc) +* [Dropbox-Uploader](https://github.com/SensorsIot/IOTstack/wiki/Misc) + +*** + diff --git a/docs/How-the-script-works.md b/docs/How-the-script-works.md new file mode 100644 index 00000000..4ce4c754 --- /dev/null +++ b/docs/How-the-script-works.md @@ -0,0 +1,5 @@ +The build script creates the ./services directory and populates it from the template file in .templates . The script then appends the text withing each service.yml file to the docker-compose.yml . When the stack is rebuild the menu doesn not overwrite the service folder if it already exists. Make sure to sync any alterations you have made to the docker-compose.yml file with the respective service.yml so that on your next build your changes pull through. + +The .gitignore file is setup such that if you do a `git pull origin master` it does not overwrite the files you have already created. Because the build script does not overwrite your service directory any changes in the .templates directory will have no affect on the services you have already made. You will need to move your service folder out to get the latest version of the template. + + diff --git a/docs/InfluxDB.md b/docs/InfluxDB.md new file mode 100644 index 00000000..2a10eb8e --- /dev/null +++ b/docs/InfluxDB.md @@ -0,0 +1,8 @@ +# References +- [Docker](https://hub.docker.com/_/influxdb) +- [Website](https://www.influxdata.com/) + +# Security +The credentials and default database name for influxdb are stored in the file called influxdb/influx.env . The default username and password is set to "nodered" for both. It is HIGHLY recommended that you change them. The environment file contains several commented out options allowing you to set several access options such as default admin user credentials as well as the default database name. Any change to the environment file will require a restart of the service. + +To access the terminal for influxdb execute `./services/influxdb/terminal.sh`. Here you can set additional parameters or create other databases. diff --git a/docs/MariaDB.md b/docs/MariaDB.md new file mode 100644 index 00000000..9b4928ee --- /dev/null +++ b/docs/MariaDB.md @@ -0,0 +1,23 @@ +# Source +* [Docker hub](https://hub.docker.com/r/linuxserver/mariadb/) +* [Webpage](https://mariadb.org/) + +# About + +MariaDB is a fork of MySQL. This is an unofficial image provided by linuxserver.io because there is no official image for arm + +# Conneting to the DB + +The port is 3306. It exists inside the docker network so you can connect via `mariadb:3306` for internal connections. For external connections use `:3306` + +![image](https://user-images.githubusercontent.com/46672225/69734358-7f030800-1137-11ea-9874-7d2c86b3d239.png) + +# Setup + +Before starting the stack edit the `./services/mariadb/mariadb.env` file and set your access details. This is optional however you will only have one shot at the preconfig. If you start the container without setting the passwords then you will have to either delete its volume directory or enter the terminal and change manually + +The env file has three commented fields for credentials, either **all three** must be commented or un-commented. You can't have only one or two, its all or nothing. + +# Terminal + +A terminal is provided to access mariadb by the cli. execute `./services/maraidb/terminal.sh`. You will need to run `mysql -uroot -p` to enter mariadbs interface \ No newline at end of file diff --git a/docs/Misc.md b/docs/Misc.md new file mode 100644 index 00000000..dca1aa99 --- /dev/null +++ b/docs/Misc.md @@ -0,0 +1,9 @@ +# Miscellaneous + +## log2ram +https://github.com/azlux/log2ram +One of the drawbacks of an sd card is that it has a limited lifespan. One way to reduce the load on the sd card is to move your log files to RAM. log2ram is a convenient tool to simply set this up. It can be installed from the miscellaneous menu. + +## Dropbox-Uploader +This a great utility to easily upload data from your PI to the cloud. https://magpi.raspberrypi.org/articles/dropbox-raspberry-pi +The MagPi has an excellent explanation of the process of setting up the Dropbox API. Dropbox-Uploader is used in the backup script. \ No newline at end of file diff --git a/docs/Mosquitto.md b/docs/Mosquitto.md new file mode 100644 index 00000000..6ea3415c --- /dev/null +++ b/docs/Mosquitto.md @@ -0,0 +1,14 @@ +# References +- [Docker](https://hub.docker.com/_/eclipse-mosquitto) +- [Website](https://mosquitto.org/) + +[Setting up passwords](https://www.youtube.com/watch?v=1msiFQT_flo) + +# Security +By default, the Mosquitto container has no password. You can leave it that way if you like but its always a good idea to secure your services. + +Step 1 +To add the password run `./services/mosquitto/terminal.sh`, I put some helper text in the script. Basically, you use the `mosquitto_passwd -c /mosquitto/config/pwfile MYUSER` command, replacing MYUSER with your username. it will then ask you to type your password and confirm it. exiting with `exit`. + +Step 2 +Edit the file called services/mosquitto/mosquitto.conf and remove the comment in front of password_file. Restart the container with `docker-compose restart mosquitto`. Type those credentials into Node-red etc. diff --git a/docs/MotionEye.md b/docs/MotionEye.md new file mode 100644 index 00000000..596625bf --- /dev/null +++ b/docs/MotionEye.md @@ -0,0 +1,43 @@ +# References + +* [Website](https://github.com/ccrisan/motioneye/wiki/Install-In-Docker) + +# About + +MotionEye is a camera/webcam package. The port is set to 8765 + +# Config + +This is the yml entry. Notice that the devices is commented out. This is because if you don't have a camera attached then it will fail to start. Uncomment if you need to. This is for a Pi camera, you will need to add additional lines for usb cameras + +```yml + motioneye: + image: "ccrisan/motioneye:master-armhf" + container_name: "motioneye" + restart: unless-stopped + ports: + - 8765:8765 + - 8081:8081 + volumes: + - /etc/localtime:/etc/localtime:ro + - ./volumes/motioneye/etc_motioneye:/etc/motioneye + - ./volumes/motioneye/var_lib_motioneye:/var/lib/motioneye + #devices: + # - "/dev/video0:/dev/video0" +``` + +# Login Details + +On first login you will be asked for login details. The default user is `admin` (all lowercase) with no password + +# Storage + +By default local camera data will be stored in `/var/lib/motioneye/camera_name` in the container which equates to the following: + +![image](https://user-images.githubusercontent.com/46672225/69735730-df934480-1139-11ea-833b-705c40ee4f8e.png) + +![image](https://user-images.githubusercontent.com/46672225/69735408-4fed9600-1139-11ea-8618-f5b6c0064f27.png) + +## Remote motioneye + +If you have connected to a remote motion eye note that the directory is on that device and has nothing to do with the container. \ No newline at end of file diff --git a/docs/Native-RTL_433.md b/docs/Native-RTL_433.md new file mode 100644 index 00000000..c20c7cd7 --- /dev/null +++ b/docs/Native-RTL_433.md @@ -0,0 +1,3 @@ +RTL_433 can be installed from the "Native install sections" + +[This video](https://www.youtube.com/watch?v=L0fSEbGEY-Q&t=386s) demonstrates how to use RTL_433 \ No newline at end of file diff --git a/docs/Networking.md b/docs/Networking.md new file mode 100644 index 00000000..00d9695e --- /dev/null +++ b/docs/Networking.md @@ -0,0 +1,27 @@ +The docker-compose instruction creates an internal network for the containers to communicate in, the ports get exposed to the PI's IP address when you want to connect from outside. It also creates a "DNS" the name being the container name. So it is important to note that when one container talks to another they talk by name. All the containers names are lowercase like nodered, influxdb... + +An easy way to find out your IP is by typing `ip address` in the terminal and look next to eth0 or wlan0 for your IP. It is highly recommended that you set a static IP for your PI or at least reserve an IP on your router so that you know it + +Check the docker-compose.yml to see which ports have been used + +![net](https://user-images.githubusercontent.com/46672225/66702353-0bcc4080-ed07-11e9-994b-62219f50b096.png) + +# Examples +- You want to connect your nodered to your mqtt server. In nodered drop an mqtt node, when you need to specify the address type `mosquitto` +- You want to connect to your influxdb from grafana. You are in the Docker network and you need to use the name of the Container. The address you specify in the grafana is `http://influxdb:8086` +- You want to connect to the web interface of grafana from your laptop. Now you are outside the container environment you type PI's IP eg 192.168.n.m:3000 + +# Ports +Many containers try to use popular ports such as 80,443,8080. For example openHAB and Adminer both want to use port 8080 for their web interface. Adminer's port has been moved 9080 to accommodate this. Please check the description of the container in the README to see if there are any changes as they may not be the same as the port you are used to. + +Port mapping is done in the docker-compose.yml file. Each service should have a section that reads like this: +``` + ports: + - HOST_PORT:CONTAINER_PORT +``` +For adminer: +``` + ports: + - 9080:8080 +``` +Port 9080 on Host Pi is mapped to port 8080 of the container. Therefore 127.0.0.1:8080 will take you to openHAB, where 127.0.0.1:9080 will take you to adminer \ No newline at end of file diff --git a/docs/NextCloud.md b/docs/NextCloud.md new file mode 100644 index 00000000..d6c55bcb --- /dev/null +++ b/docs/NextCloud.md @@ -0,0 +1,62 @@ +# DO NOT EXPOSE PORT 80 TO THE WEB + +It is a very bad idea to expose unencrypted traffic to the web. You will need to use a reverse-proxy to ensure your password is not stolen and your account hacked. + +I'm still working on getting a good encrypted reverse proxy working. However in the interim you can use a VPN tunnel like OpenVPN or Zerotier to securely connect to your private cloud + +# Backups + +Nextcloud has been excluded from the docker_backup script due to its potential size. Once I've found a better way of backing it up I will add a dedicated script for it. + +# Setup + +Next-Cloud recommends using MySQL/MariaDB for the accounts and file list. The alternative is to use SQLite however they strongly discourage using it + +This is the service yml. Notice that there are in fact two containers, one for the db and the other for the cloud itself. You will need to change the passwords **before** starting the stack (remember to change the docker-compose.yml and ./services/nextcloud/service.yml), if you dont you will need to delete the volume directory and start again. + +```yml + nextcloud: + image: nextcloud + container_name: nextcloud + ports: + - 9321:80 + volumes: + - ./volumes/nextcloud/html:/var/www/html + restart: unless-stopped + depends_on: + - nextcloud_db + + nextcloud_db: + image: linuxserver/mariadb + container_name: nextcloud_db + volumes: + - ./volumes/nextcloud/db:/config + environment: + - MYSQL_ROOT_PASSWORD=stronger_password + - MYSQL_PASSWORD=strong_password + - MYSQL_DATABASE=nextcloud + - MYSQL_USER=nextcloud + +``` + +The port is 9321 + +![image](https://user-images.githubusercontent.com/46672225/69730546-f2ede200-1130-11ea-97f4-0f4f81d08fad.png) + +click on the storage options, select maraiadb/mysql and fill in the details as follows + +![image](https://user-images.githubusercontent.com/46672225/69731273-46acfb00-1132-11ea-9b10-579bb8b3dd9a.png) + +Note that you data will be stored in `./volumes/nextcloud/html/data/{account}` + +![image](https://user-images.githubusercontent.com/46672225/69732101-b1ab0180-1133-11ea-95dc-8a2e6fb80536.png) + +Also note that file permissions are "www-data" so you cant simply copy data into this folder directly, you should use the web interface or the app. + +It would be a good idea to mount an external drive to store the data in rather than on your sd card. details to follow shortly. Something like: + +![image](https://user-images.githubusercontent.com/46672225/69873297-a3d6b700-12c0-11ea-98c9-40f358137b77.png) + +The external drive will have to be an ext4 formatted drive because smb, fat32 and NTFS can't handle linux file permissions. If the permissions aren't set to "www-data" then the container wont be able to write to the disk. + +Just a warning: If your database gets corrupted then your nextcloud is pretty much stuffed \ No newline at end of file diff --git a/docs/Node-RED.md b/docs/Node-RED.md new file mode 100644 index 00000000..316ef2de --- /dev/null +++ b/docs/Node-RED.md @@ -0,0 +1,117 @@ +# References +- [Docker](https://hub.docker.com/r/nodered/node-red) +- [website](https://nodered.org/) + +# Build warning +The Node-RED build will complain about several issues. This is completely normal behaviour. + +# SQLite +Thanks to @fragolinux the SQLite node will install now. **WARNING it will output many error and will look as if it has gotten stuck. Just give it time and it will continue.** + +# GPIO +To communicate to your Pi's GPIO you need to use the `node-red-node-pi-gpiod` node. It allowes you to connect to multiple Pis from the same nodered service. + +You need to make sure that pigpdiod is running. The recommended method is listed [here](https://github.com/node-red/node-red-nodes/tree/master/hardware/pigpiod) +You run the following command `sudo nano /etc/rc.local` and add the line `/usr/bin/pigpiod` above `exit 0` and reboot the Pi. There is an option to secure the service see the writeup for further instuctions. + +Fot the Rpi Image you will also need to update to the most recent version + +``` +sudo apt-get update +sudo apt-get install pigpio python-pigpio python3-pigpio +``` + +Drop the gpio node and use your Pi's IP. Example: 192.168.1.123 (127.0.0.1 won't work because this is the local address of every computer'.) + +# Securing Node-RED +To secure Node-RED you need a password hash. There is a terminal script `./services/nodered/terminal.sh` execute it to get into the terminal. +Copy the helper text `node -e ..... PASSWORD`, paste it and change your password to get a hash. + +Open the file `./volumes/nodered/data/settings.js` and follow the writeup on https://nodered.org/docs/user-guide/runtime/securing-node-red for further instructions + + +# Sharing files between Node-RED and the host +Containers run in a sandboxed environment they can't see the host (the Pi itself) file system. This presents a problem if you want to read a file directly to the host from inside the container. Fortunately there is a method, the containers have been set up with volume mapping. The volume maps a specific directory or file from the host file system into the container. Therefore if you write to that directory both the host and the container can see the files. + +Consider the following: + +The docker-compose.yml file shows the following for Node-RED +``` + volumes: + - ./volumes/nodered/data:/data +``` +If inside Node-RED you were to write to the `/data` folder then the host would see it in `~/IOTstack/volumes/nodered/data` (the ./volumes above implies relative to the docker-compose.yml file) + +![image](https://user-images.githubusercontent.com/46672225/68073532-e2e51b80-fd99-11e9-955e-3f27e1c57998.png) + +The flow writes the file `/data/test.txt` and it is visible in the host as `~/IOTstack/volumes/nodered/data/test.txt` + +Remember, files and directories in the volume are persistent between restarts. If you save your data elsewhere it will be destroyed should you restart. Creating a subdirectory in volume i.e. `/data/storage/` would be advised + +# Using Bluetooth +In order to allow Node-RED to access the Pi's Bluetooth module the docker-comose.yml file needs to be modified to allow it access. `network_mode: "host"` needs to be added (make sure the indentation is correct, us spaces not tabs): + +``` + nodered: + container_name: nodered + build: ./services/nodered/. + restart: unless-stopped + user: "0" + network_mode: "host" +``` +By activating host mode the Node-RED container can no longer access containers by name `http://influxdb:8086` will no longer work. Node-RED thinks it now is the host and therefore access to the following services will look as follows: +* influxdb `http://127.0.0.1:8086` +* GPIO `127.0.0.1` port `8888` +* MQTT `127.0.0.1` + +# Unused node in Protainer + +Portainer will report that the nodered image is unsed, this is normal due to the method used build the image. This is normal behavior. It is not advised to remove it as it is used as the base for the iotstack_nodered image, you will need to redownload it should you rebuild the nodered image. + +UnusedImage + +# Running the exec node against the host Pi + +Due to the isolation between containers and the host the exec node will run against the container. + +There is a solution to work around this. You can use ssh to execute a script on the pi. It requires a little setup but is possible. + +For this example I'll be running a simple script called test.sh +I create a file called test.sh in my IOTstack directory with nano + +The contents are as follows: +```bash +#!/bin/bash +echo "hello" +exit 0 +``` + +The exit 0 will stop the exec node from reporting an issue. + +Its a good idea to add the shebang at the top. make it executable with `chmod +x test.sh` + +This nodered running open the nodered terminal with `./services/nodered/terminal.sh` or `docker exec -it nodered /bin/bash` or use portainer + +create the ssh folder in the data directory (the /data directory is persistently mapped volume) + +`mkdir -p /data/ssh` + +create key, this will require naming the output file + +`ssh-keygen -f /data/ssh/nodered` put in any additional config you want key type strength +copy the key to the Pi. When asked for a password leave it blank + +copy the ssh-key to your pi + +`ssh-copy-id -i /data/ssh/nodered pi@192.168.x.x` replace with your static IP address. You will have to reply yes to the prompt. You may also see an error referring to regular expressions however you can ignore it. + +now to execute a script on the pi run `ssh -i /data/ssh/nodered pi@192.168.x.x /home/pi/IOTstack/test.sh` + +type exit to leave the terminal + +(you could also restart your pi with `ssh -i /data/ssh/nodered pi@192.168.x.x sudo reboot`) + +in node red in your exec node you can run the command `ssh -i /data/ssh/nodered pi@192.168.x.x /home/pi/IOTstack/test.sh` other the script or command of your choice + +![image](https://user-images.githubusercontent.com/46672225/70431264-55c27000-1a85-11ea-8706-b087dc39479d.png) + diff --git a/docs/Pi-hole.md b/docs/Pi-hole.md new file mode 100644 index 00000000..a7d3f77f --- /dev/null +++ b/docs/Pi-hole.md @@ -0,0 +1,7 @@ +Pi-hole is a fantastic utility to reduce ads + +The interface can be found on `"your_ip":8089/admin` + +Default password is `pihole`. This can be changed in the `~/IOTstack/services/pihole/pihole.env` file + +To enable your router to use the pihole container edit your DNS settings on your router to point to your Pi's IP address \ No newline at end of file diff --git a/docs/Plex.md b/docs/Plex.md new file mode 100644 index 00000000..e587d71b --- /dev/null +++ b/docs/Plex.md @@ -0,0 +1,11 @@ +# References +* [Homepage](https://www.plex.tv/) +* [Docker](https://hub.docker.com/r/linuxserver/plex/) + +# Web interface +The web UI can be found on `"your_ip":32400/web` + +# Mounting an external drive by UUID to the home directory +[official mounting guide](https://www.raspberrypi.org/documentation/configuration/external-storage.md) + +Create a directory in you home directory called `mnt` with a subdirectory `HDD`. Follow the instruction above to mount your external drive to `/home/pi/mnt/HDD` in you `fstab` edit your docker-compose.yml file under plex and uncomment the volumes for tv series and movies (modify the path to point to your media locations). Run `docker-compose up -d` to rebuild plex with the new volumes \ No newline at end of file diff --git a/docs/Portainer-agent.md b/docs/Portainer-agent.md new file mode 100644 index 00000000..63d392e6 --- /dev/null +++ b/docs/Portainer-agent.md @@ -0,0 +1,18 @@ +# References +- [Docker](https://hub.docker.com/r/portainer/agent) +- [Docs](https://portainer.readthedocs.io/en/stable/agent.html) + +# About + +The protainer agent is a great way to add a second docker instance to a existing portainer instance. this allows you to mananage multiple docker enviroments form one prortainer instance + +# Adding to an existing instance + +When you want to add the the agent to an existing portianer instance. + +* You go to the endpoints tab. +* Click on `Add endpoint` +* Select Agent +* Enter the name of the agent +* Enter the url of the endpoint `ip-of-agent-instance:9001` +* Click on add endpoint \ No newline at end of file diff --git a/docs/Portainer.md b/docs/Portainer.md new file mode 100644 index 00000000..f9badec8 --- /dev/null +++ b/docs/Portainer.md @@ -0,0 +1,25 @@ +# References +- [Docker](https://hub.docker.com/r/portainer/portainer/) +- [Website](https://www.portainer.io/) + +# Portainer restart by itself + +There is an issue with the armhf Portainer image where it randomly restarts. This does not affect its operation. The bug has been reported. + +# About + +Portainer is a great application for managing Docker. In your web browser navigate to `#yourip:9000`. You will be asked to choose a password. In the next window select 'Local' and connect, it shouldn't ask you this again. From here you can play around, click local, and take a look around. This can help you find unused images/containers. On the Containers section, there are 'Quick actions' to view logs and other stats. Note: This can all be done from the CLI but portainer just makes it much much easier. + +# Setup Public IP + +When you first run Portainer and navigate to the Containers list you will see that there is a clickable link to the ports however this will direct you to `0.0.0.0:port`. This is because Portainer doesn't know your IP address. This can be set in the endpoint + +![image](https://user-images.githubusercontent.com/46672225/69695462-26a31a80-10e5-11ea-991d-24b7282c8963.png) + +and set the public IP + +![image](https://user-images.githubusercontent.com/46672225/69695485-3c184480-10e5-11ea-85f7-8385ac339d76.png) + +# Forgotten password + +If you have forgotten the password you created for the container, stop the stack remove portainers volume with `sudo rm -r ./volumes/portainer` and start the stack. Your browser may get a little confused when it restarts. Just navigate to "yourip:9000" (may require more than one attempt) and create your new login details. If it doesn't ask you to connect to the 'Local' docker or shows an empty endpoint just logout and log back in and it will give you the option. From now on it should just work fine. \ No newline at end of file diff --git a/docs/PostgreSQL.md b/docs/PostgreSQL.md new file mode 100644 index 00000000..24dd1976 --- /dev/null +++ b/docs/PostgreSQL.md @@ -0,0 +1,6 @@ +# References +- [Docker](https://hub.docker.com/_/postgres) +- [Website](https://www.postgresql.org/) +# About +PostgreSQL is an SQL server, for those that need an SQL database. The database credentials can be found in the file `./volumes/postgres/postgres.env`. It is highly recommended to change the user, password and default database + diff --git a/docs/Python.md b/docs/Python.md new file mode 100644 index 00000000..2104b3a9 --- /dev/null +++ b/docs/Python.md @@ -0,0 +1,50 @@ +* [Docker hub](https://hub.docker.com/_/python) + +# Running python code in docker + +In order to run code in docker the container needs to be build from a Dockerfile. There are 2 key files in the service directory + +## services/python/requirements.txt + +Normally on your system you would install modules with pip and they would be available system wide. The container that comes off Docker hub is blank and we will have to install them and bake them into the container. Before your first run add the modules that you require to the requirements.txt, each on a new line + +``` +flask +bs4 +``` +**IMPORTANT**: Every time you alter the requirements file you will need to rebuild the container and bake in the new modules + +To build the container run `docker-compose build python`. + +## services/python/service.yml + +This is the template that gets concatenated into docker-compose.yml and there are a few things to note here + +```yml + python: + container_name: python + build: ./services/python/. + restart: unless-stopped + network_mode: host + volumes: + - ./volumes/python/app:/usr/src/app +``` + +The container runs in host network mode. This is because i have no idea which ports you want to use. The implication of this is you will not be able to connect by name to the other container and therefore if you want to connect to the mqtt service or influx you will need to use `localhost` or `127.0.0.1` because the python container "thinks" from network perspective that it is the Pi + +The container is set to restart unless stopped. Therefore if you write an application it will effectively execute in an endless loop. If you only want a run once method then you will need to comment out the "restart" section in the docker-compose.yml file and the service.yml + +# Where to put your code + +You will need to copy your code to `IOTstack/volumes/python/app`. The container is set to execute `app.py` as the main file. + +## writing to the console + +If you execute a print statement the text will appear in the console of the container. The output can be accessed by running `docker logs python` + +## writing to disk +Inside the container the working directory is `/usr/src/app` as mapped in the volume command. It would be advised to read or write any data from this directory. + +# Image clutter + +Doing multiple builds of the python image will create many unused images. These can be cleaned up inside portainer or by running `./scripts/prune-images.sh` diff --git a/docs/RPIEasy_native.md b/docs/RPIEasy_native.md new file mode 100644 index 00000000..086481d5 --- /dev/null +++ b/docs/RPIEasy_native.md @@ -0,0 +1,13 @@ +RPIEasy can now be installed under the native menu + +The installer will install any dependencies. If `~/rpieasy` exists it will update the project to its latest, if not it will clone the project + +# Running Running RPIEasy + +RPIEasy can be run by `sudo ~/rpieasy/RPIEasy.py` + +To have RPIEasy start on boot in the webui under hardware look for "RPIEasy autostart at boot" + +# Ports + +RPIEasy will select its ports from the first available one in the list (80,8080,8008). If you run Hass.io then there will be a conflict so check the next available port \ No newline at end of file diff --git a/docs/RTL_433-docker.md b/docs/RTL_433-docker.md new file mode 100644 index 00000000..0f582d69 --- /dev/null +++ b/docs/RTL_433-docker.md @@ -0,0 +1,28 @@ +Requirements, you will need to have a SDR dongle for you to be able to use RTL. I've tested this with a RTL2838 + +Make sure you can see your receiver by running `lsusb` + +```bash +$ lsusb +Bus 003 Device 001: ID 1d6b:0002 Linux Foundation 2.0 root hub +Bus 002 Device 001: ID 1d6b:0003 Linux Foundation 3.0 root hub +Bus 001 Device 004: ID 0bda:2838 Realtek Semiconductor Corp. RTL2838 DVB-T +Bus 001 Device 002: ID 2109:3431 VIA Labs, Inc. Hub +Bus 001 Device 001: ID 1d6b:0002 Linux Foundation 2.0 root hub + +``` + +Before starting the container please install RTL_433 from the native installs menu. This will setup your environment with the correct variables and programs. It is also advised to run RTL_433 to verify that it is working correctly on your system. + +The container is designed to send all detected messages over mqtt + +Edit the IOTstack/services/rtl_433/rtl_433.env file with your relevant settings for your mqtt server: +``` +MQTT_ADDRESS=mosquitto +MQTT_PORT=1833 +#MQTT_USER=myuser +#MQTT_PASSWORD=mypassword +MQTT_TOPIC=RTL_433 +``` + +the container starts with the command `rtl_433 -F mqtt:....` currently it does not filter any packets, you will need to do this in Node-RED diff --git a/docs/TasmoAdmin.md b/docs/TasmoAdmin.md new file mode 100644 index 00000000..4cb41482 --- /dev/null +++ b/docs/TasmoAdmin.md @@ -0,0 +1,9 @@ +# References +* [Homepage](https://github.com/arendst/Tasmota/wiki/TasmoAdmin) +* [Docker](https://hub.docker.com/r/raymondmm/tasmoadmin/) + +# Web interface +The web UI can be found on `"your_ip":8088` + +# Usage +(instructions to follow) \ No newline at end of file diff --git a/docs/Understanding-Containers.md b/docs/Understanding-Containers.md new file mode 100644 index 00000000..2d21bcc9 --- /dev/null +++ b/docs/Understanding-Containers.md @@ -0,0 +1 @@ +Content coming soon. \ No newline at end of file diff --git a/docs/Updating-the-Project.md b/docs/Updating-the-Project.md new file mode 100644 index 00000000..df836429 --- /dev/null +++ b/docs/Updating-the-Project.md @@ -0,0 +1,20 @@ +**If you ran the git checkout -- 'git ls-files -m' as suggested in the old wiki entry then please check your duck.sh because it removed your domain and token** + + +Periodically updates are made to project which include new or modified container template, changes to backups or additional features. As these are released your local copy of this project will become out of date. This section deals with how to bring your project to the latest published state. + +Git offers build in functionality to fetch the latest changes. + +`git pull origin master` will fetch the latest changes from GitHub without overwriting files that you have modified yourself. If you have done a local commit then your project may to handle a merge conflict. + +This can be verified by running `git status`. You can ignore if it reports duck.sh as being modified. + +![image](https://user-images.githubusercontent.com/46672225/68645804-d42d0000-0521-11ea-842f-fd0b2d22cd0e.png) + +Should you have any modified scripts or templates they can be reset to the latest version with `git checkout -- scripts/ .templates/` + +With the new latest version of the project you can now use the menu to build your stack. If there is a particular container you would like to update its template then you can select that at the overwrite option for your container. You have the choice to not to overwrite, preserve env files or to completely overwrite any changes (passwords) + +![image](https://user-images.githubusercontent.com/46672225/68646024-8fee2f80-0522-11ea-8b6e-f1d439a5be7f.png) + +After your stack had been rebuild you can run `docker-compose up -d` to pull in the latest changes. If you have not update your images in a while consider running the `./scripts/update.sh` to get the latest version of the image from Docker hub as well \ No newline at end of file diff --git a/docs/Zigbee2MQTT.md b/docs/Zigbee2MQTT.md new file mode 100644 index 00000000..df7ae54f --- /dev/null +++ b/docs/Zigbee2MQTT.md @@ -0,0 +1,34 @@ + +* [Web Guide](https://www.zigbee2mqtt.io/information/docker.html) + +# First startup + +After starting the stack check to see if there is an error due to missing device. This is because the devices are mapped differently on the Pi. If your device is not showing in the container then you can also follow the followings steps. + +If you get a startup failure open the docker-compose.yml file and under zigbee2mqtt change this: + +```yml + devices: + - /dev/ttyAMA0:/dev/ttyACM0 + #- /dev/ttyACM0:/dev/ttyACM0 +``` + +to + +```yml + devices: + #- /dev/ttyAMA0:/dev/ttyACM0 + - /dev/ttyACM0:/dev/ttyACM0 +``` + +and run docker-compose up -d again + +If the container starts then run `docker logs zigbe2mqtt` so see the log output and if your device is recognised. You may need to reset the device for docker to see it. + +To edit the configuration file `sudo nano volumes/zigbee2mqtt/data/configuration.yaml` you many need to restart the container for changes to take affect `docker-compose restart zigbee2mqtt` + +Unfortunately I don't own a zigbee device and cannot offer support on the setup you will need to follow the website instructions for further instructions https://www.zigbee2mqtt.io/ + +# terminal access + +to access the terminal run `docker exec -it zigbee2mqtt /bin/sh` or select `/bin/sh \ No newline at end of file diff --git a/docs/diyHue.md b/docs/diyHue.md new file mode 100644 index 00000000..d5722ead --- /dev/null +++ b/docs/diyHue.md @@ -0,0 +1,19 @@ +* [website](https://diyhue.org/getting-started/) + +# About + +diyHue is a utility to contol the lights in your home + +# Setup + +Before you start diyHue you will need to get your IP and MAC addresses. Run `ip addr` in the terminal + +![image](https://user-images.githubusercontent.com/46672225/69816794-c2c24400-1201-11ea-9d97-e8e03b98d9f4.png) + +Enter these values into the `./services/diyhue/diyhue.env` file + +The default username and password it `Hue` and `Hue` respectively + +# Usage + +The web interface is available on port 8070 \ No newline at end of file diff --git a/docs/openHAB.md b/docs/openHAB.md new file mode 100644 index 00000000..2fec435a --- /dev/null +++ b/docs/openHAB.md @@ -0,0 +1,5 @@ +# References +- [Docker](https://hub.docker.com/r/openhab/openhab/) +- [website](https://www.openhab.org/) + +openHAB has been added without Amazon Dashbutton binding. Port binding is `8080` for http and `8443` for https. diff --git a/docs/x2go.md b/docs/x2go.md new file mode 100644 index 00000000..d572bfe4 --- /dev/null +++ b/docs/x2go.md @@ -0,0 +1,23 @@ +x2go is an "alternative" to using VNC for a remote connection. It uses X11 forwarding over ssh to provide a desktop environment + +Reason for using: +I have a Pi 4 and I didn't buy a micro hdmi cable. You can use VNC however you are limited to a 800x600 window. + +# Installation + +Install with `sudo apt install x2goserver` + +x2go cant connect to the native Raspbian Desktop so you will need to install another with `sudo tasksel` + +![image](https://user-images.githubusercontent.com/46672225/69007692-b4df0a00-0949-11ea-82d5-09a6833df186.png) + +I chose Xfce because it is light weight. + +Install the x2go client from their [website](https://wiki.x2go.org/doku.php/download:start) + +Now I have a full-screen client + +![image](https://user-images.githubusercontent.com/46672225/69007780-0045e800-094b-11ea-9626-4947595a016e.png) + +# YouTube tutorial +[Laurence systems](https://www.youtube.com/watch?v=oSuy1TS8FGs) \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..ef1e5725 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,4 @@ +site_name: IOTstack +site_description: 'Docker stack for getting started on IOT on the Raspberry PI' +theme: + name: material From f6571aece4b24ea7afa291831614ad7860ab5156 Mon Sep 17 00:00:00 2001 From: Willem Dekker Date: Sun, 19 Apr 2020 13:05:51 +0200 Subject: [PATCH 19/69] added index.md --- docs/index.md | 198 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 docs/index.md diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..b59ba57e --- /dev/null +++ b/docs/index.md @@ -0,0 +1,198 @@ +# IOTStack + +IOTstack is a builder for docker-compose to easily make and maintain IoT stacks on the Raspberry Pi + +## Announcements + +The bulk of the README has moved to the Wiki. Please check it out [here](https://github.com/gcgarner/IOTstack/wiki) + +* 2019-12-19 Added python container, tweaked update script +* 2019-12-12 modified zigbee2mqtt template file +* 2019-12-12 Added Function to add custom containers to the stack +* 2019-12-12 PR cmskedgell: Added Homebridge +* 2019-12-12 PR 877dev: Added trimming of online backups +* 2019-12-03 BUGFIX Mosquitto: Fixed issue where mosquitto failed to start as a result of 11-28 change +* 2019-12-03 Added terminal for postgres, temporarily removed setfacl from menu +* 2019-11-28 PR @stfnhmplr added diyHue +* 2019-11-28 Fixed update notification on menu +* 2019-11-28 Fixed mosquitto logs and database not mapping correctly to volumes. Pull new template to fix +* 2019-11-28 added the option to disable swapfile by setting swappiness to 0 +* 2019-11-28 PR @stfnhmplr fixed incorrect shegang on MariaDB terminal.sh +* 2019-11-28 Added native install for RPIEasy +* 2019-11-27 Additions: NextCloud, MariaDB, MotionEye, Mozilla Webthings, blynk-server (fixed issue with selection.txt) +* 2019-11-22 BUGFIX selection.txt failed on fresh install, added pushd IOTstack to menu to ensure correct path +* 2019-11-22 Added notification into menu if project update is available +* 2019-11-20 BUGFIX influxdb backup: Placing docker_backup in crontab caused influxdb backup not to execute correctly +* 2019-11-20 BUGFIX disable swap: swapfile recreation on reboot fixed. Re-run from menu to fix. +* Node-RED: serial port. New template adds privileged which allows acces to serial devices +* EspurinoHub: is available for testing see wiki entry + +*** + +## Highlighted topics + +* [Bluetooth and Node-RED](https://github.com/gcgarner/IOTstack/wiki/Node-RED#using-bluetooth) +* [Saving files to disk inside containers](https://github.com/gcgarner/IOTstack/wiki/Node-RED#sharing-files-between-node-red-and-the-host) +* [Updating the Project](https://github.com/gcgarner/IOTstack/wiki/Updating-the-Project) + + *** + +## Coming soon + +* reverse proxy is now next on the list, I cant keep up with the ports +* Detection of arhcitecture for seperate stack options for amd64, armhf, i386 +* autocleanup of backups on cloud +* Gitea (in testing branch) +* OwnCloud + +*** + +## About + +Docker stack for getting started on IoT on the Raspberry Pi. + +This Docker stack consists of: + +* Node-RED +* Grafana +* InfluxDB +* Postgres +* Mosquitto mqtt +* Portainer +* Adminer +* openHAB +* Home Assistant (HASSIO) +* zigbee2mqtt +* Pi-Hole +* TasmoAdmin (parial wiki) +* Plex media server +* Telegraf (wiki coming soon) +* RTL_433 +* EspruinoHub (testing) +* MotionEye +* MariaDB +* Plex +* Homebridge + +In addition, there is a write-up and some scripts to get a dynamic DNS via duckdns and VPN up and running. + +Firstly what is docker? The correct question is "what are containers?". Docker is just one of the utilities to run a container. + +A Container can be thought of as ultra-minimal virtual machines, they are a collection of binaries that run in a sandbox environment. You download a preconfigured base image and create a new container. Only the differences between the base and your "VM" are stored. +Containers don't have [GUI](https://en.wikipedia.org/wiki/Graphical_user_interface)s so generally the way you interact with them is via web services or you can launch into a terminal. +One of the major advantages is that the image comes mostly preconfigured. + +There are pro's and cons for using native installs vs containers. For me, one of the best parts of containers is that it doesn't "clutter" your device. If you don't need Postgres anymore then just stop and delete the container. It will be like the container was never there. + +The container will fail if you try to run the docker and native vesions as the same time. It is best to install this on a fresh system. + +For those looking for a script that installs native applications check out [Peter Scargill's script](https://tech.scargill.net/the-script/) + +## Tested platform + +Raspberry Pi 3B and 4B Raspbian (Buster) + +### Older Pi's + +Docker will not run on a PiZero or A model 1 because of the CPU. It has not been tested on a Model 2. You can still use Peter Scargill's [script](https://tech.scargill.net/the-script/) + +## Running under a virtual machine + +For those wanting to test out the script in a Virtual Machine before installing on their Pi there are some limitations. The script is designed to work with Debian based distributions. Not all the container have x86_64 images. For example Portainer does not and will give an error when you try and start the stack. Please see the pinned issue [#29](https://github.com/gcgarner/IOTstack/issues/29), there is more info there. + +## Feature Requests + +Please direct all feature requests to [Discord](https://discord.gg/W45tD83) + +## Youtube reference + +This repo was originally inspired by Andreas Spiess's video on using some of these tools. Some containers have been added to extend its functionality. + +[YouTube video](https://www.youtube.com/watch?v=JdV4x925au0): This is an alternative approach to the setup. Be sure to watch the video for the instructions. Just note that the network addresses are different, see the wiki under Docker Networks. + +### YouTube guide + +@peyanski (Kiril) made a YouTube video on getting started using the project, check it out [here](https://youtu.be/5JMNHuHv134) + +## Download the project + +1.On the lite image you will need to install git first + +```bash +sudo apt-get install git +``` + +2.Download the repository with: + +```bash +git clone https://github.com/gcgarner/IOTstack.git ~/IOTstack +``` + +Due to some script restraints, this project needs to be stored in ~/IOTstack + +3.To enter the directory run: + +```bash +cd ~/IOTstack +``` + +## The Menu + +I've added a menu to make things easier. It is good to familiarise yourself with the installation process. +The menu can be used to install docker and build the docker-compose.yml file necessary for starting the stack. It also runs a few common commands. I do recommend you start to learn the docker and docker-compose commands if you plan on using docker in the long run. I've added several helper scripts, have a look inside. + +Navigate to the project folder and run `./menu.sh` + +### Installing from the menu + +Select the first option and follow the prompts + +### Build the docker-compose file + +docker-compose uses the `docker-compose.yml` file to configure all the services. Run through the menu to select the options you want to install. + +### Docker commands + +This menu executes shell scripts in the root of the project. It is not necessary to run them from the menu. Open up the shell script files to see what is inside and what they do. + +### Miscellaneous commands + +Some helpful commands have been added like disabling swap. + +## Running Docker commands + +From this point on make sure you are executing the commands from inside the project folder. Docker-compose commands need to be run from the folder where the docker-compose.yml is located. If you want to move the folder make sure you move the whole project folder. + +## Starting and Stopping containers + +to start the stack navigate to the project folder containing the docker-compose.yml file + +To start the stack run: +`docker-compose up -d` or `./scripts/start.sh` + +To stop: +`docker-compose stop` + +The first time you run 'start' the stack docker will download all the images for the web. Depending on how many containers you selected and your internet speed this can take a long while. + +The `docker-compose down` command stops the containers then deletes them. + +## Persistent data + +Docker allows you to map folders inside your containers to folders on the disk. This is done with the "volume" key. There are two types of volumes. Modification to the container are reflected in the volume. + +## See Wiki for further info + +[Wiki](https://github.com/gcgarner/IOTstack/wiki) + +## Add to the project + +Feel free to add your comments on features or images that you think should be added. + +## Contributions + +If you use some of the tools in the project please consider donating or contributing on their projects. It doesn't have to be monetary, reporting bugs and PRs help improve the projects for everyone. + +### Thanks + +@mrmx, @oscrx, @brianimmel, @Slyke, @AugustasV, @Paulf007, @affankingkhan, @877dev, @Paraphraser, @stfnhmplr, @peyanski, @cmskedgell From 55542d46cfeb0426087892a148d49ffa77cd907d Mon Sep 17 00:00:00 2001 From: Willem Dekker <16556575+Willem-Dekker@users.noreply.github.com> Date: Sun, 19 Apr 2020 13:06:56 +0200 Subject: [PATCH 20/69] added index.md (#2) --- docs/index.md | 198 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 docs/index.md diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..b59ba57e --- /dev/null +++ b/docs/index.md @@ -0,0 +1,198 @@ +# IOTStack + +IOTstack is a builder for docker-compose to easily make and maintain IoT stacks on the Raspberry Pi + +## Announcements + +The bulk of the README has moved to the Wiki. Please check it out [here](https://github.com/gcgarner/IOTstack/wiki) + +* 2019-12-19 Added python container, tweaked update script +* 2019-12-12 modified zigbee2mqtt template file +* 2019-12-12 Added Function to add custom containers to the stack +* 2019-12-12 PR cmskedgell: Added Homebridge +* 2019-12-12 PR 877dev: Added trimming of online backups +* 2019-12-03 BUGFIX Mosquitto: Fixed issue where mosquitto failed to start as a result of 11-28 change +* 2019-12-03 Added terminal for postgres, temporarily removed setfacl from menu +* 2019-11-28 PR @stfnhmplr added diyHue +* 2019-11-28 Fixed update notification on menu +* 2019-11-28 Fixed mosquitto logs and database not mapping correctly to volumes. Pull new template to fix +* 2019-11-28 added the option to disable swapfile by setting swappiness to 0 +* 2019-11-28 PR @stfnhmplr fixed incorrect shegang on MariaDB terminal.sh +* 2019-11-28 Added native install for RPIEasy +* 2019-11-27 Additions: NextCloud, MariaDB, MotionEye, Mozilla Webthings, blynk-server (fixed issue with selection.txt) +* 2019-11-22 BUGFIX selection.txt failed on fresh install, added pushd IOTstack to menu to ensure correct path +* 2019-11-22 Added notification into menu if project update is available +* 2019-11-20 BUGFIX influxdb backup: Placing docker_backup in crontab caused influxdb backup not to execute correctly +* 2019-11-20 BUGFIX disable swap: swapfile recreation on reboot fixed. Re-run from menu to fix. +* Node-RED: serial port. New template adds privileged which allows acces to serial devices +* EspurinoHub: is available for testing see wiki entry + +*** + +## Highlighted topics + +* [Bluetooth and Node-RED](https://github.com/gcgarner/IOTstack/wiki/Node-RED#using-bluetooth) +* [Saving files to disk inside containers](https://github.com/gcgarner/IOTstack/wiki/Node-RED#sharing-files-between-node-red-and-the-host) +* [Updating the Project](https://github.com/gcgarner/IOTstack/wiki/Updating-the-Project) + + *** + +## Coming soon + +* reverse proxy is now next on the list, I cant keep up with the ports +* Detection of arhcitecture for seperate stack options for amd64, armhf, i386 +* autocleanup of backups on cloud +* Gitea (in testing branch) +* OwnCloud + +*** + +## About + +Docker stack for getting started on IoT on the Raspberry Pi. + +This Docker stack consists of: + +* Node-RED +* Grafana +* InfluxDB +* Postgres +* Mosquitto mqtt +* Portainer +* Adminer +* openHAB +* Home Assistant (HASSIO) +* zigbee2mqtt +* Pi-Hole +* TasmoAdmin (parial wiki) +* Plex media server +* Telegraf (wiki coming soon) +* RTL_433 +* EspruinoHub (testing) +* MotionEye +* MariaDB +* Plex +* Homebridge + +In addition, there is a write-up and some scripts to get a dynamic DNS via duckdns and VPN up and running. + +Firstly what is docker? The correct question is "what are containers?". Docker is just one of the utilities to run a container. + +A Container can be thought of as ultra-minimal virtual machines, they are a collection of binaries that run in a sandbox environment. You download a preconfigured base image and create a new container. Only the differences between the base and your "VM" are stored. +Containers don't have [GUI](https://en.wikipedia.org/wiki/Graphical_user_interface)s so generally the way you interact with them is via web services or you can launch into a terminal. +One of the major advantages is that the image comes mostly preconfigured. + +There are pro's and cons for using native installs vs containers. For me, one of the best parts of containers is that it doesn't "clutter" your device. If you don't need Postgres anymore then just stop and delete the container. It will be like the container was never there. + +The container will fail if you try to run the docker and native vesions as the same time. It is best to install this on a fresh system. + +For those looking for a script that installs native applications check out [Peter Scargill's script](https://tech.scargill.net/the-script/) + +## Tested platform + +Raspberry Pi 3B and 4B Raspbian (Buster) + +### Older Pi's + +Docker will not run on a PiZero or A model 1 because of the CPU. It has not been tested on a Model 2. You can still use Peter Scargill's [script](https://tech.scargill.net/the-script/) + +## Running under a virtual machine + +For those wanting to test out the script in a Virtual Machine before installing on their Pi there are some limitations. The script is designed to work with Debian based distributions. Not all the container have x86_64 images. For example Portainer does not and will give an error when you try and start the stack. Please see the pinned issue [#29](https://github.com/gcgarner/IOTstack/issues/29), there is more info there. + +## Feature Requests + +Please direct all feature requests to [Discord](https://discord.gg/W45tD83) + +## Youtube reference + +This repo was originally inspired by Andreas Spiess's video on using some of these tools. Some containers have been added to extend its functionality. + +[YouTube video](https://www.youtube.com/watch?v=JdV4x925au0): This is an alternative approach to the setup. Be sure to watch the video for the instructions. Just note that the network addresses are different, see the wiki under Docker Networks. + +### YouTube guide + +@peyanski (Kiril) made a YouTube video on getting started using the project, check it out [here](https://youtu.be/5JMNHuHv134) + +## Download the project + +1.On the lite image you will need to install git first + +```bash +sudo apt-get install git +``` + +2.Download the repository with: + +```bash +git clone https://github.com/gcgarner/IOTstack.git ~/IOTstack +``` + +Due to some script restraints, this project needs to be stored in ~/IOTstack + +3.To enter the directory run: + +```bash +cd ~/IOTstack +``` + +## The Menu + +I've added a menu to make things easier. It is good to familiarise yourself with the installation process. +The menu can be used to install docker and build the docker-compose.yml file necessary for starting the stack. It also runs a few common commands. I do recommend you start to learn the docker and docker-compose commands if you plan on using docker in the long run. I've added several helper scripts, have a look inside. + +Navigate to the project folder and run `./menu.sh` + +### Installing from the menu + +Select the first option and follow the prompts + +### Build the docker-compose file + +docker-compose uses the `docker-compose.yml` file to configure all the services. Run through the menu to select the options you want to install. + +### Docker commands + +This menu executes shell scripts in the root of the project. It is not necessary to run them from the menu. Open up the shell script files to see what is inside and what they do. + +### Miscellaneous commands + +Some helpful commands have been added like disabling swap. + +## Running Docker commands + +From this point on make sure you are executing the commands from inside the project folder. Docker-compose commands need to be run from the folder where the docker-compose.yml is located. If you want to move the folder make sure you move the whole project folder. + +## Starting and Stopping containers + +to start the stack navigate to the project folder containing the docker-compose.yml file + +To start the stack run: +`docker-compose up -d` or `./scripts/start.sh` + +To stop: +`docker-compose stop` + +The first time you run 'start' the stack docker will download all the images for the web. Depending on how many containers you selected and your internet speed this can take a long while. + +The `docker-compose down` command stops the containers then deletes them. + +## Persistent data + +Docker allows you to map folders inside your containers to folders on the disk. This is done with the "volume" key. There are two types of volumes. Modification to the container are reflected in the volume. + +## See Wiki for further info + +[Wiki](https://github.com/gcgarner/IOTstack/wiki) + +## Add to the project + +Feel free to add your comments on features or images that you think should be added. + +## Contributions + +If you use some of the tools in the project please consider donating or contributing on their projects. It doesn't have to be monetary, reporting bugs and PRs help improve the projects for everyone. + +### Thanks + +@mrmx, @oscrx, @brianimmel, @Slyke, @AugustasV, @Paulf007, @affankingkhan, @877dev, @Paraphraser, @stfnhmplr, @peyanski, @cmskedgell From 96454c9680bb45e14b5f27053b2e73a69b7fe382 Mon Sep 17 00:00:00 2001 From: Willem Dekker Date: Sun, 19 Apr 2020 14:07:06 +0200 Subject: [PATCH 21/69] Updated styling of existing pages --- ...Accessing-your-Device-from-the-internet.md | 7 +++-- docs/Adminer.md | 5 ++-- docs/Backups.md | 25 ++++++++-------- docs/Blynk_server.md | 15 +++++----- docs/Custom.md | 1 + docs/Docker-commands.md | 3 +- docs/EspruinoHub.md | 1 + docs/Getting-Started.md | 29 ++++++++++--------- docs/Grafana.md | 7 +++-- docs/Home-Assistant.md | 9 +++--- docs/How-the-script-works.md | 1 + docs/InfluxDB.md | 5 ++-- docs/MariaDB.md | 10 +++---- docs/Mosquitto.md | 5 ++-- docs/MotionEye.md | 13 +++++---- docs/Native-RTL_433.md | 1 + docs/Networking.md | 5 ++-- docs/NextCloud.md | 7 +++-- docs/Node-RED.md | 19 ++++++------ docs/Pi-hole.md | 1 + docs/Plex.md | 7 +++-- docs/Portainer-agent.md | 7 +++-- docs/Portainer.md | 11 +++---- docs/PostgreSQL.md | 5 ++-- docs/Python.md | 15 +++++----- docs/RPIEasy_native.md | 5 ++-- docs/RTL_433-docker.md | 1 + docs/TasmoAdmin.md | 7 +++-- docs/Understanding-Containers.md | 1 + docs/Updating-the-Project.md | 1 + docs/Zigbee2MQTT.md | 6 ++-- docs/diyHue.md | 7 +++-- docs/openHAB.md | 3 +- docs/x2go.md | 5 ++-- 34 files changed, 141 insertions(+), 109 deletions(-) diff --git a/docs/Accessing-your-Device-from-the-internet.md b/docs/Accessing-your-Device-from-the-internet.md index 908a9143..b798ec71 100644 --- a/docs/Accessing-your-Device-from-the-internet.md +++ b/docs/Accessing-your-Device-from-the-internet.md @@ -1,8 +1,9 @@ +# accessing your device from the internet The challenge most of us face with remotely accessing your home network is that you don't have a static IP. From time to time the IP that your ISP assigns to you changes and it's difficult to keep up. Fortunately, there is a solution, a DynamicDNS. The section below shows you how to set up an easy to remember address that follows your public IP no matter when it changes. Secondly, how do you get into your home network? Your router has a firewall that is designed to keep the rest of the internet out of your network to protect you. Here we install a VPN and configure the firewall to only allow very secure VPN traffic in. -# DuckDNS +## DuckDNS If you want to have a dynamic DNS point to your Public IP I added a helper script. Register with duckdns.org and create a subdomain name. Then edit the `nano ~/IOTstack/duck/duck.sh` file and add your @@ -20,7 +21,7 @@ paste the following in the editor `*/5 * * * * sudo ~/IOTstack/duck/duck.sh >/de Your Public IP should be updated every five minutes -# PiVPN +## PiVPN pimylifeup.com has an excellent tutorial on how to install [PiVPN](https://pimylifeup.com/raspberry-pi-vpn-server/) In point 17 and 18 they mention using noip for their dynamic DNS. Here you can use the DuckDNS address if you created one. @@ -31,7 +32,7 @@ Once you activate your VPN (from your phone/laptop/work computer) you will effec I personally use the VPN any time I'm on public wifi, all your traffic is secure. -# Zerotier +## Zerotier https://www.zerotier.com/ Zerotier is an alternative to PiVPN that doesn't require port forwarding on your router. It does however require registering for their free tier service [here](https://my.zerotier.com/login). diff --git a/docs/Adminer.md b/docs/Adminer.md index e469e927..2df6a824 100644 --- a/docs/Adminer.md +++ b/docs/Adminer.md @@ -1,8 +1,9 @@ -# References +# Adminer +## References - [Docker](https://hub.docker.com/_/adminer) - [Website](https://www.adminer.org/) -# About +## About This is a nice tool for managing databases. Web interface has moved to port 9080. There was an issue where openHAB and Adminer were using the same ports. If you have an port conflict edit the docker-compose.yml and under the adminer service change the line to read: ``` diff --git a/docs/Backups.md b/docs/Backups.md index aa47fdcf..ddb657c9 100644 --- a/docs/Backups.md +++ b/docs/Backups.md @@ -1,9 +1,10 @@ +# Backups Because containers can easily be rebuilt from docker hub we only have to back up the data in the "volumes" directory. -# Cloud Backups -## Dropbox-Uploader +## Cloud Backups +### Dropbox-Uploader This a great utility to easily upload data from your Pi to the cloud. https://magpi.raspberrypi.org/articles/dropbox-raspberry-pi. It can be installed from the Menu under Backups. -## rclone (Google Drive) +### rclone (Google Drive) This is a service to upload to Google Drive. The config is described [here]( https://medium.com/@artur.klauser/mounting-google-drive-on-raspberry-pi-f5002c7095c2). Install it from the menu then follow the link for these sections: * Getting a Google Drive Client ID * Setting up the Rclone Configuration @@ -15,10 +16,10 @@ When naming the service in `rclone config` ensure to call it "gdrive" If you want your Google Drive to mount on every boot then follow the instructions at the bottom of the wiki page -# Influxdb +## Influxdb `~/IOTstack/scripts/backup_influxdb.sh` does a database snapshot and stores it in ~/IOTstack/backups/influxdb/db . This can be restored with the help a script (that I still need to write) -# Docker backups +## Docker backups The script `~/IOTstack/scripts/docker_backup.sh` performs the master backup for the stack. This script can be placed in a cron job to backup on a schedule. @@ -27,20 +28,20 @@ Then add `0 23 * * * ~/IOTstack/scripts/docker_backup.sh >/dev/null 2>&1` to hav This script cheats by copying the volume folder live. The correct way would be to stop the stack first then copy the volumes and restart. The cheating method shouldn't be a problem unless you have fast changing data like in influxdb. This is why the script makes a database export of influxdb and ignores its volume. -## Cloud integration +### Cloud integration The docker_backup.sh script now no longer requires modification to enable cloud backups. It now tests for the presence of and enable file in the backups folder -### Drobox-Uploader +#### Drobox-Uploader The backup tests for a file called `~/IOTstack/backups/dropbox`, if it is present it will upload to dropbox. To disable dropbox upload delete the file. To enable run `sudo touch ~/IOTstack/backups/dropbox` -### rclone +#### rclone The backup tests for a file called `~/IOTstack/backups/rclone`, if it is present it will upload to google drive. To disable rclone upload delete the file. To enable run `sudo touch ~/IOTstack/backups/rclone` -### Pruning online backups +#### Pruning online backups @877dev has added functionality to prune both local and cloud backups. For dropbox make sure you dont have any files that contain spaces in your backup directory as the script cannot handle it at this time. -## Restoring a backup +### Restoring a backup The "volumes" directory contains all the persistent data necessary to recreate the container. The docker-compose.yml and the environment files are optional as they can be regenerated with the menu. Simply copy the volumes directory into the IOTstack directory, Rebuild the stack and start. -# Added your Dropbox token incorrectly or aborted the install at the token screen +## Added your Dropbox token incorrectly or aborted the install at the token screen Make sure you are running the latest version of the project [link](https://github.com/gcgarner/IOTstack/wiki/Updating-the-Project). @@ -70,7 +71,7 @@ Ensure you **are not** running as sudo as this will store your api in the /root If you ran the command with sudo the remove the old token file if it exists with either `sudo rm /root/.dropbox_uploader` or `sudo ~/Dropbox-Uploader/dropbox_uploader.sh unlink` -# Auto-mount Gdrive with rclone +## Auto-mount Gdrive with rclone To enable rclone to mount on boot you will need to make a user service. Run the following commands diff --git a/docs/Blynk_server.md b/docs/Blynk_server.md index 7b3d7759..e255d518 100644 --- a/docs/Blynk_server.md +++ b/docs/Blynk_server.md @@ -1,3 +1,4 @@ +# Blynk server This is a custom implementation of Blynk Server ```yml @@ -21,7 +22,7 @@ When setting up the application on your mobile be sure to select custom setup [h Writeup From @877dev -**Getting started:** +## Getting started Log into admin panel at https://youripaddress:9443/admin (Use your Pi's IP address, and ignore Chrome warning). @@ -29,22 +30,22 @@ Default credentials: user:admin@blynk.cc pass:admin -**Change username and password:** +## Change username and password Click on Users > "email address" and edit email, name and password. Save changes Restarting the container using Portainer may be required to take effect. -**Setup gmail:** +## Setup gmail Optional step, useful for getting the auth token emailed to you. (To be added once confirmed working....) -**iOS/Android app setup:** +## iOS/Android app setup Login the app as per the photos [HERE](https://github.com/blynkkk/blynk-server#app-and-sketch-changes) Press "New Project" Give it a name, choose device "Raspberry Pi 3 B" so you have plenty of [virtual pins](http://help.blynk.cc/en/articles/512061-what-is-virtual-pins) available, and lastly select WiFi. Create project and the [auth token](https://docs.blynk.cc/#getting-started-getting-started-with-the-blynk-app-4-auth-token) will be emailed to you (if emails configured). You can also find the token in app under the phone app settings, or in the admin web interface by clicking Users>"email address" and scroll down to token. -**Quick usage guide for app:** +## Quick usage guide for app Press on the empty page, the widgets will appear from the right. Select your widget, let's say a button. It appears on the page, press on it to configure. @@ -55,7 +56,7 @@ To start the project running, press top right Play button. You will get an offline message, because no devices are connected to your project via the token. Enter node red..... -**Node red:** +## Node red Install node-red-contrib-blynk-ws from pallette manager Drag a "write event" node into your flow, and connect to a debug node Configure the Blynk node for the first time: @@ -64,7 +65,7 @@ Enter your [auth token](https://docs.blynk.cc/#getting-started-getting-started-w When you deploy the flow, notice the app shows connected message, as does the Blynk node. Press the button on the app, you will notice the payload is sent to the debug node. -**What next?** +## What next? Further information and advanced setup: https://github.com/blynkkk/blynk-server diff --git a/docs/Custom.md b/docs/Custom.md index b86caf44..1d84b018 100644 --- a/docs/Custom.md +++ b/docs/Custom.md @@ -1,3 +1,4 @@ +# Custom container If you have a container that you want to stop and start with the stack you can now use the custom container option. This you can use for testing or in prep for a Pull Request. You will need to create a directory for your container call `IOTstack/services/` diff --git a/docs/Docker-commands.md b/docs/Docker-commands.md index b0989987..6965c9c2 100644 --- a/docs/Docker-commands.md +++ b/docs/Docker-commands.md @@ -1,4 +1,5 @@ -# Aliases +# Docker commands +## Aliases I've added bash aliases for stopping and starting the stack. They can be installed in the docker commands menu. These commands no longer need to be executed from the IOTstack directory and can be executed in any directory diff --git a/docs/EspruinoHub.md b/docs/EspruinoHub.md index 1db2ca53..215a0de5 100644 --- a/docs/EspruinoHub.md +++ b/docs/EspruinoHub.md @@ -1,3 +1,4 @@ +# Espruinohub This is a testing container I tried it however the container keeps restarting `docker logs espruinohub` I get "BLE Broken?" but could just be i dont have any BLE devices nearby diff --git a/docs/Getting-Started.md b/docs/Getting-Started.md index da733f54..b04375ad 100644 --- a/docs/Getting-Started.md +++ b/docs/Getting-Started.md @@ -1,4 +1,5 @@ -# Download the project +# Getting started +## Download the project On the lite image you will need to install git first ``` @@ -14,28 +15,28 @@ To enter the directory run: ``` cd ~/IOTstack ``` -# The Menu +## The Menu I've added a menu to make things easier. It is good however to familiarise yourself with how things are installed. The menu can be used to install docker and then build the docker-compose.yml file necessary for starting the stack and it runs a few common commands. I do recommend you start to learn the docker and docker-compose commands if you plan using docker in the long run. I've added several helper scripts, have a look inside. Navigate to the project folder and run `./menu.sh` -## Installing from the menu +### Installing from the menu Select the first option and follow the prompts -## Build the docker-compose file +### Build the docker-compose file docker-compose uses the `docker-compose.yml` file to configure all the services. Run through the menu to select the options you want to install. -## Docker commands +### Docker commands This menu executes shell scripts in the root of the project. It is not necessary to run them from the menu. Open up the shell script files to see what is inside and what they do -## Miscellaneous commands +### Miscellaneous commands Some helpful commands have been added like disabling swap -# Running Docker commands +## Running Docker commands From this point on make sure you are executing the commands from inside the project folder. Docker-compose commands need to be run from the folder where the docker-compose.yml is located. If you want to move the folder make sure you move the whole project folder. -## Starting and Stopping containers +### Starting and Stopping containers to start the stack navigate to the project folder containing the docker-compose.yml file To start the stack run: @@ -49,13 +50,13 @@ To remove the stack: The first time you run 'start' the stack docker will download all the images for the web. Depending on how many containers you selected and your internet speed this can take a long while. -## Persistent data +### Persistent data Docker allows you to map folders inside your containers to folders on the disk. This is done with the "volume" key. There are two types of volumes. Any modification to the container reflects in the volume. -### Sharing files between the Pi and containers +#### Sharing files between the Pi and containers Have a look a the wiki on how to share files between Node-RED and the Pi. [Wiki](https://github.com/gcgarner/IOTstack/wiki/Node-RED#sharing-files-between-node-red-and-the-host) -## Updating the images +### Updating the images If a new version of a container image is available on docker hub it can be updated by a pull command. Use the `docker-compose stop` command to stop the stack @@ -68,14 +69,14 @@ Start the new stack based on the updated images `docker-compose up -d` -## Node-RED error after modifications to setup files +### Node-RED error after modifications to setup files The Node-RED image differs from the rest of the images in this project. It uses the "build" key. It uses a dockerfile for the setup to inject the nodes for pre-installation. If you get an error for Node-RED run `docker-compose build` then `docker-compose up -d` -## Deleting containers, volumes and images +### Deleting containers, volumes and images `./prune-images.sh` will remove all images not associated with a container. If you run it while the stack is up it will ignore any in-use images. If you run this while you stack is down it will delete all images and you will have to redownload all images from scratch. This command can be helpful to reclaim disk space after updating your images, just make sure to run it while your stack is running as not to delete the images in use. (your data will still be safe in your volume mapping) -## Deleting folder volumes +### Deleting folder volumes If you want to delete the influxdb data folder run the following command `sudo rm -r volumes/influxdb/`. Only the data folder is deleted leaving the env file intact. review the docker-compose.yml file to see where the file volumes are stored. You can use git to delete all files and folders to return your folder to the freshly cloned state, AS IN YOU WILL LOSE ALL YOUR DATA. diff --git a/docs/Grafana.md b/docs/Grafana.md index 198ef4ac..8194f1c6 100644 --- a/docs/Grafana.md +++ b/docs/Grafana.md @@ -1,11 +1,12 @@ -# References +# Grafana +## References - [Docker](https://hub.docker.com/r/grafana/grafana) - [Website](https://grafana.com/) -# Security +## Security Grafana's default credentials are username "admin" password "admin" it will ask you to choose a new password on boot. Go to `:3000` in your web browser. -# Overwriting grafana.ini settings +## Overwriting grafana.ini settings A list of the settings available in grafana.ini are listed [here](https://grafana.com/docs/installation/configuration/) diff --git a/docs/Home-Assistant.md b/docs/Home-Assistant.md index 4634c969..40f4d679 100644 --- a/docs/Home-Assistant.md +++ b/docs/Home-Assistant.md @@ -1,19 +1,20 @@ -# References +# Home assistant +## References - [Docker](https://hub.docker.com/r/homeassistant/home-assistant/) - [Webpage](https://www.home-assistant.io/) Hass.io is a home automation platform running on Python 3. It is able to track and control all devices at home and offer a platform for automating control. Port binding is `8123`. Hass.io is exposed to your hosts' network in order to discover devices on your LAN. That means that it does not sit inside docker's network. -# Menu installation +## Menu installation Hass.io now has a seperate installation in the menu. The old version was incorrect and should be removed. Be sure to update you project and install the correct version. You will be asked to select you device type during the installation. Hass.io is no longer dependant on the IOTstack, it has its own service for maintaining its uptime. -# Installation +## Installation The installation of Hass.io takes up to 20 minutes (depending on your internet connection). Refrain from restarting your Pi until it had come online and you are able to create a user account -# Removal +## Removal To remove Hass.io you first need to stop the service that controls it. Run the following in the terminal: diff --git a/docs/How-the-script-works.md b/docs/How-the-script-works.md index 4ce4c754..54dddd9b 100644 --- a/docs/How-the-script-works.md +++ b/docs/How-the-script-works.md @@ -1,3 +1,4 @@ +# How the script works The build script creates the ./services directory and populates it from the template file in .templates . The script then appends the text withing each service.yml file to the docker-compose.yml . When the stack is rebuild the menu doesn not overwrite the service folder if it already exists. Make sure to sync any alterations you have made to the docker-compose.yml file with the respective service.yml so that on your next build your changes pull through. The .gitignore file is setup such that if you do a `git pull origin master` it does not overwrite the files you have already created. Because the build script does not overwrite your service directory any changes in the .templates directory will have no affect on the services you have already made. You will need to move your service folder out to get the latest version of the template. diff --git a/docs/InfluxDB.md b/docs/InfluxDB.md index 2a10eb8e..4f714771 100644 --- a/docs/InfluxDB.md +++ b/docs/InfluxDB.md @@ -1,8 +1,9 @@ -# References +# InfluxDB +## References - [Docker](https://hub.docker.com/_/influxdb) - [Website](https://www.influxdata.com/) -# Security +## Security The credentials and default database name for influxdb are stored in the file called influxdb/influx.env . The default username and password is set to "nodered" for both. It is HIGHLY recommended that you change them. The environment file contains several commented out options allowing you to set several access options such as default admin user credentials as well as the default database name. Any change to the environment file will require a restart of the service. To access the terminal for influxdb execute `./services/influxdb/terminal.sh`. Here you can set additional parameters or create other databases. diff --git a/docs/MariaDB.md b/docs/MariaDB.md index 9b4928ee..0b2408ac 100644 --- a/docs/MariaDB.md +++ b/docs/MariaDB.md @@ -1,23 +1,23 @@ -# Source +## Source * [Docker hub](https://hub.docker.com/r/linuxserver/mariadb/) * [Webpage](https://mariadb.org/) -# About +## About MariaDB is a fork of MySQL. This is an unofficial image provided by linuxserver.io because there is no official image for arm -# Conneting to the DB +## Conneting to the DB The port is 3306. It exists inside the docker network so you can connect via `mariadb:3306` for internal connections. For external connections use `:3306` ![image](https://user-images.githubusercontent.com/46672225/69734358-7f030800-1137-11ea-9874-7d2c86b3d239.png) -# Setup +## Setup Before starting the stack edit the `./services/mariadb/mariadb.env` file and set your access details. This is optional however you will only have one shot at the preconfig. If you start the container without setting the passwords then you will have to either delete its volume directory or enter the terminal and change manually The env file has three commented fields for credentials, either **all three** must be commented or un-commented. You can't have only one or two, its all or nothing. -# Terminal +## Terminal A terminal is provided to access mariadb by the cli. execute `./services/maraidb/terminal.sh`. You will need to run `mysql -uroot -p` to enter mariadbs interface \ No newline at end of file diff --git a/docs/Mosquitto.md b/docs/Mosquitto.md index 6ea3415c..05a197e2 100644 --- a/docs/Mosquitto.md +++ b/docs/Mosquitto.md @@ -1,10 +1,11 @@ -# References +# Mosquitto +## References - [Docker](https://hub.docker.com/_/eclipse-mosquitto) - [Website](https://mosquitto.org/) [Setting up passwords](https://www.youtube.com/watch?v=1msiFQT_flo) -# Security +## Security By default, the Mosquitto container has no password. You can leave it that way if you like but its always a good idea to secure your services. Step 1 diff --git a/docs/MotionEye.md b/docs/MotionEye.md index 596625bf..920cada6 100644 --- a/docs/MotionEye.md +++ b/docs/MotionEye.md @@ -1,12 +1,13 @@ -# References +# MotionQye +## References * [Website](https://github.com/ccrisan/motioneye/wiki/Install-In-Docker) -# About +## About MotionEye is a camera/webcam package. The port is set to 8765 -# Config +## Config This is the yml entry. Notice that the devices is commented out. This is because if you don't have a camera attached then it will fail to start. Uncomment if you need to. This is for a Pi camera, you will need to add additional lines for usb cameras @@ -26,11 +27,11 @@ This is the yml entry. Notice that the devices is commented out. This is because # - "/dev/video0:/dev/video0" ``` -# Login Details +## Login Details On first login you will be asked for login details. The default user is `admin` (all lowercase) with no password -# Storage +## Storage By default local camera data will be stored in `/var/lib/motioneye/camera_name` in the container which equates to the following: @@ -38,6 +39,6 @@ By default local camera data will be stored in `/var/lib/motioneye/camera_name` ![image](https://user-images.githubusercontent.com/46672225/69735408-4fed9600-1139-11ea-8618-f5b6c0064f27.png) -## Remote motioneye +### Remote motioneye If you have connected to a remote motion eye note that the directory is on that device and has nothing to do with the container. \ No newline at end of file diff --git a/docs/Native-RTL_433.md b/docs/Native-RTL_433.md index c20c7cd7..659e7677 100644 --- a/docs/Native-RTL_433.md +++ b/docs/Native-RTL_433.md @@ -1,3 +1,4 @@ +# Native RTL_433 RTL_433 can be installed from the "Native install sections" [This video](https://www.youtube.com/watch?v=L0fSEbGEY-Q&t=386s) demonstrates how to use RTL_433 \ No newline at end of file diff --git a/docs/Networking.md b/docs/Networking.md index 00d9695e..cdd60e0f 100644 --- a/docs/Networking.md +++ b/docs/Networking.md @@ -1,3 +1,4 @@ +# Networking The docker-compose instruction creates an internal network for the containers to communicate in, the ports get exposed to the PI's IP address when you want to connect from outside. It also creates a "DNS" the name being the container name. So it is important to note that when one container talks to another they talk by name. All the containers names are lowercase like nodered, influxdb... An easy way to find out your IP is by typing `ip address` in the terminal and look next to eth0 or wlan0 for your IP. It is highly recommended that you set a static IP for your PI or at least reserve an IP on your router so that you know it @@ -6,12 +7,12 @@ Check the docker-compose.yml to see which ports have been used ![net](https://user-images.githubusercontent.com/46672225/66702353-0bcc4080-ed07-11e9-994b-62219f50b096.png) -# Examples +## Examples - You want to connect your nodered to your mqtt server. In nodered drop an mqtt node, when you need to specify the address type `mosquitto` - You want to connect to your influxdb from grafana. You are in the Docker network and you need to use the name of the Container. The address you specify in the grafana is `http://influxdb:8086` - You want to connect to the web interface of grafana from your laptop. Now you are outside the container environment you type PI's IP eg 192.168.n.m:3000 -# Ports +## Ports Many containers try to use popular ports such as 80,443,8080. For example openHAB and Adminer both want to use port 8080 for their web interface. Adminer's port has been moved 9080 to accommodate this. Please check the description of the container in the README to see if there are any changes as they may not be the same as the port you are used to. Port mapping is done in the docker-compose.yml file. Each service should have a section that reads like this: diff --git a/docs/NextCloud.md b/docs/NextCloud.md index d6c55bcb..ae473839 100644 --- a/docs/NextCloud.md +++ b/docs/NextCloud.md @@ -1,14 +1,15 @@ -# DO NOT EXPOSE PORT 80 TO THE WEB +# Next Cloud +## DO NOT EXPOSE PORT 80 TO THE WEB It is a very bad idea to expose unencrypted traffic to the web. You will need to use a reverse-proxy to ensure your password is not stolen and your account hacked. I'm still working on getting a good encrypted reverse proxy working. However in the interim you can use a VPN tunnel like OpenVPN or Zerotier to securely connect to your private cloud -# Backups +## Backups Nextcloud has been excluded from the docker_backup script due to its potential size. Once I've found a better way of backing it up I will add a dedicated script for it. -# Setup +## Setup Next-Cloud recommends using MySQL/MariaDB for the accounts and file list. The alternative is to use SQLite however they strongly discourage using it diff --git a/docs/Node-RED.md b/docs/Node-RED.md index 316ef2de..ee4cb179 100644 --- a/docs/Node-RED.md +++ b/docs/Node-RED.md @@ -1,14 +1,15 @@ -# References +# Node-RED +## References - [Docker](https://hub.docker.com/r/nodered/node-red) - [website](https://nodered.org/) -# Build warning +## Build warning The Node-RED build will complain about several issues. This is completely normal behaviour. -# SQLite +## SQLite Thanks to @fragolinux the SQLite node will install now. **WARNING it will output many error and will look as if it has gotten stuck. Just give it time and it will continue.** -# GPIO +## GPIO To communicate to your Pi's GPIO you need to use the `node-red-node-pi-gpiod` node. It allowes you to connect to multiple Pis from the same nodered service. You need to make sure that pigpdiod is running. The recommended method is listed [here](https://github.com/node-red/node-red-nodes/tree/master/hardware/pigpiod) @@ -23,14 +24,14 @@ sudo apt-get install pigpio python-pigpio python3-pigpio Drop the gpio node and use your Pi's IP. Example: 192.168.1.123 (127.0.0.1 won't work because this is the local address of every computer'.) -# Securing Node-RED +## Securing Node-RED To secure Node-RED you need a password hash. There is a terminal script `./services/nodered/terminal.sh` execute it to get into the terminal. Copy the helper text `node -e ..... PASSWORD`, paste it and change your password to get a hash. Open the file `./volumes/nodered/data/settings.js` and follow the writeup on https://nodered.org/docs/user-guide/runtime/securing-node-red for further instructions -# Sharing files between Node-RED and the host +## Sharing files between Node-RED and the host Containers run in a sandboxed environment they can't see the host (the Pi itself) file system. This presents a problem if you want to read a file directly to the host from inside the container. Fortunately there is a method, the containers have been set up with volume mapping. The volume maps a specific directory or file from the host file system into the container. Therefore if you write to that directory both the host and the container can see the files. Consider the following: @@ -48,7 +49,7 @@ The flow writes the file `/data/test.txt` and it is visible in the host as `~/IO Remember, files and directories in the volume are persistent between restarts. If you save your data elsewhere it will be destroyed should you restart. Creating a subdirectory in volume i.e. `/data/storage/` would be advised -# Using Bluetooth +## Using Bluetooth In order to allow Node-RED to access the Pi's Bluetooth module the docker-comose.yml file needs to be modified to allow it access. `network_mode: "host"` needs to be added (make sure the indentation is correct, us spaces not tabs): ``` @@ -64,13 +65,13 @@ By activating host mode the Node-RED container can no longer access containers b * GPIO `127.0.0.1` port `8888` * MQTT `127.0.0.1` -# Unused node in Protainer +## Unused node in Protainer Portainer will report that the nodered image is unsed, this is normal due to the method used build the image. This is normal behavior. It is not advised to remove it as it is used as the base for the iotstack_nodered image, you will need to redownload it should you rebuild the nodered image. UnusedImage -# Running the exec node against the host Pi +## Running the exec node against the host Pi Due to the isolation between containers and the host the exec node will run against the container. diff --git a/docs/Pi-hole.md b/docs/Pi-hole.md index a7d3f77f..7842f79b 100644 --- a/docs/Pi-hole.md +++ b/docs/Pi-hole.md @@ -1,3 +1,4 @@ +# Pi-hole Pi-hole is a fantastic utility to reduce ads The interface can be found on `"your_ip":8089/admin` diff --git a/docs/Plex.md b/docs/Plex.md index e587d71b..7b196995 100644 --- a/docs/Plex.md +++ b/docs/Plex.md @@ -1,11 +1,12 @@ -# References +# Plex +## References * [Homepage](https://www.plex.tv/) * [Docker](https://hub.docker.com/r/linuxserver/plex/) -# Web interface +## Web interface The web UI can be found on `"your_ip":32400/web` -# Mounting an external drive by UUID to the home directory +## Mounting an external drive by UUID to the home directory [official mounting guide](https://www.raspberrypi.org/documentation/configuration/external-storage.md) Create a directory in you home directory called `mnt` with a subdirectory `HDD`. Follow the instruction above to mount your external drive to `/home/pi/mnt/HDD` in you `fstab` edit your docker-compose.yml file under plex and uncomment the volumes for tv series and movies (modify the path to point to your media locations). Run `docker-compose up -d` to rebuild plex with the new volumes \ No newline at end of file diff --git a/docs/Portainer-agent.md b/docs/Portainer-agent.md index 63d392e6..0fbaf182 100644 --- a/docs/Portainer-agent.md +++ b/docs/Portainer-agent.md @@ -1,12 +1,13 @@ -# References +# Portainer agent +## References - [Docker](https://hub.docker.com/r/portainer/agent) - [Docs](https://portainer.readthedocs.io/en/stable/agent.html) -# About +## About The protainer agent is a great way to add a second docker instance to a existing portainer instance. this allows you to mananage multiple docker enviroments form one prortainer instance -# Adding to an existing instance +## Adding to an existing instance When you want to add the the agent to an existing portianer instance. diff --git a/docs/Portainer.md b/docs/Portainer.md index f9badec8..14afb0f1 100644 --- a/docs/Portainer.md +++ b/docs/Portainer.md @@ -1,16 +1,17 @@ -# References +# Portainer +## References - [Docker](https://hub.docker.com/r/portainer/portainer/) - [Website](https://www.portainer.io/) -# Portainer restart by itself +## Portainer restart by itself There is an issue with the armhf Portainer image where it randomly restarts. This does not affect its operation. The bug has been reported. -# About +## About Portainer is a great application for managing Docker. In your web browser navigate to `#yourip:9000`. You will be asked to choose a password. In the next window select 'Local' and connect, it shouldn't ask you this again. From here you can play around, click local, and take a look around. This can help you find unused images/containers. On the Containers section, there are 'Quick actions' to view logs and other stats. Note: This can all be done from the CLI but portainer just makes it much much easier. -# Setup Public IP +## Setup Public IP When you first run Portainer and navigate to the Containers list you will see that there is a clickable link to the ports however this will direct you to `0.0.0.0:port`. This is because Portainer doesn't know your IP address. This can be set in the endpoint @@ -20,6 +21,6 @@ and set the public IP ![image](https://user-images.githubusercontent.com/46672225/69695485-3c184480-10e5-11ea-85f7-8385ac339d76.png) -# Forgotten password +## Forgotten password If you have forgotten the password you created for the container, stop the stack remove portainers volume with `sudo rm -r ./volumes/portainer` and start the stack. Your browser may get a little confused when it restarts. Just navigate to "yourip:9000" (may require more than one attempt) and create your new login details. If it doesn't ask you to connect to the 'Local' docker or shows an empty endpoint just logout and log back in and it will give you the option. From now on it should just work fine. \ No newline at end of file diff --git a/docs/PostgreSQL.md b/docs/PostgreSQL.md index 24dd1976..8452f03b 100644 --- a/docs/PostgreSQL.md +++ b/docs/PostgreSQL.md @@ -1,6 +1,7 @@ -# References +# PostgreSQL +## References - [Docker](https://hub.docker.com/_/postgres) - [Website](https://www.postgresql.org/) -# About +## About PostgreSQL is an SQL server, for those that need an SQL database. The database credentials can be found in the file `./volumes/postgres/postgres.env`. It is highly recommended to change the user, password and default database diff --git a/docs/Python.md b/docs/Python.md index 2104b3a9..5948dcb8 100644 --- a/docs/Python.md +++ b/docs/Python.md @@ -1,10 +1,11 @@ +# Python * [Docker hub](https://hub.docker.com/_/python) -# Running python code in docker +## Running python code in docker In order to run code in docker the container needs to be build from a Dockerfile. There are 2 key files in the service directory -## services/python/requirements.txt +### services/python/requirements.txt Normally on your system you would install modules with pip and they would be available system wide. The container that comes off Docker hub is blank and we will have to install them and bake them into the container. Before your first run add the modules that you require to the requirements.txt, each on a new line @@ -16,7 +17,7 @@ bs4 To build the container run `docker-compose build python`. -## services/python/service.yml +### services/python/service.yml This is the template that gets concatenated into docker-compose.yml and there are a few things to note here @@ -34,17 +35,17 @@ The container runs in host network mode. This is because i have no idea which po The container is set to restart unless stopped. Therefore if you write an application it will effectively execute in an endless loop. If you only want a run once method then you will need to comment out the "restart" section in the docker-compose.yml file and the service.yml -# Where to put your code +## Where to put your code You will need to copy your code to `IOTstack/volumes/python/app`. The container is set to execute `app.py` as the main file. -## writing to the console +### writing to the console If you execute a print statement the text will appear in the console of the container. The output can be accessed by running `docker logs python` -## writing to disk +### writing to disk Inside the container the working directory is `/usr/src/app` as mapped in the volume command. It would be advised to read or write any data from this directory. -# Image clutter +## Image clutter Doing multiple builds of the python image will create many unused images. These can be cleaned up inside portainer or by running `./scripts/prune-images.sh` diff --git a/docs/RPIEasy_native.md b/docs/RPIEasy_native.md index 086481d5..1fb8bb8e 100644 --- a/docs/RPIEasy_native.md +++ b/docs/RPIEasy_native.md @@ -1,13 +1,14 @@ +# RPIEasy RPIEasy can now be installed under the native menu The installer will install any dependencies. If `~/rpieasy` exists it will update the project to its latest, if not it will clone the project -# Running Running RPIEasy +## Running Running RPIEasy RPIEasy can be run by `sudo ~/rpieasy/RPIEasy.py` To have RPIEasy start on boot in the webui under hardware look for "RPIEasy autostart at boot" -# Ports +## Ports RPIEasy will select its ports from the first available one in the list (80,8080,8008). If you run Hass.io then there will be a conflict so check the next available port \ No newline at end of file diff --git a/docs/RTL_433-docker.md b/docs/RTL_433-docker.md index 0f582d69..2df2016b 100644 --- a/docs/RTL_433-docker.md +++ b/docs/RTL_433-docker.md @@ -1,3 +1,4 @@ +# RTL_433 Docker Requirements, you will need to have a SDR dongle for you to be able to use RTL. I've tested this with a RTL2838 Make sure you can see your receiver by running `lsusb` diff --git a/docs/TasmoAdmin.md b/docs/TasmoAdmin.md index 4cb41482..2d2c3c1e 100644 --- a/docs/TasmoAdmin.md +++ b/docs/TasmoAdmin.md @@ -1,9 +1,10 @@ -# References +# TasmoAdmin +## References * [Homepage](https://github.com/arendst/Tasmota/wiki/TasmoAdmin) * [Docker](https://hub.docker.com/r/raymondmm/tasmoadmin/) -# Web interface +## Web interface The web UI can be found on `"your_ip":8088` -# Usage +## Usage (instructions to follow) \ No newline at end of file diff --git a/docs/Understanding-Containers.md b/docs/Understanding-Containers.md index 2d21bcc9..4b7ebee6 100644 --- a/docs/Understanding-Containers.md +++ b/docs/Understanding-Containers.md @@ -1 +1,2 @@ +# Understanding containers Content coming soon. \ No newline at end of file diff --git a/docs/Updating-the-Project.md b/docs/Updating-the-Project.md index df836429..5d0aeff0 100644 --- a/docs/Updating-the-Project.md +++ b/docs/Updating-the-Project.md @@ -1,3 +1,4 @@ +# Updating the project **If you ran the git checkout -- 'git ls-files -m' as suggested in the old wiki entry then please check your duck.sh because it removed your domain and token** diff --git a/docs/Zigbee2MQTT.md b/docs/Zigbee2MQTT.md index df7ae54f..df958533 100644 --- a/docs/Zigbee2MQTT.md +++ b/docs/Zigbee2MQTT.md @@ -1,7 +1,7 @@ - +# Zigbe2MQTT * [Web Guide](https://www.zigbee2mqtt.io/information/docker.html) -# First startup +## First startup After starting the stack check to see if there is an error due to missing device. This is because the devices are mapped differently on the Pi. If your device is not showing in the container then you can also follow the followings steps. @@ -29,6 +29,6 @@ To edit the configuration file `sudo nano volumes/zigbee2mqtt/data/configuration Unfortunately I don't own a zigbee device and cannot offer support on the setup you will need to follow the website instructions for further instructions https://www.zigbee2mqtt.io/ -# terminal access +## terminal access to access the terminal run `docker exec -it zigbee2mqtt /bin/sh` or select `/bin/sh \ No newline at end of file diff --git a/docs/diyHue.md b/docs/diyHue.md index d5722ead..6fe62078 100644 --- a/docs/diyHue.md +++ b/docs/diyHue.md @@ -1,10 +1,11 @@ +# DIY hue * [website](https://diyhue.org/getting-started/) -# About +## About diyHue is a utility to contol the lights in your home -# Setup +## Setup Before you start diyHue you will need to get your IP and MAC addresses. Run `ip addr` in the terminal @@ -14,6 +15,6 @@ Enter these values into the `./services/diyhue/diyhue.env` file The default username and password it `Hue` and `Hue` respectively -# Usage +## Usage The web interface is available on port 8070 \ No newline at end of file diff --git a/docs/openHAB.md b/docs/openHAB.md index 2fec435a..b289e48b 100644 --- a/docs/openHAB.md +++ b/docs/openHAB.md @@ -1,4 +1,5 @@ -# References +# Openhab +## References - [Docker](https://hub.docker.com/r/openhab/openhab/) - [website](https://www.openhab.org/) diff --git a/docs/x2go.md b/docs/x2go.md index d572bfe4..6983ac05 100644 --- a/docs/x2go.md +++ b/docs/x2go.md @@ -1,9 +1,10 @@ +# x2go x2go is an "alternative" to using VNC for a remote connection. It uses X11 forwarding over ssh to provide a desktop environment Reason for using: I have a Pi 4 and I didn't buy a micro hdmi cable. You can use VNC however you are limited to a 800x600 window. -# Installation +## Installation Install with `sudo apt install x2goserver` @@ -19,5 +20,5 @@ Now I have a full-screen client ![image](https://user-images.githubusercontent.com/46672225/69007780-0045e800-094b-11ea-9626-4947595a016e.png) -# YouTube tutorial +## YouTube tutorial [Laurence systems](https://www.youtube.com/watch?v=oSuy1TS8FGs) \ No newline at end of file From 7bf68e721274ded1f633baa83e31420a2c5d5634 Mon Sep 17 00:00:00 2001 From: Willem Dekker Date: Sun, 19 Apr 2020 14:21:05 +0200 Subject: [PATCH 22/69] Updated styling of existing pages --- docs/Accessing-your-Device-from-the-internet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Accessing-your-Device-from-the-internet.md b/docs/Accessing-your-Device-from-the-internet.md index b798ec71..386b3da5 100644 --- a/docs/Accessing-your-Device-from-the-internet.md +++ b/docs/Accessing-your-Device-from-the-internet.md @@ -1,4 +1,4 @@ -# accessing your device from the internet +# Accessing your device from the internet The challenge most of us face with remotely accessing your home network is that you don't have a static IP. From time to time the IP that your ISP assigns to you changes and it's difficult to keep up. Fortunately, there is a solution, a DynamicDNS. The section below shows you how to set up an easy to remember address that follows your public IP no matter when it changes. Secondly, how do you get into your home network? Your router has a firewall that is designed to keep the rest of the internet out of your network to protect you. Here we install a VPN and configure the firewall to only allow very secure VPN traffic in. From a9afc969a407adec6f73af109380906d2b04e33c Mon Sep 17 00:00:00 2001 From: SensorsIOT Date: Sun, 19 Apr 2020 15:55:34 +0200 Subject: [PATCH 23/69] Update README.md --- README.md | 199 +----------------------------------------------------- 1 file changed, 1 insertion(+), 198 deletions(-) diff --git a/README.md b/README.md index b59ba57e..1170b1ba 100644 --- a/README.md +++ b/README.md @@ -1,198 +1 @@ -# IOTStack - -IOTstack is a builder for docker-compose to easily make and maintain IoT stacks on the Raspberry Pi - -## Announcements - -The bulk of the README has moved to the Wiki. Please check it out [here](https://github.com/gcgarner/IOTstack/wiki) - -* 2019-12-19 Added python container, tweaked update script -* 2019-12-12 modified zigbee2mqtt template file -* 2019-12-12 Added Function to add custom containers to the stack -* 2019-12-12 PR cmskedgell: Added Homebridge -* 2019-12-12 PR 877dev: Added trimming of online backups -* 2019-12-03 BUGFIX Mosquitto: Fixed issue where mosquitto failed to start as a result of 11-28 change -* 2019-12-03 Added terminal for postgres, temporarily removed setfacl from menu -* 2019-11-28 PR @stfnhmplr added diyHue -* 2019-11-28 Fixed update notification on menu -* 2019-11-28 Fixed mosquitto logs and database not mapping correctly to volumes. Pull new template to fix -* 2019-11-28 added the option to disable swapfile by setting swappiness to 0 -* 2019-11-28 PR @stfnhmplr fixed incorrect shegang on MariaDB terminal.sh -* 2019-11-28 Added native install for RPIEasy -* 2019-11-27 Additions: NextCloud, MariaDB, MotionEye, Mozilla Webthings, blynk-server (fixed issue with selection.txt) -* 2019-11-22 BUGFIX selection.txt failed on fresh install, added pushd IOTstack to menu to ensure correct path -* 2019-11-22 Added notification into menu if project update is available -* 2019-11-20 BUGFIX influxdb backup: Placing docker_backup in crontab caused influxdb backup not to execute correctly -* 2019-11-20 BUGFIX disable swap: swapfile recreation on reboot fixed. Re-run from menu to fix. -* Node-RED: serial port. New template adds privileged which allows acces to serial devices -* EspurinoHub: is available for testing see wiki entry - -*** - -## Highlighted topics - -* [Bluetooth and Node-RED](https://github.com/gcgarner/IOTstack/wiki/Node-RED#using-bluetooth) -* [Saving files to disk inside containers](https://github.com/gcgarner/IOTstack/wiki/Node-RED#sharing-files-between-node-red-and-the-host) -* [Updating the Project](https://github.com/gcgarner/IOTstack/wiki/Updating-the-Project) - - *** - -## Coming soon - -* reverse proxy is now next on the list, I cant keep up with the ports -* Detection of arhcitecture for seperate stack options for amd64, armhf, i386 -* autocleanup of backups on cloud -* Gitea (in testing branch) -* OwnCloud - -*** - -## About - -Docker stack for getting started on IoT on the Raspberry Pi. - -This Docker stack consists of: - -* Node-RED -* Grafana -* InfluxDB -* Postgres -* Mosquitto mqtt -* Portainer -* Adminer -* openHAB -* Home Assistant (HASSIO) -* zigbee2mqtt -* Pi-Hole -* TasmoAdmin (parial wiki) -* Plex media server -* Telegraf (wiki coming soon) -* RTL_433 -* EspruinoHub (testing) -* MotionEye -* MariaDB -* Plex -* Homebridge - -In addition, there is a write-up and some scripts to get a dynamic DNS via duckdns and VPN up and running. - -Firstly what is docker? The correct question is "what are containers?". Docker is just one of the utilities to run a container. - -A Container can be thought of as ultra-minimal virtual machines, they are a collection of binaries that run in a sandbox environment. You download a preconfigured base image and create a new container. Only the differences between the base and your "VM" are stored. -Containers don't have [GUI](https://en.wikipedia.org/wiki/Graphical_user_interface)s so generally the way you interact with them is via web services or you can launch into a terminal. -One of the major advantages is that the image comes mostly preconfigured. - -There are pro's and cons for using native installs vs containers. For me, one of the best parts of containers is that it doesn't "clutter" your device. If you don't need Postgres anymore then just stop and delete the container. It will be like the container was never there. - -The container will fail if you try to run the docker and native vesions as the same time. It is best to install this on a fresh system. - -For those looking for a script that installs native applications check out [Peter Scargill's script](https://tech.scargill.net/the-script/) - -## Tested platform - -Raspberry Pi 3B and 4B Raspbian (Buster) - -### Older Pi's - -Docker will not run on a PiZero or A model 1 because of the CPU. It has not been tested on a Model 2. You can still use Peter Scargill's [script](https://tech.scargill.net/the-script/) - -## Running under a virtual machine - -For those wanting to test out the script in a Virtual Machine before installing on their Pi there are some limitations. The script is designed to work with Debian based distributions. Not all the container have x86_64 images. For example Portainer does not and will give an error when you try and start the stack. Please see the pinned issue [#29](https://github.com/gcgarner/IOTstack/issues/29), there is more info there. - -## Feature Requests - -Please direct all feature requests to [Discord](https://discord.gg/W45tD83) - -## Youtube reference - -This repo was originally inspired by Andreas Spiess's video on using some of these tools. Some containers have been added to extend its functionality. - -[YouTube video](https://www.youtube.com/watch?v=JdV4x925au0): This is an alternative approach to the setup. Be sure to watch the video for the instructions. Just note that the network addresses are different, see the wiki under Docker Networks. - -### YouTube guide - -@peyanski (Kiril) made a YouTube video on getting started using the project, check it out [here](https://youtu.be/5JMNHuHv134) - -## Download the project - -1.On the lite image you will need to install git first - -```bash -sudo apt-get install git -``` - -2.Download the repository with: - -```bash -git clone https://github.com/gcgarner/IOTstack.git ~/IOTstack -``` - -Due to some script restraints, this project needs to be stored in ~/IOTstack - -3.To enter the directory run: - -```bash -cd ~/IOTstack -``` - -## The Menu - -I've added a menu to make things easier. It is good to familiarise yourself with the installation process. -The menu can be used to install docker and build the docker-compose.yml file necessary for starting the stack. It also runs a few common commands. I do recommend you start to learn the docker and docker-compose commands if you plan on using docker in the long run. I've added several helper scripts, have a look inside. - -Navigate to the project folder and run `./menu.sh` - -### Installing from the menu - -Select the first option and follow the prompts - -### Build the docker-compose file - -docker-compose uses the `docker-compose.yml` file to configure all the services. Run through the menu to select the options you want to install. - -### Docker commands - -This menu executes shell scripts in the root of the project. It is not necessary to run them from the menu. Open up the shell script files to see what is inside and what they do. - -### Miscellaneous commands - -Some helpful commands have been added like disabling swap. - -## Running Docker commands - -From this point on make sure you are executing the commands from inside the project folder. Docker-compose commands need to be run from the folder where the docker-compose.yml is located. If you want to move the folder make sure you move the whole project folder. - -## Starting and Stopping containers - -to start the stack navigate to the project folder containing the docker-compose.yml file - -To start the stack run: -`docker-compose up -d` or `./scripts/start.sh` - -To stop: -`docker-compose stop` - -The first time you run 'start' the stack docker will download all the images for the web. Depending on how many containers you selected and your internet speed this can take a long while. - -The `docker-compose down` command stops the containers then deletes them. - -## Persistent data - -Docker allows you to map folders inside your containers to folders on the disk. This is done with the "volume" key. There are two types of volumes. Modification to the container are reflected in the volume. - -## See Wiki for further info - -[Wiki](https://github.com/gcgarner/IOTstack/wiki) - -## Add to the project - -Feel free to add your comments on features or images that you think should be added. - -## Contributions - -If you use some of the tools in the project please consider donating or contributing on their projects. It doesn't have to be monetary, reporting bugs and PRs help improve the projects for everyone. - -### Thanks - -@mrmx, @oscrx, @brianimmel, @Slyke, @AugustasV, @Paulf007, @affankingkhan, @877dev, @Paraphraser, @stfnhmplr, @peyanski, @cmskedgell +Documentation for the project: From 50cbf5250ac83a5106ad2bd1da92ff78fb4a0277 Mon Sep 17 00:00:00 2001 From: SensorsIOT Date: Sun, 19 Apr 2020 15:57:46 +0200 Subject: [PATCH 24/69] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1170b1ba..d6fa99b4 100644 --- a/README.md +++ b/README.md @@ -1 +1,3 @@ -Documentation for the project: +Documentation for the project: https://sensorsiot.github.io/IOTstack/ + +Video: https://youtu.be/a6mjt8tWUws From 5d0c85ab87c131e2634e4ab6ea915ba3de6704e8 Mon Sep 17 00:00:00 2001 From: Slyke Date: Sun, 19 Apr 2020 16:45:50 -0700 Subject: [PATCH 25/69] Changed each menu item to be on a separate line to make merges easier. --- menu.sh | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/menu.sh b/menu.sh index db259a6a..9d9536ff 100755 --- a/menu.sh +++ b/menu.sh @@ -28,11 +28,34 @@ declare -A cont_array=( [diyhue]="diyHue" [homebridge]="Homebridge" [python]="Python 3" +) +declare -a armhf_keys=( + "portainer" + "nodered" + "influxdb" + "grafana" + "mosquitto" + "telegraf" + "mariadb" + "postgres" + "adminer" + "openhab" + "zigbee2mqtt" + "pihole" + "plex" + "tasmoadmin" + "rtl_433" + "espruinohub" + "motioneye" + "webthings_gateway" + "blynk_server" + "nextcloud" + "diyhue" + "homebridge" + "python" + # add yours here ) -declare -a armhf_keys=("portainer" "nodered" "influxdb" "grafana" "mosquitto" "telegraf" "mariadb" "postgres" - "adminer" "openhab" "zigbee2mqtt" "pihole" "plex" "tasmoadmin" "rtl_433" "espruinohub" - "motioneye" "webthings_gateway" "blynk_server" "nextcloud" "diyhue" "homebridge" "python") sys_arch=$(uname -m) From 52ddbcb63f37113355c9fcd0b9b3b3ea1b24ac2f Mon Sep 17 00:00:00 2001 From: Slyke Date: Sun, 19 Apr 2020 23:05:31 -0700 Subject: [PATCH 26/69] Changed docker-compose yml file version to 3.6. --- menu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/menu.sh b/menu.sh index f9a0c4c9..6eabd122 100755 --- a/menu.sh +++ b/menu.sh @@ -231,7 +231,7 @@ case $mainmenu_selection in #if no container is selected then dont overwrite the docker-compose.yml file if [ -n "$container_selection" ]; then touch docker-compose.yml - echo "version: '3.7'" >docker-compose.yml + echo "version: '3.6'" >docker-compose.yml echo "services:" >>docker-compose.yml #set the ACL for the stack From 15d840bb5475ae52ff33f05af653539dd8e0846f Mon Sep 17 00:00:00 2001 From: Slyke Date: Mon, 20 Apr 2020 00:53:27 -0700 Subject: [PATCH 27/69] Update README.md Updated readme with quick install instructions so people don't have to dig for it in the documentation. --- README.md | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d6fa99b4..ae8fefe0 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,35 @@ -Documentation for the project: https://sensorsiot.github.io/IOTstack/ +# IOT Stack +IOTstack is a builder for docker-compose to easily make and maintain IoT stacks on the Raspberry Pi. -Video: https://youtu.be/a6mjt8tWUws +## Documentation for the project: + +https://sensorsiot.github.io/IOTstack/ + +## Video +https://youtu.be/a6mjt8tWUws + +## Installation +1. On the (PRi) lite image you will need to install git first + +``` +sudo apt-get install git -y +``` + +2. Download the repository with: +``` +git clone https://github.com/SensorsIot/IOTstack.git ~/IOTstack +``` + +Due to some script restraints, this project needs to be stored in ~/IOTstack + +3. To enter the directory and run menu for installation options: +``` +cd ~/IOTstack && bash ./menu.sh +``` + +4. Install docker with the menu, restart your system. + +5. Run menu again to select your build options, then start docker-compose with +``` +docker-compose up -d +``` From a7d37c05e62b7093036d2cf0d508e5804b56e3be Mon Sep 17 00:00:00 2001 From: Phill Kelley Date: Tue, 21 Apr 2020 16:46:21 +1000 Subject: [PATCH 28/69] Fixes some inconsistencies and makes other tweaks to index page Removed references to the Wiki (this IS the Wiki). Fixed typo in EspruinoHub (was EspurinoHub). Changed "sudo apt-get install git" to "sudo apt install git". My understanding is that "apt" is considered best practice. Changed the URL to download the repository from "https://github.com/gcgarner/IOTstack.git" to "https://github.com/SensorsIot/IOTstack.git". Removed the last sentence under "Running under a virtual machine". Graham closed and unpinned issue 29 so this seems redundant. Changed several URLs to refer to the new documentation structure. Added a "Networking" section below "Persistent data" to cover-off the "first timer" problem of not realising why 127.0.0.1 won't work. It's such a common problem that I thought it would be better to put it up front (and thereby explain it twice) rather than expect newbies to read the Networking page and immediately "grok" its significance. Other tweaks such as clarifying the need to install Docker on the first menu run, and consistent layout for the commands to start and stop the stack. Also harmonised "IOTstack" as the spelling and added some missing full stops. --- docs/index.md | 79 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 29 deletions(-) diff --git a/docs/index.md b/docs/index.md index b59ba57e..55c40b28 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,11 +1,9 @@ -# IOTStack +# IOTstack -IOTstack is a builder for docker-compose to easily make and maintain IoT stacks on the Raspberry Pi +IOTstack is a builder for docker-compose to easily make and maintain IoT stacks on the Raspberry Pi. ## Announcements -The bulk of the README has moved to the Wiki. Please check it out [here](https://github.com/gcgarner/IOTstack/wiki) - * 2019-12-19 Added python container, tweaked update script * 2019-12-12 modified zigbee2mqtt template file * 2019-12-12 Added Function to add custom containers to the stack @@ -25,15 +23,15 @@ The bulk of the README has moved to the Wiki. Please check it out [here](https:/ * 2019-11-20 BUGFIX influxdb backup: Placing docker_backup in crontab caused influxdb backup not to execute correctly * 2019-11-20 BUGFIX disable swap: swapfile recreation on reboot fixed. Re-run from menu to fix. * Node-RED: serial port. New template adds privileged which allows acces to serial devices -* EspurinoHub: is available for testing see wiki entry +* EspruinoHub: available for testing *** ## Highlighted topics -* [Bluetooth and Node-RED](https://github.com/gcgarner/IOTstack/wiki/Node-RED#using-bluetooth) -* [Saving files to disk inside containers](https://github.com/gcgarner/IOTstack/wiki/Node-RED#sharing-files-between-node-red-and-the-host) -* [Updating the Project](https://github.com/gcgarner/IOTstack/wiki/Updating-the-Project) +* [Bluetooth and Node-RED](https://sensorsiot.github.io/IOTstack/Node-RED#using-bluetooth) +* [Saving files to disk inside containers](https://sensorsiot.github.io/IOTstack/Node-RED#sharing-files-between-node-red-and-the-host) +* [Updating the Project](https://sensorsiot.github.io/IOTstack/Updating-the-Project/) *** @@ -90,47 +88,47 @@ For those looking for a script that installs native applications check out [Pete ## Tested platform -Raspberry Pi 3B and 4B Raspbian (Buster) +Raspberry Pi 3B and 4B Raspbian (Buster). ### Older Pi's -Docker will not run on a PiZero or A model 1 because of the CPU. It has not been tested on a Model 2. You can still use Peter Scargill's [script](https://tech.scargill.net/the-script/) +Docker will not run on a PiZero or A model 1 because of the CPU. It has not been tested on a Model 2. You can still use Peter Scargill's [script](https://tech.scargill.net/the-script/). ## Running under a virtual machine -For those wanting to test out the script in a Virtual Machine before installing on their Pi there are some limitations. The script is designed to work with Debian based distributions. Not all the container have x86_64 images. For example Portainer does not and will give an error when you try and start the stack. Please see the pinned issue [#29](https://github.com/gcgarner/IOTstack/issues/29), there is more info there. +For those wanting to test out the script in a Virtual Machine before installing on their Pi there are some limitations. The script is designed to work with Debian based distributions. Not all the container have x86_64 images. For example Portainer does not and will give an error when you try and start the stack. ## Feature Requests -Please direct all feature requests to [Discord](https://discord.gg/W45tD83) +Please direct all feature requests to [Discord](https://discord.gg/W45tD83). ## Youtube reference This repo was originally inspired by Andreas Spiess's video on using some of these tools. Some containers have been added to extend its functionality. -[YouTube video](https://www.youtube.com/watch?v=JdV4x925au0): This is an alternative approach to the setup. Be sure to watch the video for the instructions. Just note that the network addresses are different, see the wiki under Docker Networks. +[YouTube video](https://www.youtube.com/watch?v=JdV4x925au0): This is an alternative approach to the setup. Be sure to watch the video for the instructions. Just note that the network addresses are different, see [Networking](https://sensorsiot.github.io/IOTstack/Networking/). ### YouTube guide -@peyanski (Kiril) made a YouTube video on getting started using the project, check it out [here](https://youtu.be/5JMNHuHv134) +@peyanski (Kiril) made a [YouTube video](https://youtu.be/5JMNHuHv134) on getting started using the project. ## Download the project -1.On the lite image you will need to install git first +1. On the lite image you will need to install git first. ```bash -sudo apt-get install git +sudo apt install git ``` -2.Download the repository with: +2. Download the repository with: ```bash -git clone https://github.com/gcgarner/IOTstack.git ~/IOTstack +git clone https://github.com/SensorsIot/IOTstack.git ~/IOTstack ``` Due to some script restraints, this project needs to be stored in ~/IOTstack -3.To enter the directory run: +3. To enter the directory run: ```bash cd ~/IOTstack @@ -139,13 +137,13 @@ cd ~/IOTstack ## The Menu I've added a menu to make things easier. It is good to familiarise yourself with the installation process. -The menu can be used to install docker and build the docker-compose.yml file necessary for starting the stack. It also runs a few common commands. I do recommend you start to learn the docker and docker-compose commands if you plan on using docker in the long run. I've added several helper scripts, have a look inside. +The menu can be used to install docker and build the docker-compose.yml file necessary for starting the stack. It also runs a few common commands. I do recommend you start to learn the docker and docker-compose commands if you plan on using Docker in the long run. I've added several helper scripts, have a look inside. Navigate to the project folder and run `./menu.sh` -### Installing from the menu +### Installing Docker from the menu -Select the first option and follow the prompts +Select "Install Docker" and follow the prompts. This ends with a reboot. ### Build the docker-compose file @@ -165,13 +163,19 @@ From this point on make sure you are executing the commands from inside the proj ## Starting and Stopping containers -to start the stack navigate to the project folder containing the docker-compose.yml file +To start the stack: -To start the stack run: -`docker-compose up -d` or `./scripts/start.sh` +```bash +cd ~/IOTstack +docker-compose up -d +``` -To stop: -`docker-compose stop` +To stop the stack: + +```bash +cd ~/IOTstack +docker-compose stop +``` The first time you run 'start' the stack docker will download all the images for the web. Depending on how many containers you selected and your internet speed this can take a long while. @@ -181,9 +185,26 @@ The `docker-compose down` command stops the containers then deletes them. Docker allows you to map folders inside your containers to folders on the disk. This is done with the "volume" key. There are two types of volumes. Modification to the container are reflected in the volume. -## See Wiki for further info +## Networking + +One common problem newcomers face is understanding that Docker creates an internal bridged network. A process running inside one container reaches a process running inside another container by using the **name** of that other container as the domain name. For example, if a Node-Red flow needs to: + +* subscribe to an MQTT topic being managed by Mosquitto, it refers to "mosquitto" as the domain name and communicates over port 1883. +* send data to InfluxDB, it refers to "influxdb" as the domain name and communicates over port 8086. + +In particular, if you are migrating Node-Red flows from a non-Docker environment, they will probably contain references to the loopback address (127.0.0.1). Those will need to be updated. + +Similarly, if you are migrating a Grafana dashboard that relies on, say, InfluxDB as for its data. The data source will need to be updated to refer to "influxdb:8086". + +You can discover container names like this: + +```bash +cd ~/IOTstack +docker-compose ps +``` + +See also [Networking](https://sensorsiot.github.io/IOTstack/Networking/). -[Wiki](https://github.com/gcgarner/IOTstack/wiki) ## Add to the project From e82dac88d4705d5aef3a3d831c65db856d3b3c7a Mon Sep 17 00:00:00 2001 From: Slyke Date: Mon, 20 Apr 2020 23:47:21 -0700 Subject: [PATCH 29/69] Fixed typo in readme (PRi --> RPi) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ae8fefe0..f5c5e131 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ https://sensorsiot.github.io/IOTstack/ https://youtu.be/a6mjt8tWUws ## Installation -1. On the (PRi) lite image you will need to install git first +1. On the (RPi) lite image you will need to install git first ``` sudo apt-get install git -y From 7889207a2cc61a515363815cae3554737eac1630 Mon Sep 17 00:00:00 2001 From: Slyke Date: Mon, 20 Apr 2020 23:53:20 -0700 Subject: [PATCH 30/69] Added docker version check --- menu.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/menu.sh b/menu.sh index 7b2c6d82..c079734f 100755 --- a/menu.sh +++ b/menu.sh @@ -180,6 +180,25 @@ else fi fi +#--------------------------------------------------------------------------------------------------- +# Docker updates +echo "checking docker version" +SERVER_VERSION=$(docker version -f "{{.Server.Version}}") +SERVER_VERSION_MAJOR=$(echo "$SERVER_VERSION"| cut -d'.' -f 1) +SERVER_VERSION_MINOR=$(echo "$SERVER_VERSION"| cut -d'.' -f 2) +SERVER_VERSION_BUILD=$(echo "$SERVER_VERSION"| cut -d'.' -f 3) + +if [ "${SERVER_VERSION_MAJOR}" -ge 18 ] && \ + [ "${SERVER_VERSION_MINOR}" -ge 2 ] && \ + [ "${SERVER_VERSION_BUILD}" -ge 0 ]; then + echo "Docker version >= 18.2.0. You are good to go." +else + echo "" + echo "Docker version less than 18.02.0 consider upgrading or you may experience issues" + echo "Upgrade by typing: 'sudo apt upgrade docker docker-compose'" + sleep 2 +fi + #--------------------------------------------------------------------------------------------------- # Menu system starts here # Display main menu From 47fde77aab0efb005d9712cd207f6dba85fde175 Mon Sep 17 00:00:00 2001 From: Slyke Date: Tue, 21 Apr 2020 17:59:44 -0700 Subject: [PATCH 31/69] Update README.md Added migration steps --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index f5c5e131..a64cf201 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,16 @@ # IOT Stack IOTstack is a builder for docker-compose to easily make and maintain IoT stacks on the Raspberry Pi. + ## Documentation for the project: https://sensorsiot.github.io/IOTstack/ + ## Video https://youtu.be/a6mjt8tWUws + ## Installation 1. On the (RPi) lite image you will need to install git first @@ -33,3 +36,14 @@ cd ~/IOTstack && bash ./menu.sh ``` docker-compose up -d ``` + + +## Migrating from the old repo? +``` +cd ~/IOTstack/ +git remote set-url origin https://github.com/SensorsIot/IOTstack.git +git pull origin master +docker-compose down +./menu.sh +docker-compose up -d +``` From b36b7b3bc8bbd730dbbf70c07d777e5c9235455c Mon Sep 17 00:00:00 2001 From: Slyke Date: Wed, 22 Apr 2020 00:22:09 -0700 Subject: [PATCH 32/69] Added a check to see if docker is installed before attempting to get its version --- menu.sh | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/menu.sh b/menu.sh index 0f297364..c16f4904 100755 --- a/menu.sh +++ b/menu.sh @@ -182,21 +182,25 @@ fi #--------------------------------------------------------------------------------------------------- # Docker updates -echo "checking docker version" -SERVER_VERSION=$(docker version -f "{{.Server.Version}}") -SERVER_VERSION_MAJOR=$(echo "$SERVER_VERSION"| cut -d'.' -f 1) -SERVER_VERSION_MINOR=$(echo "$SERVER_VERSION"| cut -d'.' -f 2) -SERVER_VERSION_BUILD=$(echo "$SERVER_VERSION"| cut -d'.' -f 3) - -if [ "${SERVER_VERSION_MAJOR}" -ge 18 ] && \ - [ "${SERVER_VERSION_MINOR}" -ge 2 ] && \ - [ "${SERVER_VERSION_BUILD}" -ge 0 ]; then - echo "Docker version >= 18.2.0. You are good to go." +if command_exists docker; then + echo "checking docker version" + SERVER_VERSION=$(docker version -f "{{.Server.Version}}") + SERVER_VERSION_MAJOR=$(echo "$SERVER_VERSION"| cut -d'.' -f 1) + SERVER_VERSION_MINOR=$(echo "$SERVER_VERSION"| cut -d'.' -f 2) + SERVER_VERSION_BUILD=$(echo "$SERVER_VERSION"| cut -d'.' -f 3) + + if [ "${SERVER_VERSION_MAJOR}" -ge 18 ] && \ + [ "${SERVER_VERSION_MINOR}" -ge 2 ] && \ + [ "${SERVER_VERSION_BUILD}" -ge 0 ]; then + echo "Docker version >= 18.2.0. You are good to go." + else + echo "" + echo "Docker version less than 18.02.0 consider upgrading or you may experience issues" + echo "Upgrade by typing: 'sudo apt upgrade docker docker-compose'" + sleep 2 + fi else - echo "" - echo "Docker version less than 18.02.0 consider upgrading or you may experience issues" - echo "Upgrade by typing: 'sudo apt upgrade docker docker-compose'" - sleep 2 + echo "docker not installed" fi #--------------------------------------------------------------------------------------------------- From e95298fb8e1582eaa01b70efa972d769d9c205f0 Mon Sep 17 00:00:00 2001 From: Gabriele Pongelli Date: Wed, 22 Apr 2020 09:53:30 +0200 Subject: [PATCH 33/69] Removed bookmark from transmission volumes.yml --- .templates/transmission/volumes.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.templates/transmission/volumes.yml b/.templates/transmission/volumes.yml index 597c494b..655a4708 100644 --- a/.templates/transmission/volumes.yml +++ b/.templates/transmission/volumes.yml @@ -4,18 +4,18 @@ driver_opts: o: bind type: none - device: + device: /home/${USER}/IOTstack/.volumes/transmission/downloads transm_watch_volume: driver: local driver_opts: o: bind type: none - device: + device: /home/${USER}/IOTstack/.volumes/transmission/watch transm_config_volume: driver: local driver_opts: o: bind type: none - device: + device: /home/${USER}/IOTstack/.volumes/transmission/config From 19369898b12474c8d8a81fb11cb2bb245b5c881c Mon Sep 17 00:00:00 2001 From: Slyke Date: Fri, 24 Apr 2020 22:59:50 -0700 Subject: [PATCH 34/69] Create default directories on build --- .templates/transmission/build.sh | 7 +++++++ .templates/transmission/volumes.yml | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100755 .templates/transmission/build.sh diff --git a/.templates/transmission/build.sh b/.templates/transmission/build.sh new file mode 100755 index 00000000..f7e5a4ee --- /dev/null +++ b/.templates/transmission/build.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# create directories for named volumes +TRANSMISSION_BASEDIR=.volumes/transmission +mkdir -p $TRANSMISSION_BASEDIR/downloads +mkdir -p $TRANSMISSION_BASEDIR/watch +mkdir -p $TRANSMISSION_BASEDIR/config diff --git a/.templates/transmission/volumes.yml b/.templates/transmission/volumes.yml index 655a4708..2b6ae64c 100644 --- a/.templates/transmission/volumes.yml +++ b/.templates/transmission/volumes.yml @@ -4,18 +4,18 @@ driver_opts: o: bind type: none - device: /home/${USER}/IOTstack/.volumes/transmission/downloads + device: .volumes/transmission/downloads transm_watch_volume: driver: local driver_opts: o: bind type: none - device: /home/${USER}/IOTstack/.volumes/transmission/watch + device: .volumes/transmission/watch transm_config_volume: driver: local driver_opts: o: bind type: none - device: /home/${USER}/IOTstack/.volumes/transmission/config + device: .volumes/transmission/config From 4976473040675d408246e564f242564350f07ed3 Mon Sep 17 00:00:00 2001 From: Slyke Date: Sat, 25 Apr 2020 07:45:47 +0100 Subject: [PATCH 35/69] Added requirements.txt for Python 3 --- .templates/python/requirements.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .templates/python/requirements.txt diff --git a/.templates/python/requirements.txt b/.templates/python/requirements.txt new file mode 100644 index 00000000..e69de29b From 694778fc37f493d7b66cb4bac9a81596b5e9ce6e Mon Sep 17 00:00:00 2001 From: Slyke Date: Sun, 26 Apr 2020 03:12:22 -0700 Subject: [PATCH 36/69] Added YAML merging Python script, with menu script modifications. Added minor changes and bug fixes to backup. --- .gitignore | 5 +- .tmp/.gitkeep | 0 compose-override.yml | 4 ++ menu.sh | 116 ++++++++++++++++++++------------------- scripts/docker_backup.sh | 13 +++-- scripts/yaml_merge.py | 36 ++++++++++++ 6 files changed, 113 insertions(+), 61 deletions(-) create mode 100755 .tmp/.gitkeep create mode 100755 compose-override.yml create mode 100755 scripts/yaml_merge.py diff --git a/.gitignore b/.gitignore index 5f09a9ea..6add4474 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,8 @@ /services/ /volumes/ /backups/ +/.tmp/* docker-compose.yml -.outofdate \ No newline at end of file +.outofdate + +!.gitkeep diff --git a/.tmp/.gitkeep b/.tmp/.gitkeep new file mode 100755 index 00000000..e69de29b diff --git a/compose-override.yml b/compose-override.yml new file mode 100755 index 00000000..4e07442e --- /dev/null +++ b/compose-override.yml @@ -0,0 +1,4 @@ +services: + mosquitto: + ports: + - 1882:1882 diff --git a/menu.sh b/menu.sh index 994f4941..4aba1102 100755 --- a/menu.sh +++ b/menu.sh @@ -3,6 +3,16 @@ #get path of menu correct pushd ~/IOTstack +# Consts/vars +TMP_DOCKER_COMPOSE_YML=./.tmp/docker-compose.tmp.yml +DOCKER_COMPOSE_YML=./docker-compose.yml +DOCKER_COMPOSE_OVERRIDE_YML=./compose-override.yml + +COMPOSE_VERSION="3.6" +DOCKER_VERSION_MAJOR=18 +DOCKER_VERSION_MINOR=2 +DOCKER_VERSION_BUILD=0 + declare -A cont_array=( [portainer]="Portainer" [nodered]="Node-RED" @@ -79,6 +89,7 @@ docker_setfacl() { [ -d ./services ] || mkdir ./services [ -d ./volumes ] || mkdir ./volumes [ -d ./backups ] || mkdir ./backups + [ -d ./tmp ] || mkdir ./tmp #give current user rwx on the volumes and backups [ $(getfacl ./volumes | grep -c "default:user:$USER") -eq 0 ] && sudo setfacl -Rdm u:$USER:rwx ./volumes @@ -108,47 +119,47 @@ function yml_builder() { [ -d ./services/ ] || mkdir ./services/ - if [ -d ./services/$1 ]; then - #directory already exists prompt user to overwrite - sevice_overwrite=$(whiptail --radiolist --title "Overwrite Option" --notags \ - "$1 service directory has been detected, use [SPACEBAR] to select you overwrite option" 20 78 12 \ - "none" "Do not overwrite" "ON" \ - "env" "Preserve Environment and Config files" "OFF" \ - "full" "Pull full service from template" "OFF" \ - 3>&1 1>&2 2>&3) - - case $sevice_overwrite in - - "full") - echo "...pulled full $1 from template" - rsync -a -q .templates/$1/ services/$1/ --exclude 'build.sh' - ;; - "env") - echo "...pulled $1 excluding env file" - rsync -a -q .templates/$1/ services/$1/ --exclude 'build.sh' --exclude '$1.env' --exclude '*.conf' - ;; - "none") - echo "...$1 service not overwritten" - ;; - - esac + if [ -d ./services/$1 ]; then + #directory already exists prompt user to overwrite + sevice_overwrite=$(whiptail --radiolist --title "Overwrite Option" --notags \ + "$1 service directory has been detected, use [SPACEBAR] to select you overwrite option" 20 78 12 \ + "none" "Do not overwrite" "ON" \ + "env" "Preserve Environment and Config files" "OFF" \ + "full" "Pull full service from template" "OFF" \ + 3>&1 1>&2 2>&3) - else - mkdir ./services/$1 + case $sevice_overwrite in + + "full") echo "...pulled full $1 from template" rsync -a -q .templates/$1/ services/$1/ --exclude 'build.sh' - fi + ;; + "env") + echo "...pulled $1 excluding env file" + rsync -a -q .templates/$1/ services/$1/ --exclude 'build.sh' --exclude '$1.env' --exclude '*.conf' + ;; + "none") + echo "...$1 service not overwritten" + ;; + + esac + + else + mkdir ./services/$1 + echo "...pulled full $1 from template" + rsync -a -q .templates/$1/ services/$1/ --exclude 'build.sh' + fi #if an env file exists check for timezone [ -f "./services/$1/$1.env" ] && timezones ./services/$1/$1.env - # if a volumes.yml exists, append to overall volumes.yml file - [ -f "./services/$1/volumes.yml" ] && cat "./services/$1/volumes.yml" >> docker-volumes.yml + # if a volumes.yml exists, append to overall volumes.yml file + [ -f "./services/$1/volumes.yml" ] && cat "./services/$1/volumes.yml" >> docker-volumes.yml #add new line then append service - echo "" >>docker-compose.yml - cat $service >>docker-compose.yml + echo "" >> $TMP_DOCKER_COMPOSE_YML + cat $service >> $TMP_DOCKER_COMPOSE_YML #test for post build if [ -f ./.templates/$1/build.sh ]; then @@ -194,9 +205,9 @@ SERVER_VERSION_MAJOR=$(echo "$SERVER_VERSION"| cut -d'.' -f 1) SERVER_VERSION_MINOR=$(echo "$SERVER_VERSION"| cut -d'.' -f 2) SERVER_VERSION_BUILD=$(echo "$SERVER_VERSION"| cut -d'.' -f 3) -if [ "${SERVER_VERSION_MAJOR}" -ge 18 ] && \ - [ "${SERVER_VERSION_MINOR}" -ge 2 ] && \ - [ "${SERVER_VERSION_BUILD}" -ge 0 ]; then +if [ "${SERVER_VERSION_MAJOR}" -ge $DOCKER_VERSION_MAJOR ] && \ + [ "${SERVER_VERSION_MINOR}" -ge $DOCKER_VERSION_MINOR ] && \ + [ "${SERVER_VERSION_BUILD}" -ge $DOCKER_VERSION_BUILD ]; then echo "Docker version >= 18.2.0. You are good to go." else echo "" @@ -279,9 +290,12 @@ case $mainmenu_selection in #if no container is selected then dont overwrite the docker-compose.yml file if [ -n "$container_selection" ]; then - touch docker-compose.yml - echo "version: '3.6'" >docker-compose.yml - echo "services:" >>docker-compose.yml + touch $TMP_DOCKER_COMPOSE_YML + echo "services:" > $TMP_DOCKER_COMPOSE_YML + + # Uncomment once sort_keys is available in Pyton->yaml + # echo "version: '$COMPOSE_VERSION'" > $TMP_DOCKER_COMPOSE_YML + # echo "services:" >> $TMP_DOCKER_COMPOSE_YML #set the ACL for the stack #docker_setfacl @@ -298,25 +312,17 @@ case $mainmenu_selection in echo "$container" >>./services/selection.txt done - # add custom containers - if [ -f ./services/custom.txt ]; then - if (whiptail --title "Custom Container detected" --yesno "custom.txt has been detected do you want to add these containers to the stack?" 20 78); then - mapfile -t containers <<<$(cat ./services/custom.txt) - for container in "${containers[@]}"; do - echo "Adding $container container" - yml_builder "$container" - done - fi - fi - - # if a container needs volume, put it at the end of docker-compose - if [ -f docker-volumes.yml ]; then - echo "" >> docker-compose.yml - echo "volumes:" >> docker-compose.yml - cat docker-volumes.yml >> docker-compose.yml - rm docker-volumes.yml + if [ -f "$DOCKER_COMPOSE_OVERRIDE_YML" ]; then + echo "merging docker overrides with docker-compose.yaml" + python ./scripts/yaml_merge.py $TMP_DOCKER_COMPOSE_YML $DOCKER_COMPOSE_OVERRIDE_YML $DOCKER_COMPOSE_YML + else + echo "no override found, using docker-compose.yaml" + cp $TMP_DOCKER_COMPOSE_YML $DOCKER_COMPOSE_YML fi + # Prepend compose version after merging/copying, so that merging doesn't move it to the bottom (alphabetically ordered). + echo -e "version: '$COMPOSE_VERSION'\n$(cat $DOCKER_COMPOSE_YML)" > $DOCKER_COMPOSE_YML # Remove once sort_keys works in Python->yaml. + echo "docker-compose successfully created" echo "run 'docker-compose up -d' to start the stack" else @@ -496,4 +502,4 @@ case $mainmenu_selection in esac -popd +popd > /dev/null 2>&1 diff --git a/scripts/docker_backup.sh b/scripts/docker_backup.sh index bf4db358..9df1d843 100755 --- a/scripts/docker_backup.sh +++ b/scripts/docker_backup.sh @@ -1,6 +1,7 @@ #!/bin/bash pushd ~/IOTstack +USER=$(whoami) [ -d ./backups ] || mkdir ./backups @@ -9,6 +10,10 @@ echo "./docker-compose.yml" >list.txt echo "./services/" >>list.txt echo "./volumes/" >>list.txt +if [ -f "./compose-override.yml" ]; then + echo "./compose-override.yml" >>list.txt +fi + #if influxdb is running if [ $(docker ps | grep -c influxdb) -gt 0 ]; then ./scripts/backup_influxdb.sh @@ -30,12 +35,12 @@ sudo tar -czf \ rm list.txt #set permission for backup files -sudo chown pi:pi ./backups/backup* +sudo chown $USER:$USER ./backups/backup* #create local logfile and append the latest backup file to it echo "backup saved to ./backups/$backupfile" sudo touch $logfile -sudo chown pi:pi $logfile +sudo chown $USER:$USER $logfile echo $backupfile >>$logfile #show size of archive file @@ -67,7 +72,7 @@ if [ -f ./backups/dropbox ]; then #write files to be deleted to dropbox logfile sudo touch $dropboxlog - sudo chown pi:pi $dropboxlog + sudo chown $USER:$USER $dropboxlog echo $files | tr " " "\n" >$dropboxlog #delete files from dropbox as per logfile @@ -81,9 +86,7 @@ if [ -f ./backups/dropbox ]; then $dropboxuploader delete $dropboxfolder/$file done < "$input" fi - echo "backups deleted from dropbox" >>$dropboxlog - fi diff --git a/scripts/yaml_merge.py b/scripts/yaml_merge.py new file mode 100755 index 00000000..713a15e5 --- /dev/null +++ b/scripts/yaml_merge.py @@ -0,0 +1,36 @@ +import sys +import yaml + +if len(sys.argv) < 4: + print("Error: Not enough args") + print("Usage:") + print(" yaml_merge.py [inputFile] [mergeFile] [outputFile]") + print("") + print("Example:") + print(" yaml_merge.py ./.tmp/docker-compose.tmp.yml ./compose-override.yml ./docker-compose.yml") + sys.exit(1) + +pathTempDockerCompose = sys.argv[1] +pathOverride = sys.argv[2] +pathOutput = sys.argv[3] + +def mergeYaml(priorityYaml, extensionYaml): + if isinstance(priorityYaml,dict) and isinstance(extensionYaml,dict): + for k,v in extensionYaml.iteritems(): + if k not in priorityYaml: + priorityYaml[k] = v + else: + priorityYaml[k] = mergeYaml(priorityYaml[k],v) + return priorityYaml + +with open(r'%s' % pathTempDockerCompose) as fileTempDockerCompose: + yamlTempDockerCompose = yaml.load(fileTempDockerCompose) + +with open(r'%s' % pathOverride) as fileOverride: + yamlOverride = yaml.load(fileOverride) + +mergedYaml = mergeYaml(yamlOverride, yamlTempDockerCompose) + +with open(r'%s' % pathOutput, 'w') as outputFile: + # yaml.dump(mergedYaml, outputFile, default_flow_style=False, sort_keys=False) # TODO: 'sort_keys' not available in this version of Python/yaml + yaml.dump(mergedYaml, outputFile, default_flow_style=False) # Gotta pretty this up for human consumption From 18010cd5ecaeef00efb7bb28364dc198b9c6f7f0 Mon Sep 17 00:00:00 2001 From: Slyke Date: Sun, 26 Apr 2020 03:25:50 -0700 Subject: [PATCH 37/69] Removed testing override file and useless comment --- .templates/python/app/requirements.txt | 0 compose-override.yml | 4 ---- scripts/yaml_merge.py | 2 +- 3 files changed, 1 insertion(+), 5 deletions(-) create mode 100755 .templates/python/app/requirements.txt delete mode 100755 compose-override.yml diff --git a/.templates/python/app/requirements.txt b/.templates/python/app/requirements.txt new file mode 100755 index 00000000..e69de29b diff --git a/compose-override.yml b/compose-override.yml deleted file mode 100755 index 4e07442e..00000000 --- a/compose-override.yml +++ /dev/null @@ -1,4 +0,0 @@ -services: - mosquitto: - ports: - - 1882:1882 diff --git a/scripts/yaml_merge.py b/scripts/yaml_merge.py index 713a15e5..924f79c8 100755 --- a/scripts/yaml_merge.py +++ b/scripts/yaml_merge.py @@ -33,4 +33,4 @@ def mergeYaml(priorityYaml, extensionYaml): with open(r'%s' % pathOutput, 'w') as outputFile: # yaml.dump(mergedYaml, outputFile, default_flow_style=False, sort_keys=False) # TODO: 'sort_keys' not available in this version of Python/yaml - yaml.dump(mergedYaml, outputFile, default_flow_style=False) # Gotta pretty this up for human consumption + yaml.dump(mergedYaml, outputFile, default_flow_style=False) From 186c2c0e4df64667ab2d535ff16ec510ec8c6a0e Mon Sep 17 00:00:00 2001 From: Slyke Date: Sun, 26 Apr 2020 03:42:45 -0700 Subject: [PATCH 38/69] Added loader option to avoid errors on some systems --- scripts/yaml_merge.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/yaml_merge.py b/scripts/yaml_merge.py index 924f79c8..1e004b26 100755 --- a/scripts/yaml_merge.py +++ b/scripts/yaml_merge.py @@ -32,5 +32,5 @@ def mergeYaml(priorityYaml, extensionYaml): mergedYaml = mergeYaml(yamlOverride, yamlTempDockerCompose) with open(r'%s' % pathOutput, 'w') as outputFile: - # yaml.dump(mergedYaml, outputFile, default_flow_style=False, sort_keys=False) # TODO: 'sort_keys' not available in this version of Python/yaml - yaml.dump(mergedYaml, outputFile, default_flow_style=False) + # yaml.dump(mergedYaml, outputFile, Loader=yaml.FullLoader, default_flow_style=False, sort_keys=False) # TODO: 'sort_keys' not available in this version of Python/yaml + yaml.dump(mergedYaml, outputFile, Loader=yaml.FullLoader, default_flow_style=False) From ba686ad1680d8fe6197f31a04ef3b4927dc4d5cc Mon Sep 17 00:00:00 2001 From: Gabriele Pongelli Date: Sun, 26 Apr 2020 22:08:41 +0200 Subject: [PATCH 39/69] added timescaledb --- .templates/timescaledb/service.yml | 10 ++++++++++ .templates/timescaledb/timescaledb.env | 3 +++ 2 files changed, 13 insertions(+) create mode 100644 .templates/timescaledb/service.yml create mode 100644 .templates/timescaledb/timescaledb.env diff --git a/.templates/timescaledb/service.yml b/.templates/timescaledb/service.yml new file mode 100644 index 00000000..f1bac4a2 --- /dev/null +++ b/.templates/timescaledb/service.yml @@ -0,0 +1,10 @@ + timescaledb: + container_name: timescaledb + image: timescale/timescaledb:latest-pg12 + restart: unless-stopped + env_file: + - ./services/timescaledb/timescaledb.env + ports: + - 5432:5432 + volumes: + - ./volumes/timescaledb/data:/var/lib/postgresql/data diff --git a/.templates/timescaledb/timescaledb.env b/.templates/timescaledb/timescaledb.env new file mode 100644 index 00000000..b492b311 --- /dev/null +++ b/.templates/timescaledb/timescaledb.env @@ -0,0 +1,3 @@ +POSTGRES_USER=postuser +POSTGRES_PASSWORD=postpassword +POSTGRES_DB=postdb \ No newline at end of file From 1ce3b57e2191791417e09ff327b3abd93fa67f5c Mon Sep 17 00:00:00 2001 From: Gabriele Pongelli Date: Sun, 26 Apr 2020 22:36:05 +0200 Subject: [PATCH 40/69] Add timescaledb in menu.sh --- menu.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/menu.sh b/menu.sh index 994f4941..c9506915 100755 --- a/menu.sh +++ b/menu.sh @@ -12,6 +12,7 @@ declare -A cont_array=( [grafana]="Grafana" [mosquitto]="Eclipse-Mosquitto" [postgres]="Postgres" + [timescaledb]="Timescaledb" [mariadb]="MariaDB (MySQL fork)" [adminer]="Adminer" [openhab]="openHAB" @@ -42,7 +43,8 @@ declare -a armhf_keys=( "telegraf" "mariadb" "postgres" - "transmission" + "timescaledb" + "transmission" "adminer" "openhab" "zigbee2mqtt" From 78ae80f47e68281c512c2b8f684e77293b6e168c Mon Sep 17 00:00:00 2001 From: Slyke Date: Sun, 26 Apr 2020 23:26:35 -0700 Subject: [PATCH 41/69] Added in Python, pip and PyYaml version checking --- compose-override.yml | 0 menu.sh | 103 ++++++++++++++++++++++++++++++++++++------ scripts/yaml_merge.py | 64 ++++++++++++++++++-------- 3 files changed, 133 insertions(+), 34 deletions(-) create mode 100755 compose-override.yml diff --git a/compose-override.yml b/compose-override.yml new file mode 100755 index 00000000..e69de29b diff --git a/menu.sh b/menu.sh index 4aba1102..e5e5de78 100755 --- a/menu.sh +++ b/menu.sh @@ -8,10 +8,19 @@ TMP_DOCKER_COMPOSE_YML=./.tmp/docker-compose.tmp.yml DOCKER_COMPOSE_YML=./docker-compose.yml DOCKER_COMPOSE_OVERRIDE_YML=./compose-override.yml +# Minimum Software Versions COMPOSE_VERSION="3.6" -DOCKER_VERSION_MAJOR=18 -DOCKER_VERSION_MINOR=2 -DOCKER_VERSION_BUILD=0 +REQ_DOCKER_VERSION_MAJOR=18 +REQ_DOCKER_VERSION_MINOR=2 +REQ_DOCKER_VERSION_BUILD=0 + +REQ_PYTHON_VERSION_MAJOR=3 +REQ_PYTHON_VERSION_MINOR=6 +REQ_PYTHON_VERSION_BUILD=9 + +REQ_PYYAML_VERSION_MAJOR=5 +REQ_PYYAML_VERSION_MINOR=3 +REQ_PYYAML_VERSION_BUILD=1 declare -A cont_array=( [portainer]="Portainer" @@ -109,9 +118,68 @@ password_dialog() { #test=$( password_dialog ) function command_exists() { - command -v "$@" >/dev/null 2>&1 + command -v "$@" > /dev/null 2>&1 +} + +function install_python3_and_deps() { + if (whiptail --title "Python 3 and Dependencies" --yesno "Python 3 (v3.6.9) or later, PyYaml 5.3.1 and pip3 is required for compose-overrides.yml file to merge into the docker-compose.yml file. Install now?" 20 78); then + sudo apt install -y python3-pip python3-dev + if [ $? -eq 0 ]; then + PYTHON_VERSION_GOOD="true" + else + echo "Failed to install Python" + exit 1 + fi + pip3 install -U pyyaml==5.3.1 + if [ $? -eq 0 ]; then + PYYAML_VERSION_GOOD="true" + else + echo "Failed to install Python" + exit 1 + fi + fi +} + +function do_python3_pip() { + PYTHON_VERSION_GOOD="false" + PYYAML_VERSION_GOOD="false" + + if command_exists python3 && command_exists pip3; then + PYTHON_VERSION=$(python3 --version) + echo "Python Version: $PYTHON_VERSION" + PYTHON_VERSION_MAJOR=$(echo "$PYTHON_VERSION"| cut -d' ' -f 2 | cut -d'.' -f 1) + PYTHON_VERSION_MINOR=$(echo "$PYTHON_VERSION"| cut -d'.' -f 2) + PYTHON_VERSION_BUILD=$(echo "$PYTHON_VERSION"| cut -d'.' -f 3) + + if [ "${PYTHON_VERSION_MAJOR}" -eq $REQ_PYTHON_VERSION_MAJOR ] && \ + [ "${PYTHON_VERSION_MINOR}" -eq $REQ_PYTHON_VERSION_MINOR ] && \ + [ "${PYTHON_VERSION_BUILD}" -ge $REQ_PYTHON_VERSION_BUILD ]; then + PYTHON_VERSION_GOOD="true" + else + echo "Python is outdated." + install_python3_and_deps + fi + + PYYAML_VERSION=$(python3 ./scripts/yaml_merge.py --pyyaml-version) + echo "PyYaml Version: $PYYAML_VERSION" + PYYAML_VERSION_MAJOR=$(echo "$PYYAML_VERSION"| cut -d' ' -f 2 | cut -d'.' -f 1) + PYYAML_VERSION_MINOR=$(echo "$PYYAML_VERSION"| cut -d'.' -f 2) + PYYAML_VERSION_BUILD=$(echo "$PYYAML_VERSION"| cut -d'.' -f 3) + + if [ "${PYYAML_VERSION_MAJOR}" -ge $REQ_PYYAML_VERSION_MAJOR ] && \ + [ "${PYYAML_VERSION_MINOR}" -ge $REQ_PYYAML_VERSION_MINOR ] && \ + [ "${PYYAML_VERSION_BUILD}" -ge $REQ_PYYAML_VERSION_BUILD ]; then + PYYAML_VERSION_GOOD="true" + else + echo "PyYaml is outdated." + install_python3_and_deps + fi + else + install_python3_and_deps + fi } + #function copies the template yml file to the local service folder and appends to the docker-compose.yml file function yml_builder() { @@ -205,9 +273,9 @@ SERVER_VERSION_MAJOR=$(echo "$SERVER_VERSION"| cut -d'.' -f 1) SERVER_VERSION_MINOR=$(echo "$SERVER_VERSION"| cut -d'.' -f 2) SERVER_VERSION_BUILD=$(echo "$SERVER_VERSION"| cut -d'.' -f 3) -if [ "${SERVER_VERSION_MAJOR}" -ge $DOCKER_VERSION_MAJOR ] && \ - [ "${SERVER_VERSION_MINOR}" -ge $DOCKER_VERSION_MINOR ] && \ - [ "${SERVER_VERSION_BUILD}" -ge $DOCKER_VERSION_BUILD ]; then +if [ "${SERVER_VERSION_MAJOR}" -ge $REQ_DOCKER_VERSION_MAJOR ] && \ + [ "${SERVER_VERSION_MINOR}" -ge $REQ_DOCKER_VERSION_MINOR ] && \ + [ "${SERVER_VERSION_BUILD}" -ge $REQ_DOCKER_VERSION_BUILD ]; then echo "Docker version >= 18.2.0. You are good to go." else echo "" @@ -291,11 +359,11 @@ case $mainmenu_selection in #if no container is selected then dont overwrite the docker-compose.yml file if [ -n "$container_selection" ]; then touch $TMP_DOCKER_COMPOSE_YML - echo "services:" > $TMP_DOCKER_COMPOSE_YML + # echo "services:" > $TMP_DOCKER_COMPOSE_YML # Uncomment once sort_keys is available in Pyton->yaml - # echo "version: '$COMPOSE_VERSION'" > $TMP_DOCKER_COMPOSE_YML - # echo "services:" >> $TMP_DOCKER_COMPOSE_YML + echo "version: '$COMPOSE_VERSION'" > $TMP_DOCKER_COMPOSE_YML + echo "services:" >> $TMP_DOCKER_COMPOSE_YML #set the ACL for the stack #docker_setfacl @@ -313,15 +381,22 @@ case $mainmenu_selection in done if [ -f "$DOCKER_COMPOSE_OVERRIDE_YML" ]; then - echo "merging docker overrides with docker-compose.yaml" - python ./scripts/yaml_merge.py $TMP_DOCKER_COMPOSE_YML $DOCKER_COMPOSE_OVERRIDE_YML $DOCKER_COMPOSE_YML + do_python3_pip + + if [ "$PYTHON_VERSION_GOOD" == "true" ] && [ "$PYYAML_VERSION_GOOD" == "true" ]; then + echo "merging docker overrides with docker-compose.yml" + python3 ./scripts/yaml_merge.py $TMP_DOCKER_COMPOSE_YML $DOCKER_COMPOSE_OVERRIDE_YML $DOCKER_COMPOSE_YML + else + echo "incorrect python or dependency versions, aborting override and using docker-compose.yml" + cp $TMP_DOCKER_COMPOSE_YML $DOCKER_COMPOSE_YML + fi else - echo "no override found, using docker-compose.yaml" + echo "no override found, using docker-compose.yml" cp $TMP_DOCKER_COMPOSE_YML $DOCKER_COMPOSE_YML fi # Prepend compose version after merging/copying, so that merging doesn't move it to the bottom (alphabetically ordered). - echo -e "version: '$COMPOSE_VERSION'\n$(cat $DOCKER_COMPOSE_YML)" > $DOCKER_COMPOSE_YML # Remove once sort_keys works in Python->yaml. + # echo -e "version: '$COMPOSE_VERSION'\n$(cat $DOCKER_COMPOSE_YML)" > $DOCKER_COMPOSE_YML # Remove once sort_keys works in Python->yaml. echo "docker-compose successfully created" echo "run 'docker-compose up -d' to start the stack" diff --git a/scripts/yaml_merge.py b/scripts/yaml_merge.py index 1e004b26..af9dee8c 100755 --- a/scripts/yaml_merge.py +++ b/scripts/yaml_merge.py @@ -1,6 +1,17 @@ import sys +import traceback import yaml +if sys.argv[1] == "--pyyaml-version": + try: + print("pyyaml", yaml.__version__) + sys.exit(0) + except SystemExit: + sys.exit(0) + except: + print("could not get pyyaml version") + sys.exit(3) + if len(sys.argv) < 4: print("Error: Not enough args") print("Usage:") @@ -8,29 +19,42 @@ print("") print("Example:") print(" yaml_merge.py ./.tmp/docker-compose.tmp.yml ./compose-override.yml ./docker-compose.yml") - sys.exit(1) + sys.exit(4) + +try: + pathTempDockerCompose = sys.argv[1] + pathOverride = sys.argv[2] + pathOutput = sys.argv[3] -pathTempDockerCompose = sys.argv[1] -pathOverride = sys.argv[2] -pathOutput = sys.argv[3] + def mergeYaml(priorityYaml, defaultYaml): + if isinstance(priorityYaml, dict) and isinstance(defaultYaml, dict): + for k, v in defaultYaml.iteritems(): + if k not in priorityYaml: + priorityYaml[k] = v + else: + priorityYaml[k] = mergeYaml(priorityYaml[k], v) + return defaultYaml -def mergeYaml(priorityYaml, extensionYaml): - if isinstance(priorityYaml,dict) and isinstance(extensionYaml,dict): - for k,v in extensionYaml.iteritems(): - if k not in priorityYaml: - priorityYaml[k] = v - else: - priorityYaml[k] = mergeYaml(priorityYaml[k],v) - return priorityYaml + with open(r'%s' % pathTempDockerCompose) as fileTempDockerCompose: + yamlTempDockerCompose = yaml.load(fileTempDockerCompose, Loader=yaml.SafeLoader) -with open(r'%s' % pathTempDockerCompose) as fileTempDockerCompose: - yamlTempDockerCompose = yaml.load(fileTempDockerCompose) + with open(r'%s' % pathOverride) as fileOverride: + yamlOverride = yaml.load(fileOverride, Loader=yaml.SafeLoader) -with open(r'%s' % pathOverride) as fileOverride: - yamlOverride = yaml.load(fileOverride) + mergedYaml = mergeYaml(yamlOverride, yamlTempDockerCompose) -mergedYaml = mergeYaml(yamlOverride, yamlTempDockerCompose) + with open(r'%s' % pathOutput, 'w') as outputFile: + yaml.dump(mergedYaml, outputFile, default_flow_style=False, sort_keys=False) -with open(r'%s' % pathOutput, 'w') as outputFile: - # yaml.dump(mergedYaml, outputFile, Loader=yaml.FullLoader, default_flow_style=False, sort_keys=False) # TODO: 'sort_keys' not available in this version of Python/yaml - yaml.dump(mergedYaml, outputFile, Loader=yaml.FullLoader, default_flow_style=False) + sys.exit(0) +except SystemExit: + sys.exit(0) +except: + print("Something went wrong: ") + print(sys.exc_info()) + print(traceback.print_exc()) + print("") + print("") + print("PyYaml Version: ", yaml.__version__) + print("") + sys.exit(2) From 038be7df2cfcfbd77aaa45ca98e431036f2b3eca Mon Sep 17 00:00:00 2001 From: Slyke Date: Sun, 26 Apr 2020 23:56:25 -0700 Subject: [PATCH 42/69] Added simplified version checking. Added whiptail for docker version --- menu.sh | 76 ++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/menu.sh b/menu.sh index e5e5de78..a3caa0fa 100755 --- a/menu.sh +++ b/menu.sh @@ -10,17 +10,9 @@ DOCKER_COMPOSE_OVERRIDE_YML=./compose-override.yml # Minimum Software Versions COMPOSE_VERSION="3.6" -REQ_DOCKER_VERSION_MAJOR=18 -REQ_DOCKER_VERSION_MINOR=2 -REQ_DOCKER_VERSION_BUILD=0 - -REQ_PYTHON_VERSION_MAJOR=3 -REQ_PYTHON_VERSION_MINOR=6 -REQ_PYTHON_VERSION_BUILD=9 - -REQ_PYYAML_VERSION_MAJOR=5 -REQ_PYYAML_VERSION_MINOR=3 -REQ_PYYAML_VERSION_BUILD=1 +REQ_DOCKER_VERSION=18.2.0 +REQ_PYTHON_VERSION=3.6.9 +REQ_PYYAML_VERSION_=5.3.1 declare -A cont_array=( [portainer]="Portainer" @@ -121,6 +113,37 @@ function command_exists() { command -v "$@" > /dev/null 2>&1 } +function minimum_version_check() { + # minimum_version_check required_version current_major current_minor current_build + # minimum_version_check "1.2.3" 1 2 3 + REQ_MIN_VERSION_MAJOR=$(echo "$1"| cut -d' ' -f 2 | cut -d'.' -f 1) + REQ_MIN_VERSION_MINOR=$(echo "$1"| cut -d' ' -f 2 | cut -d'.' -f 2) + REQ_MIN_VERSION_BUILD=$(echo "$1"| cut -d' ' -f 2 | cut -d'.' -f 3) + + CURR_VERSION_MAJOR=$2 + CURR_VERSION_MINOR=$3 + CURR_VERSION_BUILD=$4 + + VERSION_GOOD="false" + + if [ "${CURR_VERSION_MAJOR}" -ge $REQ_MIN_VERSION_MAJOR ]; then + VERSION_GOOD="true" + fi + + if [ "${CURR_VERSION_MAJOR}" -ge $REQ_MIN_VERSION_MAJOR ] && \ + [ "${CURR_VERSION_MINOR}" -ge $REQ_MIN_VERSION_MINOR ]; then + VERSION_GOOD="true" + fi + + if [ "${CURR_VERSION_MAJOR}" -ge $REQ_MIN_VERSION_MAJOR ] && \ + [ "${CURR_VERSION_MINOR}" -ge $REQ_MIN_VERSION_MINOR ] && \ + [ "${CURR_VERSION_BUILD}" -ge $REQ_MIN_VERSION_BUILD ]; then + VERSION_GOOD="true" + fi + + echo "$VERSION_GOOD" +} + function install_python3_and_deps() { if (whiptail --title "Python 3 and Dependencies" --yesno "Python 3 (v3.6.9) or later, PyYaml 5.3.1 and pip3 is required for compose-overrides.yml file to merge into the docker-compose.yml file. Install now?" 20 78); then sudo apt install -y python3-pip python3-dev @@ -151,9 +174,7 @@ function do_python3_pip() { PYTHON_VERSION_MINOR=$(echo "$PYTHON_VERSION"| cut -d'.' -f 2) PYTHON_VERSION_BUILD=$(echo "$PYTHON_VERSION"| cut -d'.' -f 3) - if [ "${PYTHON_VERSION_MAJOR}" -eq $REQ_PYTHON_VERSION_MAJOR ] && \ - [ "${PYTHON_VERSION_MINOR}" -eq $REQ_PYTHON_VERSION_MINOR ] && \ - [ "${PYTHON_VERSION_BUILD}" -ge $REQ_PYTHON_VERSION_BUILD ]; then + if [ "$(minimum_version_check $REQ_PYTHON_VERSION $PYTHON_VERSION_MAJOR $PYTHON_VERSION_MINOR $PYTHON_VERSION_BUILD )" == "true" ]; then PYTHON_VERSION_GOOD="true" else echo "Python is outdated." @@ -166,9 +187,7 @@ function do_python3_pip() { PYYAML_VERSION_MINOR=$(echo "$PYYAML_VERSION"| cut -d'.' -f 2) PYYAML_VERSION_BUILD=$(echo "$PYYAML_VERSION"| cut -d'.' -f 3) - if [ "${PYYAML_VERSION_MAJOR}" -ge $REQ_PYYAML_VERSION_MAJOR ] && \ - [ "${PYYAML_VERSION_MINOR}" -ge $REQ_PYYAML_VERSION_MINOR ] && \ - [ "${PYYAML_VERSION_BUILD}" -ge $REQ_PYYAML_VERSION_BUILD ]; then + if [ "$(minimum_version_check $REQ_PYYAML_VERSION $PYYAML_VERSION_MAJOR $PYYAML_VERSION_MINOR $PYYAML_VERSION_BUILD )" == "true" ]; then PYYAML_VERSION_GOOD="true" else echo "PyYaml is outdated." @@ -268,20 +287,17 @@ fi #--------------------------------------------------------------------------------------------------- # Docker updates echo "checking docker version" -SERVER_VERSION=$(docker version -f "{{.Server.Version}}") -SERVER_VERSION_MAJOR=$(echo "$SERVER_VERSION"| cut -d'.' -f 1) -SERVER_VERSION_MINOR=$(echo "$SERVER_VERSION"| cut -d'.' -f 2) -SERVER_VERSION_BUILD=$(echo "$SERVER_VERSION"| cut -d'.' -f 3) - -if [ "${SERVER_VERSION_MAJOR}" -ge $REQ_DOCKER_VERSION_MAJOR ] && \ - [ "${SERVER_VERSION_MINOR}" -ge $REQ_DOCKER_VERSION_MINOR ] && \ - [ "${SERVER_VERSION_BUILD}" -ge $REQ_DOCKER_VERSION_BUILD ]; then - echo "Docker version >= 18.2.0. You are good to go." +DOCKER_VERSION=$(docker version -f "{{.Server.Version}}") +DOCKER_VERSION_MAJOR=$(echo "$DOCKER_VERSION"| cut -d'.' -f 1) +DOCKER_VERSION_MINOR=$(echo "$DOCKER_VERSION"| cut -d'.' -f 2) +DOCKER_VERSION_BUILD=$(echo "$DOCKER_VERSION"| cut -d'.' -f 3) + +if [ "$(minimum_version_check $REQ_DOCKER_VERSION $DOCKER_VERSION_MAJOR $DOCKER_VERSION_MINOR $DOCKER_VERSION_BUILD )" == "true" ]; then + echo "Docker version >= $REQ_DOCKER_VERSION. You are good to go." else - echo "" - echo "Docker version less than 18.02.0 consider upgrading or you may experience issues" - echo "Upgrade by typing: 'sudo apt upgrade docker docker-compose'" - sleep 2 + if (whiptail --title "Docker and Docker-Compose Version Issue" --yesno "Docker version is currently $DOCKER_VERSION which is less than $REQ_DOCKER_VERSION consider upgrading or you may experience issues. You can manually upgrade by typing 'sudo apt upgrade docker docker-compose'. Attempt to upgrade now?" 20 78); then + sudo apt upgrade docker docker-compose + fi fi #--------------------------------------------------------------------------------------------------- From 35bf6a32f73e7f82b4d5b4bc143aa9ed3124da7f Mon Sep 17 00:00:00 2001 From: Slyke Date: Mon, 27 Apr 2020 00:17:19 -0700 Subject: [PATCH 43/69] Added better version messaging for python and deps --- menu.sh | 70 ++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 20 deletions(-) diff --git a/menu.sh b/menu.sh index a3caa0fa..07d99628 100755 --- a/menu.sh +++ b/menu.sh @@ -12,7 +12,7 @@ DOCKER_COMPOSE_OVERRIDE_YML=./compose-override.yml COMPOSE_VERSION="3.6" REQ_DOCKER_VERSION=18.2.0 REQ_PYTHON_VERSION=3.6.9 -REQ_PYYAML_VERSION_=5.3.1 +REQ_PYYAML_VERSION=5.3.1 declare -A cont_array=( [portainer]="Portainer" @@ -123,29 +123,58 @@ function minimum_version_check() { CURR_VERSION_MAJOR=$2 CURR_VERSION_MINOR=$3 CURR_VERSION_BUILD=$4 + + VERSION_GOOD="Unknown" - VERSION_GOOD="false" + if [ -z "$CURR_VERSION_MAJOR" ]; then + echo "$VERSION_GOOD" + return 1 + fi + + if [ -z "$CURR_VERSION_MINOR" ]; then + echo "$VERSION_GOOD" + return 1 + fi + + if [ -z "$CURR_VERSION_BUILD" ]; then + echo "$VERSION_GOOD" + return 1 + fi if [ "${CURR_VERSION_MAJOR}" -ge $REQ_MIN_VERSION_MAJOR ]; then VERSION_GOOD="true" + echo "$VERSION_GOOD" + return 0 + else + VERSION_GOOD="false" fi if [ "${CURR_VERSION_MAJOR}" -ge $REQ_MIN_VERSION_MAJOR ] && \ [ "${CURR_VERSION_MINOR}" -ge $REQ_MIN_VERSION_MINOR ]; then VERSION_GOOD="true" + echo "$VERSION_GOOD" + return 0 + else + VERSION_GOOD="false" fi if [ "${CURR_VERSION_MAJOR}" -ge $REQ_MIN_VERSION_MAJOR ] && \ [ "${CURR_VERSION_MINOR}" -ge $REQ_MIN_VERSION_MINOR ] && \ [ "${CURR_VERSION_BUILD}" -ge $REQ_MIN_VERSION_BUILD ]; then VERSION_GOOD="true" + echo "$VERSION_GOOD" + return 0 + else + VERSION_GOOD="false" fi echo "$VERSION_GOOD" } function install_python3_and_deps() { - if (whiptail --title "Python 3 and Dependencies" --yesno "Python 3 (v3.6.9) or later, PyYaml 5.3.1 and pip3 is required for compose-overrides.yml file to merge into the docker-compose.yml file. Install now?" 20 78); then + CURR_PYTHON_VER="${1:-Unknown}" + CURR_PYYAML_VER="${2:-Unknown}" + if (whiptail --title "Python 3 and Dependencies" --yesno "Python 3.6.9 or later (Current = $CURR_PYTHON_VER), PyYaml 5.3.1 or later (Current = $CURR_PYYAML_VER) and pip3 is required for compose-overrides.yml file to merge into the docker-compose.yml file. Install these now?" 20 78); then sudo apt install -y python3-pip python3-dev if [ $? -eq 0 ]; then PYTHON_VERSION_GOOD="true" @@ -169,36 +198,42 @@ function do_python3_pip() { if command_exists python3 && command_exists pip3; then PYTHON_VERSION=$(python3 --version) - echo "Python Version: $PYTHON_VERSION" + echo "Python Version: ${PYTHON_VERSION:-Unknown}" PYTHON_VERSION_MAJOR=$(echo "$PYTHON_VERSION"| cut -d' ' -f 2 | cut -d'.' -f 1) PYTHON_VERSION_MINOR=$(echo "$PYTHON_VERSION"| cut -d'.' -f 2) PYTHON_VERSION_BUILD=$(echo "$PYTHON_VERSION"| cut -d'.' -f 3) - if [ "$(minimum_version_check $REQ_PYTHON_VERSION $PYTHON_VERSION_MAJOR $PYTHON_VERSION_MINOR $PYTHON_VERSION_BUILD )" == "true" ]; then - PYTHON_VERSION_GOOD="true" - else - echo "Python is outdated." - install_python3_and_deps - fi - PYYAML_VERSION=$(python3 ./scripts/yaml_merge.py --pyyaml-version) - echo "PyYaml Version: $PYYAML_VERSION" + PYYAML_VERSION="${PYYAML_VERSION:-Unknown}" PYYAML_VERSION_MAJOR=$(echo "$PYYAML_VERSION"| cut -d' ' -f 2 | cut -d'.' -f 1) PYYAML_VERSION_MINOR=$(echo "$PYYAML_VERSION"| cut -d'.' -f 2) PYYAML_VERSION_BUILD=$(echo "$PYYAML_VERSION"| cut -d'.' -f 3) - if [ "$(minimum_version_check $REQ_PYYAML_VERSION $PYYAML_VERSION_MAJOR $PYYAML_VERSION_MINOR $PYYAML_VERSION_BUILD )" == "true" ]; then + if [ "$(minimum_version_check $REQ_PYTHON_VERSION $PYTHON_VERSION_MAJOR $PYTHON_VERSION_MINOR $PYTHON_VERSION_BUILD)" == "true" ]; then + PYTHON_VERSION_GOOD="true" + else + echo "Python is outdated." + install_python3_and_deps "$PYTHON_VERSION_MAJOR.$PYTHON_VERSION_MINOR.$PYTHON_VERSION_BUILD" "$PYYAML_VERSION_MAJOR.$PYYAML_VERSION_MINOR.$PYYAML_VERSION_BUILD" + return 1 + fi + echo "PyYaml Version: $PYYAML_VERSION" + if [ "$(minimum_version_check $REQ_PYYAML_VERSION $PYYAML_VERSION_MAJOR $PYYAML_VERSION_MINOR $PYYAML_VERSION_BUILD)" == "true" ]; then PYYAML_VERSION_GOOD="true" else echo "PyYaml is outdated." - install_python3_and_deps + if [ "$PYYAML_VERSION" != "Unknown" ]; then + install_python3_and_deps "$PYTHON_VERSION_MAJOR.$PYTHON_VERSION_MINOR.$PYTHON_VERSION_BUILD" "$PYYAML_VERSION_MAJOR.$PYYAML_VERSION_MINOR.$PYYAML_VERSION_BUILD" + else + install_python3_and_deps "$PYTHON_VERSION_MAJOR.$PYTHON_VERSION_MINOR.$PYTHON_VERSION_BUILD" + fi + return 1 fi else install_python3_and_deps + return 1 fi } - #function copies the template yml file to the local service folder and appends to the docker-compose.yml file function yml_builder() { @@ -375,9 +410,7 @@ case $mainmenu_selection in #if no container is selected then dont overwrite the docker-compose.yml file if [ -n "$container_selection" ]; then touch $TMP_DOCKER_COMPOSE_YML - # echo "services:" > $TMP_DOCKER_COMPOSE_YML - # Uncomment once sort_keys is available in Pyton->yaml echo "version: '$COMPOSE_VERSION'" > $TMP_DOCKER_COMPOSE_YML echo "services:" >> $TMP_DOCKER_COMPOSE_YML @@ -411,9 +444,6 @@ case $mainmenu_selection in cp $TMP_DOCKER_COMPOSE_YML $DOCKER_COMPOSE_YML fi - # Prepend compose version after merging/copying, so that merging doesn't move it to the bottom (alphabetically ordered). - # echo -e "version: '$COMPOSE_VERSION'\n$(cat $DOCKER_COMPOSE_YML)" > $DOCKER_COMPOSE_YML # Remove once sort_keys works in Python->yaml. - echo "docker-compose successfully created" echo "run 'docker-compose up -d' to start the stack" else From 53d715e886263c4ddfc144c590149d60c329d918 Mon Sep 17 00:00:00 2001 From: Slyke Date: Mon, 27 Apr 2020 23:08:47 -0700 Subject: [PATCH 44/69] Updated readme with instructions on how to select experimental features --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index a64cf201..07b58478 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,14 @@ cd ~/IOTstack && bash ./menu.sh docker-compose up -d ``` +## Experimental Features +Want to have the latest and greatest features? Switch to the experimental branch: +``` +git pull && git checkout experimental +./menu.sh +``` + +Do note that the experimental branch may be broken, or may break your setup, so ensure you have a good backup, and please report any issues. ## Migrating from the old repo? ``` From bcb42c333c144864a6d58456113ed52650575024 Mon Sep 17 00:00:00 2001 From: Slyke Date: Tue, 28 Apr 2020 01:55:14 -0700 Subject: [PATCH 45/69] Fixed merge conflict issue --- menu.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/menu.sh b/menu.sh index 7ee61e40..b60f39d6 100755 --- a/menu.sh +++ b/menu.sh @@ -334,6 +334,7 @@ if command_exists docker; then if (whiptail --title "Docker and Docker-Compose Version Issue" --yesno "Docker version is currently $DOCKER_VERSION which is less than $REQ_DOCKER_VERSION consider upgrading or you may experience issues. You can manually upgrade by typing 'sudo apt upgrade docker docker-compose'. Attempt to upgrade now?" 20 78); then sudo apt upgrade docker docker-compose fi + fi else echo "docker not installed" fi From 0b306042f07a74d4dbc97cae9f37d37d55d2dbc0 Mon Sep 17 00:00:00 2001 From: Slyke Date: Tue, 28 Apr 2020 02:43:50 -0700 Subject: [PATCH 46/69] Added some more nodered modules. Removed override file. Added branch selection --- .templates/nodered/build.sh | 8 ++++++-- compose-override.yml | 0 menu.sh | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) delete mode 100755 compose-override.yml diff --git a/.templates/nodered/build.sh b/.templates/nodered/build.sh index ba5ab374..276cb5a5 100755 --- a/.templates/nodered/build.sh +++ b/.templates/nodered/build.sh @@ -6,7 +6,13 @@ node_selection=$(whiptail --title "Node-RED nodes" --checklist --separate-output "Use the [SPACEBAR] to select the nodes you want preinstalled" 20 78 12 -- \ "node-red-node-pi-gpiod" " " "ON" \ "node-red-dashboard" " " "ON" \ + "node-red-contrib-influxdb" " " "ON" \ + "node-red-contrib-boolean-logic" " " "ON" \ + "node-red-node-rbe" " " "ON" \ + "node-red-configurable-ping" " " "ON" \ "node-red-node-openweathermap" " " "OFF" \ + "node-red-contrib-discord" " " "OFF" \ + "node-red-node-email" " " "on" \ "node-red-node-google" " " "OFF" \ "node-red-node-emoncms" " " "OFF" \ "node-red-node-geofence" " " "OFF" \ @@ -15,7 +21,6 @@ node_selection=$(whiptail --title "Node-RED nodes" --checklist --separate-output "node-red-node-smooth" " " "OFF" \ "node-red-node-darksky" " " "OFF" \ "node-red-node-sqlite" " " "OFF" \ - "node-red-contrib-influxdb" " " "ON" \ "node-red-contrib-config" " " "OFF" \ "node-red-contrib-grove" " " "OFF" \ "node-red-contrib-diode" " " "OFF" \ @@ -29,7 +34,6 @@ node_selection=$(whiptail --title "Node-RED nodes" --checklist --separate-output "node-red-contrib-isonline" " " "OFF" \ "node-red-contrib-npm" " " "OFF" \ "node-red-contrib-file-function" " " "OFF" \ - "node-red-contrib-boolean-logic" " " "OFF" \ "node-red-contrib-home-assistant-websocket" " " "OFF" \ "node-red-contrib-blynk-ws" " " "OFF" \ "node-red-contrib-owntracks" " " "OFF" \ diff --git a/compose-override.yml b/compose-override.yml deleted file mode 100755 index e69de29b..00000000 diff --git a/menu.sh b/menu.sh index b60f39d6..cc2c9420 100755 --- a/menu.sh +++ b/menu.sh @@ -3,6 +3,8 @@ #get path of menu correct pushd ~/IOTstack +CURRENT_BRANCH=${1:-$(git name-rev --name-only HEAD)} + # Consts/vars TMP_DOCKER_COMPOSE_YML=./.tmp/docker-compose.tmp.yml DOCKER_COMPOSE_YML=./docker-compose.yml @@ -602,7 +604,7 @@ case $mainmenu_selection in ;; "update") echo "Pulling latest project file from Github.com ---------------------------------------------" - git pull origin master + git pull origin $CURRENT_BRANCH echo "git status ------------------------------------------------------------------------------" git status ;; From 18bdefee1a22efbe7b29e5e434f019e3a7b95a64 Mon Sep 17 00:00:00 2001 From: DIY techie Date: Sun, 3 May 2020 12:05:22 +0200 Subject: [PATCH 47/69] port changes and devices commented --- .templates/deconz/service.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.templates/deconz/service.yml b/.templates/deconz/service.yml index 41339439..c4d343c7 100644 --- a/.templates/deconz/service.yml +++ b/.templates/deconz/service.yml @@ -4,18 +4,18 @@ restart: unless-stopped network_mode: bridge ports: - - '80:80' - - '443:443' + - '8090:80' + - '4439:443' - '5901:5900' volumes: - ./volumes/deconz/:/root/.local/share/dresden-elektronik/deCONZ devices: - #ConBee II: - - /dev/ttyACM0 - #ConBee: - # - /dev/ttyUSB0 - #RaspBee: - # - /dev/ttyAMA0 or /dev/ttyS0 + #ConBee II uncomment next line + #- /dev/ttyACM0 + #ConBee uncomment next line + #- /dev/ttyUSB0 + #RaspBee uncomment next line + #- /dev/ttyAMA0 or /dev/ttyS0 environment: - DECONZ_VNC_MODE=1 - DECONZ_VNC_PASSWORD=changeme From f3be535df707e06858c2d8d9b4d69586e5b8d3df Mon Sep 17 00:00:00 2001 From: DIY techie Date: Sun, 3 May 2020 12:34:43 +0200 Subject: [PATCH 48/69] devices commented out --- .templates/deconz/service.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.templates/deconz/service.yml b/.templates/deconz/service.yml index c4d343c7..9199adc3 100644 --- a/.templates/deconz/service.yml +++ b/.templates/deconz/service.yml @@ -9,7 +9,7 @@ - '5901:5900' volumes: - ./volumes/deconz/:/root/.local/share/dresden-elektronik/deCONZ - devices: + #devices: #ConBee II uncomment next line #- /dev/ttyACM0 #ConBee uncomment next line From 1a0c310551ccfbe5da1c0932c09002de8d37606c Mon Sep 17 00:00:00 2001 From: DIY techie Date: Tue, 5 May 2020 22:21:08 +0200 Subject: [PATCH 49/69] Create deconz.md --- docs/deconz.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 docs/deconz.md diff --git a/docs/deconz.md b/docs/deconz.md new file mode 100644 index 00000000..25641b3a --- /dev/null +++ b/docs/deconz.md @@ -0,0 +1,20 @@ +# deCONZ +## References +- [Docker](https://hub.docker.com/r/marthoc/deconz) +- [Website](https://github.com/dresden-elektronik/deconz-rest-plugin/blob/master/README.md) + +## Installing Phoscon (the part of deCONZ that lets you add devices and setup the gateway) +All devices are commented out of the docker-compose template by default to avoid errors. After adding deCONZ to the stack, comment in your device and physically plug in the relevant device (e.g. ConBee II) BEFORE running `docker-compose up -d` again. + + + +## Viewing the Zigbee mesh +The Zigbee mesh can be seen using VNC (port 8090). The default password is "changeme". + +By default, VNC container has no password. You can leave it that way if you like but its always a good idea to secure your services. + +## Connecting deCONZ and Node-RED +(port 8090) + +Step 2 +Edit the file called services/mosquitto/mosquitto.conf and remove the comment in front of password_file. Restart the container with `docker-compose restart mosquitto`. Type those credentials into Node-red etc. From fc30deee4ff5742b6b1da6e0280e8688cd508435 Mon Sep 17 00:00:00 2001 From: DIY techie Date: Thu, 7 May 2020 00:29:15 +0200 Subject: [PATCH 50/69] Update deconz.md --- docs/deconz.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/docs/deconz.md b/docs/deconz.md index 25641b3a..60038481 100644 --- a/docs/deconz.md +++ b/docs/deconz.md @@ -3,18 +3,27 @@ - [Docker](https://hub.docker.com/r/marthoc/deconz) - [Website](https://github.com/dresden-elektronik/deconz-rest-plugin/blob/master/README.md) -## Installing Phoscon (the part of deCONZ that lets you add devices and setup the gateway) -All devices are commented out of the docker-compose template by default to avoid errors. After adding deCONZ to the stack, comment in your device and physically plug in the relevant device (e.g. ConBee II) BEFORE running `docker-compose up -d` again. +## Installing deCONZ +1) Plug in you ConBee II, ConBee or Raspbee (if no device is plugged in, deCONZ will not be properly installed) +2) Add deCONZ to your stack using `cd ~/IOTstack && bash ./menu.sh`and select deCONZ +3) Edit the deconz service.yml using `sudo nano ~/IOTstack/services/deconz/service.yml` - activate your device by removing the "#" in front of "devices" and the "#" in front of your specific device. Save the changes using "ctrl+x", "y" and "enter". +4) Build the stack again using `cd ~/IOTstack && bash ./menu.sh` to apply the changes and select "Do not overwrite". +5) Run `docker-compose up -d` to build the deCONZ container. +These steps are also shown in the gif below: +![installing deCONZ](https://github.com/DIYtechie/resources/blob/master/images/installing%20deconz%20-%20short%20version.gif?raw=true) -## Viewing the Zigbee mesh -The Zigbee mesh can be seen using VNC (port 8090). The default password is "changeme". +deCONZ should now be available at http://:8090/ -By default, VNC container has no password. You can leave it that way if you like but its always a good idea to secure your services. +## Viewing the deCONZ Zigbee mesh +The Zigbee mesh can be viewed using VNC (port 5901). The default VNC password is "changeme". ## Connecting deCONZ and Node-RED -(port 8090) +1) Install [node-red-contrib-deconz](https://flows.nodered.org/node/node-red-contrib-deconz) via the manage palette menu (if not already pre-installed with Node-RED) +2) Open a deCONZ in node, select "Add new deconz-server" and type in the IP-address of your IOTstack pc (Rpi) and type in port 8090 (not websocket port) +3) Open deCONZ (http://:8090/) and open settings>gateway>advanced. Click "Authenticate app". +4) Go back to your Node-RED instance and click the magick:get setting button in the server node. The API and websocket port will be automatically inserted. Click "Add" and you are good to go. -Step 2 -Edit the file called services/mosquitto/mosquitto.conf and remove the comment in front of password_file. Restart the container with `docker-compose restart mosquitto`. Type those credentials into Node-red etc. +![installing deCONZ](https://github.com/DIYtechie/resources/blob/master/images/deconz%20authenticate.png?raw=true) +![installing deCONZ](https://github.com/DIYtechie/resources/blob/master/images/deconz%20node%20red%20config.png?raw=true) From c3f39e8d98fc98048011777ef276f64658270ff8 Mon Sep 17 00:00:00 2001 From: DIY techie Date: Thu, 7 May 2020 00:31:10 +0200 Subject: [PATCH 51/69] Update deconz.md --- docs/deconz.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/deconz.md b/docs/deconz.md index 60038481..5b0e265f 100644 --- a/docs/deconz.md +++ b/docs/deconz.md @@ -14,7 +14,7 @@ These steps are also shown in the gif below: ![installing deCONZ](https://github.com/DIYtechie/resources/blob/master/images/installing%20deconz%20-%20short%20version.gif?raw=true) -deCONZ should now be available at http://:8090/ +deCONZ should now be available at http://ip.address.of.your.IOTstack.pc.here:8090/ ## Viewing the deCONZ Zigbee mesh The Zigbee mesh can be viewed using VNC (port 5901). The default VNC password is "changeme". @@ -22,7 +22,7 @@ The Zigbee mesh can be viewed using VNC (port 5901). The default VNC password is ## Connecting deCONZ and Node-RED 1) Install [node-red-contrib-deconz](https://flows.nodered.org/node/node-red-contrib-deconz) via the manage palette menu (if not already pre-installed with Node-RED) 2) Open a deCONZ in node, select "Add new deconz-server" and type in the IP-address of your IOTstack pc (Rpi) and type in port 8090 (not websocket port) -3) Open deCONZ (http://:8090/) and open settings>gateway>advanced. Click "Authenticate app". +3) Open deCONZ (http://ip.address.of.your.IOTstack.pc.here:8090/) and open settings>gateway>advanced. Click "Authenticate app". 4) Go back to your Node-RED instance and click the magick:get setting button in the server node. The API and websocket port will be automatically inserted. Click "Add" and you are good to go. ![installing deCONZ](https://github.com/DIYtechie/resources/blob/master/images/deconz%20authenticate.png?raw=true) From 609c6c956bccce6fdbbb22d39405458ed54d17b5 Mon Sep 17 00:00:00 2001 From: LePirlouit Date: Thu, 7 May 2020 10:40:36 +0200 Subject: [PATCH 52/69] :pencil2: fixing typo --- .templates/pihole/pihole.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.templates/pihole/pihole.env b/.templates/pihole/pihole.env index 2f768070..a64878d2 100644 --- a/.templates/pihole/pihole.env +++ b/.templates/pihole/pihole.env @@ -5,7 +5,7 @@ WEBPASSWORD=pihole #DNSSEC=false #DNS_BOGUS_PRIV=True #CONDITIONAL_FORWARDING=False -#CONDITIONAL_FORWARDING_IP=your_router_ip_here (only if CONDITIONAL_FORWARDING=ture) +#CONDITIONAL_FORWARDING_IP=your_router_ip_here (only if CONDITIONAL_FORWARDING=true) #CONDITIONAL_FORWARDING_DOMAIN=optional #CONDITIONAL_FORWARDING_REVERSE=optional #ServerIP=your_Pi's_IP_here << recommended @@ -13,4 +13,4 @@ WEBPASSWORD=pihole #VIRTUAL_HOST=$ServerIP #IPv6=True INTERFACE=eth0 -#DNSMASQ_LISTENING=local \ No newline at end of file +#DNSMASQ_LISTENING=local From 8cbdba6e3e6f1f37c0baf8f33adc561b4624109e Mon Sep 17 00:00:00 2001 From: DIY techie Date: Sat, 9 May 2020 09:23:54 +0200 Subject: [PATCH 53/69] Update deconz.md --- docs/deconz.md | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/docs/deconz.md b/docs/deconz.md index 5b0e265f..45442f28 100644 --- a/docs/deconz.md +++ b/docs/deconz.md @@ -3,17 +3,7 @@ - [Docker](https://hub.docker.com/r/marthoc/deconz) - [Website](https://github.com/dresden-elektronik/deconz-rest-plugin/blob/master/README.md) -## Installing deCONZ -1) Plug in you ConBee II, ConBee or Raspbee (if no device is plugged in, deCONZ will not be properly installed) -2) Add deCONZ to your stack using `cd ~/IOTstack && bash ./menu.sh`and select deCONZ -3) Edit the deconz service.yml using `sudo nano ~/IOTstack/services/deconz/service.yml` - activate your device by removing the "#" in front of "devices" and the "#" in front of your specific device. Save the changes using "ctrl+x", "y" and "enter". -4) Build the stack again using `cd ~/IOTstack && bash ./menu.sh` to apply the changes and select "Do not overwrite". -5) Run `docker-compose up -d` to build the deCONZ container. - -These steps are also shown in the gif below: - -![installing deCONZ](https://github.com/DIYtechie/resources/blob/master/images/installing%20deconz%20-%20short%20version.gif?raw=true) - +## Accessing Phoscon ui deCONZ should now be available at http://ip.address.of.your.IOTstack.pc.here:8090/ ## Viewing the deCONZ Zigbee mesh From c584129eb2a80f696917e7cfaed81dfddeb30ab7 Mon Sep 17 00:00:00 2001 From: DIY techie Date: Sat, 9 May 2020 09:26:06 +0200 Subject: [PATCH 54/69] make empty this ensures that no deCONZ container is created if a device is not plugged in (the user has chosen cancel) or selected from the deconz menu --- .templates/deconz/service.yml | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/.templates/deconz/service.yml b/.templates/deconz/service.yml index 9199adc3..8b137891 100644 --- a/.templates/deconz/service.yml +++ b/.templates/deconz/service.yml @@ -1,26 +1 @@ - deconz: - image: marthoc/deconz - container_name: deconz - restart: unless-stopped - network_mode: bridge - ports: - - '8090:80' - - '4439:443' - - '5901:5900' - volumes: - - ./volumes/deconz/:/root/.local/share/dresden-elektronik/deCONZ - #devices: - #ConBee II uncomment next line - #- /dev/ttyACM0 - #ConBee uncomment next line - #- /dev/ttyUSB0 - #RaspBee uncomment next line - #- /dev/ttyAMA0 or /dev/ttyS0 - environment: - - DECONZ_VNC_MODE=1 - - DECONZ_VNC_PASSWORD=changeme - - DEBUG_INFO=1 - - DEBUG_APS=0 - - DEBUG_ZCL=0 - - DEBUG_ZDP=0 - - DEBUG_OTAU=0 + From b80b21e04032538c5c4aa4272cf998a61adeaf1e Mon Sep 17 00:00:00 2001 From: DIY techie Date: Sat, 9 May 2020 09:28:28 +0200 Subject: [PATCH 55/69] create deCONZ device config --- .templates/deconz/build.sh | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .templates/deconz/build.sh diff --git a/.templates/deconz/build.sh b/.templates/deconz/build.sh new file mode 100644 index 00000000..4dcf8f69 --- /dev/null +++ b/.templates/deconz/build.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +#deCONZ device configuration + +device_selection=$(whiptail --radiolist --title "deCONZ device configuration" --notags \ + "Use the [SPACEBAR] to select your Zigbee device from the list below and make sure it is plugged in (if not, press [ESC])." 20 78 12 \ + "ConBeeII" "ConBee II " "ON" \ + "ConBee" "ConBee " "OFF" \ + "RaspBee" "RaspBee " "OFF" \ + 3>&1 1>&2 2>&3) + + case $device_selection in + + "ConBeeII") + echo "...copied ConBee II config from template" + echo "" >>docker-compose.yml + cat .templates/deconz/service_conbee_II.yml >>docker-compose.yml + ;; + "ConBee") + echo "...copied ConBee config from template" + echo "" >>docker-compose.yml + cat .templates/deconz/service_conbee.yml >>docker-compose.yml + ;; + "RaspBee") + echo "...copied RaspBee config from template" + echo "" >>docker-compose.yml + cat .templates/deconz/service_raspbee.yml >>docker-compose.yml + ;; + esac From 9fc4d6780025c185e2d931a77363cba9a23d9a0b Mon Sep 17 00:00:00 2001 From: DIY techie Date: Sat, 9 May 2020 09:29:16 +0200 Subject: [PATCH 56/69] Create service_conbee.yml --- .templates/deconz/service_conbee.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .templates/deconz/service_conbee.yml diff --git a/.templates/deconz/service_conbee.yml b/.templates/deconz/service_conbee.yml new file mode 100644 index 00000000..c23a9a84 --- /dev/null +++ b/.templates/deconz/service_conbee.yml @@ -0,0 +1,21 @@ + deconz: + image: marthoc/deconz + container_name: deconz + restart: unless-stopped + ports: + - '8090:80' + - '443:443' + - '5901:5900' + volumes: + - ./volumes/deconz/:/root/.local/share/dresden-elektronik/deCONZ + devices: + #ConBee: + - /dev/ttyUSB0 + environment: + - DECONZ_VNC_MODE=1 + - DECONZ_VNC_PASSWORD=changeme + - DEBUG_INFO=1 + - DEBUG_APS=0 + - DEBUG_ZCL=0 + - DEBUG_ZDP=0 + - DEBUG_OTAU=0 From c2d231c6aedb123a4a44c48f10dc2f45e4b8e29d Mon Sep 17 00:00:00 2001 From: DIY techie Date: Sat, 9 May 2020 09:29:58 +0200 Subject: [PATCH 57/69] Create service_conbee_II.yml --- .templates/deconz/service_conbee_II.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .templates/deconz/service_conbee_II.yml diff --git a/.templates/deconz/service_conbee_II.yml b/.templates/deconz/service_conbee_II.yml new file mode 100644 index 00000000..da9d99e6 --- /dev/null +++ b/.templates/deconz/service_conbee_II.yml @@ -0,0 +1,21 @@ + deconz: + image: marthoc/deconz + container_name: deconz + restart: unless-stopped + ports: + - '8090:80' + - '443:443' + - '5901:5900' + volumes: + - ./volumes/deconz/:/root/.local/share/dresden-elektronik/deCONZ + devices: + #ConBee II: + - /dev/ttyACM0 + environment: + - DECONZ_VNC_MODE=1 + - DECONZ_VNC_PASSWORD=changeme + - DEBUG_INFO=1 + - DEBUG_APS=0 + - DEBUG_ZCL=0 + - DEBUG_ZDP=0 + - DEBUG_OTAU=0 From d09497e809ace7ad8fad35dca055f528e3106866 Mon Sep 17 00:00:00 2001 From: DIY techie Date: Sat, 9 May 2020 09:30:52 +0200 Subject: [PATCH 58/69] Create service_raspbee.yml --- .templates/deconz/service_raspbee.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .templates/deconz/service_raspbee.yml diff --git a/.templates/deconz/service_raspbee.yml b/.templates/deconz/service_raspbee.yml new file mode 100644 index 00000000..4aedca66 --- /dev/null +++ b/.templates/deconz/service_raspbee.yml @@ -0,0 +1,22 @@ + deconz: + image: marthoc/deconz + container_name: deconz + restart: unless-stopped + ports: + - '8090:80' + - '443:443' + - '5901:5900' + volumes: + - ./volumes/deconz/:/root/.local/share/dresden-elektronik/deCONZ + devices: + #RaspBee: + - /dev/ttyAMA0 + #alternatively - /dev/ttyS0 + environment: + - DECONZ_VNC_MODE=1 + - DECONZ_VNC_PASSWORD=changeme + - DEBUG_INFO=1 + - DEBUG_APS=0 + - DEBUG_ZCL=0 + - DEBUG_ZDP=0 + - DEBUG_OTAU=0 From a483b4317404f4f777f906a712867a1e23208163 Mon Sep 17 00:00:00 2001 From: DIY techie Date: Sat, 9 May 2020 13:58:29 +0200 Subject: [PATCH 59/69] fixing typos and simplifying with gif --- docs/deconz.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/deconz.md b/docs/deconz.md index 45442f28..92ff6f09 100644 --- a/docs/deconz.md +++ b/docs/deconz.md @@ -3,17 +3,18 @@ - [Docker](https://hub.docker.com/r/marthoc/deconz) - [Website](https://github.com/dresden-elektronik/deconz-rest-plugin/blob/master/README.md) -## Accessing Phoscon ui -deCONZ should now be available at http://ip.address.of.your.IOTstack.pc.here:8090/ +## Pre-installation +Before running the command that creates the deconz Docker container (`docker-compose up -d`), you may need to add your Linux user to the dialout group, which allows the user access to serial devices (i.e. Conbee/Conbee II/RaspBee): + +`sudo usermod -a -G dialout pi` (pi user being used as an example) + +## Accessing the Phoscon UI +The Phoscon UI is available using port 8090 (http://your.local.ip.address:8090/) ## Viewing the deCONZ Zigbee mesh The Zigbee mesh can be viewed using VNC (port 5901). The default VNC password is "changeme". ## Connecting deCONZ and Node-RED -1) Install [node-red-contrib-deconz](https://flows.nodered.org/node/node-red-contrib-deconz) via the manage palette menu (if not already pre-installed with Node-RED) -2) Open a deCONZ in node, select "Add new deconz-server" and type in the IP-address of your IOTstack pc (Rpi) and type in port 8090 (not websocket port) -3) Open deCONZ (http://ip.address.of.your.IOTstack.pc.here:8090/) and open settings>gateway>advanced. Click "Authenticate app". -4) Go back to your Node-RED instance and click the magick:get setting button in the server node. The API and websocket port will be automatically inserted. Click "Add" and you are good to go. +Install [node-red-contrib-deconz](https://flows.nodered.org/node/node-red-contrib-deconz) via the "Manage palette" menu in Node-RED (if it is not already pre-installed) and follow the 2 simple steps in the video below: -![installing deCONZ](https://github.com/DIYtechie/resources/blob/master/images/deconz%20authenticate.png?raw=true) -![installing deCONZ](https://github.com/DIYtechie/resources/blob/master/images/deconz%20node%20red%20config.png?raw=true) +![installing deCONZ](https://github.com/DIYtechie/resources/blob/master/images/Setup%20deCONZ%20in%20Node-RED.gif?raw=true) From 9fadb354bf41428b587de38fc32ea9acff2fd68f Mon Sep 17 00:00:00 2001 From: Robert Csakany Date: Mon, 11 May 2020 00:21:24 +0200 Subject: [PATCH 60/69] Add Dozzle container. --- .templates/dozzle/service.yml | 9 +++++++++ .templates/mosquitto/mosquitto.conf | 4 +++- menu.sh | 2 ++ 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 .templates/dozzle/service.yml diff --git a/.templates/dozzle/service.yml b/.templates/dozzle/service.yml new file mode 100644 index 00000000..74fb7602 --- /dev/null +++ b/.templates/dozzle/service.yml @@ -0,0 +1,9 @@ + dozzle: + container_name: dozzle + image: amir20/dozzle:latest + restart: unless-stopped + network_mode: host + ports: + - 8888:8080 + volumes: + - /var/run/docker.sock:/var/run/docker.sock diff --git a/.templates/mosquitto/mosquitto.conf b/.templates/mosquitto/mosquitto.conf index 5827737d..39ca6df4 100644 --- a/.templates/mosquitto/mosquitto.conf +++ b/.templates/mosquitto/mosquitto.conf @@ -1,6 +1,8 @@ persistence true persistence_location /mosquitto/data/ -log_dest file /mosquitto/log/mosquitto.log +#log_dest file /mosquitto/log/mosquitto.log +# To avoid flash wearing +log_dest stdout #Uncomment to enable passwords #password_file /mosquitto/config/pwfile diff --git a/menu.sh b/menu.sh index fc611972..2cbb8fa8 100755 --- a/menu.sh +++ b/menu.sh @@ -32,6 +32,7 @@ declare -A cont_array=( [homebridge]="Homebridge" [python]="Python 3" [gitea]="Gitea" + [dozzle]="Dozzle" ) declare -a armhf_keys=( @@ -62,6 +63,7 @@ declare -a armhf_keys=( "homebridge" "python" "gitea" + "dozzle" # add yours here ) sys_arch=$(uname -m) From e102a7e648544af03259ab924cc543e640f94da1 Mon Sep 17 00:00:00 2001 From: Robert Csakany Date: Mon, 11 May 2020 00:57:54 +0200 Subject: [PATCH 61/69] Fixing tablulator --- .templates/dozzle/service.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.templates/dozzle/service.yml b/.templates/dozzle/service.yml index 74fb7602..2c9c5217 100644 --- a/.templates/dozzle/service.yml +++ b/.templates/dozzle/service.yml @@ -5,5 +5,5 @@ network_mode: host ports: - 8888:8080 - volumes: + volumes: - /var/run/docker.sock:/var/run/docker.sock From ae04776554d7a3c6889f143aa1a91d3ffbcaf915 Mon Sep 17 00:00:00 2001 From: Phill Kelley Date: Sat, 16 May 2020 14:11:39 +1000 Subject: [PATCH 62/69] Expansion of Grafana documentation page. Inspired by discussion at https://github.com/gcgarner/IOTstack/issues/185 and https://github.com/gcgarner/IOTstack/issues/131 and similar. Covers setting the time-zone for the container. Recommends NOT attempting to change either the admin username or password via environment variables and explains why. Also includes how to recover from a lost admin password. Discusses the how-to of overriding Grafana settings using the example of enabling anonymous access. Explains how to establish a "clean slate" state (a bit quicker than completely deleting and re-installing IOTstack). --- docs/Grafana.md | 231 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 216 insertions(+), 15 deletions(-) diff --git a/docs/Grafana.md b/docs/Grafana.md index 8194f1c6..824dd8e8 100644 --- a/docs/Grafana.md +++ b/docs/Grafana.md @@ -1,27 +1,228 @@ # Grafana + ## References + - [Docker](https://hub.docker.com/r/grafana/grafana) - [Website](https://grafana.com/) -## Security -Grafana's default credentials are username "admin" password "admin" it will ask you to choose a new password on boot. Go to `:3000` in your web browser. +## Setting your time-zone + +The default *~/IOTstack/services/grafana/grafana.env* contains this line: + +``` +#TZ=Africa/Johannesburg +``` + +Uncomment that line and change the right hand side to [your own timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones). + + +## Security + +If Grafana has just been installed but has **never** been launched then the following will be true: + +* The folder *~/IOTstack/volumes/grafana* will not exist; and +* The file *~/IOTstack/services/grafana/grafana.env* will contain these lines: + + ``` + #GF_SECURITY_ADMIN_USER=admin + #GF_SECURITY_ADMIN_PASSWORD=admin + ``` + +You should see those lines as **documentation** rather than something you are being invited to edit. It is telling you that the default administative user for Grafana is "admin" and that the default password for that user is "admin". + +If you do not change anything then, when you bring up the stack and use a browser to connect to your Raspberry Pi on port 3000, Grafana will: + +* Expect you to login as user "admin" with password "admin"; and then +* Force you to change the default password to something else. + +Thereafter, you will login as "admin" with whatever password you chose. You can change the administrator's password as often as you like via the web UI (*profile* button, *change password* tab). + +This method (of **not** touching these two keys in *grafana.env*) is the recommended approach. *Please* try to resist the temptation to fiddle! + +### I want a different admin username (not recommended) + +If, before you bring up the stack for the first time, you do this: + +``` +GF_SECURITY_ADMIN_USER=jack +#GF_SECURITY_ADMIN_PASSWORD=admin +``` + +then, when you bring up the stack and connect on port 3000, Grafana will: + +* Expect you to login as user "jack" with password "admin"; and then +* Force you to change the default password to something else. + +Thereafter, "jack" will be the Grafana administrator and you will login with the password you chose, until you decide to change the password to something else via the web UI. + +Don't think you can come back later and tweak the Grafana administrator name in the environment variables. It doesn't work that way. It's a one-shot. + +### I want a different default admin password (not recommended) + +Well, first off, the two methods above both make you set a different password on first login so there probably isn't much point to this. + +But, if you *really* insist… + +If, before you bring up the stack for the first time, you do this: + +``` +#GF_SECURITY_ADMIN_USER=admin +GF_SECURITY_ADMIN_PASSWORD=jack +``` + +then, when you bring up the stack and use a browser to connect to your Raspberry Pi on port 3000, Grafana will expect you to login as user "admin" with password "jack". + +Grafana will not force you to change the password on first login but you will still be able to change it via the web UI. + +But don't think you can come back later and change the password in the environment variables. It doesn't work that way. It's a one-shot. + +### I want to change everything (not recommended) + +If, before you bring up the stack for the first time, you do this: + +``` +GF_SECURITY_ADMIN_USER=bill +GF_SECURITY_ADMIN_PASSWORD=jack +``` + +then, when you bring up the stack and use a browser to connect to your Raspberry Pi on port 3000, Grafana will expect you to login as user "bill" with password "jack". + +Grafana will not force you to change the password on first login but you will still be able to change it via the web UI. + +But don't think you can come back later and tweak either the username or password in the environment variables. It doesn't work that way. It's a one-shot. + +### Distilling it down + +**Before** Grafana is launched for the first time: + +* *GF\_SECURITY\_ADMIN\_USER* has a default value of "admin". You *can* explicitly set it to "admin" or some other value. Whatever option you choose then that's the account name of Grafana's administrative user. But choosing any value other than "admin" is probably a bad idea. +* *GF\_SECURITY\_ADMIN\_PASSWORD* has a default value of "admin". You can explicitly set it to "admin" or some other value. If its value is "admin" then you will be forced to change it the first time you login to Grafana. If its value is something other than "admin" then that will be the password until you change it via the web UI. + +These two environment keys **only** work for the **first** launch of Grafana. Once Grafana has been launched, you can **never** change either the username or the password by editing *grafana.env*. + +For this reason, it is better to leave *grafana.env* in its shrink-wrapped state. Your first login is as "admin/admin" and then you set the password you actually want when Grafana prompts you to change it. + +### HELP – I forgot my Grafana admin password! + +Assuming your IOTstack is up, the magic incantation is: + +``` +$ docker exec grafana grafana-cli --homepath "/usr/share/grafana" admin reset-admin-password "admin" +``` + +Then, use a browser to connect to your Raspberry Pi on port 3000. Grafana will: + +* Expect you login as user "admin" with password "admin"; and then +* Force you to change the default password to something else. + +This magic incantation assumes that your administrative username is "admin". If you ignored the advice above and changed the administator username to something else then all bets are off. It might work anyway but we haven't tested it. Sorry. But that's why we said changing the username was not recommended. + +## Overriding Grafana settings + +Grafana documentation contains [a list of settings](https://grafana.com/docs/installation/configuration/). Settings are described in terms of how they appear in ".ini" files. + +An example of the sort of thing you might want to do is to enable anonymous access to your Grafana dashboards. The [Grafana documentation](https://grafana.com/docs/grafana/latest/auth/overview/#anonymous-authentication) describes this in ".ini" format as: + +``` +[auth.anonymous] +enabled = true + +# Organization name that should be used for unauthenticated users +org_name = Main Org. + +# Role for unauthenticated users, other valid values are `Editor` and `Admin` +org_role = Viewer +``` + +".ini" format is not really appropriate in a Docker context. Instead, you use environment variables to override Docker's settings. Environment variables are placed in *~/IOTstack/services/grafana/grafana.env*. + +You need to convert ".ini" format to environment variable syntax. The rules are: + +* Start with "GF_", then +* Append the \[section name\], replacing any periods with underscores, then +* Append the section key "as is", then +* Append an "=", then +* Append the right hand side in quotes. + +Applying those rules gets you: + +``` +GF_AUTH_ANONYMOUS_ENABLED="true" +GF_AUTH_ANONYMOUS_ORG_NAME="Main Org." +GF_AUTH_ANONYMOUS_ORG_ROLE="Viewer" +``` + +> It is not strictly necessary to encapsulate every right hand side value in quotes. In the above, both "true" and "Viewer" would work without quotes, whereas "Main Org." needs quotes because of the embedded space. + +After you have changed *~/IOTstack/services/grafana/grafana.env*, you need to propagate the changes into the Grafana container: + +``` +$ cd ~/IOTstack +$ docker-compose stop grafana +$ docker-compose up -d +``` + +> In theory, the second command could be omitted, or both the second and third commands could be replaced with "docker-compose restart grafana" but experience suggests stopping the container is more reliable. + +A slightly more real-world example would involve choosing a different default organisation name for anonymous access. This example uses "ChezMoi". + +First, the environment key needs to be set to that value: + +``` +GF_AUTH_ANONYMOUS_ORG_NAME=ChezMoi +``` + +Then that change needs to be propagated into the Grafana container as explained above. + +Next, Grafana needs to be told that "ChezMoi" is the default organisation: + +1. Use your browser to login to Grafana as an administrator. +2. From the "Server Admin" slide-out menu on the left hand side, choose "Orgs". +3. In the list that appears, click on "Main Org". This opens an editing panel. +4. Change the "Name" field to "ChezMoi" and click "Update". +5. Sign-out of Grafana. You will be taken back to the login screen. Your URL bar will look something like this: + + ``` + http://myhost.mydomain.com:3000/login + ``` +6. Edit the URL to remove the "login" suffix and press return. If all your changes were applied successfully, you will have anonymous access and the URL will look something like this: + + ``` + http://myhost.mydomain.com:3000/?orgId=1 + ``` + +## HELP – I made a mess! + +"I made a bit of a mess with Grafana. First time user. Steep learning curve. False starts, many. Mistakes, unavoidable. Been there, done that. But now I **really** need to start from a clean slate. And, yes, I understand there is no *undo* for this." + +Begin by stopping Grafana: + +``` +$ cd ~/IOTstack +$ docker-compose stop grafana +``` + +You have two options: + +1. Destroy your settings and dashboards but retain any plugins you may have installed: + + ``` + $ sudo rm ~/IOTstack/volumes/grafana/data/grafana.db + ``` + +2. Nuke everything (triple-check this command **before** you hit return): -## Overwriting grafana.ini settings + ``` + $ sudo rm -rf ~/IOTstack/volumes/grafana/data + ``` -A list of the settings available in grafana.ini are listed [here](https://grafana.com/docs/installation/configuration/) +This is where you should edit *~/IOTstack/services/grafana/grafana.env* to correct any problems (such as choosing an administrative username other than "admin"). -To overwrite a setting edit the IOTstack/services/grafana/grafana.env file. The format is `GF__` +When you are ready, bring Grafana back up again: -An example would be: ``` -GF_PATHS_DATA=/var/lib/grafana -GF_PATHS_LOGS=/var/log/grafana -# [SERVER] -GF_SERVER_ROOT_URL=http://localhost:3000/grafana -GF_SERVER_SERVE_FROM_SUB_PATH=true -# [SECURITY] -GF_SECURITY_ADMIN_USER=admin -GF_SECURITY_ADMIN_PASSWORD=admin +$ cd ~/IOTstack +$ docker-compose up -d ``` -After the alterations run `docker-compose up -d` to pull them in +Grafana will automatically recreate everything it needs. You will be able to login as "admin/admin". From 0b71c431e950cdc21df3ec0545e61158b5805119 Mon Sep 17 00:00:00 2001 From: Phill Kelley Date: Mon, 18 May 2020 17:41:54 +1000 Subject: [PATCH 63/69] Removes pin of Grafana to version 6.3.6. Grafana is pinned to version 6.3.6. There is no obvious reason. I removed the pinning and Grafana advanced to 6.7.3. Some of the Grafana user interface had moved around (eg the top level contained a panel for the Grafana blog rather than the list of installed panels) but things like that are to be expected. There were one or two cosmetic changes in some of my dashboards (eg font size changes in bar charts) but the panels all worked out-of-the-box and there was nothing that suggested any need for caution in removing the pinning to 6.3.6. --- .templates/grafana/service.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.templates/grafana/service.yml b/.templates/grafana/service.yml index 538c9858..8303096a 100644 --- a/.templates/grafana/service.yml +++ b/.templates/grafana/service.yml @@ -1,6 +1,6 @@ grafana: container_name: grafana - image: grafana/grafana:6.3.6 + image: grafana/grafana restart: unless-stopped user: "0" ports: From 64f755f1e7b3b3281a1ab6ae010413f6d44156f6 Mon Sep 17 00:00:00 2001 From: DIY techie Date: Thu, 21 May 2020 15:34:00 +0200 Subject: [PATCH 64/69] Added troubleshooting text and other amendments --- docs/deconz.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/deconz.md b/docs/deconz.md index 92ff6f09..dec44af2 100644 --- a/docs/deconz.md +++ b/docs/deconz.md @@ -3,18 +3,29 @@ - [Docker](https://hub.docker.com/r/marthoc/deconz) - [Website](https://github.com/dresden-elektronik/deconz-rest-plugin/blob/master/README.md) -## Pre-installation -Before running the command that creates the deconz Docker container (`docker-compose up -d`), you may need to add your Linux user to the dialout group, which allows the user access to serial devices (i.e. Conbee/Conbee II/RaspBee): +## Troubleshooting +Make sure your Conbee/Conbee II/RaspBee gateway is connected. If your gateway is not detected, or no lights can be paired, try moving the device to another usb port, reboot your computer and build the stack from the menu again `cd ~/IOTstack && bash ./menu.sh` (select "Pull full service from template" if prompted). The device must be plugged in when the deCONZ Docker container is being built. -`sudo usermod -a -G dialout pi` (pi user being used as an example) +Before running `docker-compose up -d`, make sure your Linux user is part of the dialout group, which allows the user access to serial devices (i.e. Conbee/Conbee II/RaspBee). If you are not certain, simply add your user to the dialout group by running the following command (username "pi" being used as an example): + +`sudo usermod -a -G dialout pi` + +Now run `docker-compose up -d`. + +If you are still experiencing issues, run `docker-compose down` and then `docker-compose up -d` again. + +Use a 0.5-1m usb extension cable with ConBee (II) to avoid wifi and bluetooth noise/interference from your Raspberry Pi (recommended by the manufacturer and often the solution to poor performance). ## Accessing the Phoscon UI The Phoscon UI is available using port 8090 (http://your.local.ip.address:8090/) ## Viewing the deCONZ Zigbee mesh -The Zigbee mesh can be viewed using VNC (port 5901). The default VNC password is "changeme". +The Zigbee mesh can be viewed using VNC on port 5901. The default VNC password is "changeme". ## Connecting deCONZ and Node-RED -Install [node-red-contrib-deconz](https://flows.nodered.org/node/node-red-contrib-deconz) via the "Manage palette" menu in Node-RED (if it is not already pre-installed) and follow the 2 simple steps in the video below: +Install [node-red-contrib-deconz](https://flows.nodered.org/node/node-red-contrib-deconz) via the "Manage palette" menu in Node-RED (if not already installed) and follow these 2 simple steps (also shown in the video below): + +Step 1: In the Phoscon UI, Go to Settings > Gateway > Advanced and click "Authenticate app". +Step 2: In Node-RED, open a deCONZ node, select "Add new deonz-server", insert your ip adress and port 8090 and click "Get settings". Click "Add", "Done" and "Deploy". Your device list will not be updated before deploying. ![installing deCONZ](https://github.com/DIYtechie/resources/blob/master/images/Setup%20deCONZ%20in%20Node-RED.gif?raw=true) From d39ae55c721bea38c0f61f9b5065331d952adcc5 Mon Sep 17 00:00:00 2001 From: DIY techie Date: Thu, 21 May 2020 15:36:28 +0200 Subject: [PATCH 65/69] Update deconz.md --- docs/deconz.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/deconz.md b/docs/deconz.md index dec44af2..c77ba898 100644 --- a/docs/deconz.md +++ b/docs/deconz.md @@ -4,15 +4,13 @@ - [Website](https://github.com/dresden-elektronik/deconz-rest-plugin/blob/master/README.md) ## Troubleshooting -Make sure your Conbee/Conbee II/RaspBee gateway is connected. If your gateway is not detected, or no lights can be paired, try moving the device to another usb port, reboot your computer and build the stack from the menu again `cd ~/IOTstack && bash ./menu.sh` (select "Pull full service from template" if prompted). The device must be plugged in when the deCONZ Docker container is being built. +Make sure your Conbee/Conbee II/RaspBee gateway is connected. If your gateway is not detected, or no lights can be paired, try moving the device to another usb port, reboot your computer and build the stack from the menu again `cd ~/IOTstack && bash ./menu.sh` (select "Pull full service from template" if prompted). The gateway must be plugged in when the deCONZ Docker container is being built. -Before running `docker-compose up -d`, make sure your Linux user is part of the dialout group, which allows the user access to serial devices (i.e. Conbee/Conbee II/RaspBee). If you are not certain, simply add your user to the dialout group by running the following command (username "pi" being used as an example): +Before running `docker-compose up -d`, make sure your Linux user is part of the dialout group, which allows the user access to serial devices (i.e. Conbee/Conbee II/RaspBee). If you are not certain, simply add your user to the dialout group by running the following command (username "pi" being used as an example): `sudo usermod -a -G dialout pi` -`sudo usermod -a -G dialout pi` +Now run `docker-compose up -d` to build the stack. -Now run `docker-compose up -d`. - -If you are still experiencing issues, run `docker-compose down` and then `docker-compose up -d` again. +If you are still experiencing issues, run `docker-compose down` to remove all containers from the stack and then `docker-compose up -d` to build them again. Use a 0.5-1m usb extension cable with ConBee (II) to avoid wifi and bluetooth noise/interference from your Raspberry Pi (recommended by the manufacturer and often the solution to poor performance). From f4ad7b9930dd8439d3d52e519cb067a036fcf1ab Mon Sep 17 00:00:00 2001 From: DIY techie Date: Thu, 21 May 2020 15:39:26 +0200 Subject: [PATCH 66/69] Update deconz.md --- docs/deconz.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/deconz.md b/docs/deconz.md index c77ba898..7b684ed1 100644 --- a/docs/deconz.md +++ b/docs/deconz.md @@ -24,6 +24,7 @@ The Zigbee mesh can be viewed using VNC on port 5901. The default VNC password i Install [node-red-contrib-deconz](https://flows.nodered.org/node/node-red-contrib-deconz) via the "Manage palette" menu in Node-RED (if not already installed) and follow these 2 simple steps (also shown in the video below): Step 1: In the Phoscon UI, Go to Settings > Gateway > Advanced and click "Authenticate app". + Step 2: In Node-RED, open a deCONZ node, select "Add new deonz-server", insert your ip adress and port 8090 and click "Get settings". Click "Add", "Done" and "Deploy". Your device list will not be updated before deploying. ![installing deCONZ](https://github.com/DIYtechie/resources/blob/master/images/Setup%20deCONZ%20in%20Node-RED.gif?raw=true) From ebe29dbbfcea42181867004abf98d831fd971480 Mon Sep 17 00:00:00 2001 From: DIY techie Date: Thu, 21 May 2020 15:39:53 +0200 Subject: [PATCH 67/69] Update deconz.md --- docs/deconz.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/deconz.md b/docs/deconz.md index 7b684ed1..b7c7335a 100644 --- a/docs/deconz.md +++ b/docs/deconz.md @@ -27,4 +27,5 @@ Step 1: In the Phoscon UI, Go to Settings > Gateway > Advanced and click "Authen Step 2: In Node-RED, open a deCONZ node, select "Add new deonz-server", insert your ip adress and port 8090 and click "Get settings". Click "Add", "Done" and "Deploy". Your device list will not be updated before deploying. + ![installing deCONZ](https://github.com/DIYtechie/resources/blob/master/images/Setup%20deCONZ%20in%20Node-RED.gif?raw=true) From 1cc95dbeca2ce1d893f8fe494fe0634f361d2f46 Mon Sep 17 00:00:00 2001 From: DIY techie Date: Thu, 21 May 2020 15:46:43 +0200 Subject: [PATCH 68/69] Clean up deleted unnecessary echo's --- .templates/deconz/build.sh | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.templates/deconz/build.sh b/.templates/deconz/build.sh index 4dcf8f69..7ef8f32c 100644 --- a/.templates/deconz/build.sh +++ b/.templates/deconz/build.sh @@ -2,8 +2,8 @@ #deCONZ device configuration -device_selection=$(whiptail --radiolist --title "deCONZ device configuration" --notags \ - "Use the [SPACEBAR] to select your Zigbee device from the list below and make sure it is plugged in (if not, press [ESC])." 20 78 12 \ +device_selection=$(whiptail --radiolist --title "Select deCONZ gateway" --notags \ + "Use the [SPACEBAR] to select your deCONZ gateway from the list below AND MAKE SURE IT IS PLUGGED IN (if not, press [ESC])." 20 78 12 \ "ConBeeII" "ConBee II " "ON" \ "ConBee" "ConBee " "OFF" \ "RaspBee" "RaspBee " "OFF" \ @@ -12,18 +12,15 @@ device_selection=$(whiptail --radiolist --title "deCONZ device configuration" -- case $device_selection in "ConBeeII") - echo "...copied ConBee II config from template" - echo "" >>docker-compose.yml cat .templates/deconz/service_conbee_II.yml >>docker-compose.yml + echo "...copied ConBee II config from template" ;; "ConBee") - echo "...copied ConBee config from template" - echo "" >>docker-compose.yml cat .templates/deconz/service_conbee.yml >>docker-compose.yml + echo "...copied ConBee config from template" ;; "RaspBee") - echo "...copied RaspBee config from template" - echo "" >>docker-compose.yml cat .templates/deconz/service_raspbee.yml >>docker-compose.yml + echo "...copied RaspBee config from template" ;; esac From 01eb79a72a2a3ceb7036473319f85fb593c29f17 Mon Sep 17 00:00:00 2001 From: Slyke Date: Fri, 29 May 2020 16:12:22 -0700 Subject: [PATCH 69/69] Fixed bug where yaml merging was overwriting deconz services --- .templates/deconz/build.sh | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/.templates/deconz/build.sh b/.templates/deconz/build.sh index 7ef8f32c..d4b5d33c 100644 --- a/.templates/deconz/build.sh +++ b/.templates/deconz/build.sh @@ -2,6 +2,12 @@ #deCONZ device configuration +DOCKER_COMPOSE_PATH=./.tmp/docker-compose.tmp.yml + +if [[ ! -f $DOCKER_COMPOSE_PATH ]]; then + echo "[deCONZ] Warning: $DOCKER_COMPOSE_PATH does not exist." +fi + device_selection=$(whiptail --radiolist --title "Select deCONZ gateway" --notags \ "Use the [SPACEBAR] to select your deCONZ gateway from the list below AND MAKE SURE IT IS PLUGGED IN (if not, press [ESC])." 20 78 12 \ "ConBeeII" "ConBee II " "ON" \ @@ -12,15 +18,27 @@ device_selection=$(whiptail --radiolist --title "Select deCONZ gateway" --notags case $device_selection in "ConBeeII") - cat .templates/deconz/service_conbee_II.yml >>docker-compose.yml - echo "...copied ConBee II config from template" + if [[ ! -f ./.templates/deconz/service_conbee_II.yml ]]; then + echo "Error: ./.templates/deconz/service_conbee_II.yml does not exist." + else + cat ./.templates/deconz/service_conbee_II.yml >> $DOCKER_COMPOSE_PATH + echo "...copied ConBee II config from template" + fi ;; "ConBee") - cat .templates/deconz/service_conbee.yml >>docker-compose.yml - echo "...copied ConBee config from template" + if [[ ! -f ./.templates/deconz/service_conbee.yml ]]; then + echo "Error: ./.templates/deconz/service_conbee.yml does not exist." + else + cat ./.templates/deconz/service_conbee.yml >> $DOCKER_COMPOSE_PATH + echo "...copied ConBee config from template" + fi ;; "RaspBee") - cat .templates/deconz/service_raspbee.yml >>docker-compose.yml - echo "...copied RaspBee config from template" + if [[ ! -f ./.templates/deconz/service_raspbee.yml ]]; then + echo "Error: ./.templates/deconz/service_raspbee.yml does not exist." + else + cat ./.templates/deconz/service_raspbee.yml >> $DOCKER_COMPOSE_PATH + echo "...copied RaspBee config from template" + fi ;; - esac + esac \ No newline at end of file