Merge pull request #60 from graphite-project/DZ-RedisTagDB

Introducing Redis TagDB

Author: deniszh

Dockerfile

@@ -23,10 +23,10 @@ RUN python3 -m pip install --upgrade virtualenv virtualenv-tools && \
   . /opt/graphite/bin/activate && \
   python3 -m pip install --upgrade pip && \
   pip3 install django==1.11.15 && \
-  pip3 install fadvise && \
   pip3 install msgpack-python && \
   pip3 install gunicorn && \
   pip3 install fadvise && \
+  pip3 install redis && \
   pip3 install msgpack-python && \
   pip3 install django-statsd-mozilla
 
@@ -95,16 +95,16 @@ RUN apt-get update --fix-missing \
     && apt-get -y upgrade \
     && apt-get install --yes --no-install-recommends \
     git \
+    redis \
     nginx \
-    python-flup \
-    python-pip \
-    python-ldap \
+    python3-pip \
+    python3-ldap \
     expect \
     memcached \
     sqlite3 \
     libcairo2 \
-    python-cairo \
-    python-rrdtool && \
+    python3-cairo \
+    python3-rrdtool && \
     apt-get clean && \
     apt-get autoremove --yes  && \
     rm -rf /var/lib/apt/lists/*
@@ -117,6 +117,9 @@ RUN rm /etc/nginx/sites-enabled/default
 ADD conf/etc/nginx/nginx.conf /etc/nginx/nginx.conf
 ADD conf/etc/nginx/sites-enabled/graphite-statsd.conf /etc/nginx/sites-enabled/graphite-statsd.conf
 
+# config redis
+ADD conf/etc/redis/redis.conf /etc/redis/redis.conf
+
 # logging support
 RUN mkdir -p /var/log/carbon /var/log/graphite /var/log/nginx /var/log/graphite/
 ADD conf/etc/logrotate.d/graphite-statsd /etc/logrotate.d/graphite-statsd
@@ -127,6 +130,8 @@ ADD conf/etc/service/carbon-aggregator/run /etc/service/carbon-aggregator/run
 ADD conf/etc/service/graphite/run /etc/service/graphite/run
 ADD conf/etc/service/statsd/run /etc/service/statsd/run
 ADD conf/etc/service/nginx/run /etc/service/nginx/run
+ADD conf/etc/service/redis/run /etc/service/redis/run
+RUN chmod 0755 /etc/service/*/run
 
 # default conf setup
 ADD conf /etc/graphite-statsd/conf
@@ -139,7 +144,7 @@ RUN chmod +x /usr/local/bin/manage.sh && /usr/local/bin/django_admin_init.exp
 
 # defaults
 EXPOSE 80 2003-2004 2023-2024 8080 8125 8125/udp 8126
-VOLUME ["/opt/graphite/conf", "/opt/graphite/storage", "/opt/graphite/webapp/graphite/functions/custom", "/etc/nginx", "/opt/statsd", "/etc/logrotate.d", "/var/log"]
+VOLUME ["/opt/graphite/conf", "/opt/graphite/storage", "/opt/graphite/webapp/graphite/functions/custom", "/etc/nginx", "/opt/statsd", "/etc/logrotate.d", "/var/log", "/var/lib/redis"]
 WORKDIR /
 ENV HOME /root
 ENV STATSD_INTERFACE udp

README.md

@@ -66,6 +66,7 @@ DOCKER ASSIGNED   | /etc/nginx                 | nginx config
 DOCKER ASSIGNED   | /opt/statsd                | statsd config
 DOCKER ASSIGNED   | /etc/logrotate.d           | logrotate config
 DOCKER ASSIGNED   | /var/log                   | log files
+DOCKER ASSIGNED   | /var/lib/redis             | Redis TagDB data (optional)
 
 ### Base Image
 
@@ -144,6 +145,21 @@ Additional environment variables can be set to adjust performance.
 * GRAPHITE_FIND_CACHE_DURATION: (0) Time to cache remote metric find results
 * GRAPHITE_STATSD_HOST: ("127.0.0.1") If set, django_statsd.middleware.GraphiteRequestTimingMiddleware and django_statsd.middleware.GraphiteMiddleware will be enabled.
 
+## TagDB
+Graphite stores tag information in a separate tag database (TagDB). Please check [tags documentation](https://graphite.readthedocs.io/en/latest/tags.html) for details.
+
+* GRAPHITE_TAGDB: ('graphite.tags.localdatabase.LocalDatabaseTagDB') TagDB is a pluggable store, by default it uses the local SQLite database.
+* REDIS_TAGDB: (false) if set to true will use local Redis instance to store tags. 
+* GRAPHITE_TAGDB_CACHE_DURATION: (60) Time to cache seriesByTag results.
+* GRAPHITE_TAGDB_AUTOCOMPLETE_LIMIT: (100) Autocomplete default result limit.
+* GRAPHITE_TAGDB_REDIS_HOST: ('localhost') Redis TagDB host
+* GRAPHITE_TAGDB_REDIS_PORT: (6379) Redis TagDB port
+* GRAPHITE_TAGDB_REDIS_DB: (0) Redis TagDB database number
+* GRAPHITE_TAGDB_HTTP_URL: ('') URL for HTTP TagDB
+* GRAPHITE_TAGDB_HTTP_USER: ('') Username for HTTP TagDB
+* GRAPHITE_TAGDB_HTTP_PASSWORD: ('') Password for HTTP TagDB
+* GRAPHITE_TAGDB_HTTP_AUTOCOMPLETE: (false) Does the remote TagDB support autocomplete?
+
 ## Change the Configuration
 
 Read up on Graphite's [post-install tasks](https://graphite.readthedocs.org/en/latest/install.html#post-install-tasks).

conf/etc/redis/redis.conf

@@ -0,0 +1,44 @@
+daemonize no
+pidfile /var/run/redis/redis-server.pid
+port 6379
+bind 127.0.0.1
+timeout 0
+tcp-keepalive 0
+loglevel notice
+logfile /var/log/redis/redis-server.log
+databases 16
+save 900 1
+save 300 10
+save 60 10000
+stop-writes-on-bgsave-error yes
+rdbcompression yes
+rdbchecksum yes
+dbfilename dump.rdb
+dir /var/lib/redis
+slave-serve-stale-data yes
+slave-read-only yes
+repl-disable-tcp-nodelay no
+slave-priority 100
+appendonly yes
+appendfilename "appendonly.aof"
+appendfsync everysec
+no-appendfsync-on-rewrite no
+auto-aof-rewrite-percentage 100
+auto-aof-rewrite-min-size 64mb
+lua-time-limit 5000
+slowlog-log-slower-than 10000
+slowlog-max-len 128
+notify-keyspace-events ""
+hash-max-ziplist-entries 512
+hash-max-ziplist-value 64
+list-max-ziplist-entries 512
+list-max-ziplist-value 64
+set-max-intset-entries 512
+zset-max-ziplist-entries 128
+zset-max-ziplist-value 64
+activerehashing yes
+client-output-buffer-limit normal 0 0 0
+client-output-buffer-limit slave 256mb 64mb 60
+client-output-buffer-limit pubsub 32mb 8mb 60
+hz 10
+aof-rewrite-incremental-fsync yes

conf/etc/service/redis/run

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+[[ -n ${REDIS_TAGDB} ]] || exit 1
+exec /usr/bin/redis-server /etc/redis/redis.conf

conf/opt/graphite/webapp/graphite/local_settings.py

@@ -392,8 +392,13 @@
 # TagDB Settings #
 #####################################
 # Tag Database
-#TAGDB = 'graphite.tags.localdatabase.LocalDatabaseTagDB'
-TAGDB = os.environ.get('GRAPHITE_TAGDB', '')
+
+# set TAGDB to Redis if REDIS_TAGDB env var is set
+_REDIS_TAGDB = os.environ.get('REDIS_TAGDB', 'false').lower() in ['1', 'true', 'yes']
+
+# default TAGDB is local database. Set to '' to disable
+TAGDB = 'graphite.tags.redis.RedisTagDB' if _REDIS_TAGDB else \
+    os.environ.get('GRAPHITE_TAGDB', 'graphite.tags.localdatabase.LocalDatabaseTagDB')
 
 # Time to cache seriesByTag results
 TAGDB_CACHE_DURATION = int(os.environ.get('GRAPHITE_TAGDB_CACHE_DURATION') or 60)
@@ -402,9 +407,9 @@
 TAGDB_AUTOCOMPLETE_LIMIT = int(os.environ.get('GRAPHITE_TAGDB_AUTOCOMPLETE_LIMIT') or 100)
 
 # Settings for Redis TagDB
-#TAGDB_REDIS_HOST = 'localhost'
-#TAGDB_REDIS_PORT = 6379
-#TAGDB_REDIS_DB = 0
+TAGDB_REDIS_HOST = os.environ.get('GRAPHITE_TAGDB_REDIS_HOST', 'localhost')
+TAGDB_REDIS_PORT = int(os.environ.get('GRAPHITE_TAGDB_REDIS_PORT') or 6379)
+TAGDB_REDIS_DB = int(os.environ.get('GRAPHITE_TAGDB_REDIS_DB') or 0)
 
 # Settings for HTTP TagDB
 TAGDB_HTTP_URL = os.environ.get('GRAPHITE_TAGDB_HTTP_URL', '')