99import requests
1010import yaml
1111from environs import Env , EnvError
12- from tenacity import retry , stop_after_attempt , wait_random
12+ from requests .exceptions import HTTPError
13+ from tenacity import (
14+ before_log ,
15+ retry ,
16+ retry_if_exception_type ,
17+ stop_after_attempt ,
18+ wait_fixed ,
19+ wait_random ,
20+ )
1321from yaml .loader import SafeLoader
1422
15- RETRY_SLEEP_DURATION_SEC = 15
16-
1723logging .basicConfig (level = "INFO" )
1824logger = logging .getLogger ()
1925
2531env = Env ()
2632env .read_env ("./../.env" , recurse = False )
2733
34+ SUPPORTED_GRAYLOG_MAJOR_VERSION = 6
35+
36+ GRAYLOG_BASE_DOMAIN = "https://monitoring." + env .str ("MACHINE_FQDN" ) + "/graylog"
37+ GRAYLOG_WAIT_ONLINE_TIMEOUT_SEC = env .int ("GRAYLOG_WAIT_ONLINE_TIMEOUT_SEC" )
38+ REQUESTS_AUTH = (env .str ("SERVICES_USER" ), env .str ("SERVICES_PASSWORD" ))
2839
29- def log_attempt_number (retry_state ):
30- """return the result of the last call attempt"""
31- print (f"Retrying: { retry_state .attempt_number } )
40+ GRAYLOG_LOG_MAX_DAYS_IN_STORAGE = env .int ("GRAYLOG_LOG_MAX_DAYS_IN_STORAGE" )
41+ GRAYLOG_LOG_MIN_DAYS_IN_STORAGE = env .int ("GRAYLOG_LOG_MIN_DAYS_IN_STORAGE" )
3242
3343
3444@retry (
35- stop = stop_after_attempt (10 ),
36- wait = wait_random (min = 1 , max = 10 ),
37- after = log_attempt_number ,
45+ stop = stop_after_attempt (GRAYLOG_WAIT_ONLINE_TIMEOUT_SEC / 5 ),
46+ wait = wait_fixed (5 ),
47+ retry = retry_if_exception_type (HTTPError ),
48+ before = before_log (logger , logging .INFO ),
3849)
39- def check_graylog_online ():
40- _url = "https://monitoring." + env .str ("MACHINE_FQDN" ) + "/graylog/api/users"
41- _urlhed = {"Content-Type" : "application/json" , "Accept" : "application/json" }
42- _session = requests .Session ()
43- _session .auth = (
44- "admin" ,
45- env .str ("SERVICES_PASSWORD" ),
46- )
47- _r = _session .get (_url , headers = _urlhed , verify = False )
48- if _r .status_code != 401 and str (_r .status_code ) != "200" :
49- print (_r .status_code )
50- sleep (RETRY_SLEEP_DURATION_SEC )
51- raise RuntimeError ("Could not connect to graylog." )
52- return True
50+ def wait_graylog_is_online ():
51+ _r = requests .get (GRAYLOG_BASE_DOMAIN + "/api/system" , auth = REQUESTS_AUTH )
52+
53+ if _r .status_code == 401 :
54+ raise TypeError (f"Graylog unauthorized HTTP response: { _r } )
55+
56+ _r .raise_for_status ()
57+ logger .info ("Graylog is online" )
58+
59+
60+ def validate_graylog_version_is_supported ():
61+ _r = requests .get (GRAYLOG_BASE_DOMAIN + "/api/system" , auth = REQUESTS_AUTH )
62+ _r .raise_for_status ()
63+
64+ graylog_version = _r .json ()["version" ]
65+ major_version = int (graylog_version .split ("." )[0 ])
66+
67+ if major_version != SUPPORTED_GRAYLOG_MAJOR_VERSION :
68+ raise TypeError (
69+ f"Graylog major version { major_version }
70+ f"Supported version is { SUPPORTED_GRAYLOG_MAJOR_VERSION }
71+ )
5372
5473
5574@retry (stop = stop_after_attempt (5 ), wait = wait_random (min = 1 , max = 10 ))
@@ -59,7 +78,6 @@ def get_graylog_inputs(_session, _headers, _url):
5978 # DEBUG
6079 if _r .status_code == 200 :
6180 print ("Successfully send GET /api/system/inputs" )
62- print ("Graylog is online :)" )
6381 return _r
6482 error_message = (
6583 "Error while sending GET /api/system/inputs. Status code of the request : "
@@ -121,54 +139,46 @@ def configure_email_notifications(_session, _headers):
121139
122140
123141def configure_log_retention (_session , _headers ):
124- try :
125- _url = (
126- "https://monitoring."
127- + env .str ("MACHINE_FQDN" )
128- + "/graylog/api/system/indices/index_sets"
129- )
130- _r = _session .get (_url , headers = _headers , verify = False )
131- index_of_interest = [
132- index
133- for index in _r .json ()["index_sets" ]
134- if index ["title" ] == "Default index set"
135- ][0 ]
136- index_of_interest [
137- "rotation_strategy_class"
138- ] = "org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategy"
139- # Rotate logs every day
140- index_of_interest ["rotation_strategy" ] = {
141- "rotation_period" : "P1D" ,
142- "type" : "org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategyConfig" ,
143- }
144- index_of_interest ["retention_strategy" ] = {
145- "max_number_of_indices" : str (env .str ("GRAYLOG_RETENTION_TIME_DAYS" )),
146- "type" : "org.graylog2.indexer.retention.strategies.DeletionRetentionStrategyConfig" ,
147- }
148- _url = (
149- "https://monitoring."
150- + env .str ("MACHINE_FQDN" )
151- + "/graylog/api/system/indices/index_sets"
152- )
153- raw_data = json .dumps (index_of_interest )
154- _r = _session .put (
155- _url + "/" + str (index_of_interest ["id" ]),
156- headers = _headers ,
157- data = raw_data ,
158- verify = False ,
159- )
160- if _r .status_code == 200 :
161- print ("Log retention time successfully updated !" )
162- else :
163- print (
164- "Error updating log retention time! Status code of the request : "
165- + str (_r .status_code )
166- + " "
167- + r .text
168- )
169- except EnvError :
142+ _url = (
143+ "https://monitoring."
144+ + env .str ("MACHINE_FQDN" )
145+ + "/graylog/api/system/indices/index_sets"
146+ )
147+ _r = _session .get (_url , headers = _headers , verify = False )
148+ index_of_interest = [
149+ index
150+ for index in _r .json ()["index_sets" ]
151+ if index ["title" ] == "Default index set"
152+ ][0 ]
153+
154+ # https://graylog.org/post/understanding-data-tiering-in-60-seconds/
155+ # https://community.graylog.org/t/graylog-6-0-data-tiering-for-rotation-and-retention/33302
156+ index_of_interest ["use_legacy_rotation" ] = False
157+ index_of_interest ["data_tiering" ] = {
158+ "type" : "hot_only" ,
159+ "index_lifetime_min" : f"P{ GRAYLOG_LOG_MIN_DAYS_IN_STORAGE } ,
160+ "index_lifetime_max" : f"P{ GRAYLOG_LOG_MAX_DAYS_IN_STORAGE } ,
161+ }
162+ _url = (
163+ "https://monitoring."
164+ + env .str ("MACHINE_FQDN" )
165+ + "/graylog/api/system/indices/index_sets"
166+ )
167+ raw_data = json .dumps (index_of_interest )
168+ _r = _session .put (
169+ _url + "/" + str (index_of_interest ["id" ]),
170+ headers = _headers ,
171+ data = raw_data ,
172+ verify = False ,
173+ )
174+ if _r .status_code == 200 :
175+ print ("Log retention time successfully updated !" )
176+ else :
170177 print (
171- "Setting retention time: GRAYLOG_RETENTION_TIME_DAYS not set or failed, default retention is used..."
178+ "Error updating log retention time! Status code of the request : "
179+ + str (_r .status_code )
180+ + " "
181+ + r .text
172182 )
173183
174184 try :
@@ -380,21 +390,13 @@ def install_content_pack_revision(content_pack):
380390
381391
382392if __name__ == "__main__" :
383- print (
384- "Waiting for graylog to run for provisioning. This can take up to some minutes, please be patient..."
385- )
386- try :
387- check_graylog_online ()
388- except RuntimeError as e :
389- print (e )
390- print ("Exception or: Graylog is still not online." )
391- print ("Graylog script will now stop." )
392- sys .exit (1 )
393+ wait_graylog_is_online ()
394+ validate_graylog_version_is_supported ()
393395
394396 session = requests .Session ()
395397 session .verify = False
396398 session .auth = (
397- "admin" ,
399+ env . str ( "SERVICES_USER" ) ,
398400 env .str ("SERVICES_PASSWORD" ),
399401 ) # Graylog username is always "admin"
400402 hed = {
0 commit comments