Skip to content

Commit eca0380

Browse files
author
Dougal Ballantyne
committed
Changing nodewatcher to run constantly.
1 parent 5c48bba commit eca0380

File tree

2 files changed

+94
-84
lines changed

2 files changed

+94
-84
lines changed

‎cloudformation/cfncluster.cfn.json‎

Lines changed: 81 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -1167,6 +1167,51 @@
11671167
},
11681168
"deployConfigFiles" : {
11691169
"files" : {
1170+
"/tmp/cwlogs/cfn-logs.conf" : {
1171+
"content" : {
1172+
"Fn::If" : [
1173+
"CloudWatchLogs",
1174+
[
1175+
"",
1176+
[
1177+
"[general]\n",
1178+
"state_file= /var/awslogs/agent-state\n",
1179+
"[/var/log/cloud-init.log]\n",
1180+
"file = /var/log/cloud-init.log\n",
1181+
"log_group_name = ",
1182+
{
1183+
"Ref" : "CfnClusterLogs"
1184+
},
1185+
"\n",
1186+
"log_stream_name = {instance_id}/cloud-init.log\n",
1187+
"datetime_format = %d/%b/%Y:%H:%M:%S\n",
1188+
"[/var/log/cfn-init.log]\n",
1189+
"file = /var/log/cfn-init.log\n",
1190+
"log_group_name = ",
1191+
{
1192+
"Ref" : "CfnClusterLogs"
1193+
},
1194+
"\n",
1195+
"log_stream_name = {instance_id}/cfn-init.log\n",
1196+
"datetime_format = %d/%b/%Y:%H:%M:%S\n",
1197+
"[/var/log/cfn-wire.log]\n",
1198+
"file = /var/log/cfn-wire.log\n",
1199+
"log_group_name = ",
1200+
{
1201+
"Ref" : "CfnClusterLogs"
1202+
},
1203+
"\n",
1204+
"log_stream_name = {instance_id}/cfn-wire.log\n",
1205+
"datetime_format = %d/%b/%Y:%H:%M:%S\n"
1206+
]
1207+
],
1208+
" "
1209+
]
1210+
},
1211+
"mode" : "000444",
1212+
"owner" : "root",
1213+
"group" : "root"
1214+
},
11701215
"/opt/cfncluster/cfnconfig" : {
11711216
"content" : {
11721217
"Fn::Join" : [
@@ -1288,47 +1333,6 @@
12881333
"owner" : "root",
12891334
"group" : "root"
12901335
},
1291-
"/tmp/cwlogs/cfn-logs.conf" : {
1292-
"content" : {
1293-
"Fn::Join" : [
1294-
"",
1295-
[
1296-
"[general]\n",
1297-
"state_file= /var/awslogs/agent-state\n",
1298-
"[/var/log/cloud-init.log]\n",
1299-
"file = /var/log/cloud-init.log\n",
1300-
"log_group_name = ",
1301-
{
1302-
"Ref" : "CfnClusterLogs"
1303-
},
1304-
"\n",
1305-
"log_stream_name = {instance_id}/cloud-init.log\n",
1306-
"datetime_format = %d/%b/%Y:%H:%M:%S\n",
1307-
"[/var/log/cfn-init.log]\n",
1308-
"file = /var/log/cfn-init.log\n",
1309-
"log_group_name = ",
1310-
{
1311-
"Ref" : "CfnClusterLogs"
1312-
},
1313-
"\n",
1314-
"log_stream_name = {instance_id}/cfn-init.log\n",
1315-
"datetime_format = %d/%b/%Y:%H:%M:%S\n",
1316-
"[/var/log/cfn-wire.log]\n",
1317-
"file = /var/log/cfn-wire.log\n",
1318-
"log_group_name = ",
1319-
{
1320-
"Ref" : "CfnClusterLogs"
1321-
},
1322-
"\n",
1323-
"log_stream_name = {instance_id}/cfn-wire.log\n",
1324-
"datetime_format = %d/%b/%Y:%H:%M:%S\n"
1325-
]
1326-
]
1327-
},
1328-
"mode" : "000444",
1329-
"owner" : "root",
1330-
"group" : "root"
1331-
},
13321336
"/opt/cfncluster/cfncluster_supervisord.conf" : {
13331337
"content" : {
13341338
"Fn::Join" : [
@@ -1709,39 +1713,43 @@
17091713
},
17101714
"/tmp/cwlogs/cfn-logs.conf" : {
17111715
"content" : {
1712-
"Fn::Join" : [
1713-
"",
1716+
"Fn::If" : [
1717+
"CloudWatchLogs",
17141718
[
1715-
"[general]\n",
1716-
"state_file= /var/awslogs/agent-state\n",
1717-
"[/var/log/cloud-init.log]\n",
1718-
"file = /var/log/cloud-init.log\n",
1719-
"log_group_name = ",
1720-
{
1721-
"Ref" : "CfnClusterLogs"
1722-
},
1723-
"\n",
1724-
"log_stream_name = {instance_id}/cloud-init.log\n",
1725-
"datetime_format = %d/%b/%Y:%H:%M:%S\n",
1726-
"[/var/log/cfn-init.log]\n",
1727-
"file = /var/log/cfn-init.log\n",
1728-
"log_group_name = ",
1729-
{
1730-
"Ref" : "CfnClusterLogs"
1731-
},
1732-
"\n",
1733-
"log_stream_name = {instance_id}/cfn-init.log\n",
1734-
"datetime_format = %d/%b/%Y:%H:%M:%S\n",
1735-
"[/var/log/cfn-wire.log]\n",
1736-
"file = /var/log/cfn-wire.log\n",
1737-
"log_group_name = ",
1738-
{
1739-
"Ref" : "CfnClusterLogs"
1740-
},
1741-
"\n",
1742-
"log_stream_name = {instance_id}/cfn-wire.log\n",
1743-
"datetime_format = %d/%b/%Y:%H:%M:%S\n"
1744-
]
1719+
"",
1720+
[
1721+
"[general]\n",
1722+
"state_file= /var/awslogs/agent-state\n",
1723+
"[/var/log/cloud-init.log]\n",
1724+
"file = /var/log/cloud-init.log\n",
1725+
"log_group_name = ",
1726+
{
1727+
"Ref" : "CfnClusterLogs"
1728+
},
1729+
"\n",
1730+
"log_stream_name = {instance_id}/cloud-init.log\n",
1731+
"datetime_format = %d/%b/%Y:%H:%M:%S\n",
1732+
"[/var/log/cfn-init.log]\n",
1733+
"file = /var/log/cfn-init.log\n",
1734+
"log_group_name = ",
1735+
{
1736+
"Ref" : "CfnClusterLogs"
1737+
},
1738+
"\n",
1739+
"log_stream_name = {instance_id}/cfn-init.log\n",
1740+
"datetime_format = %d/%b/%Y:%H:%M:%S\n",
1741+
"[/var/log/cfn-wire.log]\n",
1742+
"file = /var/log/cfn-wire.log\n",
1743+
"log_group_name = ",
1744+
{
1745+
"Ref" : "CfnClusterLogs"
1746+
},
1747+
"\n",
1748+
"log_stream_name = {instance_id}/cfn-wire.log\n",
1749+
"datetime_format = %d/%b/%Y:%H:%M:%S\n"
1750+
]
1751+
],
1752+
" "
17451753
]
17461754
},
17471755
"mode" : "000444",

‎node/src/nodewatcher/nodewatcher.py‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@
1818
import dateutil.parser
1919
import urllib2
2020
import ConfigParser
21-
import logging #http://docs.python.org/2/howto/logging.html
2221
import boto.ec2.autoscale
2322
import os
23+
import time
2424
import sys
2525
import tempfile
2626

@@ -111,15 +111,17 @@ def selfTerminate(asg):
111111

112112
s = loadSchedulerModule(scheduler)
113113

114-
jobs = getJobs(s, hostname)
115-
print jobs
116-
if jobs == True:
117-
print('Instance has active jobs. Exiting')
118-
sys.exit(0)
114+
while True:
115+
time.sleep(60)
116+
jobs = getJobs(s, hostname)
117+
print jobs
118+
if jobs == True:
119+
print('Instance has active jobs.')
120+
continue
119121

120-
conn = boto.ec2.connect_to_region(region)
121-
hour_percentile = getHourPercentile(instance_id,conn)
122-
print('Percent of hour used: %d' % hour_percentile)
122+
conn = boto.ec2.connect_to_region(region)
123+
hour_percentile = getHourPercentile(instance_id,conn)
124+
print('Percent of hour used: %d' % hour_percentile)
123125

124-
if hour_percentile > 95:
125-
selfTerminate(asg)
126+
if hour_percentile > 95:
127+
selfTerminate(asg)

0 commit comments

Comments
 (0)