Skip to content

Commit 6d5991e

Browse files
sahilsuneja1sahilsuneja1
authored
Merge pull request #335 from hitomitak/metrics_k8s
Add a function to get ip address for metrics crawler in k8s environment
2 parents ccb5e14 + 44d752a commit 6d5991e

File tree

12 files changed

+624
-419
lines changed

12 files changed

+624
-419
lines changed

‎crawler/plugins/applications/apache/apache_container_crawler.py‎

Lines changed: 45 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,19 @@
1+
from utils.namespace import run_as_another_namespace
12
import logging
2-
3-
3+
import json
4+
import utils.misc
45
import dockercontainer
56
from icrawl_plugin import IContainerCrawler
67
from plugins.applications.apache import apache_crawler
7-
from utils.crawler_exceptions import CrawlError
8+
from requests.exceptions import ConnectionError
89

910
logger = logging.getLogger('crawlutils')
1011

1112

1213
class ApacheContainerCrawler(IContainerCrawler):
1314
feature_type = 'application'
1415
feature_key = 'apache'
16+
default_port = 80
1517

1618
def get_feature(self):
1719
return self.feature_key
@@ -20,32 +22,44 @@ def crawl(self, container_id=None, **kwargs):
2022

2123
c = dockercontainer.DockerContainer(container_id)
2224

23-
# check image name
24-
if c.image_name.find("httpd") == -1:
25-
26-
logger.error("%s is not %s container",
27-
c.image_name,
28-
self.feature_key)
29-
raise CrawlError("%s does not have expected name for %s (name=%s)",
30-
container_id,
31-
self.feature_key,
32-
c.image_name)
33-
34-
# extract IP and Port information
35-
ip = c.get_container_ip()
36-
ports = c.get_container_ports()
37-
38-
# crawl all candidate ports
39-
for port in ports:
40-
try:
41-
metrics = apache_crawler.retrieve_metrics(ip, port)
42-
except CrawlError:
43-
logger.error("can't find metrics endpoint at http://%s:%s",
44-
ip,
45-
port)
46-
continue
47-
return [(self.feature_key, metrics, self.feature_type)]
25+
port = None
26+
27+
if "annotation.io.kubernetes.container.ports" in\
28+
c.inspect['Config']['Labels']:
29+
30+
ports = c.inspect['Config']['Labels'][
31+
'annotation.io.kubernetes.container.ports']
32+
33+
ports = json.loads(ports)
4834

49-
raise CrawlError("%s has no accessible endpoint for %s",
50-
container_id,
51-
self.feature_key)
35+
else:
36+
ports = c.get_container_ports()
37+
38+
for each_port in ports:
39+
tmp_port = None
40+
if "containerPort" in each_port:
41+
tmp_port = int(each_port['containerPort'])
42+
else:
43+
tmp_port = int(each_port)
44+
45+
if tmp_port == self.default_port:
46+
port = tmp_port
47+
48+
if not port:
49+
return
50+
51+
state = c.inspect['State']
52+
pid = str(state['Pid'])
53+
ips = run_as_another_namespace(
54+
pid, ['net'], utils.misc.get_host_ip4_addresses)
55+
56+
for each_ip in ips:
57+
if each_ip != "127.0.0.1":
58+
ip = each_ip
59+
break
60+
try:
61+
metrics = apache_crawler.retrieve_metrics(ip, port)
62+
return [(self.feature_key, metrics, self.feature_type)]
63+
except:
64+
logger.info("apache does not listen on port:%d", port)
65+
raise ConnectionError("apache does not listen on port:%d", port)

‎crawler/plugins/applications/db2/db2_container_crawler.py‎

Lines changed: 56 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,24 @@
11
import logging
2-
32
import dockercontainer
43
from icrawl_plugin import IContainerCrawler
54
from plugins.applications.db2 import db2_crawler
6-
from utils.crawler_exceptions import CrawlError
5+
from utils.namespace import run_as_another_namespace
6+
import json
7+
import utils.misc
8+
from requests.exceptions import ConnectionError
79

810
logger = logging.getLogger('crawlutils')
911

1012

1113
class DB2ContainerCrawler(IContainerCrawler):
1214
feature_type = 'application'
1315
feature_key = 'db2'
16+
default_port = 50000
1417

1518
def get_feature(self):
1619
return self.feature_key
1720

18-
def crawl(self, container_id=None, **kwargs):
21+
def get_opt(self, kwargs):
1922
password = "db2inst1"
2023
user = "db2inst1-pwd"
2124
db = "sample"
@@ -29,37 +32,57 @@ def crawl(self, container_id=None, **kwargs):
2932
if "db" in kwargs:
3033
db = kwargs["db"]
3134

35+
return password, user, db
36+
37+
def crawl(self, container_id=None, **kwargs):
38+
39+
password, user, db = self.get_opt(kwargs)
3240
c = dockercontainer.DockerContainer(container_id)
3341

34-
# check image name
35-
if c.image_name.find(self.feature_key) == -1:
36-
logger.error("%s is not %s container",
37-
c.image_name,
38-
self.feature_key)
39-
raise CrawlError("%s does not have expected name for %s (name=%s)",
40-
container_id,
41-
self.feature_key,
42-
c.image_name)
43-
44-
# extract IP and Port information
45-
ip = c.get_container_ip()
46-
ports = c.get_container_ports()
47-
48-
# crawl all candidate ports
42+
port = None
43+
44+
if "annotation.io.kubernetes.container.ports" in\
45+
c.inspect['Config']['Labels']:
46+
47+
ports = c.inspect['Config']['Labels'][
48+
'annotation.io.kubernetes.container.ports']
49+
50+
ports = json.loads(ports)
51+
52+
else:
53+
ports = c.get_container_ports()
54+
4955
for each_port in ports:
50-
try:
51-
metrics = db2_crawler.retrieve_metrics(
52-
host=ip,
53-
user=user,
54-
password=password,
55-
db=db,
56-
)
57-
except CrawlError:
58-
logger.error("can't find metrics endpoint at %s db %s",
59-
ip, db)
60-
continue
61-
return [(self.feature_key, metrics, self.feature_type)]
56+
tmp_port = None
57+
if "containerPort" in each_port:
58+
tmp_port = int(each_port['containerPort'])
59+
else:
60+
tmp_port = int(each_port)
61+
62+
if tmp_port == self.default_port:
63+
port = tmp_port
6264

63-
raise CrawlError("%s has no accessible endpoint for %s",
64-
container_id,
65-
self.feature_key)
65+
if not port:
66+
return
67+
68+
state = c.inspect['State']
69+
pid = str(state['Pid'])
70+
ips = run_as_another_namespace(
71+
pid, ['net'], utils.misc.get_host_ip4_addresses)
72+
73+
for each_ip in ips:
74+
if each_ip != "127.0.0.1":
75+
ip = each_ip
76+
break
77+
78+
try:
79+
metrics = db2_crawler.retrieve_metrics(
80+
host=ip,
81+
user=user,
82+
password=password,
83+
db=db,
84+
)
85+
return [(self.feature_key, metrics, self.feature_type)]
86+
except:
87+
logger.info("db2 does not listen on port:%d", port)
88+
raise ConnectionError("db2 does not listen on port:%d", port)

‎crawler/plugins/applications/liberty/liberty_container_crawler.py‎

Lines changed: 55 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
import logging
2-
32
import dockercontainer
43
from icrawl_plugin import IContainerCrawler
54
from plugins.applications.liberty import liberty_crawler
6-
from utils.crawler_exceptions import CrawlError
5+
from utils.namespace import run_as_another_namespace
6+
import json
7+
import utils.misc
8+
from requests.exceptions import ConnectionError
79

810
logger = logging.getLogger('crawlutils')
911

@@ -16,7 +18,7 @@ class LibertyContainerCrawler(IContainerCrawler):
1618
def get_feature(self):
1719
return self.feature_key
1820

19-
def crawl(self, container_id=None, **kwargs):
21+
def get_opt(self, kwargs):
2022
password = "password"
2123
user = "user"
2224

@@ -26,32 +28,56 @@ def crawl(self, container_id=None, **kwargs):
2628
if "user" in kwargs:
2729
user = kwargs["user"]
2830

31+
return password, user
32+
33+
def crawl(self, container_id=None, **kwargs):
34+
35+
password, user = self.get_opt(kwargs)
2936
c = dockercontainer.DockerContainer(container_id)
3037

31-
# check image name
32-
if c.image_name.find(self.feature_key) == -1:
33-
logger.error("%s is not %s container",
34-
c.image_name,
35-
self.feature_key)
36-
raise CrawlError("%s does not have expected name for %s (name=%s)",
37-
container_id,
38-
self.feature_key,
39-
c.image_name)
40-
41-
# extract IP and Port information
42-
ip = c.get_container_ip()
43-
ports = c.get_container_ports()
44-
45-
# crawl all candidate ports
38+
port = None
39+
40+
if "annotation.io.kubernetes.container.ports" in\
41+
c.inspect['Config']['Labels']:
42+
43+
ports = c.inspect['Config']['Labels'][
44+
'annotation.io.kubernetes.container.ports']
45+
46+
ports = json.loads(ports)
47+
48+
else:
49+
ports = c.get_container_ports()
50+
4651
for each_port in ports:
47-
return liberty_crawler.retrieve_metrics(
48-
host=ip,
49-
port=each_port,
50-
user=user,
51-
password=password,
52-
feature_type=self.feature_type
53-
)
54-
55-
raise CrawlError("%s has no accessible endpoint for %s",
56-
container_id,
57-
self.feature_key)
52+
tmp_port = None
53+
if "containerPort" in each_port:
54+
tmp_port = int(each_port['containerPort'])
55+
else:
56+
tmp_port = int(each_port)
57+
58+
if tmp_port == self.default_port:
59+
port = tmp_port
60+
61+
if not port:
62+
return
63+
64+
state = c.inspect['State']
65+
pid = str(state['Pid'])
66+
ips = run_as_another_namespace(
67+
pid, ['net'], utils.misc.get_host_ip4_addresses)
68+
69+
for each_ip in ips:
70+
if each_ip != "127.0.0.1":
71+
ip = each_ip
72+
break
73+
74+
try:
75+
return liberty_crawler.retrieve_metrics(
76+
host=ip,
77+
port=port,
78+
user=user,
79+
password=password,
80+
feature_type=self.feature_type)
81+
except:
82+
logger.info("liberty does not listen on port:%d", port)
83+
raise ConnectionError("liberty does not listen on port:%d", port)

0 commit comments

Comments
 (0)