Skip to content

Commit d0722d8

Browse files
sebastienbonamisebastienbonami
authored
Several improvements in the cloudstats table (#107)
1 parent 88eb220 commit d0722d8

File tree

4 files changed

+154
-46
lines changed

4 files changed

+154
-46
lines changed

‎Containerfile.dev‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@ RUN sed -i "/'django.middleware.security.SecurityMiddleware',/a \ \ \ \ 'whiteno
2424

2525
RUN cat <<'EOF' > cloud.yml
2626
projects:
27-
someproject:
27+
null:
2828
members:
29-
- someuser
29+
- null
3030
EOF
3131

3232
RUN python3.12 manage.py collectstatic --noinput

‎cloudstats/templates/cloudstats/index.html‎

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,32 +30,60 @@ <h4>{% translate "Projects" %}</h4>
3030
Total physical cores: {{physical_cores | floatformat:0}}<br>
3131
Total physical memory: {{physical_memory | floatformat:0}} GB<br>
3232

33+
<br><i>Note: <b>Quota</b> and <b>Running</b> = latest value; <b>Used</b> = last month's average</i>
3334
<table class="table table-striped">
3435
<thead class="thead-dark">
3536
<tr>
3637
<th>{% translate "Project" %}</th>
37-
<th>{% translate "Running cores" %}</th>
38-
<th>{% translate "Used cores" %}</th>
39-
<th>{% translate "Running memory (GB)" %}</th>
40-
<th>{% translate "Used memory (GB)" %}</th>
38+
<th>{% translate "Quota Cores" %}</th>
39+
<th>{% translate "Running Cores" %}</th>
40+
<th>{% translate "Used Cores" %}</th>
41+
<th>{% translate "Quota Memory (GB)" %}</th>
42+
<th>{% translate "Running Memory (GB)" %}</th>
43+
<th>{% translate "Used Memory (GB)" %}</th>
4144
<th>{% translate "Running GPUs" %}</th>
42-
<th>{% translate "Block capacity (GB)" %}</th>
45+
<th>{% translate "Quota Block (GB)" %}</th>
46+
<th>{% translate "Running Block (GB)" %}</th>
47+
<th>{% translate "Used Block (GB)" %}</th>
48+
<th>{% translate "Quota Object (GB)" %}</th>
49+
<th>{% translate "Used Object (GB)" %}</th>
50+
<th>{% translate "Quota CephFS (GB)" %}</th>
51+
<th>{% translate "Running CephFS (GB)" %}</th>
4352
</tr>
4453
</thead>
4554
<tbody>
4655
{% for project in all_projects %}
4756
<tr>
4857
<td><a href="{{ project.id }}">{{project.name | anonymize }}</a></td>
49-
<td>{{project.cores | floatformat:1}}</td>
58+
<td class="delimiter">{{project.quota_cores | floatformat:1}}</td>
59+
<td>{{project.running_cores | floatformat:1}}</td>
5060
<td>{{project.used_cores | floatformat:1}}</td>
51-
<td>{{project.memory | floatformat:1}}</td>
61+
<td class="delimiter">{{project.quota_memory | floatformat:1}}</td>
62+
<td>{{project.running_memory | floatformat:1}}</td>
5263
<td>{{project.used_memory | floatformat:1}}</td>
53-
<td>{{project.gpu_qty | floatformat:1}}</td>
54-
<td>{{project.block_capacity | floatformat:1}}</td>
64+
<td class="delimiter">{{project.running_gpus | floatformat:1}}</td>
65+
<td class="delimiter">{{project.quota_block | floatformat:1}}</td>
66+
<td>{{project.running_block | floatformat:1}}</td>
67+
<td>{{project.used_block | floatformat:1 }}</td>
68+
<td class="delimiter">{{project.quota_object | floatformat:1 }}</td>
69+
<td>{{project.used_object | floatformat:1 }}</td>
70+
<td class="delimiter">{{project.quota_cephfs | floatformat:1 }}</td>
71+
<td>{{project.running_cephfs | floatformat:1 }}</td>
5572
</tr>
5673
{% endfor %}
5774
</tbody>
5875
</table>
76+
77+
<style>
78+
.table thead th {
79+
position: sticky;
80+
top: 47px;
81+
}
82+
83+
.table td.delimiter {
84+
border-left: 1px solid currentColor;
85+
}
86+
</style>
5987
{% endif %}
6088

6189
{% endblock content %}

‎cloudstats/views.py‎

Lines changed: 115 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,19 @@ def index(request):
2121
context['all_projects'] = []
2222
all_projects = {}
2323

24+
query_quota_cores = 'openstack_quota_compute_cores{{ {filter} }}'.format(
25+
filter=prom.get_filter('cloudstats'),
26+
)
27+
stats_quota_cores = prom.query_last(query_quota_cores)
28+
for line in stats_quota_cores:
29+
all_projects[line['metric']['project_name']] = {'quota_cores': float(line['value'][1])}
30+
2431
query_count_cores = 'count(libvirtd_domain_vcpu_time{{ {filter} }}) by (project_name)'.format(
2532
filter=prom.get_filter('cloudstats'),
2633
)
27-
stats_avg_cores = prom.query_prometheus_multiple(query_count_cores, datetime.now() - timedelta(days=31), datetime.now(), step='1d')
28-
for line in stats_avg_cores:
29-
all_projects[line['metric']['project_name']] = {'cores': statistics.mean(line['y'])}
34+
stats_running_cores = prom.query_last(query_count_cores)
35+
for line in stats_running_cores:
36+
all_projects[line['metric']['project_name']]['running_cores'] = float(line['value'][1])
3037

3138
query_used_cores = 'sum(rate(libvirtd_domain_vcpu_time{{ {filter} }}[1h])/1000/1000/1000) by (project_name)'.format(
3239
filter=prom.get_filter('cloudstats'),
@@ -35,12 +42,19 @@ def index(request):
3542
for line in stats_avg_used_cores:
3643
all_projects[line['metric']['project_name']]['used_cores'] = statistics.mean(line['y'])
3744

38-
query_memory = 'sum(libvirtd_domain_balloon_current{{ {filter} }}/1024/1024) by (project_name)'.format(
45+
query_quota_memory = 'openstack_quota_compute_ram{{ {filter} }}/1024'.format(
46+
filter=prom.get_filter('cloudstats'),
47+
)
48+
stats_quota_memory = prom.query_last(query_quota_memory)
49+
for line in stats_quota_memory:
50+
all_projects[line['metric']['project_name']]['quota_memory'] = float(line['value'][1])
51+
52+
query_running_memory = 'sum(libvirtd_domain_balloon_current{{ {filter} }}/1024/1024) by (project_name)'.format(
3953
filter=prom.get_filter('cloudstats'),
4054
)
41-
stats_memory = prom.query_prometheus_multiple(query_memory, datetime.now() - timedelta(days=31), datetime.now(), step='1d')
42-
for line in stats_memory:
43-
all_projects[line['metric']['project_name']]['memory'] = statistics.mean(line['y'])
55+
stats_running_memory = prom.query_last(query_running_memory)
56+
for line in stats_running_memory:
57+
all_projects[line['metric']['project_name']]['running_memory'] = float(line['value'][1])
4458

4559
query_used_memory = 'sum((libvirtd_domain_balloon_current{{ {filter} }} - libvirtd_domain_balloon_usable{{ {filter} }})/1024/1024) by (project_name)'.format(
4660
filter=prom.get_filter('cloudstats'),
@@ -50,56 +64,122 @@ def index(request):
5064
all_projects[line['metric']['project_name']]['used_memory'] = statistics.mean(line['y'])
5165

5266
# infer the number of gpus from the instance_name
53-
query_gpus = 'count(libvirtd_domain_balloon_current{{ {filter} }}) by (project_name, instance_type)'.format(
67+
query_running_gpus = 'count(libvirtd_domain_balloon_current{{ {filter} }}) by (project_name, instance_type)'.format(
5468
filter=prom.get_filter('cloudstats'),
5569
)
56-
stats_gpus = prom.query_prometheus_multiple(query_gpus, datetime.now() - timedelta(days=31), datetime.now(), step='1d')
57-
for line in stats_gpus:
70+
stats_running_gpus = prom.query_last(query_running_gpus)
71+
for line in stats_running_gpus:
5872
try:
5973
gpu_qty = settings.CLOUD_INSTANCE_TYPE[line['metric']['instance_type']]['gpu']
6074
except KeyError:
6175
gpu_qty = 0
6276

63-
if 'gpu_qty' in all_projects[line['metric']['project_name']]:
64-
all_projects[line['metric']['project_name']]['gpu_qty'] += statistics.mean(line['y']) * gpu_qty
77+
if 'running_gpus' in all_projects[line['metric']['project_name']]:
78+
all_projects[line['metric']['project_name']]['running_gpus'] += float(line['value'][1]) * gpu_qty
6579
else:
66-
all_projects[line['metric']['project_name']]['gpu_qty'] = statistics.mean(line['y']) * gpu_qty
80+
all_projects[line['metric']['project_name']]['running_gpus'] = float(line['value'][1]) * gpu_qty
81+
82+
query_quota_block = 'openstack_quota_volume_gigabytes{{ {filter} }}'.format(
83+
filter=prom.get_filter('cloudstats'),
84+
)
85+
stats_quota_block = prom.query_last(query_quota_block)
86+
for line in stats_quota_block:
87+
all_projects[line['metric']['project_name']]['quota_block'] = float(line['value'][1])
88+
89+
query_running_block = 'sum(libvirtd_domain_block_capacity{{ {filter} }}/1024/1024/1024) by (project_name)'.format(
90+
filter=prom.get_filter('cloudstats'),
91+
)
92+
stats_running_block = prom.query_last(query_running_block)
93+
for line in stats_running_block:
94+
all_projects[line['metric']['project_name']]['running_block'] = float(line['value'][1])
95+
96+
query_used_block = 'sum(libvirtd_domain_block_allocation{{ {filter} }}/1024/1024/1024) by (project_name)'.format(
97+
filter=prom.get_filter('cloudstats'),
98+
)
99+
stats_avg_used_block = prom.query_prometheus_multiple(query_used_block, datetime.now() - timedelta(days=31), datetime.now(), step='1d')
100+
for line in stats_avg_used_block:
101+
all_projects[line['metric']['project_name']]['used_block'] = statistics.mean(line['y'])
102+
103+
query_quota_object = 'openstack_quota_object_storage_x_account_meta_quota_bytes{{ {filter} }}/1024/1024/1024'.format(
104+
filter=prom.get_filter('cloudstats'),
105+
)
106+
stats_quota_object = prom.query_last(query_quota_object)
107+
for line in stats_quota_object:
108+
all_projects[line['metric']['project_name']]['quota_object'] = float(line['value'][1])
109+
110+
query_used_object = 'openstack_quota_object_storage_x_account_bytes_used{{ {filter} }}/1024/1024/1024'.format(
111+
filter=prom.get_filter('cloudstats'),
112+
)
113+
stats_avg_used_object = prom.query_prometheus_multiple(query_used_object, datetime.now() - timedelta(days=31), datetime.now(), step='1d')
114+
for line in stats_avg_used_object:
115+
all_projects[line['metric']['project_name']]['used_object'] = statistics.mean(line['y'])
116+
117+
query_quota_cephfs = 'openstack_quota_sharefs_gigabytes{{ {filter} }}'.format(
118+
filter=prom.get_filter('cloudstats'),
119+
)
120+
stats_quota_cephfs = prom.query_last(query_quota_cephfs)
121+
for line in stats_quota_cephfs:
122+
all_projects[line['metric']['project_name']]['quota_cephfs'] = float(line['value'][1])
67123

68-
query_block_capacity = 'sum(libvirtd_domain_block_capacity{{ {filter} }}/1024/1024/1024) by (project_name)'.format(
124+
query_running_cephfs = 'openstack_quota_sharefs_gigabytes_used{{ {filter} }}'.format(
69125
filter=prom.get_filter('cloudstats'),
70126
)
71-
stats_block_capacity = prom.query_prometheus_multiple(query_block_capacity, datetime.now() - timedelta(days=31), datetime.now(), step='1d')
72-
for line in stats_block_capacity:
73-
all_projects[line['metric']['project_name']]['block_capacity'] = statistics.mean(line['y'])
127+
stats_running_cephfs = prom.query_last(query_running_cephfs)
128+
for line in stats_running_cephfs:
129+
all_projects[line['metric']['project_name']]['running_cephfs'] = float(line['value'][1])
74130

75-
context['total_projects'] = {'cores': 0, 'used_cores': 0, 'memory': 0, 'used_memory': 0, 'gpu_qty': 0, 'block_capacity': 0}
131+
context['total_projects'] = {'quota_cores': 0, 'running_cores': 0, 'used_cores': 0, 'quota_memory': 0, 'running_memory': 0, 'used_memory': 0, 'running_gpus': 0, 'quota_block': 0, 'running_block': 0, 'used_block': 0, 'quota_object': 0, 'used_object': 0, 'quota_cephfs': 0, 'running_cephfs': 0}
76132
for project in sorted(all_projects):
77133
context['all_projects'].append({
78134
'id': project,
79135
'name': project,
80-
'cores': all_projects[project]['cores'],
81-
'used_cores': all_projects[project]['used_cores'],
82-
'memory': all_projects[project]['memory'],
83-
'used_memory': all_projects[project]['used_memory'],
84-
'gpu_qty': all_projects[project]['gpu_qty'],
85-
'block_capacity': all_projects[project]['block_capacity'],
136+
'quota_cores': all_projects[project].get('quota_cores', 0),
137+
'running_cores': all_projects[project].get('running_cores', 0),
138+
'used_cores': all_projects[project].get('used_cores', 0),
139+
'quota_memory': all_projects[project].get('quota_memory', 0),
140+
'running_memory': all_projects[project].get('running_memory', 0),
141+
'used_memory': all_projects[project].get('used_memory', 0),
142+
'running_gpus': all_projects[project].get('running_gpus', 0),
143+
'quota_block': all_projects[project].get('quota_block', 0),
144+
'running_block': all_projects[project].get('running_block', 0),
145+
'used_block': all_projects[project].get('used_block', 0),
146+
'quota_object': all_projects[project].get('quota_object', 0),
147+
'used_object': all_projects[project].get('used_object', 0),
148+
'quota_cephfs': all_projects[project].get('quota_cephfs', 0),
149+
'running_cephfs': all_projects[project].get('running_cephfs', 0),
86150
})
87-
context['total_projects']['cores'] += all_projects[project]['cores']
88-
context['total_projects']['used_cores'] += all_projects[project]['used_cores']
89-
context['total_projects']['memory'] += all_projects[project]['memory']
90-
context['total_projects']['used_memory'] += all_projects[project]['used_memory']
91-
context['total_projects']['gpu_qty'] += all_projects[project]['gpu_qty']
92-
context['total_projects']['block_capacity'] += all_projects[project]['block_capacity']
151+
context['total_projects']['quota_cores'] += all_projects[project].get('quota_cores', 0)
152+
context['total_projects']['running_cores'] += all_projects[project].get('running_cores', 0)
153+
context['total_projects']['used_cores'] += all_projects[project].get('used_cores', 0)
154+
context['total_projects']['quota_memory'] += all_projects[project].get('quota_memory', 0)
155+
context['total_projects']['running_memory'] += all_projects[project].get('running_memory', 0)
156+
context['total_projects']['used_memory'] += all_projects[project].get('used_memory', 0)
157+
context['total_projects']['running_gpus'] += all_projects[project].get('running_gpus', 0)
158+
context['total_projects']['quota_block'] += all_projects[project].get('quota_block', 0)
159+
context['total_projects']['running_block'] += all_projects[project].get('running_block', 0)
160+
context['total_projects']['used_block'] += all_projects[project].get('used_block', 0)
161+
context['total_projects']['quota_object'] += all_projects[project].get('quota_object', 0)
162+
context['total_projects']['used_object'] += all_projects[project].get('used_object', 0)
163+
context['total_projects']['quota_cephfs'] += all_projects[project].get('quota_cephfs', 0)
164+
context['total_projects']['running_cephfs'] += all_projects[project].get('running_cephfs', 0)
93165

94166
context['all_projects'].append({
95167
'id': 'total',
96-
'name': _('Total'),
97-
'cores': context['total_projects']['cores'],
168+
'name': _('TOTAL'),
169+
'quota_cores': context['total_projects']['quota_cores'],
170+
'running_cores': context['total_projects']['running_cores'],
98171
'used_cores': context['total_projects']['used_cores'],
99-
'memory': context['total_projects']['memory'],
172+
'quota_memory': context['total_projects']['quota_memory'],
173+
'running_memory': context['total_projects']['running_memory'],
100174
'used_memory': context['total_projects']['used_memory'],
101-
'gpu_qty': context['total_projects']['gpu_qty'],
102-
'block_capacity': context['total_projects']['block_capacity'],
175+
'running_gpus': context['total_projects']['running_gpus'],
176+
'quota_block': context['total_projects']['quota_block'],
177+
'running_block': context['total_projects']['running_block'],
178+
'used_block': context['total_projects']['used_block'],
179+
'quota_object': context['total_projects']['quota_object'],
180+
'used_object': context['total_projects']['used_object'],
181+
'quota_cephfs': context['total_projects']['quota_cephfs'],
182+
'running_cephfs': context['total_projects']['running_cephfs'],
103183
})
104184

105185
# Grab the hypervisors hostnames

‎docs/cloudstats.png‎

-1.45 KB
Loading

0 commit comments

Comments
 (0)