Skip to content

Commit 533c8e5

Browse files
markgoddardmarkgoddard
authored
Merge pull request #38 from stackhpc/upstream/xena-2023-02-13
Synchronise xena with upstream
2 parents 024e507 + 185f9be commit 533c8e5

File tree

6 files changed

+331
-124
lines changed

6 files changed

+331
-124
lines changed

‎cloudkitty/dataframe.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@
3737
voluptuous.Required('price', default=0):
3838
voluptuous.Coerce(str),
3939
},
40-
voluptuous.Required('groupby'): vutils.DictTypeValidator(str, str),
41-
voluptuous.Required('metadata'): vutils.DictTypeValidator(str, str),
40+
voluptuous.Required('groupby'): voluptuous.Coerce(dict),
41+
voluptuous.Required('metadata'): voluptuous.Coerce(dict),
4242
})
4343

4444

‎cloudkitty/rating/pyscripts/__init__.py‎

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,15 @@
1313
# License for the specific language governing permissions and limitations
1414
# under the License.
1515
#
16+
from cloudkitty import dataframe
1617
from cloudkitty import rating
1718
from cloudkitty.rating.pyscripts.controllers import root as root_api
1819
from cloudkitty.rating.pyscripts.db import api as pyscripts_db_api
1920

21+
from oslo_log import log as logging
22+
23+
LOG = logging.getLogger(__name__)
24+
2025

2126
class PyScripts(rating.RatingProcessorBase):
2227
"""PyScripts rating module.
@@ -33,20 +38,17 @@ class PyScripts(rating.RatingProcessorBase):
3338
db_api = pyscripts_db_api.get_instance()
3439

3540
def __init__(self, tenant_id=None):
41+
# current scripts loaded to memory
3642
self._scripts = {}
43+
3744
self.load_scripts_in_memory()
3845
super(PyScripts, self).__init__(tenant_id)
3946

4047
def load_scripts_in_memory(self):
4148
db = pyscripts_db_api.get_instance()
4249
scripts_uuid_list = db.list_scripts()
43-
# Purge old entries
44-
scripts_to_purge = []
45-
for script_uuid in self._scripts.keys():
46-
if script_uuid not in scripts_uuid_list:
47-
scripts_to_purge.append(script_uuid)
48-
for script_uuid in scripts_to_purge:
49-
del self._scripts[script_uuid]
50+
self.purge_removed_scripts(scripts_uuid_list)
51+
5052
# Load or update script
5153
for script_uuid in scripts_uuid_list:
5254
script_db = db.get_script(uuid=script_uuid)
@@ -67,11 +69,31 @@ def load_scripts_in_memory(self):
6769
'code': code,
6870
'checksum': checksum})
6971

72+
def purge_removed_scripts(self, scripts_uuid_list):
73+
scripts_to_purge = self.get_all_script_to_remove(scripts_uuid_list)
74+
self.remove_purged_scripts(scripts_to_purge)
75+
76+
def get_all_script_to_remove(self, new_scripts_uuid_list):
77+
scripts_to_purge = []
78+
for script_uuid in self._scripts.keys():
79+
if script_uuid not in new_scripts_uuid_list:
80+
scripts_to_purge.append(script_uuid)
81+
return scripts_to_purge
82+
83+
def remove_purged_scripts(self, scripts_to_purge):
84+
for script_uuid in scripts_to_purge:
85+
LOG.info("Removing script [%s] from the script list to execute.",
86+
self._scripts[script_uuid])
87+
88+
del self._scripts[script_uuid]
89+
7090
def reload_config(self):
7191
"""Reload the module's configuration.
7292
7393
"""
94+
LOG.debug("Executing the reload of configurations.")
7495
self.load_scripts_in_memory()
96+
LOG.debug("Configurations reloaded.")
7597

7698
def start_script(self, code, data):
7799
context = {'data': data}
@@ -80,5 +102,14 @@ def start_script(self, code, data):
80102

81103
def process(self, data):
82104
for script in self._scripts.values():
83-
data = self.start_script(script['code'], data)
105+
data_dict = data.as_dict(mutable=True)
106+
LOG.debug("Executing pyscript [%s] with data [%s].",
107+
script, data_dict)
108+
109+
data_output = self.start_script(script['code'], data_dict)
110+
111+
LOG.debug("Result [%s] for processing with pyscript [%s] with "
112+
"data [%s].", data_output, script, data_dict)
113+
114+
data = dataframe.DataFrame.from_dict(data_output)
84115
return data

‎cloudkitty/tests/gabbi/gabbits/v1-storage.yaml‎

Lines changed: 40 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -75,35 +75,35 @@ tests:
7575
$.dataframes[0].resources[0].volume: "1"
7676
$.dataframes[0].resources[0].rating: "1.337"
7777
$.dataframes[0].resources[0].service: "cpu"
78-
$.dataframes[0].resources[0].desc.dummy: 'True'
79-
$.dataframes[0].resources[0].desc.fake_meta: '1.0'
78+
$.dataframes[0].resources[0].desc.dummy: True
79+
$.dataframes[0].resources[0].desc.fake_meta: 1.0
8080
$.dataframes[1].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
8181
$.dataframes[1].begin: "2015-01-04T13:00:00"
8282
$.dataframes[1].end: "2015-01-04T14:00:00"
8383
$.dataframes[1].resources.`len`: 1
8484
$.dataframes[1].resources[0].volume: "1"
8585
$.dataframes[1].resources[0].rating: "1.337"
8686
$.dataframes[1].resources[0].service: "cpu"
87-
$.dataframes[1].resources[0].desc.dummy: 'True'
88-
$.dataframes[1].resources[0].desc.fake_meta: '1.0'
87+
$.dataframes[1].resources[0].desc.dummy: True
88+
$.dataframes[1].resources[0].desc.fake_meta: 1.0
8989
$.dataframes[2].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
9090
$.dataframes[2].begin: "2015-01-04T13:00:00"
9191
$.dataframes[2].end: "2015-01-04T14:00:00"
9292
$.dataframes[2].resources.`len`: 1
9393
$.dataframes[2].resources[0].volume: "1"
9494
$.dataframes[2].resources[0].rating: "0.121"
9595
$.dataframes[2].resources[0].service: "image.size"
96-
$.dataframes[2].resources[0].desc.dummy: 'True'
97-
$.dataframes[2].resources[0].desc.fake_meta: '1.0'
96+
$.dataframes[2].resources[0].desc.dummy: True
97+
$.dataframes[2].resources[0].desc.fake_meta: 1.0
9898
$.dataframes[3].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
9999
$.dataframes[3].begin: "2015-01-04T13:00:00"
100100
$.dataframes[3].end: "2015-01-04T14:00:00"
101101
$.dataframes[3].resources.`len`: 1
102102
$.dataframes[3].resources[0].volume: "1"
103103
$.dataframes[3].resources[0].rating: "0.121"
104104
$.dataframes[3].resources[0].service: "image.size"
105-
$.dataframes[3].resources[0].desc.dummy: 'True'
106-
$.dataframes[3].resources[0].desc.fake_meta: '1.0'
105+
$.dataframes[3].resources[0].desc.dummy: True
106+
$.dataframes[3].resources[0].desc.fake_meta: 1.0
107107

108108
- name: fetch data for the second tenant
109109
url: /v1/storage/dataframes
@@ -121,35 +121,35 @@ tests:
121121
$.dataframes[0].resources[0].volume: "1"
122122
$.dataframes[0].resources[0].rating: "1.337"
123123
$.dataframes[0].resources[0].service: "cpu"
124-
$.dataframes[0].resources[0].desc.dummy: 'True'
125-
$.dataframes[0].resources[0].desc.fake_meta: '1.0'
124+
$.dataframes[0].resources[0].desc.dummy: True
125+
$.dataframes[0].resources[0].desc.fake_meta: 1.0
126126
$.dataframes[1].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
127127
$.dataframes[1].begin: "2015-01-04T13:00:00"
128128
$.dataframes[1].end: "2015-01-04T14:00:00"
129129
$.dataframes[1].resources.`len`: 1
130130
$.dataframes[1].resources[0].volume: "1"
131131
$.dataframes[1].resources[0].rating: "1.337"
132132
$.dataframes[1].resources[0].service: "cpu"
133-
$.dataframes[1].resources[0].desc.dummy: 'True'
134-
$.dataframes[1].resources[0].desc.fake_meta: '1.0'
133+
$.dataframes[1].resources[0].desc.dummy: True
134+
$.dataframes[1].resources[0].desc.fake_meta: 1.0
135135
$.dataframes[2].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
136136
$.dataframes[2].begin: "2015-01-04T13:00:00"
137137
$.dataframes[2].end: "2015-01-04T14:00:00"
138138
$.dataframes[2].resources.`len`: 1
139139
$.dataframes[2].resources[0].volume: "1"
140140
$.dataframes[2].resources[0].rating: "0.121"
141141
$.dataframes[2].resources[0].service: "image.size"
142-
$.dataframes[2].resources[0].desc.dummy: 'True'
143-
$.dataframes[2].resources[0].desc.fake_meta: '1.0'
142+
$.dataframes[2].resources[0].desc.dummy: True
143+
$.dataframes[2].resources[0].desc.fake_meta: 1.0
144144
$.dataframes[3].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
145145
$.dataframes[3].begin: "2015-01-04T13:00:00"
146146
$.dataframes[3].end: "2015-01-04T14:00:00"
147147
$.dataframes[3].resources.`len`: 1
148148
$.dataframes[3].resources[0].volume: "1"
149149
$.dataframes[3].resources[0].rating: "0.121"
150150
$.dataframes[3].resources[0].service: "image.size"
151-
$.dataframes[3].resources[0].desc.dummy: 'True'
152-
$.dataframes[3].resources[0].desc.fake_meta: '1.0'
151+
$.dataframes[3].resources[0].desc.dummy: True
152+
$.dataframes[3].resources[0].desc.fake_meta: 1.0
153153

154154

155155
- name: fetch data for multiple tenants
@@ -167,71 +167,71 @@ tests:
167167
$.dataframes[0].resources[0].volume: "1"
168168
$.dataframes[0].resources[0].rating: "1.337"
169169
$.dataframes[0].resources[0].service: "cpu"
170-
$.dataframes[0].resources[0].desc.dummy: 'True'
171-
$.dataframes[0].resources[0].desc.fake_meta: '1.0'
170+
$.dataframes[0].resources[0].desc.dummy: True
171+
$.dataframes[0].resources[0].desc.fake_meta: 1.0
172172
$.dataframes[1].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
173173
$.dataframes[1].begin: "2015-01-04T13:00:00"
174174
$.dataframes[1].end: "2015-01-04T14:00:00"
175175
$.dataframes[1].resources.`len`: 1
176176
$.dataframes[1].resources[0].volume: "1"
177177
$.dataframes[1].resources[0].rating: "1.337"
178178
$.dataframes[1].resources[0].service: "cpu"
179-
$.dataframes[1].resources[0].desc.dummy: 'True'
180-
$.dataframes[1].resources[0].desc.fake_meta: '1.0'
179+
$.dataframes[1].resources[0].desc.dummy: True
180+
$.dataframes[1].resources[0].desc.fake_meta: 1.0
181181
$.dataframes[2].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
182182
$.dataframes[2].begin: "2015-01-04T13:00:00"
183183
$.dataframes[2].end: "2015-01-04T14:00:00"
184184
$.dataframes[2].resources.`len`: 1
185185
$.dataframes[2].resources[0].volume: "1"
186186
$.dataframes[2].resources[0].rating: "0.121"
187187
$.dataframes[2].resources[0].service: "image.size"
188-
$.dataframes[2].resources[0].desc.dummy: 'True'
189-
$.dataframes[2].resources[0].desc.fake_meta: '1.0'
188+
$.dataframes[2].resources[0].desc.dummy: True
189+
$.dataframes[2].resources[0].desc.fake_meta: 1.0
190190
$.dataframes[3].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
191191
$.dataframes[3].begin: "2015-01-04T13:00:00"
192192
$.dataframes[3].end: "2015-01-04T14:00:00"
193193
$.dataframes[3].resources.`len`: 1
194194
$.dataframes[3].resources[0].volume: "1"
195195
$.dataframes[3].resources[0].rating: "0.121"
196196
$.dataframes[3].resources[0].service: "image.size"
197-
$.dataframes[3].resources[0].desc.dummy: 'True'
198-
$.dataframes[3].resources[0].desc.fake_meta: '1.0'
197+
$.dataframes[3].resources[0].desc.dummy: True
198+
$.dataframes[3].resources[0].desc.fake_meta: 1.0
199199
$.dataframes[0].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
200200
$.dataframes[0].begin: "2015-01-04T13:00:00"
201201
$.dataframes[0].end: "2015-01-04T14:00:00"
202202
$.dataframes[0].resources.`len`: 1
203203
$.dataframes[0].resources[0].volume: "1"
204204
$.dataframes[0].resources[0].rating: "1.337"
205205
$.dataframes[0].resources[0].service: "cpu"
206-
$.dataframes[0].resources[0].desc.dummy: 'True'
207-
$.dataframes[0].resources[0].desc.fake_meta: '1.0'
206+
$.dataframes[0].resources[0].desc.dummy: True
207+
$.dataframes[0].resources[0].desc.fake_meta: 1.0
208208
$.dataframes[1].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
209209
$.dataframes[1].begin: "2015-01-04T13:00:00"
210210
$.dataframes[1].end: "2015-01-04T14:00:00"
211211
$.dataframes[1].resources.`len`: 1
212212
$.dataframes[1].resources[0].volume: "1"
213213
$.dataframes[1].resources[0].rating: "1.337"
214214
$.dataframes[1].resources[0].service: "cpu"
215-
$.dataframes[1].resources[0].desc.dummy: 'True'
216-
$.dataframes[1].resources[0].desc.fake_meta: '1.0'
215+
$.dataframes[1].resources[0].desc.dummy: True
216+
$.dataframes[1].resources[0].desc.fake_meta: 1.0
217217
$.dataframes[2].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
218218
$.dataframes[2].begin: "2015-01-04T13:00:00"
219219
$.dataframes[2].end: "2015-01-04T14:00:00"
220220
$.dataframes[2].resources.`len`: 1
221221
$.dataframes[2].resources[0].volume: "1"
222222
$.dataframes[2].resources[0].rating: "0.121"
223223
$.dataframes[2].resources[0].service: "image.size"
224-
$.dataframes[2].resources[0].desc.dummy: 'True'
225-
$.dataframes[2].resources[0].desc.fake_meta: '1.0'
224+
$.dataframes[2].resources[0].desc.dummy: True
225+
$.dataframes[2].resources[0].desc.fake_meta: 1.0
226226
$.dataframes[3].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
227227
$.dataframes[3].begin: "2015-01-04T13:00:00"
228228
$.dataframes[3].end: "2015-01-04T14:00:00"
229229
$.dataframes[3].resources.`len`: 1
230230
$.dataframes[3].resources[0].volume: "1"
231231
$.dataframes[3].resources[0].rating: "0.121"
232232
$.dataframes[3].resources[0].service: "image.size"
233-
$.dataframes[3].resources[0].desc.dummy: 'True'
234-
$.dataframes[3].resources[0].desc.fake_meta: '1.0'
233+
$.dataframes[3].resources[0].desc.dummy: True
234+
$.dataframes[3].resources[0].desc.fake_meta: 1.0
235235

236236
- name: fetch data filtering on cpu service and tenant
237237
url: /v1/storage/dataframes
@@ -250,17 +250,17 @@ tests:
250250
$.dataframes[0].resources[0].volume: "1"
251251
$.dataframes[0].resources[0].rating: "1.337"
252252
$.dataframes[0].resources[0].service: "cpu"
253-
$.dataframes[0].resources[0].desc.dummy: 'True'
254-
$.dataframes[0].resources[0].desc.fake_meta: '1.0'
253+
$.dataframes[0].resources[0].desc.dummy: True
254+
$.dataframes[0].resources[0].desc.fake_meta: 1.0
255255
$.dataframes[1].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
256256
$.dataframes[1].begin: "2015-01-04T13:00:00"
257257
$.dataframes[1].end: "2015-01-04T14:00:00"
258258
$.dataframes[1].resources.`len`: 1
259259
$.dataframes[1].resources[0].volume: "1"
260260
$.dataframes[1].resources[0].rating: "1.337"
261261
$.dataframes[1].resources[0].service: "cpu"
262-
$.dataframes[1].resources[0].desc.dummy: 'True'
263-
$.dataframes[1].resources[0].desc.fake_meta: '1.0'
262+
$.dataframes[1].resources[0].desc.dummy: True
263+
$.dataframes[1].resources[0].desc.fake_meta: 1.0
264264

265265
- name: fetch data filtering on image service and tenant
266266
url: /v1/storage/dataframes
@@ -279,17 +279,17 @@ tests:
279279
$.dataframes[0].resources[0].volume: "1"
280280
$.dataframes[0].resources[0].rating: "0.121"
281281
$.dataframes[0].resources[0].service: "image.size"
282-
$.dataframes[0].resources[0].desc.dummy: 'True'
283-
$.dataframes[0].resources[0].desc.fake_meta: '1.0'
282+
$.dataframes[0].resources[0].desc.dummy: True
283+
$.dataframes[0].resources[0].desc.fake_meta: 1.0
284284
$.dataframes[1].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
285285
$.dataframes[1].begin: "2015-01-04T13:00:00"
286286
$.dataframes[1].end: "2015-01-04T14:00:00"
287287
$.dataframes[1].resources.`len`: 1
288288
$.dataframes[1].resources[0].volume: "1"
289289
$.dataframes[1].resources[0].rating: "0.121"
290290
$.dataframes[1].resources[0].service: "image.size"
291-
$.dataframes[1].resources[0].desc.dummy: 'True'
292-
$.dataframes[1].resources[0].desc.fake_meta: '1.0'
291+
$.dataframes[1].resources[0].desc.dummy: True
292+
$.dataframes[1].resources[0].desc.fake_meta: 1.0
293293

294294
- name: fetch data filtering on service with no data and tenant
295295
url: /v1/storage/dataframes

0 commit comments

Comments
 (0)