Skip to content

Commit 480ea38

Browse files
pslestangalistarle
authored andcommitted
Implement glance-download internal plugin
Add a new import method called glance-download that implements a glance to glance download in a multi-region cloud with a federated Keystone. This method will copy the image data and selected metadata to the target glance, checking that the downloaded size match the "size" image attribute in the source glance. Implements: blueprint glance-download-import Co-Authored-By: Victor Coutellier <[email protected]> Change-Id: Ic51c5fd87caf04d38aeaf758ad2d0e2f28098e4d
1 parent 653d52e commit 480ea38

15 files changed

+1039
-278
lines changed

glance/api/v2/images.py

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import os
1919
import re
2020
import urllib.parse as urlparse
21+
import uuid
2122

2223
from castellan.common import exception as castellan_exception
2324
from castellan import key_manager
@@ -332,9 +333,10 @@ def import_image(self, req, image_id, body):
332333
msg = _("Only images with status active can be targeted for "
333334
"copying")
334335
raise exception.Conflict(msg)
335-
if image.status != 'queued' and import_method == 'web-download':
336+
if (image.status != 'queued' and
337+
import_method in ['web-download', 'glance-download']):
336338
msg = _("Image needs to be in 'queued' state to use "
337-
"'web-download' method")
339+
"'%s' method") % import_method
338340
raise exception.Conflict(msg)
339341
if (image.status != 'uploading' and
340342
import_method == 'glance-direct'):
@@ -347,6 +349,23 @@ def import_image(self, req, image_id, body):
347349
if not getattr(image, 'disk_format', None):
348350
msg = _("'disk_format' needs to be set before import")
349351
raise exception.Conflict(msg)
352+
if import_method == 'glance-download':
353+
if 'glance_region' not in body.get('method'):
354+
msg = _("'glance_region' needs to be set for "
355+
"glance-download import method")
356+
raise webob.exc.HTTPBadRequest(explanation=msg)
357+
if 'glance_image_id' not in body.get('method'):
358+
msg = _("'glance_image_id' needs to be set for "
359+
"glance-download import method")
360+
raise webob.exc.HTTPBadRequest(explanation=msg)
361+
try:
362+
uuid.UUID(body['method']['glance_image_id'])
363+
except ValueError:
364+
msg = (_("Remote image id does not look like a UUID: %s")
365+
% body['method']['glance_image_id'])
366+
raise webob.exc.HTTPBadRequest(explanation=msg)
367+
if 'glance_service_interface' not in body.get('method'):
368+
body.get('method')['glance_service_interface'] = 'public'
350369

351370
# NOTE(danms): For copy-image only, we check policy to decide
352371
# if the user should be able to do this. Otherwise, we forbid
Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
# Copyright 2018 Red Hat, Inc.
2+
# Copyright 2022 OVHCloud
3+
# All Rights Reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License"); you may
6+
# not use this file except in compliance with the License. You may obtain
7+
# a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14+
# License for the specific language governing permissions and limitations
15+
# under the License.
16+
17+
import abc
18+
19+
import glance_store as store_api
20+
from glance_store import backend
21+
from oslo_config import cfg
22+
from oslo_log import log as logging
23+
import six
24+
from taskflow import task
25+
26+
from glance.common import exception
27+
from glance.i18n import _, _LE
28+
29+
LOG = logging.getLogger(__name__)
30+
31+
CONF = cfg.CONF
32+
33+
34+
@six.add_metaclass(abc.ABCMeta)
35+
class BaseDownload(task.Task):
36+
37+
default_provides = 'file_uri'
38+
39+
def __init__(self, task_id, task_type, action_wrapper, stores,
40+
plugin_name):
41+
self.task_id = task_id
42+
self.task_type = task_type
43+
self.image_id = action_wrapper.image_id
44+
self.action_wrapper = action_wrapper
45+
self.stores = stores
46+
self._path = None
47+
self.plugin_name = plugin_name or 'Download'
48+
super(BaseDownload, self).__init__(
49+
name='%s-%s-%s' % (task_type, self.plugin_name, task_id))
50+
51+
# NOTE(abhishekk): Use reserved 'os_glance_staging_store' for
52+
# staging the data, the else part will be removed once old way
53+
# of configuring store is deprecated.
54+
if CONF.enabled_backends:
55+
self.store = store_api.get_store_from_store_identifier(
56+
'os_glance_staging_store')
57+
else:
58+
if CONF.node_staging_uri is None:
59+
msg = (_("%(task_id)s of %(task_type)s not configured "
60+
"properly. Missing node_staging_uri: %(work_dir)s") %
61+
{'task_id': self.task_id,
62+
'task_type': self.task_type,
63+
'work_dir': CONF.node_staging_uri})
64+
raise exception.BadTaskConfiguration(msg)
65+
66+
self.store = self._build_store()
67+
68+
def _build_store(self):
69+
# NOTE(flaper87): Due to the nice glance_store api (#sarcasm), we're
70+
# forced to build our own config object, register the required options
71+
# (and by required I mean *ALL* of them, even the ones we don't want),
72+
# and create our own store instance by calling a private function.
73+
# This is certainly unfortunate but it's the best we can do until the
74+
# glance_store refactor is done. A good thing is that glance_store is
75+
# under our team's management and it gates on Glance so changes to
76+
# this API will (should?) break task's tests.
77+
# TODO(abhishekk): After removal of backend module from glance_store
78+
# need to change this to use multi_backend module.
79+
conf = cfg.ConfigOpts()
80+
try:
81+
backend.register_opts(conf)
82+
except cfg.DuplicateOptError:
83+
pass
84+
85+
conf.set_override('filesystem_store_datadir',
86+
CONF.node_staging_uri[7:],
87+
group='glance_store')
88+
89+
# NOTE(flaper87): Do not even try to judge me for this... :(
90+
# With the glance_store refactor, this code will change, until
91+
# that happens, we don't have a better option and this is the
92+
# least worst one, IMHO.
93+
store = store_api.backend._load_store(conf, 'file')
94+
95+
if store is None:
96+
msg = (_("%(task_id)s of %(task_type)s not configured "
97+
"properly. Could not load the filesystem store") %
98+
{'task_id': self.task_id, 'task_type': self.task_type})
99+
raise exception.BadTaskConfiguration(msg)
100+
101+
store.configure()
102+
return store
103+
104+
def revert(self, result, **kwargs):
105+
LOG.error(_LE('Task: %(task_id)s failed to import image '
106+
'%(image_id)s to the filesystem.'),
107+
{'task_id': self.task_id,
108+
'image_id': self.image_id})
109+
# NOTE(abhishekk): Revert image state back to 'queued' as
110+
# something went wrong.
111+
# NOTE(danms): If we failed to stage the image, then none
112+
# of the _ImportToStore() tasks could have run, so we need
113+
# to move all stores out of "importing" and into "failed".
114+
with self.action_wrapper as action:
115+
action.set_image_attribute(status='queued')
116+
action.remove_importing_stores(self.stores)
117+
action.add_failed_stores(self.stores)
118+
119+
# NOTE(abhishekk): Deleting partial image data from staging area
120+
if self._path is not None:
121+
LOG.debug(('Deleting image %(image_id)s from staging '
122+
'area.'), {'image_id': self.image_id})
123+
try:
124+
if CONF.enabled_backends:
125+
store_api.delete(self._path, None)
126+
else:
127+
store_api.delete_from_backend(self._path)
128+
except Exception:
129+
LOG.exception(_LE("Error reverting web/glance download "
130+
"task: %(task_id)s"), {
131+
'task_id': self.task_id})
Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
# Copyright 2022 OVHCloud
2+
# All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License"); you may
5+
# not use this file except in compliance with the License. You may obtain
6+
# a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
# License for the specific language governing permissions and limitations
14+
# under the License.
15+
16+
import urllib.request
17+
18+
from oslo_config import cfg
19+
from oslo_log import log as logging
20+
from oslo_utils import encodeutils
21+
from oslo_utils import excutils
22+
from taskflow.patterns import linear_flow as lf
23+
24+
from glance.async_.flows._internal_plugins import base_download
25+
from glance.async_ import utils
26+
from glance.common import exception
27+
from glance.common import utils as common_utils
28+
from glance.i18n import _, _LI, _LE
29+
30+
LOG = logging.getLogger(__name__)
31+
32+
CONF = cfg.CONF
33+
34+
35+
class _DownloadGlanceImage(base_download.BaseDownload):
36+
37+
def __init__(self, context, task_id, task_type, action_wrapper, stores,
38+
glance_region, glance_image_id, glance_service_interface):
39+
self.context = context
40+
self.glance_region = glance_region
41+
self.glance_image_id = glance_image_id
42+
self.glance_service_interface = glance_service_interface
43+
super(_DownloadGlanceImage,
44+
self).__init__(task_id, task_type, action_wrapper, stores,
45+
'GlanceDownload')
46+
47+
def execute(self, image_size):
48+
"""Create temp file into store and return path to it
49+
50+
:param image_size: Glance Image Size retrieved from ImportMetadata task
51+
"""
52+
try:
53+
glance_endpoint = utils.get_glance_endpoint(
54+
self.context,
55+
self.glance_region,
56+
self.glance_service_interface)
57+
image_download_url = '%s/v2/images/%s/file' % (
58+
glance_endpoint, self.glance_image_id)
59+
if not common_utils.validate_import_uri(image_download_url):
60+
LOG.debug("Processed URI for glance-download does not pass "
61+
"filtering: %s", image_download_url)
62+
msg = (_("Processed URI for glance-download does not pass "
63+
"filtering: %s") % image_download_url)
64+
raise exception.ImportTaskError(msg)
65+
LOG.info(_LI("Downloading glance image %s"), image_download_url)
66+
token = self.context.auth_token
67+
request = urllib.request.Request(image_download_url,
68+
headers={'X-Auth-Token': token})
69+
data = urllib.request.urlopen(request)
70+
except Exception as e:
71+
with excutils.save_and_reraise_exception():
72+
LOG.error(
73+
_LE("Task %(task_id)s failed with exception %(error)s"), {
74+
"error": encodeutils.exception_to_unicode(e),
75+
"task_id": self.task_id
76+
})
77+
78+
self._path, bytes_written = self.store.add(self.image_id, data, 0)[0:2]
79+
if bytes_written != image_size:
80+
msg = (_("Task %(task_id)s failed because downloaded data "
81+
"size %(data_size)i is different from expected %("
82+
"expected)i") %
83+
{"task_id": self.task_id, "data_size": bytes_written,
84+
"expected": image_size})
85+
raise exception.ImportTaskError(msg)
86+
return self._path
87+
88+
89+
def get_flow(**kwargs):
90+
"""Return task flow for no-op.
91+
92+
:param context: request context
93+
:param task_id: Task ID.
94+
:param task_type: Type of the task.
95+
:param image_repo: Image repository used.
96+
:param image_id: Image ID
97+
:param source_region: Source region name
98+
"""
99+
context = kwargs.get('context')
100+
task_id = kwargs.get('task_id')
101+
task_type = kwargs.get('task_type')
102+
action_wrapper = kwargs.get('action_wrapper')
103+
stores = kwargs.get('backend', [None])
104+
# glance-download parameters
105+
import_req = kwargs.get('import_req')
106+
method = import_req.get('method')
107+
glance_region = method.get('glance_region')
108+
glance_image_id = method.get('glance_image_id')
109+
glance_service_interface = method.get('glance_service_interface')
110+
111+
return lf.Flow(task_type).add(
112+
_DownloadGlanceImage(context, task_id, task_type, action_wrapper,
113+
stores, glance_region, glance_image_id,
114+
glance_service_interface),
115+
)

0 commit comments

Comments
 (0)