|
| 1 | +# This file is provided by Epic Games, Inc. and is subject to the license |
| 2 | +# file included in this repository. |
| 3 | + |
| 4 | +""" |
| 5 | +This hook is used override some of the functionality of the :class:`~sgtk.bootstrap.ToolkitManager`. |
| 6 | +
|
| 7 | +It will be instantiated only after a configuration has been selected by the :class:`~sgtk.bootstrap.ToolkitManager`. |
| 8 | +Therefore, this hook will not be invoked to download a configuration. However, the Toolkit Core, |
| 9 | +applications, frameworks and engines can be downloaded through the hook. |
| 10 | +""" |
| 11 | + |
| 12 | +import os |
| 13 | +import zipfile |
| 14 | +import json |
| 15 | +import platform |
| 16 | +import re |
| 17 | + |
| 18 | +from sgtk import get_hook_baseclass |
| 19 | + |
| 20 | + |
| 21 | +_SIX_IMPORT_WARNING = ( |
| 22 | + "Unable to import six.moves from tk-core, this can happen " |
| 23 | + "if an old version of tk-core < 0.19.1 is used in a site " |
| 24 | + "pipeline configuration. Falling back on using urllib2." |
| 25 | +) |
| 26 | + |
| 27 | + |
| 28 | +class Bootstrap(get_hook_baseclass()): |
| 29 | + """ |
| 30 | + Override the bootstrap core hook to cache some bundles ourselves. |
| 31 | + http://developer.shotgunsoftware.com/tk-core/core.html#bootstrap.Bootstrap |
| 32 | + """ |
| 33 | + # List of github repos for which we download releases, with a github token to |
| 34 | + # do the download if the repo is private |
| 35 | + _download_release_from_github = [ |
| 36 | + ("ue4plugins/tk-framework-unrealqt", ""), |
| 37 | + ("GPLgithub/tk-framework-unrealqt", ""), |
| 38 | + ] |
| 39 | + |
| 40 | + def can_cache_bundle(self, descriptor): |
| 41 | + """ |
| 42 | + Indicates if a bundle can be cached by the :meth:`populate_bundle_cache_entry` method. |
| 43 | +
|
| 44 | + This method is invoked when the bootstrap manager wants to cache a bundle used by a configuration. |
| 45 | +
|
| 46 | + .. note:: This method is not called if the bundle is already cached so it |
| 47 | + can't be used to update an existing cached bundle. |
| 48 | +
|
| 49 | + :param descriptor: Descriptor of the bundle that needs to be cached. |
| 50 | +
|
| 51 | + :returns: ``True`` if the bundle can be cached with this hook, ``False`` |
| 52 | + if not. |
| 53 | + :rtype: bool |
| 54 | + :raises RuntimeError: If six.moves is not available. |
| 55 | + """ |
| 56 | + descd = descriptor.get_dict() |
| 57 | + # Some descriptors like shotgun descriptors don't have a path: ignore |
| 58 | + # them. |
| 59 | + if not descd.get("path"): |
| 60 | + return False |
| 61 | + return bool(self._should_download_release(descd["path"])) |
| 62 | + |
| 63 | + def populate_bundle_cache_entry(self, destination, descriptor, **kwargs): |
| 64 | + """ |
| 65 | + Populates an entry from the bundle cache. |
| 66 | +
|
| 67 | + This method will be invoked for every bundle for which :meth:`can_cache_bundle` |
| 68 | + returned ``True``. The hook is responsible for writing the bundle inside |
| 69 | + the destination folder. If an exception is raised by this method, the files |
| 70 | + will be deleted from disk and the bundle cache will be left intact. |
| 71 | +
|
| 72 | + It has to properly copy all the files or the cache for this bundle |
| 73 | + will be left in an inconsistent state. |
| 74 | +
|
| 75 | + :param str destination: Folder where the bundle needs to be written. Note |
| 76 | + that this is not the final destination folder inside the bundle cache. |
| 77 | +
|
| 78 | + :param descriptor: Descriptor of the bundle that needs to be cached. |
| 79 | + """ |
| 80 | + # This logic can be removed once we can assume tk-core is > v0.19.1 not |
| 81 | + # just in configs but also in the bundled Shotgun.app. |
| 82 | + try: |
| 83 | + from tank_vendor.six.moves.urllib import request as url2 |
| 84 | + from tank_vendor.six.moves.urllib import error as error_url2 |
| 85 | + except ImportError as e: |
| 86 | + self.logger.warning(_SIX_IMPORT_WARNING) |
| 87 | + self.logger.debug("%s" % e, exc_info=True) |
| 88 | + # Fallback on using urllib2 |
| 89 | + import urllib2 as url2 |
| 90 | + import urllib2 as error_url2 |
| 91 | + |
| 92 | + descd = descriptor.get_dict() |
| 93 | + version = descriptor.version |
| 94 | + self.logger.info("Treating %s" % descd) |
| 95 | + specs = self._should_download_release(descd["path"]) |
| 96 | + if not specs: |
| 97 | + raise RuntimeError("Don't know how to download %s" % descd) |
| 98 | + name = specs[0] |
| 99 | + token = specs[1] |
| 100 | + try: |
| 101 | + if self.shotgun.config.proxy_handler: |
| 102 | + # Re-use proxy settings from the Shotgun connection |
| 103 | + opener = url2.build_opener( |
| 104 | + self.parent.shotgun.config.proxy_handler, |
| 105 | + ) |
| 106 | + url2.install_opener(opener) |
| 107 | + |
| 108 | + # Retrieve the release from the tag |
| 109 | + url = "https://api.github.com/repos/%s/releases/tags/%s" % (name, version) |
| 110 | + request = url2.Request(url) |
| 111 | + # Add the authorization token if we have one (private repos) |
| 112 | + if token: |
| 113 | + request.add_header("Authorization", "token %s" % token) |
| 114 | + request.add_header("Accept", "application/vnd.github.v3+json") |
| 115 | + try: |
| 116 | + response = url2.urlopen(request) |
| 117 | + except error_url2.URLError as e: |
| 118 | + if hasattr(e, "code"): |
| 119 | + if e.code == 404: |
| 120 | + self.logger.error("Release %s does not exists" % version) |
| 121 | + elif e.code == 401: |
| 122 | + self.logger.error("Not authorised to access release %s." % version) |
| 123 | + raise |
| 124 | + response_d = json.loads(response.read()) |
| 125 | + # Look up for suitable assets for this platform. Assets names |
| 126 | + # follow this convention: |
| 127 | + # <version>-py<python version>-<platform>.zip |
| 128 | + # We download and extract all assets for any Python version for |
| 129 | + # the current platform and version. We're assuming that the cached |
| 130 | + # config for a user will never be shared between machines with |
| 131 | + # different os. |
| 132 | + pname = { |
| 133 | + "Darwin": "osx", |
| 134 | + "Linux": "linux", |
| 135 | + "Windows": "win" |
| 136 | + }.get(platform.system()) |
| 137 | + |
| 138 | + if not pname: |
| 139 | + raise ValueError("Unsupported platform %s" % platform.system()) |
| 140 | + |
| 141 | + extracted = [] |
| 142 | + for asset in response_d["assets"]: |
| 143 | + name = asset["name"] |
| 144 | + m = re.match( |
| 145 | + "%s-py\d.\d-%s.zip" % (version, pname), |
| 146 | + name |
| 147 | + ) |
| 148 | + if m: |
| 149 | + # Download the asset payload |
| 150 | + self._download_zip_github_asset( |
| 151 | + asset, |
| 152 | + destination, |
| 153 | + token |
| 154 | + ) |
| 155 | + extracted.append(asset) |
| 156 | + |
| 157 | + if not extracted: |
| 158 | + raise RuntimeError( |
| 159 | + "Couldn't retrieve a suitable asset from %s" % [ |
| 160 | + a["name"] for a in response_d["assets"] |
| 161 | + ] |
| 162 | + ) |
| 163 | + self.logger.info( |
| 164 | + "Extracted files: %s from %s" % ( |
| 165 | + os.listdir(destination), |
| 166 | + ",".join([a["name"] for a in extracted]) |
| 167 | + ) |
| 168 | + ) |
| 169 | + except Exception as e: |
| 170 | + # Log the exception with the backtrace because TK obfuscates it. |
| 171 | + self.logger.exception(e) |
| 172 | + raise |
| 173 | + |
| 174 | + def _should_download_release(self, desc_path): |
| 175 | + """ |
| 176 | + Return a repo name and a token if the given descriptor path should be downloaded |
| 177 | + from a github release. |
| 178 | +
|
| 179 | + :param str desc_path: A Toolkit descriptor path. |
| 180 | + :returns: A name, token tuple or ``None``. |
| 181 | + """ |
| 182 | + for name, token in self._download_release_from_github: |
| 183 | + if "[email protected]:%s.git" % name == desc_path: |
| 184 | + return name, token |
| 185 | + return None |
| 186 | + |
| 187 | + def _download_zip_github_asset(self, asset, destination, token): |
| 188 | + """ |
| 189 | + Download the zipped github asset and extract it into the given destination |
| 190 | + folder. |
| 191 | +
|
| 192 | + Assets can be retrieved with the releases github REST api endpoint. |
| 193 | + https://developer.github.com/v3/repos/releases/#get-a-release-by-tag-name |
| 194 | +
|
| 195 | + :param str asset: A Github asset dictionary. |
| 196 | + :param str destination: Full path to a folder where to extract the downloaded |
| 197 | + zipped archive. The folder is created if it does not |
| 198 | + exist. |
| 199 | + :param str token: A Github OAuth or personal token. |
| 200 | + """ |
| 201 | + try: |
| 202 | + from tank_vendor.six.moves.urllib import request as url2 |
| 203 | + except ImportError as e: |
| 204 | + self.logger.warning(_SIX_IMPORT_WARNING) |
| 205 | + self.logger.debug("%s" % e, exc_info=True) |
| 206 | + # Fallback on using urllib2 |
| 207 | + import urllib2 as url2 |
| 208 | + # If we have a token use a basic auth handler |
| 209 | + # just a http handler otherwise |
| 210 | + if token: |
| 211 | + passman = url2.HTTPPasswordMgrWithDefaultRealm() |
| 212 | + passman.add_password( |
| 213 | + None, |
| 214 | + asset["url"], |
| 215 | + token, |
| 216 | + token |
| 217 | + ) |
| 218 | + auth_handler = url2.HTTPBasicAuthHandler(passman) |
| 219 | + else: |
| 220 | + auth_handler = url2.HTTPHandler() |
| 221 | + |
| 222 | + if self.shotgun.config.proxy_handler: |
| 223 | + # Re-use proxy settings from the Shotgun connection |
| 224 | + opener = url2.build_opener( |
| 225 | + self.parent.shotgun.config.proxy_handler, |
| 226 | + auth_handler |
| 227 | + ) |
| 228 | + else: |
| 229 | + opener = url2.build_opener(auth_handler) |
| 230 | + |
| 231 | + url2.install_opener(opener) |
| 232 | + request = url2.Request(asset["url"]) |
| 233 | + if token: |
| 234 | + # We will be redirected and the Auth shouldn't be in the header |
| 235 | + # for the redirection. |
| 236 | + request.add_unredirected_header("Authorization", "token %s" % token) |
| 237 | + request.add_header("Accept", "application/octet-stream") |
| 238 | + response = url2.urlopen(request) |
| 239 | + if not os.path.exists(destination): |
| 240 | + self.logger.info("Creating %s" % destination) |
| 241 | + os.makedirs(destination) |
| 242 | + tmp_file = os.path.join(destination, asset["name"]) |
| 243 | + with open(tmp_file, "wb") as f: |
| 244 | + f.write(response.read()) |
| 245 | + with zipfile.ZipFile(tmp_file, "r") as zip_ref: |
| 246 | + zip_ref.extractall(destination) |
0 commit comments