diff --git a/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py b/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py index ec1e301..0fe02ec 100644 --- a/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py +++ b/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py @@ -2,18 +2,12 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- - +import functools import logging import os.path import requests import shutil import xml.etree.ElementTree as ET -try: - # Python 3 - from urllib.request import Request, urlopen -except ImportError: - # Python 2 - from urllib2 import Request, urlopen from .configuration import get_repository, get_stagedir, get_repository_shortname from mvnfeed.cli.common.config import AUTHORIZATION, URL, load_config @@ -113,7 +107,7 @@ def _transfer_single_artifact(name, from_repository, to_repository, stage_dir, t # let's always download POM files in case we need to process the parent POM # once again or upload the children dependencies. - outfile = os.path.join(stage_dir, file2transfer['name']) + outfile = os.path.join(stage_dir, group_id + '-' + file2transfer['name']) _download_file(from_repository, artifact_relativepath, outfile) if not os.path.exists(outfile): logging.info('%s was not downloaded. Skipping', outfile) @@ -196,12 +190,22 @@ def _java_artifacts(artifact_fullname, artifact_type, artifact_path, transfer_de 'path': artifact_path, 'target': True }, + { + 'name': artifact_fullname + '.jar.sha1', + 'path': artifact_path, + 'target': False + }, { 'name': artifact_fullname + '.pom', 'path': artifact_path, 'transfer_deps': transfer_deps, 'target': False }, + { + 'name': artifact_fullname + '.pom.sha1', + 'path': artifact_path, + 'target': False + }, { 'name': artifact_fullname + '-tests.jar', 'path': artifact_path, @@ -212,6 +216,11 @@ def _java_artifacts(artifact_fullname, artifact_type, artifact_path, transfer_de 'path': artifact_path, 'target': False }, + { + 'name': artifact_fullname + '-sources.jar.sha1', + 'path': artifact_path, + 'target': False + }, { 'name': artifact_fullname + '-javadoc.jar', 'path': artifact_path, @@ -233,6 +242,11 @@ def _untyped_artifacts(artifact_fullname, artifact_type, artifact_path, transfer 'path': artifact_path, 'transfer_deps': transfer_deps, 'target': False + }, + { + 'name': artifact_fullname + '.pom.sha1', + 'path': artifact_path, + 'target': False } ] @@ -244,7 +258,7 @@ def _findNodeValue(node, name): return foundNode.text -def _download_file(from_repository, path, filename, length=16*1024): +def _download_file(from_repository, path, filename): """ Stores the path into the given filename. """ @@ -256,20 +270,23 @@ def _download_file(from_repository, path, filename, length=16*1024): url = _append_url(from_repository[URL], path) logging.debug('downloading from %s', url) - try: - request = Request(url) - if AUTHORIZATION in from_repository and from_repository[AUTHORIZATION]: - logging.debug('authorization header added') - request.add_header('Authorization', from_repository[AUTHORIZATION]) - else: - logging.debug('no authorization configured') + if AUTHORIZATION in from_repository and from_repository[AUTHORIZATION]: + logging.debug('authorization header added') + headers = {'Authorization': from_repository[AUTHORIZATION]} + else: + logging.debug('no authorization configured') + headers = {} - response = urlopen(request) - with open(filename, 'wb') as file: - shutil.copyfileobj(response, file, length) + try: + with requests.get(url, headers=headers, stream=True) as response: + if not response.ok: + logging.debug('exception while downloading (expected): %s', response.text) + return + response.raw.read = functools.partial(response.raw.read, decode_content=True) + with open(filename, 'wb') as file: + shutil.copyfileobj(response.raw, file) except Exception as ex: - logging.debug('exception while downloading (expected): %s', ex) - None + logging.error('exception while downloading: %s', ex) def _already_uploaded(to_repository, path): @@ -319,7 +336,7 @@ def _upload_file(to_repository, path, filename): try: with open(filename, 'rb') as file: - response = requests.put(url, files={filename: file}, headers=headers) + response = requests.put(url, data=file.read(), headers=headers) if not response.ok: logging.error('error while uploading of %s: %s', path, response.text) return True