From aacde60fd6c942c30582f29672d5102fb5aae188 Mon Sep 17 00:00:00 2001 From: James Annesley Date: Tue, 14 May 2024 11:27:21 +0100 Subject: [PATCH 1/5] upload shasum files and cut content disposition writing for all files --- .../mvnfeed/cli/transfer/transfer.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py b/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py index ec1e301..3a10554 100644 --- a/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py +++ b/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py @@ -196,12 +196,22 @@ def _java_artifacts(artifact_fullname, artifact_type, artifact_path, transfer_de 'path': artifact_path, 'target': True }, + { + 'name': artifact_fullname + '.jar.sha1', + 'path': artifact_path, + 'target': False + }, { 'name': artifact_fullname + '.pom', 'path': artifact_path, 'transfer_deps': transfer_deps, 'target': False }, + { + 'name': artifact_fullname + '.pom.sha1', + 'path': artifact_path, + 'target': False + }, { 'name': artifact_fullname + '-tests.jar', 'path': artifact_path, @@ -233,6 +243,11 @@ def _untyped_artifacts(artifact_fullname, artifact_type, artifact_path, transfer 'path': artifact_path, 'transfer_deps': transfer_deps, 'target': False + }, + { + 'name': artifact_fullname + '.pom.sha1', + 'path': artifact_path, + 'target': False } ] @@ -319,7 +334,7 @@ def _upload_file(to_repository, path, filename): try: with open(filename, 'rb') as file: - response = requests.put(url, files={filename: file}, headers=headers) + response = requests.put(url, data=file.read(), headers=headers) if not response.ok: logging.error('error while uploading of %s: %s', path, response.text) return True From 5e08ff5103b1775335780c1a5d3230f926e10873 Mon Sep 17 00:00:00 2001 From: James Annesley Date: Thu, 16 May 2024 15:38:19 +0100 Subject: [PATCH 2/5] Fix redirect by swapping in requests for downloads --- .../mvnfeed/cli/transfer/transfer.py | 32 ++++++++----------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py b/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py index ec1e301..9fcbcc5 100644 --- a/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py +++ b/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py @@ -2,18 +2,12 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- - +import functools import logging import os.path import requests import shutil import xml.etree.ElementTree as ET -try: - # Python 3 - from urllib.request import Request, urlopen -except ImportError: - # Python 2 - from urllib2 import Request, urlopen from .configuration import get_repository, get_stagedir, get_repository_shortname from mvnfeed.cli.common.config import AUTHORIZATION, URL, load_config @@ -244,7 +238,7 @@ def _findNodeValue(node, name): return foundNode.text -def _download_file(from_repository, path, filename, length=16*1024): +def _download_file(from_repository, path, filename): """ Stores the path into the given filename. """ @@ -256,20 +250,20 @@ def _download_file(from_repository, path, filename, length=16*1024): url = _append_url(from_repository[URL], path) logging.debug('downloading from %s', url) - try: - request = Request(url) - if AUTHORIZATION in from_repository and from_repository[AUTHORIZATION]: - logging.debug('authorization header added') - request.add_header('Authorization', from_repository[AUTHORIZATION]) - else: - logging.debug('no authorization configured') + if AUTHORIZATION in from_repository and from_repository[AUTHORIZATION]: + logging.debug('authorization header added') + headers = {'Authorization': from_repository[AUTHORIZATION]} + else: + logging.debug('no authorization configured') + headers = {} - response = urlopen(request) - with open(filename, 'wb') as file: - shutil.copyfileobj(response, file, length) + try: + with requests.get(url, headers=headers, stream=True) as response: + response.raw.read = functools.partial(response.raw.read, decode_content=True) + with open(filename, 'wb') as file: + shutil.copyfileobj(response.raw, file) except Exception as ex: logging.debug('exception while downloading (expected): %s', ex) - None def _already_uploaded(to_repository, path): From a8e8375a16ae5e18a54514fa106b47d11e4c99d7 Mon Sep 17 00:00:00 2001 From: James Annesley Date: Thu, 16 May 2024 16:25:32 +0100 Subject: [PATCH 3/5] Make cached download filenames unique using group id --- .../mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py b/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py index ec1e301..610ea7c 100644 --- a/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py +++ b/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py @@ -113,7 +113,7 @@ def _transfer_single_artifact(name, from_repository, to_repository, stage_dir, t # let's always download POM files in case we need to process the parent POM # once again or upload the children dependencies. - outfile = os.path.join(stage_dir, file2transfer['name']) + outfile = os.path.join(stage_dir, group_id + '-' + file2transfer['name']) _download_file(from_repository, artifact_relativepath, outfile) if not os.path.exists(outfile): logging.info('%s was not downloaded. Skipping', outfile) From fce33940b582a557df4dce291d2c9c458c23fde2 Mon Sep 17 00:00:00 2001 From: James Annesley Date: Thu, 16 May 2024 16:46:18 +0100 Subject: [PATCH 4/5] upload shasum source file --- .../mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py b/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py index 3a10554..38c8062 100644 --- a/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py +++ b/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py @@ -222,6 +222,11 @@ def _java_artifacts(artifact_fullname, artifact_type, artifact_path, transfer_de 'path': artifact_path, 'target': False }, + { + 'name': artifact_fullname + '-sources.jar.sha1', + 'path': artifact_path, + 'target': False + }, { 'name': artifact_fullname + '-javadoc.jar', 'path': artifact_path, From 9a2432fd6d1471fc38e233cefc33064579355b83 Mon Sep 17 00:00:00 2001 From: James Annesley Date: Thu, 16 May 2024 17:19:36 +0100 Subject: [PATCH 5/5] Handle 40x errors --- .../mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py b/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py index 9fcbcc5..c85fe77 100644 --- a/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py +++ b/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py @@ -259,11 +259,14 @@ def _download_file(from_repository, path, filename): try: with requests.get(url, headers=headers, stream=True) as response: + if not response.ok: + logging.debug('exception while downloading (expected): %s', response.text) + return response.raw.read = functools.partial(response.raw.read, decode_content=True) with open(filename, 'wb') as file: shutil.copyfileobj(response.raw, file) except Exception as ex: - logging.debug('exception while downloading (expected): %s', ex) + logging.error('exception while downloading: %s', ex) def _already_uploaded(to_repository, path):