Merge pull request #187 from reportportal/develop

Release

Author: HardNorth

CHANGELOG.md

@@ -1,8 +1,15 @@
 # Changelog
 
 ## [Unreleased]
+### Added
+- Ability to pass client instance in `RPLogHandler` constructor, by @HardNorth
+- Issue [#179](https://github.com/reportportal/client-Python/issues/179): batch logging request payload size tracking, by @HardNorth
 ### Fixed
-- Issue [#182](https://github.com/reportportal/client-Python/issues/182): logger crash on attachments, by @HardNorth
+- Issue [#184](https://github.com/reportportal/client-Python/issues/184): early logger initialization exception, by @dagansandler
+
+## [5.2.2]
+### Fixed
+- Issue [#182](https://github.com/reportportal/client-Python/issues/182): logger crash on empty client, by @HardNorth
 
 ## [5.2.1]
 ### Fixed

pyproject.toml

@@ -4,5 +4,6 @@ requires = [
   "setuptools>=40.0",
   "setuptools-scm",
   "wheel",
+  "importlib-metadata"
 ]
 build-backend = "setuptools.build_meta"

reportportal_client/_local/__init__.py

@@ -1,4 +1,6 @@
-#  Copyright (c) 2022 https://reportportal.io .
+"""Report Portal client context storing and retrieving module."""
+
+#  Copyright (c) 2022 EPAM Systems
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
@@ -10,7 +12,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License
-"""Report Portal client context storing and retrieving module."""
+
 from threading import local
 
 __INSTANCES = local()

reportportal_client/_local/__init__.pyi

@@ -1,4 +1,4 @@
-#  Copyright (c) 2022 https://reportportal.io .
+#  Copyright (c) 2022 EPAM Systems
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at

reportportal_client/client.py

@@ -20,7 +20,7 @@
 from requests.adapters import HTTPAdapter, Retry
 
 from ._local import set_current
-from .core.log_manager import LogManager
+from .core.log_manager import LogManager, MAX_LOG_BATCH_PAYLOAD_SIZE
 from .core.rp_requests import (
     HttpRequest,
     ItemStartRequest,
@@ -58,24 +58,28 @@ def __init__(self,
                  max_pool_size=50,
                  launch_id=None,
                  http_timeout=(10, 10),
+                 log_batch_payload_size=MAX_LOG_BATCH_PAYLOAD_SIZE,
                  **_):
         """Initialize required attributes.
 
-        :param endpoint:             Endpoint of the report portal service
-        :param project:              Project name to report to
-        :param token:                Authorization token
-        :param log_batch_size:       Option to set the maximum number of
-                                     logs that can be processed in one batch
-        :param is_skipped_an_issue:  Option to mark skipped tests as not
-                                     'To Investigate' items on the server side
-        :param verify_ssl:           Option to skip ssl verification
-        :param max_pool_size:        Option to set the maximum number of
-                                     connections to save the pool.
-        :param launch_id:            a launch id to use instead of starting own
-                                     one
-        :param http_timeout:         a float in seconds for connect and read
-                                     timeout. Use a Tuple to specific connect
-                                     and read separately.
+        :param endpoint:               Endpoint of the report portal service
+        :param project:                Project name to report to
+        :param token:                  Authorization token
+        :param log_batch_size:         Option to set the maximum number of
+                                       logs that can be processed in one batch
+        :param is_skipped_an_issue:    Option to mark skipped tests as not
+                                       'To Investigate' items on the server
+                                       side
+        :param verify_ssl:             Option to skip ssl verification
+        :param max_pool_size:          Option to set the maximum number of
+                                       connections to save the pool.
+        :param launch_id:              a launch id to use instead of starting
+                                       own one
+        :param http_timeout:           a float in seconds for connect and read
+                                       timeout. Use a Tuple to specific connect
+                                       and read separately.
+        :param log_batch_payload_size: maximum size in bytes of logs that can
+                                       be processed in one batch
         """
         set_current(self)
         self._batch_logs = []
@@ -89,6 +93,7 @@ def __init__(self,
         self.is_skipped_an_issue = is_skipped_an_issue
         self.launch_id = launch_id
         self.log_batch_size = log_batch_size
+        self.log_batch_payload_size = log_batch_payload_size
         self.token = token
         self.verify_ssl = verify_ssl
         self.http_timeout = http_timeout
@@ -109,7 +114,8 @@ def __init__(self,
 
         self._log_manager = LogManager(
             self.endpoint, self.session, self.api_v2, self.launch_id,
-            self.project, log_batch_size=log_batch_size)
+            self.project, max_entry_number=log_batch_size,
+            max_payload_size=log_batch_payload_size)
 
     def finish_launch(self,
                       end_time,

reportportal_client/client.pyi

@@ -20,6 +20,7 @@ class RPClient:
     is_skipped_an_issue: bool = ...
     launch_id: Text = ...
     log_batch_size: int = ...
+    log_batch_payload_size: int = ...
     project: Text = ...
     token: Text = ...
     verify_ssl: bool = ...
@@ -37,7 +38,8 @@ class RPClient:
             retries: int = ...,
             max_pool_size: int = ...,
             launch_id: Text = ...,
-            http_timeout: Union[float, Tuple[float, float]] = ...
+            http_timeout: Union[float, Tuple[float, float]] = ...,
+            log_batch_payload_size: int = ...
     ) -> None: ...
 
     def finish_launch(self,

reportportal_client/core/__init__.py

@@ -1 +1,14 @@
+#  Copyright (c) 2022 EPAM Systems
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License
+
 """This package contains core reportportal-client modules."""

reportportal_client/core/log_manager.py

@@ -1,22 +1,24 @@
-"""This module contains management functionality for processing logs.
-
-Copyright (c) 2018 https://reportportal.io .
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-https://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
+"""This module contains management functionality for processing logs."""
+
+#  Copyright (c) 2022 EPAM Systems
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License
 
 import logging
 from threading import Lock
 
 from six.moves import queue
 
+from reportportal_client import helpers
 from reportportal_client.core.rp_requests import (
     HttpRequest,
     RPFile,
@@ -28,31 +30,38 @@
 
 logger = logging.getLogger(__name__)
 
+MAX_LOG_BATCH_PAYLOAD_SIZE = 65000000
+
 
 class LogManager(object):
     """Manager of the log items."""
 
     def __init__(self, rp_url, session, api_version, launch_id, project_name,
-                 log_batch_size=20, verify_ssl=True):
+                 max_entry_number=20, verify_ssl=True,
+                 max_payload_size=MAX_LOG_BATCH_PAYLOAD_SIZE):
         """Initialize instance attributes.
 
-        :param rp_url:         Report portal URL
-        :param session:        HTTP Session object
-        :param api_version:    RP API version
-        :param launch_id:      Parent launch UUID
-        :param project_name:   RP project name
-        :param log_batch_size: The amount of log objects that need to be
-                               gathered before processing
-        :param verify_ssl:     Indicates that it is necessary to verify SSL
-                               certificates within HTTP request
+        :param rp_url:           Report portal URL
+        :param session:          HTTP Session object
+        :param api_version:      RP API version
+        :param launch_id:        Parent launch UUID
+        :param project_name:     RP project name
+        :param max_entry_number: The amount of log objects that need to be
+                                 gathered before processing
+        :param verify_ssl:       Indicates that it is necessary to verify SSL
+                                 certificates within HTTP request
+        :param max_payload_size: maximum size in bytes of logs that can be
+                                 processed in one batch
         """
         self._lock = Lock()
-        self._logs_batch = []
+        self._batch = []
+        self._payload_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH
         self._worker = None
         self.api_version = api_version
         self.queue = queue.PriorityQueue()
         self.launch_id = launch_id
-        self.log_batch_size = log_batch_size
+        self.max_entry_number = max_entry_number
+        self.max_payload_size = max_payload_size
         self.project_name = project_name
         self.rp_url = rp_url
         self.session = session
@@ -63,24 +72,31 @@ def __init__(self, rp_url, session, api_version, launch_id, project_name,
             .format(rp_url=rp_url, version=self.api_version,
                     project_name=self.project_name))
 
-    def _log_process(self, log_req):
-        """Process the given log request.
-
-        :param log_req: RPRequestLog object
-        """
-        self._logs_batch.append(log_req)
-        if len(self._logs_batch) >= self.log_batch_size:
-            self._send_batch()
-
     def _send_batch(self):
         """Send existing batch logs to the worker."""
-        batch = RPLogBatch(self._logs_batch)
+        batch = RPLogBatch(self._batch)
         http_request = HttpRequest(
             self.session.post, self._log_endpoint, files=batch.payload,
             verify_ssl=self.verify_ssl)
         batch.http_request = http_request
         self._worker.send(batch)
-        self._logs_batch.clear()
+        self._batch = []
+        self._payload_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH
+
+    def _log_process(self, log_req):
+        """Process the given log request.
+
+        :param log_req: RPRequestLog object
+        """
+        with self._lock:
+            rq_size = log_req.multipart_size
+            if self._payload_size + rq_size >= self.max_payload_size:
+                if len(self._batch) > 0:
+                    self._send_batch()
+            self._batch.append(log_req)
+            self._payload_size += rq_size
+            if len(self._batch) >= self.max_entry_number:
+                self._send_batch()
 
     def log(self, time, message=None, level=None, attachment=None,
             item_id=None):
@@ -111,7 +127,7 @@ def stop(self):
         """Send last batches to the worker followed by the stop command."""
         if self._worker:
             with self._lock:
-                if self._logs_batch:
+                if self._batch:
                     self._send_batch()
                 logger.debug('Waiting for worker {0} to complete'
                              'processing batches.'.format(self._worker))

reportportal_client/core/log_manager.pyi

@@ -1,50 +1,72 @@
+#  Copyright (c) 2022 EPAM Systems
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License
+
 from logging import Logger
-from requests import Session
 from threading import Lock
+from typing import Dict, List, Optional, Text
 
+from requests import Session
 from six.moves import queue
 
 from reportportal_client.core.rp_requests import (
-    HttpRequest as HttpRequest,
-    RPFile as RPFile,
-    RPLogBatch as RPLogBatch,
     RPRequestLog as RPRequestLog
 )
 from reportportal_client.core.worker import APIWorker as APIWorker
-from typing import Dict, List, Optional, Text
 
 logger: Logger
 
+MAX_LOG_BATCH_PAYLOAD_SIZE: int
+
+
 class LogManager:
     _lock: Lock = ...
     _log_endpoint: Text = ...
-    _logs_batch: List = ...
+    _batch: List = ...
+    _payload_size: int = ...
     _worker: Optional[APIWorker] = ...
     api_version: Text = ...
     queue: queue.PriorityQueue = ...
     launch_id: Text = ...
-    log_batch_size: int = ...
+    max_entry_number: int = ...
     project_name: Text = ...
     rp_url: Text = ...
     session: Session = ...
     verify_ssl: bool = ...
+    max_payload_size: int = ...
+
     def __init__(self,
                  rp_url: Text,
                  session: Session,
                  api_version: Text,
                  launch_id: Text,
                  project_name: Text,
-                 log_batch_size: int = ...,
-                 verify_ssl: bool = ...) -> None: ...
+                 max_entry_number: int = ...,
+                 verify_ssl: bool = ...,
+                 max_payload_size: int = ...) -> None: ...
 
     def _log_process(self, log_req: RPRequestLog) -> None: ...
+
     def _send_batch(self) -> None: ...
+
     def log(self,
             time: Text,
             message: Optional[Text] = ...,
             level: Optional[Text] = ...,
             attachment: Optional[Dict] = ...,
             item_id: Optional[Text] = ...) -> None: ...
+
     def start(self) -> None: ...
+
     def stop(self) -> None: ...
+
     def stop_force(self) -> None: ...

reportportal_client/core/rp_file.py

@@ -1,5 +1,18 @@
 """This module contains classes representing RP file object."""
 
+#  Copyright (c) 2022 EPAM Systems
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License
+
 import uuid