1- """This module contains management functionality for processing logs.
2-
3- Copyright (c) 2018 https://reportportal.io .
4- Licensed under the Apache License, Version 2.0 (the "License");
5- you may not use this file except in compliance with the License.
6- You may obtain a copy of the License at
7- https://www.apache.org/licenses/LICENSE-2.0
8- Unless required by applicable law or agreed to in writing, software
9- distributed under the License is distributed on an "AS IS" BASIS,
10- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11- See the License for the specific language governing permissions and
12- limitations under the License.
13- """
1+ """This module contains management functionality for processing logs."""
2+
3+ # Copyright (c) 2022 EPAM Systems
4+ # Licensed under the Apache License, Version 2.0 (the "License");
5+ # you may not use this file except in compliance with the License.
6+ # You may obtain a copy of the License at
7+ #
8+ # https://www.apache.org/licenses/LICENSE-2.0
9+ #
10+ # Unless required by applicable law or agreed to in writing, software
11+ # distributed under the License is distributed on an "AS IS" BASIS,
12+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+ # See the License for the specific language governing permissions and
14+ # limitations under the License
1415
1516import logging
1617from threading import Lock
1718
1819from six .moves import queue
1920
21+ from reportportal_client import helpers
2022from reportportal_client .core .rp_requests import (
2123 HttpRequest ,
2224 RPFile ,
2830
2931logger = logging .getLogger (__name__ )
3032
33+ MAX_LOG_BATCH_PAYLOAD_SIZE = 65000000
34+
3135
3236class LogManager (object ):
3337 """Manager of the log items."""
3438
3539 def __init__ (self , rp_url , session , api_version , launch_id , project_name ,
36- log_batch_size = 20 , verify_ssl = True ):
40+ max_entry_number = 20 , verify_ssl = True ,
41+ max_payload_size = MAX_LOG_BATCH_PAYLOAD_SIZE ):
3742 """Initialize instance attributes.
3843
39- :param rp_url: Report portal URL
40- :param session: HTTP Session object
41- :param api_version: RP API version
42- :param launch_id: Parent launch UUID
43- :param project_name: RP project name
44- :param log_batch_size: The amount of log objects that need to be
45- gathered before processing
46- :param verify_ssl: Indicates that it is necessary to verify SSL
47- certificates within HTTP request
44+ :param rp_url: Report portal URL
45+ :param session: HTTP Session object
46+ :param api_version: RP API version
47+ :param launch_id: Parent launch UUID
48+ :param project_name: RP project name
49+ :param max_entry_number: The amount of log objects that need to be
50+ gathered before processing
51+ :param verify_ssl: Indicates that it is necessary to verify SSL
52+ certificates within HTTP request
53+ :param max_payload_size: maximum size in bytes of logs that can be
54+ processed in one batch
4855 """
4956 self ._lock = Lock ()
50- self ._logs_batch = []
57+ self ._batch = []
58+ self ._payload_size = helpers .TYPICAL_MULTIPART_FOOTER_LENGTH
5159 self ._worker = None
5260 self .api_version = api_version
5361 self .queue = queue .PriorityQueue ()
5462 self .launch_id = launch_id
55- self .log_batch_size = log_batch_size
63+ self .max_entry_number = max_entry_number
64+ self .max_payload_size = max_payload_size
5665 self .project_name = project_name
5766 self .rp_url = rp_url
5867 self .session = session
@@ -63,24 +72,31 @@ def __init__(self, rp_url, session, api_version, launch_id, project_name,
6372 .format (rp_url = rp_url , version = self .api_version ,
6473 project_name = self .project_name ))
6574
66- def _log_process (self , log_req ):
67- """Process the given log request.
68-
69- :param log_req: RPRequestLog object
70- """
71- self ._logs_batch .append (log_req )
72- if len (self ._logs_batch ) >= self .log_batch_size :
73- self ._send_batch ()
74-
7575 def _send_batch (self ):
7676 """Send existing batch logs to the worker."""
77- batch = RPLogBatch (self ._logs_batch )
77+ batch = RPLogBatch (self ._batch )
7878 http_request = HttpRequest (
7979 self .session .post , self ._log_endpoint , files = batch .payload ,
8080 verify_ssl = self .verify_ssl )
8181 batch .http_request = http_request
8282 self ._worker .send (batch )
83- self ._logs_batch .clear ()
83+ self ._batch = []
84+ self ._payload_size = helpers .TYPICAL_MULTIPART_FOOTER_LENGTH
85+
86+ def _log_process (self , log_req ):
87+ """Process the given log request.
88+
89+ :param log_req: RPRequestLog object
90+ """
91+ with self ._lock :
92+ rq_size = log_req .multipart_size
93+ if self ._payload_size + rq_size >= self .max_payload_size :
94+ if len (self ._batch ) > 0 :
95+ self ._send_batch ()
96+ self ._batch .append (log_req )
97+ self ._payload_size += rq_size
98+ if len (self ._batch ) >= self .max_entry_number :
99+ self ._send_batch ()
84100
85101 def log (self , time , message = None , level = None , attachment = None ,
86102 item_id = None ):
@@ -111,7 +127,7 @@ def stop(self):
111127 """Send last batches to the worker followed by the stop command."""
112128 if self ._worker :
113129 with self ._lock :
114- if self ._logs_batch :
130+ if self ._batch :
115131 self ._send_batch ()
116132 logger .debug ('Waiting for worker {0} to complete'
117133 'processing batches.' .format (self ._worker ))
0 commit comments