removed unnecessary logs

Author: femalves

scan_explorer_service/manifest_factory.py

@@ -22,15 +22,13 @@ def create_manifest(self, item: Union[Article, Collection]):
         for range in self.create_range(item):
             manifest.add_range(range)
 
-        current_app.logger.debug(f"Created manifest {manifest}")
         return manifest
 
     def create_sequence(self, item: Union[Article, Collection]):
         sequence: Sequence = self.sequence()
         for page in item.pages:
             sequence.add_canvas(self.get_or_create_canvas(page)) 
 
-        current_app.logger.debug(f"Sequence {sequence}")
         return sequence
 
     def create_range(self, item: Union[Article, Collection]):
@@ -41,13 +39,11 @@ def create_range(self, item: Union[Article, Collection]):
         for page in item.pages:
             range.add_canvas(self.get_or_create_canvas(page))
 
-        current_app.logger.debug(f"Range {[range]}")
         return [range]
 
     def get_canvas_dict(self) -> Dict[str, Canvas]:
         if not hasattr(self, 'canvas_dict'):
             self.canvas_dict = {}
-        current_app.logger.debug(f"Canvas dict {self.canvas_dict}")
         return self.canvas_dict
 
     def get_or_create_canvas(self, page: Page):
@@ -68,7 +64,6 @@ def get_or_create_canvas(self, page: Page):
         annotation.on = canvas.id
         canvas.add_annotation(annotation)
         canvas_dict[page.id] = canvas
-        current_app.logger.debug(f"Canvas {canvas}")
         return canvas
 
     def create_image_annotation(self, page: Page):
@@ -82,7 +77,6 @@ def create_image_annotation(self, page: Page):
         image.format = page.format
         image.height = page.height
         image.width = page.width
-        current_app.logger.debug(f"Annotation {annotation}")
         return annotation
 
     def add_search_service(self, manifest: Manifest, search_url: str):

scan_explorer_service/models.py

@@ -151,17 +151,14 @@ def __init__(self, **kwargs):
     @property
     def image_url(self):
         image_api_url = url_for_proxy('proxy.image_proxy', path=self.image_path)
-        current_app.logger.debug(f'image api url: {image_api_url}')
         return image_api_url
 
     @property
     def image_path(self):
         separator = current_app.config.get('IMAGE_API_SLASH_SUB', '%2F')
         image_path = separator.join(self.image_path_basic[0]) 
-        current_app.logger.debug(f'color type: {self.color_type}')
         if self.color_type != PageColor.BW:
             image_path += '.tif'
-        current_app.logger.debug(f'image path: {image_path}')
         return image_path 
 
     @property
@@ -170,15 +167,13 @@ def image_path_basic(self):
         image_path = [self.collection.type, self.collection.journal, self.collection.volume]
         image_path = [item.replace('.', '_') for item in image_path]
         image_path = ['bitmaps'] + image_path + ['600', self.name]
-        current_app.logger.debug(f'image path basic: {image_path}')
         if self.color_type != PageColor.BW:
             image_format = '.tif'
         return image_path, image_format
 
     @property
     def thumbnail_url(self):
         url = f'{self.image_url}/square/480,480/0/{self.image_color_quality}.jpg'
-        current_app.logger.debug('thumbnail url: ' + url)
         return url
 
     @property

scan_explorer_service/open_search.py

@@ -74,7 +74,6 @@ def append_highlight(query: dict):
 
 def es_search(query: dict) -> Iterator[str]:
     es = opensearchpy.OpenSearch(current_app.config.get('OPEN_SEARCH_URL'))
-    current_app.logger.debug(f"Query search: {query}")
     resp = es.search(index=current_app.config.get(
         'OPEN_SEARCH_INDEX'), body=query)
     return resp
@@ -150,9 +149,6 @@ def page_ocr_os_search(collection_id: str, page_number:int):
 def aggregate_search(qs: str, aggregate_field, page, limit, sort):
     qs = qs.replace("&", "+")
     query = create_query_string_query(qs)
-    current_app.logger.debug(f"query: {query}")
     query = append_aggregate(query, aggregate_field, page, limit, sort)
-    current_app.logger.debug(f"query with aggregate: {query}")
     es_result = es_search(query)
-    current_app.logger.debug(f"es_result: {es_result}")
     return es_result

scan_explorer_service/tests/test_proxy.py

@@ -167,7 +167,6 @@ def test_fetch_object(self, mock_read_object_s3):
 
     @patch('scan_explorer_service.views.image_proxy.fetch_object')
     def test_pdf_save_success_article(self, mock_fetch_object):
-        # mock_read_object_s3.return_value = b'my_image_name'
         mock_fetch_object.return_value = b'my_image_name'
 
         data = {

scan_explorer_service/utils/db_utils.py

@@ -80,22 +80,18 @@ def page_overwrite(session, page):
 def article_thumbnail(session, id):
     page = session.query(Page).join(Article, Page.articles).filter(
                 Article.id == id).order_by(Page.volume_running_page_num.asc()).first()
-    current_app.logger.debug(f'article thumbnail {page}')
     return page.thumbnail_url
 
 def collection_thumbnail(session, id):
     page = session.query(Page).filter(Page.collection_id == id).order_by(
         Page.volume_running_page_num.asc()).first()
-    current_app.logger.debug(f'collection thumbnail {page.thumbnail_url}')
     return page.thumbnail_url
 
 def page_thumbnail(session, id):
     page = session.query(Page).filter(Page.id == id).one()
-    current_app.logger.debug(f'page thumbnail {page.thumbnail_url}')
     return page.thumbnail_url
 
 def item_thumbnail(session, id, type):
-    current_app.logger.debug(f'Getting item thumbnail: id {id} type {type}')
     if type == 'page':
         return page_thumbnail(session, id)
     elif type == 'article':

scan_explorer_service/utils/s3_utils.py

@@ -24,20 +24,16 @@ def write_object_s3(self, file_bytes, object_name):
         try:
             response = self.bucket.put_object(Body=file_bytes, Key=object_name)
         except (ClientError, ParamValidationError) as e:
-            current_app.logger.info.exception(e)
+            current_app.logger.exception(f"Error writing object {object_name}: {str(e)}")
             raise e
         return response.e_tag
 
     def read_object_s3(self, object_name):
         try:
-            current_app.logger.debug(f"Attempting to download object: {object_name}")
             with io.BytesIO() as s3_obj:
                 self.bucket.download_fileobj(object_name, s3_obj)
-                current_app.logger.debug(f"Object downloaded successfully: {object_name}")
                 s3_obj.seek(0)
                 s3_file = s3_obj.read()
-                current_app.logger.debug(f"Read {len(s3_file)} bytes from object: {object_name}")
-                current_app.logger.debug(f"First 100 bytes of file content: {s3_file[:100]}")
                 return s3_file
         except Exception as e:
             current_app.logger.exception(f"Unexpected error reading object {object_name}: {str(e)}")

scan_explorer_service/utils/search_utils.py

@@ -55,44 +55,34 @@ class OrderOptions(str, enum.Enum):
 
 def parse_query_args(args):
     qs = re.sub(':\s*', ':', args.get('q', '', str))
-    current_app.logger.debug(f'qs {qs}')
 
     qs, qs_dict = parse_query_string(qs)
 
-    current_app.logger.debug(f'qs {qs}, qs_dict {qs_dict}')
 
 
     page = args.get('page', 1, int)
     limit = args.get('limit', 10, int)
     sort_raw = args.get('sort')
     sort = parse_sorting_option(sort_raw)
-    current_app.logger.debug(f'qs {qs}, qs_dict {qs_dict}, sort {sort}')
     return qs, qs_dict, page, limit, sort
 
 def parse_query_string(qs):
     qs_to_split = qs.replace('[', '"[').replace(']',']"')
-    current_app.logger.debug(f'qs to split {qs_to_split}')
     qs_arr = [q for q in shlex.split(qs_to_split) if ':' in q]
-    current_app.logger.debug(f'qs arr {qs_arr}')
     qs_dict = {}
     qs_only_free = qs
-    current_app.logger.debug(f'qs only free {qs_only_free}')
 
     for kv in qs_arr:
         kv_arr = kv.split(':', maxsplit=1)
-        current_app.logger.debug(f'kv_arr {kv_arr}')
         #Remove all parameter from the original search to be able to handle the free search
         qs_only_free = qs_only_free.replace(kv, "")
-        current_app.logger.debug(f'qs_only_free {qs_only_free}')
 
         if len(kv_arr) == 2:
             qs_dict[kv_arr[0].lower()] = kv_arr[1].strip()
             #If the option have quotes we remove them from the free. Previous removal would than have failed
             alt_kv = kv_arr[0] + ':"' + kv_arr[1] + '"'
             qs_only_free = qs_only_free.replace(alt_kv, '')
-            current_app.logger.debug(f'kv_arr == 2. alt_kv {alt_kv}, qs_only_free {qs_only_free}')
 
-    current_app.logger.debug(f'qs dict {qs_dict}')    
     check_query(qs_dict)
     #Adds a () around each free search to force OS to look for each individual entry against all default fields
     for parameter in re.split('\s+', qs_only_free):
@@ -106,7 +96,6 @@ def parse_query_string(qs):
         # To ensure only the strings after the colon are replaced and no partial replacements are made 
         insensitive_replace = re.compile(r'(?<=:)\b' + re.escape(qs_dict[key]) + r'\b', re.IGNORECASE)
         qs = insensitive_replace.sub(qs_dict[key], qs)
-    current_app.logger.debug(f'qs: {qs} and qs dict: {qs_dict}')    
     return qs, qs_dict
 
 def parse_sorting_option(sort_input: str):
@@ -146,14 +135,12 @@ def check_page_color(qs_dict: dict):
         page_color = qs_dict[SearchOptions.PageColor.value]
         valid_types = [p.name for p in PageColor]
         if page_color in valid_types:
-            current_app.logger.debug("Page color {page_color} is valid")
             return
 
         # Check lowercased and updated to cased
         for p in PageColor:
             if page_color.replace('"','').lower() == p.name.lower():
                 qs_dict[SearchOptions.PageColor.value] = p.name
-                current_app.logger.debug("Page color {qs_dict[SearchOptions.PageColor.value]} changed to {p.name}")
                 return
         raise Exception("%s is not a valid page color, %s is possible choices"% (page_color, str(valid_types)))
 

scan_explorer_service/views/image_proxy.py

@@ -11,8 +11,6 @@
 from scan_explorer_service.utils.utils import url_for_proxy
 import re
 import io 
-import cProfile
-import pstats
 import sys
 
 bp_proxy = Blueprint('proxy', __name__, url_prefix='/image')
@@ -29,14 +27,8 @@ def image_proxy(path):
     req_headers['X-Forwarded-Path'] = current_app.config.get('PROXY_PREFIX').rstrip('/') + '/image'
 
     encoded_url = re.sub(r"[+&]", "%2B", req_url)
-    
-    current_app.logger.debug(f'req_url: {encoded_url}, params: {request.args}, headers: {req_headers}, data: {request.form}')
-
     r = requests.request(request.method, encoded_url, params=request.args, stream=True,
-                         headers=req_headers, allow_redirects=False, data=request.form)
-    
-    current_app.logger.debug(f"Response status code: {r.status_code}")
-    
+                         headers=req_headers, allow_redirects=False, data=request.form)      
     excluded_headers = ['content-encoding','content-length', 'transfer-encoding', 'connection']
     headers = [(name, value) for (name, value) in r.headers.items() if name.lower() not in excluded_headers]
 
@@ -53,23 +45,15 @@ def image_proxy_thumbnail():
     """Helper to generate the correct url for a thumbnail given an ID and type"""
     try:
         id = request.args.get('id').replace(" ", "+")
-        current_app.logger.debug(f'id {id}')
         type = request.args.get('type')
-        current_app.logger.debug(f'type {type}')
 
         with current_app.session_scope() as session:
-            thumbnail_path = item_thumbnail(session, id, type)
-            
-            current_app.logger.debug(f'thumbnail path {thumbnail_path}')
-            
+            thumbnail_path = item_thumbnail(session, id, type)              
             path = urlparse.urlparse(thumbnail_path).path
-            current_app.logger.debug(f'path {path}')
 
             remove = urlparse.urlparse(url_for_proxy('proxy.image_proxy', path='')).path
-            current_app.logger.debug(f'remove {remove}')
 
             path = path.replace(remove, '')
-            current_app.logger.debug(f'replace {path}')
 
             return image_proxy(path)
     except Exception as e:
@@ -81,10 +65,8 @@ def get_item(session, id):
                 session.query(Article).filter(Article.id == id).one_or_none()
                 or session.query(Collection).filter(Collection.id == id).one_or_none())
     if not item: 
-        current_app.logger.debug(f'Item with id {id} not found')
         raise Exception("ID: " + id + " not found")
 
-    current_app.logger.debug(f'Item retrieved successfully {item}')
     return item 
 
 
@@ -98,7 +80,6 @@ def get_pages(item, session, page_start, page_end, page_limit):
         query = session.query(Page).filter(Page.collection_id == item.id, 
             Page.volume_running_page_num >= page_start, 
             Page.volume_running_page_num <= page_end).order_by(Page.volume_running_page_num).limit(page_limit)
-    current_app.logger.info(f"Got pages {page_start}-{page_end}: {query}") 
     return query 
 
 
@@ -112,7 +93,6 @@ def fetch_images(session, item, page_start, page_end, page_limit, memory_limit):
         n_pages += 1
 
         current_app.logger.debug(f"Generating image for page: {n_pages}") 
-        current_app.logger.debug(f'Id: {page.id}, Volume_page: {page.volume_running_page_num}, memory: {memory_sum}')
         if n_pages > page_limit:
             break
         if memory_sum > memory_limit:
@@ -122,37 +102,28 @@ def fetch_images(session, item, page_start, page_end, page_limit, memory_limit):
         object_name = '/'.join(image_path)
         object_name += format
 
-        current_app.logger.debug(f"Image path: {object_name}")
         im_data = fetch_object(object_name, 'AWS_BUCKET_NAME_IMAGE')
         memory_sum += sys.getsizeof(im_data)
 
         yield im_data
 
 
 def fetch_object(object_name, bucket_name):
-    current_app.logger.debug(f"Using bucket: {bucket_name}")
     file_content = S3Provider(current_app.config, bucket_name).read_object_s3(object_name)
-    current_app.logger.debug(f"File content type: {type(file_content)}, length: {len(file_content) if file_content else 'None'}")
     if not file_content:
         current_app.logger.error(f"Failed to fetch content for {object_name}. File might be empty.")
         raise ValueError(f"File content is empty for {object_name}")
-    current_app.logger.debug(f"Successfully fetched object from S3 bucket: {object_name}")
 
     return file_content
 
 
 def fetch_article(item, memory_limit):
     try:
-        current_app.logger.debug(f"Item is an article: {item.id}")
-
         object_name = f'{item.id}.pdf'.lower()
-        current_app.logger.debug(f"object name: {object_name}")
 
         full_path = f'pdfs/{object_name}'
-        current_app.logger.debug(f"full path: {full_path}")
 
         file_content = fetch_object(full_path, 'AWS_BUCKET_NAME_PDF')
-        current_app.logger.debug(f"File content type in fetch_article: {type(file_content)}, length: {len(file_content) if file_content else 'None'}")
 
         if len(file_content) > memory_limit:
             current_app.logger.error(f"Memory limit reached: {len(file_content)} > {memory_limit}") 
@@ -173,8 +144,6 @@ def fetch_article(item, memory_limit):
 def generate_pdf(item, session, page_start, page_end, page_limit, memory_limit): 
     if isinstance(item, Article):
         response = fetch_article(item, memory_limit)
-        current_app.logger.debug(f"Item is an article")
-        current_app.logger.debug(f"response fetch article: {response}")
         if response:
             return response
         else:
@@ -189,38 +158,18 @@ def generate_pdf(item, session, page_start, page_end, page_limit, memory_limit):
 def pdf_save():
     """Generate a PDF from pages"""
     try:
-        profiler = cProfile.Profile()
-        profiler.enable()
-        
-
         id = request.args.get('id')
         page_start = request.args.get('page_start', 1, int)
         page_end = request.args.get('page_end', math.inf, int)
         memory_limit = current_app.config.get("IMAGE_PDF_MEMORY_LIMIT")
         page_limit = current_app.config.get("IMAGE_PDF_PAGE_LIMIT")
 
-        current_app.logger.debug(f"pdf ID: {id}, page_start: {page_start}, page_end: {page_end}, memory_limit: {memory_limit}, page_limit: {page_limit}")
-
         with current_app.session_scope() as session:
 
             item = get_item(session, id) 
             current_app.logger.debug(f"Item retrieved successfully: {item.id}")
 
             response = generate_pdf(item, session, page_start, page_end, page_limit, memory_limit)
-            current_app.logger.debug(f"Response pdf save: {response}")
-
-            profiler.disable()
-            
-            # Log the profiling information
-            log_buffer = io.StringIO()
-            profiler_stats = pstats.Stats(profiler, stream=log_buffer)
-            profiler_stats.strip_dirs().sort_stats('cumulative', 'calls').print_stats(20)
- 
-            formatted_stats = log_buffer.getvalue().splitlines()
-
-            current_app.logger.debug(f'==================Profiling information========================: \n')
-            for line in formatted_stats:
-                current_app.logger.debug(line)
 
             return response 
     except Exception as e: