feat: redesign the log module (#190)

fix #183

Author: timerring

logs/burningLog/.gitkeep

@@ -1,7 +1,7 @@
 # kill the previous scanSegments process
 kill -9 $(ps aux | grep 'src.burn.scan' | grep -v grep | awk '{print $2}')
 # start the scanSegments process
-nohup python -m src.burn.scan > $BILIVE_PATH/logs/scanLog/scan-$(date +%Y%m%d-%H%M%S).log 2>&1 &
+python -m src.burn.scan
 # Check if the last command was successful
 if [ $? -eq 0 ]; then
     echo "success"

logs/mergeLog/.gitkeep

@@ -5,6 +5,7 @@
 from src.config import DanmakuFactory_PATH
 import os
 from src.utils.removeEmojis import *
+from src.log.logger import scan_log
 
 def get_resolution(in_video_path):
     """Return the resolution of video
@@ -23,10 +24,10 @@ def get_resolution(in_video_path):
             check=True
         )
         resolution = result.stdout.strip()
-        print("The video resolution is " + resolution, flush=True)
+        scan_log.info("The video resolution is " + resolution)
         return resolution
     except subprocess.CalledProcessError as e:
-        print(f"Error: {e.stderr}", flush=True)
+        scan_log.error(f"Error: {e.stderr}")
         return '1920x1080'
 
 def process_danmakus(in_xml_path, resolution):
@@ -72,8 +73,14 @@ def process_danmakus(in_xml_path, resolution):
             subtitle_font_size = '16'
             subtitle_margin_v = '60'
         # Convert danmakus to ass file
-        subprocess.run([DanmakuFactory_PATH, "-o", in_ass_path, "-i", in_xml_path, "--resolution", resolution, "--msgboxsize", boxsize, "--msgboxfontsize", boxfont, "-S", danmakufont, "--ignore-warnings"])
+        try:
+            result = subprocess.run(
+                [DanmakuFactory_PATH, "-o", in_ass_path, "-i", in_xml_path, "--resolution", resolution, "--msgboxsize", boxsize, "--msgboxfontsize", boxfont, "-S", danmakufont, "--ignore-warnings"],
+                check=True, capture_output=True, text=True)
+            scan_log.debug(f"DanmakuFactory output: {result.stdout}")
+        except subprocess.CalledProcessError as e:
+            scan_log.error(f"Error: {e.stderr}")
         # Remove emojis from ass danmakus (the ffmpeg does not support emojis)
         remove_emojis(in_ass_path)
-        print(f"The {in_ass_path} has been processed.", flush=True)
+        scan_log.info(f"The emojis of {in_ass_path} has been removed.")
         return subtitle_font_size, subtitle_margin_v

logs/scanLog/.gitkeep

@@ -1,7 +1,8 @@
 # Copyright (c) 2024 bilive.
 
 import subprocess
-from src.config import WHISPER_LOG_PATH, SRC_DIR
+from src.config import SRC_DIR
+from src.log.logger import scan_log
 import os
 
 # Generate the srt file via whisper model
@@ -10,5 +11,10 @@ def generate_subtitles(in_video_path):
     Args:
         in_video_path: str, the path of video
     """
-    with open(WHISPER_LOG_PATH, 'a') as wlog:
-        subprocess.run(['python', os.path.join(SRC_DIR, 'subtitle', 'generate.py'), in_video_path], stdout=wlog)
+    try:
+        subprocess.run(
+            ['python', os.path.join(SRC_DIR, 'subtitle', 'generate.py'), in_video_path],
+            stdout=subprocess.DEVNULL
+        )
+    except subprocess.CalledProcessError as e:
+        scan_log.error(f"Generate subtitles failed: {e.stderr}")

logs/uploadLog/.gitkeep

@@ -10,6 +10,7 @@
 from src.autoslice.slice_video import slice_video, inject_metadata, zhipu_glm_4v_plus_generate_title
 from src.autoslice.calculate_density import extract_dialogues, calculate_density, format_time
 from src.upload.extract_video_info import get_video_info
+from src.log.logger import scan_log
 import queue
 import threading
 import time
@@ -31,7 +32,7 @@ def check_file_size(file_path):
 
 def render_video_only(video_path):
     if not os.path.exists(video_path):
-        print(f"File {video_path} does not exist.")
+        scan_log.error(f"File {video_path} does not exist.")
         return
 
     original_video_path = str(video_path)
@@ -47,9 +48,9 @@ def render_video_only(video_path):
         # Process the danmakus to ass and remove emojis
         subtitle_font_size, subtitle_margin_v = process_danmakus(xml_path, video_resolution)
     except TypeError as e:
-        print(f"TypeError: {e} - Check the return value of process_danmakus")
+        scan_log.error(f"TypeError: {e} - Check the return value of process_danmakus")
     except FileNotFoundError as e:
-        print(f"FileNotFoundError: {e} - Check if the file exists")
+        scan_log.error(f"FileNotFoundError: {e} - Check if the file exists")
 
     # Generate the srt file via whisper model
     if GPU_EXIST:
@@ -58,7 +59,7 @@ def render_video_only(video_path):
 
     # Burn danmaku or subtitles into the videos 
     render_video(original_video_path, format_video_path, subtitle_font_size, subtitle_margin_v)
-    print("complete danamku burning and wait for uploading!", flush=True)
+    scan_log.info("Complete danamku burning and wait for uploading!")
 
     if AUTO_SLICE:
         if check_file_size(format_video_path) > MIN_VIDEO_SIZE:
@@ -67,7 +68,7 @@ def render_video_only(video_path):
             dialogues = extract_dialogues(ass_path)
             max_start_time, max_density = calculate_density(dialogues)
             formatted_time = format_time(max_start_time)
-            print(f"The 30-second window with the highest density starts at {formatted_time} seconds with {max_density} danmakus.", flush=True)
+            scan_log.info(f"The 30-second window with the highest density starts at {formatted_time} seconds with {max_density} danmakus.")
             slice_video(format_video_path, max_start_time, slice_video_path)
             glm_title = zhipu_glm_4v_plus_generate_title(slice_video_path, artist)
             slice_video_flv_path = slice_video_path[:-4] + '.flv'
@@ -86,7 +87,7 @@ def render_video_only(video_path):
     with open(f"{SRC_DIR}/upload/uploadVideoQueue.txt", "a") as file:
         file.write(f"{format_video_path}\n")
         if AUTO_SLICE:
-            print("complete slice video and wait for uploading!", flush=True)
+            scan_log.info("Complete slice video and wait for uploading!")
             slice_video_path = format_video_path[:-4] + '_slice.mp4'
             slice_video_flv_path = slice_video_path[:-4] + '.flv'
             file.write(f"{slice_video_flv_path}\n")
@@ -106,7 +107,7 @@ def monitor_queue(self):
                 try:
                     render_video_only(video_path)
                 except Exception as e:
-                    print(f"Error processing video {video_path}: {e}", flush=True)
+                    scan_log.error(f"Error processing video {video_path}: {e}")
             else:
                 time.sleep(1)
 

scan.sh

@@ -3,11 +3,12 @@
 import argparse
 import os
 import subprocess
-from src.config import GPU_EXIST, SRC_DIR, MERGE_LOG_PATH
+from src.config import GPU_EXIST, SRC_DIR
 from src.burn.generate_danmakus import get_resolution, process_danmakus
 from src.burn.generate_subtitles import generate_subtitles
 from src.burn.render_video import render_video
 from src.upload.extract_video_info import get_video_info
+from src.log.logger import scan_log
 
 def normalize_video_path(filepath):
     """Normalize the video path to upload
@@ -28,9 +29,15 @@ def merge_videos(in_final_video, title, artist, date, merge_list):
     'ffmpeg', '-f', 'concat', '-safe', '0', '-i', merge_list, '-metadata', f'title={title}', '-metadata', f'artist={artist}', '-metadata', f'date={date}', '-use_wallclock_as_timestamps', '1', 
     '-c', 'copy', in_final_video
     ]
-    with open(MERGE_LOG_PATH, 'a') as mlog:
-        subprocess.run(merge_command, stdout=mlog, stderr=subprocess.STDOUT)
-    subprocess.run(['rm', merge_list])
+    try:
+        scan_log.info("Begin merging videos...")
+        result = subprocess.run(merge_command, check=True, capture_output=True, text=True)
+        scan_log.debug(f"FFmpeg output: {result.stdout}")
+        if result.stderr:
+            scan_log.debug(f"FFmpeg debug: {result.stderr}")
+    except subprocess.CalledProcessError as e:
+        scan_log.error(f"Error: {e.stderr}")
+    subprocess.run(['rm', merge_list], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
 
 def render_and_merge(video_path_list):
     title = ''
@@ -48,8 +55,8 @@ def render_and_merge(video_path_list):
             if output_video_path == '':
                 title, artist, date = get_video_info(stripped_line)
                 output_video_path = normalize_video_path(stripped_line)
-                print("The output video is " + output_video_path)
-                subprocess.run(['mkdir', tmp])
+                scan_log.info("The output video is " + output_video_path)
+                subprocess.run(['mkdir', tmp], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
 
             video_to_be_merged = tmp + video_name
             original_video_path = stripped_line
@@ -70,7 +77,7 @@ def render_and_merge(video_path_list):
                 open(merge_list, 'w').close() 
             with open(merge_list, 'a') as f:
                 f.write(f"file '{video_to_be_merged}'\n")
-            print("complete danamku burning and wait for uploading!", flush=True)
+            scan_log.info("Complete danamku burning and wait for uploading!")
 
         for remove_path in [original_video_path, xml_path, ass_path, srt_path, jsonl_path]:
             if os.path.exists(remove_path):

src/burn/generate_danmakus.py

@@ -1,7 +1,8 @@
 # Copyright (c) 2024 bilive.
 
 import subprocess
-from src.config import GPU_EXIST, BURN_LOG_PATH
+from src.config import GPU_EXIST
+from src.log.logger import scan_log
 import os
 
 def render_video(in_video_path, out_video_path, in_subtitle_font_size, in_subtitle_margin_v):
@@ -16,29 +17,45 @@ def render_video(in_video_path, out_video_path, in_subtitle_font_size, in_subtit
     if GPU_EXIST:
         in_srt_path = in_video_path[:-4] + '.srt'
         if os.path.isfile(in_ass_path):
-            print("wisper danmaku")
+            scan_log.info("Current Mode: GPU with danmaku")
             command = [
                 'ffmpeg', '-y', '-hwaccel', 'cuda', '-c:v', 'h264_cuvid', '-i', in_video_path,
                 '-c:v', 'h264_nvenc', '-vf', f"subtitles={in_srt_path}:force_style='Fontsize={in_subtitle_font_size},MarginV={in_subtitle_margin_v}',subtitles={in_ass_path}", out_video_path
             ]
-            with open(BURN_LOG_PATH, 'a') as blog:
-                subprocess.run(command, stdout=blog, stderr=subprocess.STDOUT)
+            try:
+                result = subprocess.run(command, check=True, capture_output=True, text=True)
+                scan_log.debug(f"FFmpeg output: {result.stdout}")
+                if result.stderr:
+                    scan_log.debug(f"FFmpeg debug: {result.stderr}")
+            except subprocess.CalledProcessError as e:
+                scan_log.error(f"Error: {e.stderr}")
+            
         else:
-            print("wisper no danmaku")
+            scan_log.info("Current Mode: GPU without danmaku")
             command_no_danmaku = [
                 'ffmpeg', '-y', '-hwaccel', 'cuda', '-c:v', 'h264_cuvid', '-i', in_video_path,
                 '-c:v', 'h264_nvenc', '-vf', f"subtitles={in_srt_path}:force_style='Fontsize={in_subtitle_font_size},MarginV={in_subtitle_margin_v}'", out_video_path
             ]
-            with open(BURN_LOG_PATH, 'a') as blog:
-                subprocess.run(command_no_danmaku, stdout=blog, stderr=subprocess.STDOUT)
+            try:
+                result = subprocess.run(command_no_danmaku, check=True, capture_output=True, text=True)
+                scan_log.debug(f"FFmpeg output: {result.stdout}")
+                if result.stderr:
+                    scan_log.debug(f"FFmpeg debug: {result.stderr}")
+            except subprocess.CalledProcessError as e:
+                scan_log.error(f"Error: {e.stderr}")
     else:
         if os.path.isfile(in_ass_path):
-            print("no gpu danmaku")
+            scan_log.info("Current Mode: CPU with danmaku")
             command_without_gpu = [
                 'ffmpeg', '-y', '-i', in_video_path, '-vf', f'ass={in_ass_path}', '-preset', 'ultrafast', out_video_path
             ]
-            with open(BURN_LOG_PATH, 'a') as blog:
-                subprocess.run(command_without_gpu, stdout=blog, stderr=subprocess.STDOUT)
+            try:
+                result = subprocess.run(command_without_gpu, check=True, capture_output=True, text=True)
+                scan_log.debug(f"FFmpeg output: {result.stdout}")
+                if result.stderr:
+                    scan_log.debug(f"FFmpeg debug: {result.stderr}")
+            except subprocess.CalledProcessError as e:
+                scan_log.error(f"Error: {e.stderr}")
         else:
-            print("no gpu no danmaku")
-            subprocess.run(['mv', in_video_path, out_video_path])
+            scan_log.info("Current Mode: CPU without danmaku")
+            subprocess.run(['mv', in_video_path, out_video_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)