feat: introduce auto slice video (#238)

fix #233
Introduce the auto-slice-video package.
Update the zhipu model to glm-4v-plus-0111.

Author: timerring

.dockerignore

@@ -6,6 +6,7 @@ pnpm-lock.yaml
 **/__pycache__
 test/*
 Videos/*
+!Videos/.gitkeep
 settings-production.toml
 startRecord-production.sh
 src/utils/cookies.json

.gitignore

@@ -350,6 +350,7 @@ $RECYCLE.BIN/
 # End of https://www.toptal.com/developers/gitignore/api/intellij+all,python,pycharm+all,macos,windows
 # Data&test
 Videos/
+!Videos/.gitkeep
 test/
 .cache
 .vscode/

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "src/danmaku/DanmakuConvert"]
 	path = src/danmaku/DanmakuConvert
 	url = https://github.com/timerring/DanmakuConvert.git
+[submodule "src/autoslice/auto_slice_video"]
+	path = src/autoslice/auto_slice_video
+	url = [email protected]:timerring/auto-slice-video.git

src/autoslice/__init__.py

@@ -2,4 +2,7 @@
 
 import sys
 import os
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from .auto_slice_video.autosv import slice_video_by_danmaku
+
+__all__ = ["slice_video_by_danmaku"]

src/autoslice/auto_slice_video

@@ -0,0 +1,23 @@
+# Copyright (c) 2024 bilive.
+
+import subprocess
+from src.log.logger import scan_log
+
+# https://stackoverflow.com/questions/64849478/cant-insert-stream-metadata-into-mp4
+def inject_metadata(video_path, generate_title, output_path):
+    """Slice the video using ffmpeg."""
+    command = [
+        'ffmpeg',
+        '-i', video_path,
+        '-metadata:g', f'generate={generate_title}',
+        '-c:v', 'copy',
+        '-c:a', 'copy',
+        output_path
+    ]
+    try:
+        result = subprocess.run(command, check=True, capture_output=True, text=True)
+        scan_log.debug(f"FFmpeg output: {result.stdout}")
+        if result.stderr:
+            scan_log.debug(f"FFmpeg debug: {result.stderr}")
+    except subprocess.CalledProcessError as e:
+        scan_log.error(f"Error: {e.stderr}")

src/autoslice/calculate_density.py

@@ -0,0 +1,35 @@
+# Copyright (c) 2024 bilive.
+
+import base64
+from src.config import Your_API_KEY
+from zhipuai import ZhipuAI
+from src.log.logger import scan_log
+
+def zhipu_glm_4v_plus_generate_title(video_path, artist):
+    with open(video_path, 'rb') as video_file:
+        video_base = base64.b64encode(video_file.read()).decode('utf-8')
+
+    client = ZhipuAI(api_key=Your_API_KEY)
+    response = client.chat.completions.create(
+        model="glm-4v-plus-0111",
+        messages=[
+        {
+            "role": "user",
+            "content": [
+            {
+                "type": "video_url",
+                "video_url": {
+                    "url" : video_base
+                }
+            },
+            {
+                "type": "text",
+                "text": f"视频是{artist}的直播的切片，请根据该视频中的内容及弹幕信息，为这段视频起一个调皮并且吸引眼球的标题，注意标题中如果有“主播”请替换成{artist}"
+            }
+            ]
+        }
+        ]
+    )
+    scan_log.info(f"Prompt: 视频是{artist}的直播的切片，请根据该视频中的内容及弹幕信息，为这段视频起一个调皮并且吸引眼球的标题，注意标题中如果有“主播”请替换成{artist}")
+    scan_log.info(f"生成的切片标题为: {response.choices[0].message.content}")
+    return response.choices[0].message.content.replace("《", "").replace("》", "")

src/autoslice/inject_metadata.py

@@ -3,12 +3,13 @@
 import argparse
 import os
 import subprocess
-from src.config import GPU_EXIST, SRC_DIR, MODEL_TYPE, AUTO_SLICE, SLICE_DURATION, MIN_VIDEO_SIZE
+from src.config import GPU_EXIST, SRC_DIR, MODEL_TYPE, AUTO_SLICE, SLICE_DURATION, MIN_VIDEO_SIZE, SLICE_NUM, SLICE_OVERLAP, SLICE_STEP
 from src.danmaku.generate_danmakus import get_resolution, process_danmakus
 from src.subtitle.generate_subtitles import generate_subtitles
 from src.burn.render_command import render_command
-from src.autoslice.slice_video import slice_video, inject_metadata, zhipu_glm_4v_plus_generate_title
-from src.autoslice.calculate_density import extract_dialogues, calculate_density, format_time
+from autoslice import slice_video_by_danmaku
+from src.autoslice.inject_metadata import inject_metadata
+from src.autoslice.zhipu_sdk import zhipu_glm_4v_plus_generate_title
 from src.upload.extract_video_info import get_video_info
 from src.log.logger import scan_log
 
@@ -61,16 +62,18 @@ def render_video(video_path):
     if AUTO_SLICE:
         if check_file_size(format_video_path) > MIN_VIDEO_SIZE:
             title, artist, date = get_video_info(format_video_path)
-            slice_video_path = format_video_path[:-4] + '_slice.mp4'
-            dialogues = extract_dialogues(ass_path)
-            max_start_time, max_density = calculate_density(dialogues)
-            formatted_time = format_time(max_start_time)
-            scan_log.info(f"The 30-second window with the highest density starts at {formatted_time} seconds with {max_density} danmakus.")
-            slice_video(format_video_path, max_start_time, slice_video_path)
-            glm_title = zhipu_glm_4v_plus_generate_title(slice_video_path, artist)
-            slice_video_flv_path = slice_video_path[:-4] + '.flv'
-            inject_metadata(slice_video_path, glm_title, slice_video_flv_path)
-            os.remove(slice_video_path)
+            slices_path = slice_video_by_danmaku(ass_path, format_video_path, SLICE_DURATION, SLICE_NUM, SLICE_OVERLAP, SLICE_STEP)
+            for slice_path in slices_path:
+                try:
+                    glm_title = zhipu_glm_4v_plus_generate_title(slice_path, artist)
+                    slice_video_flv_path = slice_path[:-4] + '.flv'
+                    inject_metadata(slice_path, glm_title, slice_video_flv_path)
+                    os.remove(slice_path)
+                    with open(f"{SRC_DIR}/upload/uploadVideoQueue.txt", "a") as file:
+                        scan_log.info(f"Complete {slice_video_flv_path} and wait for uploading!")
+                        file.write(f"{slice_video_flv_path}\n")
+                except Exception as e:
+                    scan_log.error(f"Error in {slice_path}: {e}")
 
     # Delete relative files
     for remove_path in [original_video_path, xml_path, ass_path, srt_path, jsonl_path]:
@@ -82,9 +85,4 @@ def render_video(video_path):
     # os.rename(original_video_path, test_path)
 
     with open(f"{SRC_DIR}/upload/uploadVideoQueue.txt", "a") as file:
-        file.write(f"{format_video_path}\n")
-        if AUTO_SLICE:
-            scan_log.info("Complete slice video and wait for uploading!")
-            slice_video_path = format_video_path[:-4] + '_slice.mp4'
-            slice_video_flv_path = slice_video_path[:-4] + '.flv'
-            file.write(f"{slice_video_flv_path}\n")
+        file.write(f"{format_video_path}\n")