Merged branch test into main

Author: andrey18106

.pylintrc

@@ -69,7 +69,8 @@ disable=W0703,
         R0401,
         W0611,
         C0415,
-        W0602
+        W0602,
+        R0913
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

CHANGELOG.md

@@ -2,6 +2,19 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.1.5 - 2021-10-12]
+
+### Added
+
+- Added some action loaders on Details page
+- Added links to the list of target directories
+
+### Fixed
+
+- Fixed video processing broken by previous update.
+- Fixed pip version parsing, install now works with last pip.
+- Fixed processing files from other NC instances and remote shares.
+
 ## [0.1.4 - 2021-10-09]
 
 ### Fixed

appinfo/info.xml

@@ -17,7 +17,7 @@ This app allows to find duplicate or similar 📸📹 photos and videos
 Quick start guide and further information in our [Wiki](https://github.com/andrey18106/mediadc/wiki).
 	]]>
 	</description>
-	<version>0.1.4</version>
+	<version>0.1.5</version>
 	<licence>agpl</licence>
 	<author mail="[email protected]" homepage="https://github.com/andrey18106">Andrey Borysenko</author>
 	<author mail="[email protected]" homepage="https://github.com/bigcat88">Alexander Piskun</author>

js/mediadc-main.js

@@ -87,10 +87,12 @@ def occ_call(occ_task, *params, decode: bool = True) -> [bool, Union[str, bytes]
     success, result = php_call(OCC, '--no-warnings', occ_task, *params, decode=decode)
     if not success:
         return False, result
-    clear_result = re.sub(r'.*app.*require.*upgrade.*\n?', '', result, flags=re.IGNORECASE)
-    clear_result = re.sub(r'.*occ.*upgrade.*command.*\n?', '', clear_result, flags=re.IGNORECASE)
-    clear_result = clear_result.strip('\n')
-    return True, clear_result
+    if decode:
+        clear_result = re.sub(r'.*app.*require.*upgrade.*\n?', '', result, flags=re.IGNORECASE)
+        clear_result = re.sub(r'.*occ.*upgrade.*command.*\n?', '', clear_result, flags=re.IGNORECASE)
+        clear_result = clear_result.strip('\n')
+        return True, clear_result
+    return True, result
 
 
 def get_cloud_config_value(value_name: str) -> [bool, str]:

js/mediadc-main.js.map

@@ -120,10 +120,10 @@ def process_video_hash(algo: str, hash_size: int, file_info: dict, data_dir: str
             full_path = get_file_full_path(data_dir, file_info['storage'], file_info['path'])
             if not full_path:
                 break
-            video_info = ffprobe_get_video_info(full_path)
+            video_info = ffprobe_get_video_info(full_path, None)
             if not video_info:
                 break
-            if not do_hash_video(algo, hash_size, video_info, file_info, full_path):
+            if not do_hash_video(algo, hash_size, video_info, file_info, full_path, None):
                 raise InvalidVideo
             return
         if file_info['size'] > remote_filesize_limit:
@@ -133,10 +133,10 @@ def process_video_hash(algo: str, hash_size: int, file_info: dict, data_dir: str
         data = request_file_from_php(file_info)
         if len(data) == 0:
             return
-        video_info = ffprobe_get_video_info(data)
+        video_info = ffprobe_get_video_info(None, data)
         if not video_info.get('fast_start', False):
             raise InvalidVideo
-        if not do_hash_video(algo, hash_size, video_info, file_info, data):
+        if not do_hash_video(algo, hash_size, video_info, file_info, None, data):
             raise InvalidVideo
     except Exception as exception_info:
         store_err_video_hash(file_info['fileid'], video_info.get('duration', 0),
@@ -146,15 +146,16 @@ def process_video_hash(algo: str, hash_size: int, file_info: dict, data_dir: str
             print(f"Exception({exception_name}): `{file_info['path']}`:\n`{str(traceback.format_exc())}`")
 
 
-def do_hash_video(algo: str, hash_size: int, video_info: dict, file_info: dict, path_or_data) -> bool:
+def do_hash_video(algo: str, hash_size: int, video_info: dict, file_info: dict, path, data) -> bool:
+    """Accepts path(bytes/str) or data for processing in memory."""
     if video_info['duration'] < MinVideoDuration_ms:
         return False
-    first_timestamp = get_first_timestamp(video_info, path_or_data)
+    first_timestamp = get_first_timestamp(video_info, path, data)
     if first_timestamp == -1:
         return False
     frames_timestamps = build_times_for_hashes(video_info['duration'], first_timestamp)
     ex = ()
-    res = get_frames(frames_timestamps, path_or_data, *ex)
+    res = get_frames(frames_timestamps, path, data, *ex)
     if not res[0]:
         return False
     if any(len(x) == 0 for x in res[1:]):
@@ -192,19 +193,22 @@ def get_max_first_frame_time(duration_ms) -> int:
     return max_timestamp
 
 
-def get_first_timestamp(video_info: dict, data_or_filepath) -> int:
+def get_first_timestamp(video_info: dict, path, data) -> int:
+    """Accepts path(bytes/str) or data for processing in memory."""
     max_timestamp = get_max_first_frame_time(video_info['duration'])
     ffmpeg_input_params = ['-hide_banner', '-loglevel', 'fatal', '-an', '-sn', '-dn', '-to', f'{max_timestamp}ms']
-    if isinstance(data_or_filepath, str):
-        result, err = stub_call_ff('ffmpeg', *ffmpeg_input_params, '-i', data_or_filepath,
+    if path is not None:
+        result, err = stub_call_ff('ffmpeg', *ffmpeg_input_params, '-i', path,
                                    '-f', 'rawvideo', '-s', f'{FirstFrameResolution}x{FirstFrameResolution}',
                                    '-pix_fmt', 'rgb24', 'pipe:'
                                    )
-    else:
+    elif data is not None:
         result, err = stub_call_ff('ffmpeg', *ffmpeg_input_params, '-i', 'pipe:0',
                                    '-f', 'rawvideo', '-s', f'{FirstFrameResolution}x{FirstFrameResolution}',
                                    '-pix_fmt', 'rgb24', 'pipe:1',
-                                   stdin_data=data_or_filepath)
+                                   stdin_data=data)
+    else:
+        raise ValueError("`path` or `data` argument must be specified.")
     if err:
         print('DEBUG:', err)
         return -1
@@ -223,22 +227,25 @@ def get_first_timestamp(video_info: dict, data_or_filepath) -> int:
     return 0
 
 
-def get_frames(timestamps: list, data_or_filepath, *ffmpeg_out_params) -> list:
+def get_frames(timestamps: list, path, data, *ffmpeg_out_params) -> list:
+    """Accepts path(bytes/str) or data for processing in memory."""
     ret = [False]
     for _ in range(len(timestamps)):
         ret.append(b'')
+    if path is None and data is None:
+        raise ValueError("`path` or `data` argument must be specified.")
     ffmpeg_input_params = ['-hide_banner', '-loglevel', 'fatal', '-an', '-sn', '-dn']
     for index, timestamp in enumerate(timestamps):
-        if isinstance(data_or_filepath, str):
+        if path is not None:
             result, err = stub_call_ff('ffmpeg', *ffmpeg_input_params,
-                                       '-ss', f'{timestamp}ms', '-i', data_or_filepath,
+                                       '-ss', f'{timestamp}ms', '-i', path,
                                        '-f', 'image2', '-c:v', 'bmp', '-frames', '1', *ffmpeg_out_params, 'pipe:'
                                        )
         else:
             result, err = stub_call_ff('ffmpeg', *ffmpeg_input_params,
                                        '-ss', f'{timestamp}ms', '-i', 'pipe:0',
                                        '-f', 'image2', '-c:v', 'bmp', '-frames', '1', *ffmpeg_out_params, 'pipe:1',
-                                       stdin_data=data_or_filepath)
+                                       stdin_data=data)
         if err:
             print('DEBUG:', err)
             return ret

js/mediadc-node_modules_moment_locale_sync_recursive_-src_views_CollectorDetails_vue.js

@@ -87,21 +87,23 @@ def is_moov_at_start(std_log) -> bool:
     return False
 
 
-def ffprobe_get_video_info(path_or_data) -> dict:
-    """Accepts bytes/path. Returns {} or {duration:X ms}. If input is bytes also returns `fast_start` flag."""
-    if isinstance(path_or_data, str):
+def ffprobe_get_video_info(path, data) -> dict:
+    """Accepts path(bytes/str) or data. Returns {} or {duration:X ms}. For data input also returns `fast_start` flag."""
+    if path is not None:
         result, err = stub_call_ff('ffprobe', '-hide_banner', '-loglevel', 'fatal', '-print_format', 'json',
                                    '-show_entries', 'format=duration',
-                                   path_or_data)
-    else:
+                                   path)
+    elif data is not None:
         result, err = stub_call_ff('ffprobe', '-hide_banner', '-loglevel', 'trace', '-print_format', 'json',
                                    '-show_entries', 'format=duration', '-i', 'pipe:0',
-                                   stdin_data=path_or_data, ignore_errors=True)
+                                   stdin_data=data, ignore_errors=True)
+    else:
+        raise ValueError("`path` or `data` argument must be specified.")
     if err:
         print(err)
         return {}
     ret = ffprobe_parse_results(result)
-    if isinstance(path_or_data, bytes):
+    if path is None:
         if ret['duration'] != 0:
             ret['fast_start'] = is_moov_at_start(result.stderr)
     return ret