From 46698b3f69e7defb18d1600c194dd59b6c7edeaa Mon Sep 17 00:00:00 2001 From: bakerboy448 <55419169+bakerboy448@users.noreply.github.com> Date: Tue, 15 Apr 2025 18:26:12 -0500 Subject: [PATCH 1/3] feat(dupe.sh) - revamp and add summary etc inspired by DAPS Co-authored-by: Drazzilb --- dupe.sh | 59 ++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/dupe.sh b/dupe.sh index 9c8dd6c..76794f9 100755 --- a/dupe.sh +++ b/dupe.sh @@ -19,24 +19,57 @@ else echo ".env file not found in script directory ($ENV_PATH)" fi -# Variables +# Default Variables JDUPES_OUTPUT_LOG=${JDUPES_OUTPUT_LOG:-"/mnt/data/jdupes.log"} JDUPES_SOURCE_DIR=${JDUPES_SOURCE_DIR:-"/mnt/data/media/"} JDUPES_DESTINATION_DIR=${JDUPES_DESTINATION_DIR:-"/mnt/data/torrents/"} JDUPES_HASH_DB=${JDUPES_HASH_DB:-"/.config/jdupes_hashdb"} -## Secret Variables JDUPES_COMMAND=${JDUPES_COMMAND:-"/usr/bin/jdupes"} JDUPES_EXCLUDE_DIRS=${JDUPES_EXCLUDE_DIRS:-"-X nostr:.RecycleBin -X nostr:.trash"} JDUPES_INCLUDE_EXT=${JDUPES_INCLUDE_EXT:-"mp4,mkv,avi"} +DEBUG=${DEBUG:-"false"} -# Logging the start of the operation -timestamp=$(date +"%Y-%m-%d %H:%M:%S") -echo "[$timestamp] Duplicate search started for $JDUPES_SOURCE_DIR and $JDUPES_DESTINATION_DIR." >>"$JDUPES_OUTPUT_LOG" -echo "command is" -# Running jdupes with the loaded environment variables -echo "$JDUPES_COMMAND" "$JDUPES_EXCLUDE_DIRS" -X onlyext:"$JDUPES_INCLUDE_EXT" -L -r -Z -y "$JDUPES_HASH_DB" "$JDUPES_SOURCE_DIR" "$JDUPES_DESTINATION_DIR" -"$JDUPES_COMMAND" "$JDUPES_EXCLUDE_DIRS" -X onlyext:"$JDUPES_INCLUDE_EXT" -L -r -Z -y "$JDUPES_HASH_DB" "$JDUPES_SOURCE_DIR" "$JDUPES_DESTINATION_DIR" >>"$JDUPES_OUTPUT_LOG" - -# Logging the completion of the operation -timestamp=$(date +"%Y-%m-%d %H:%M:%S") -echo "[$timestamp] Duplicate search completed for $JDUPES_SOURCE_DIR and $JDUPES_DESTINATION_DIR." >>"$JDUPES_OUTPUT_LOG" +find_duplicates() { + local log_file="$JDUPES_OUTPUT_LOG" + local start=$(date +%s) + echo "[$(date +"%Y-%m-%d %H:%M:%S")] Duplicate search started" | tee -a "$log_file" + + if [ "$DEBUG" == "true" ]; then + echo "Running jdupes with:" | tee -a "$log_file" + echo "$JDUPES_COMMAND $JDUPES_EXCLUDE_DIRS -X onlyext:$JDUPES_INCLUDE_EXT -r -M -y $JDUPES_HASH_DB $JDUPES_SOURCE_DIR $JDUPES_DESTINATION_DIR" | tee -a "$log_file" + fi + + local results + results=$("$JDUPES_COMMAND" $JDUPES_EXCLUDE_DIRS -X onlyext:"$JDUPES_INCLUDE_EXT" -r -M -y "$JDUPES_HASH_DB" "$JDUPES_SOURCE_DIR" "$JDUPES_DESTINATION_DIR") + + if [[ $results != *"No duplicates found."* ]]; then + "$JDUPES_COMMAND" $JDUPES_EXCLUDE_DIRS -X onlyext:"$JDUPES_INCLUDE_EXT" -r -L -y "$JDUPES_HASH_DB" "$JDUPES_SOURCE_DIR" "$JDUPES_DESTINATION_DIR" >>"$log_file" + fi + + if [ "$DEBUG" == "true" ]; then + echo -e "jdupes output: ${results}" | tee -a "$log_file" + fi + + parse_jdupes_output "$results" "$log_file" + echo "[$(date +"%Y-%m-%d %H:%M:%S")] Duplicate search completed" | tee -a "$log_file" +} + +parse_jdupes_output() { + local results="$1" + local log_file="$2" + + if [[ $results != *"No duplicates found."* ]]; then + field_message="❌ Unlinked files discovered..." + parsed_log=$(echo "$results" | awk -F/ '{print $NF}' | sort -u) + else + field_message="✅ No unlinked files discovered..." + parsed_log="No hardlinks created" + fi + + if [ "$DEBUG" == "true" ]; then + echo -e "$field_message" | tee -a "$log_file" + echo -e "Parsed log: ${parsed_log}" | tee -a "$log_file" + fi +} + +find_duplicates From d40facf0b787d109e20ee6121e6063ea20f21bfb Mon Sep 17 00:00:00 2001 From: bakerboy448 <55419169+bakerboy448@users.noreply.github.com> Date: Tue, 15 Apr 2025 18:27:26 -0500 Subject: [PATCH 2/3] codespell --- dupe.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dupe.sh b/dupe.sh index 76794f9..b8444c5 100755 --- a/dupe.sh +++ b/dupe.sh @@ -40,10 +40,10 @@ find_duplicates() { fi local results - results=$("$JDUPES_COMMAND" $JDUPES_EXCLUDE_DIRS -X onlyext:"$JDUPES_INCLUDE_EXT" -r -M -y "$JDUPES_HASH_DB" "$JDUPES_SOURCE_DIR" "$JDUPES_DESTINATION_DIR") + results=$("$JDUPES_COMMAND" "$JDUPES_EXCLUDE_DIRS" -X onlyext:"$JDUPES_INCLUDE_EXT" -r -M -y "$JDUPES_HASH_DB" "$JDUPES_SOURCE_DIR" "$JDUPES_DESTINATION_DIR") if [[ $results != *"No duplicates found."* ]]; then - "$JDUPES_COMMAND" $JDUPES_EXCLUDE_DIRS -X onlyext:"$JDUPES_INCLUDE_EXT" -r -L -y "$JDUPES_HASH_DB" "$JDUPES_SOURCE_DIR" "$JDUPES_DESTINATION_DIR" >>"$log_file" + "$JDUPES_COMMAND" "$JDUPES_EXCLUDE_DIRS" -X onlyext:"$JDUPES_INCLUDE_EXT" -r -L -y "$JDUPES_HASH_DB" "$JDUPES_SOURCE_DIR" "$JDUPES_DESTINATION_DIR" >>"$log_file" fi if [ "$DEBUG" == "true" ]; then From ee1961e88a01e977f3b2b75ee4785bce18e8ae95 Mon Sep 17 00:00:00 2001 From: bakerboy448 <55419169+bakerboy448@users.noreply.github.com> Date: Tue, 15 Apr 2025 18:30:40 -0500 Subject: [PATCH 3/3] fix runtime calc --- dupe.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dupe.sh b/dupe.sh index b8444c5..3cea68e 100755 --- a/dupe.sh +++ b/dupe.sh @@ -31,7 +31,7 @@ DEBUG=${DEBUG:-"false"} find_duplicates() { local log_file="$JDUPES_OUTPUT_LOG" - local start=$(date +%s) + local start_time=$(date +%s) echo "[$(date +"%Y-%m-%d %H:%M:%S")] Duplicate search started" | tee -a "$log_file" if [ "$DEBUG" == "true" ]; then @@ -51,7 +51,9 @@ find_duplicates() { fi parse_jdupes_output "$results" "$log_file" - echo "[$(date +"%Y-%m-%d %H:%M:%S")] Duplicate search completed" | tee -a "$log_file" + local finish_time=$(date +%s) + local run_time=$((finish_time - start_time)) + echo "[$(date +"%Y-%m-%d %H:%M:%S")] Duplicate search completed in ${run_time}s" | tee -a "$log_file" } parse_jdupes_output() {