Merge pull request #457 from JeromeMartinez/compute-output-hash

Option for computing the hash of the output file while checking it.

Author: JeromeMartinez

Project/GNU/CLI/Makefile.am

@@ -76,7 +76,7 @@ man1_MANS = ../../../Source/CLI/rawcooked.1
 
 AM_TESTS_FD_REDIRECT = 9>&2
 
-TESTS = test/test1.sh test/test1b.sh test/test2.sh test/test3.sh test/avi.sh test/pcm.sh test/reversibilityfile.sh test/paddingbits.sh test/check.sh test/legacy.sh test/multiple.sh test/valgrind.sh test/overwrite.sh test/increasingdigitcount.sh test/gaps.sh test/slices.sh test/framerate.sh test/notfound.sh test/version.sh
+TESTS = test/test1.sh test/test1b.sh test/test2.sh test/test3.sh test/avi.sh test/pcm.sh test/reversibilityfile.sh test/paddingbits.sh test/check.sh test/legacy.sh test/multiple.sh test/valgrind.sh test/overwrite.sh test/increasingdigitcount.sh test/gaps.sh test/slices.sh test/framerate.sh  test/hash.sh test/notfound.sh test/version.sh
 
 TESTING_DIR = test/TestingFiles
 

Project/GNU/CLI/test/hash.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+script_path="${PWD}/test"
+. ${script_path}/helpers.sh
+
+test="hash"
+
+while read file ; do
+    if [ ! -e "${files_path}/${file}" ] ; then
+        echo "NOK: ${test}/${file}, file not found" >&${fd}
+        status=1
+        continue
+    fi
+
+    output="${files_path}/${file}.mkv"
+
+    # --output-version 1
+    run_rawcooked --file --all --compute-output-hash "${files_path}/${file}"
+    check_success "file rejected at input" "file accepted at input"
+    
+    computed_md5=$(echo "${cmd_stdout}" | sed -nE 's/.*MD5 is ([a-f0-9]{32}).*/\1/p')
+    output_md5=$(${md5cmd} "${output}" | cut -d' ' -f1)
+    
+    # check result
+    if [ "${computed_md5}" != "${output_md5}" ] ; then
+        echo "NOK: ${test}/${file}, computed and output md5 differs, computed: ${computed_md5}, output: ${output_md5} " >&${fd}
+        status=1
+        clean
+        continue
+    fi
+
+    clean
+
+    # --output-version 2
+    run_rawcooked --file --all --compute-output-hash --output-version 2 "${files_path}/${file}"
+    check_success "file rejected at input" "file accepted at input"
+    
+    computed_md5=$(echo "${cmd_stdout}" | sed -nE 's/.*MD5 is ([a-f0-9]{32}).*/\1/p')
+    output_md5=$(${md5cmd} "${output}" | cut -d' ' -f1)
+    
+    # check result
+    if [ "${computed_md5}" != "${output_md5}" ] ; then
+        echo "NOK: ${test}/${file}, computed and output md5 differs, computed: ${computed_md5}, output: ${output_md5} " >&${fd}
+        status=1
+        clean
+        continue
+    fi
+
+    clean
+done < "${script_path}/hash.txt"
+
+exit ${status}

Project/GNU/CLI/test/hash.txt

@@ -0,0 +1,6 @@
+Features/AV_Package/818_DCDM_P3_IA_FIC_000918/818_OV
+Formats/TIFF/Flavors/Raw_RGBA_16_U_LE/testsrc_Raw_RGBA_16_LE.tiff
+Formats/AVI/Flavors/v210_44100_16_1_S_LE/v210_44100_16_1_S_LE.avi
+Formats/AIFF/Flavors/44100_8_1_S/44100_8_1_S.aiff
+Formats/AIFF/Flavors/44100_8_1_U/44100_8_1_U.aiff
+Formats/WAV/Flavors/44100_16_1_S_LE/44100_16_1_S_LE.wav

Source/CLI/Global.cpp

@@ -34,6 +34,13 @@ int global::SetOutputFileName(const char* FileName)
     return 0;
 }
 
+//---------------------------------------------------------------------------
+int global::SetComputeOutputHash()
+{
+    Actions.set(Action_ComputeOutputHash, true);
+    return 0;
+}
+
 //---------------------------------------------------------------------------
 int global::SetBinName(const char* FileName)
 {
@@ -304,6 +311,8 @@ int global::SetAll(bool Value)
         return ReturnValue;
     if (int ReturnValue = SetHash(Value))
         return ReturnValue;
+    if (int ReturnValue = SetComputeOutputHash())
+        return ReturnValue;
     return 0;
 }
 
@@ -726,6 +735,12 @@ int global::ManageCommandLine(const char* argv[], int argc)
             if (Value)
                 return Value;
         }
+        else if (strcmp(argv[i], "--compute-output-hash") == 0)
+        {
+            int Value = SetComputeOutputHash();
+            if (Value)
+                return Value;
+        }
         else if (strcmp(argv[i], "--quick-check") == 0)
         {
             int Value = SetQuickCheck();

Source/CLI/Global.h

@@ -85,6 +85,7 @@ class global
     int SetVersion(const char* Value);
     int SetOption(const char* argv[], int& i, int argc);
     int SetOutputFileName(const char* FileName);
+    int SetComputeOutputHash();
     int SetBinName(const char* FileName);
     int SetLicenseKey(const char* Key, bool Add);
     int SetSubLicenseId(uint64_t Id);

Source/CLI/Help.cpp

@@ -109,6 +109,9 @@ ReturnValue Help(const char* Name)
         "              ${Input}.mkv if the input is a folder, or ${Input}.RAWcooked if\n"
         "              input is a file, such as a DPX.\n"
         "\n"
+        "       --compute-output-hash\n"
+        "              Compute the hash of the output file while checking it.\n"
+        "\n"
         "       --output-version value\n"
         "              Set the version of the output..\n"
         "              The default output value is 1.\n"

Source/CLI/rawcooked.1

@@ -67,6 +67,9 @@ Set the name of the output file or folder to \fIvalue\fR.
 .br
 The default output value is opposite to the input. Expect \fI${Input}.mkv\fR if the input is a folder, or \fI${Input}.RAWcooked\fR if input is a file, such as a DPX.
 .TP
+.B --compute-output-hash
+Compute the hash of the output file while checking it.
+.TP
 .B --output-version \fIvalue\fR
 Set the version of the output.
 .br

Source/Lib/Compressed/Matroska/Matroska.cpp

@@ -26,6 +26,10 @@
 #include <iomanip>
 #include <iostream>
 #include <thread>
+extern "C"
+{
+#include "md5.h"
+}
 //---------------------------------------------------------------------------
 
 //---------------------------------------------------------------------------
@@ -71,6 +75,35 @@ static_assert(error::type_Max == sizeof(ErrorTexts) / sizeof(const char**), Inco
 
 using namespace matroska_issue;
 
+//---------------------------------------------------------------------------
+// Hash
+
+struct output_hash {
+    const uint8_t* Buffer;
+    MD5_CTX MD5;
+    uint64_t Offset = 0;
+    bool Disabled = false;
+
+    output_hash(const uint8_t* Buffer_)
+        : Buffer (Buffer_)
+    {
+        MD5_Init(&MD5);
+    }
+
+    void Update(size_t Size)
+    {
+        MD5_Update(&MD5, Buffer + Offset, Size);
+        Offset += Size;
+    }
+
+    md5 MD5_Final()
+    {
+        md5 MD5_Result;
+        ::MD5_Final(MD5_Result.data(), &MD5);
+        return MD5_Result;
+    }
+};
+
 //---------------------------------------------------------------------------
 // Matroska parser
 
@@ -307,6 +340,8 @@ void matroska::ParseBuffer()
 
     Buffer_Offset = 0;
     Level = 0;
+    if ((Actions[Action_Decode] || Actions[Action_Check]) && Actions[Action_ComputeOutputHash])
+        OutputHash = new output_hash(Buffer.Data());
 
     // Progress indicator
     Cluster_Timestamp = 0;
@@ -356,8 +391,18 @@ void matroska::ParseBuffer()
             }
         }
 
+        auto NeedRemap = Buffer_Offset - Buffer_Offset_LowerLimit >= 0x100000; // TODO: when multi-threaded frame decoding is implemented, we need to check that all thread don't need anymore memory below this value 
+        
+        // Compute MD5 of the whole file
+        if (OutputHash && !OutputHash->Disabled)
+        {
+            auto Hash_Size = Buffer_Offset - OutputHash->Offset;
+            if (NeedRemap || Hash_Size >= 0x100000)
+                OutputHash->Update(Hash_Size);
+        }
+
         // Check if we can indicate the system that we'll not need anymore memory below this value, without indicating it too much
-        if (Buffer_Offset > Buffer_Offset_LowerLimit + 1024 * 1024 && Buffer_Offset < Buffer.Size()) // TODO: when multi-threaded frame decoding is implemented, we need to check that all thread don't need anymore memory below this value 
+        if (NeedRemap)
         {
             FileMap->Remap(Buffer_Offset, Buffer_Offset + 256 * 1024 * 1024);
             Buffer = *FileMap;
@@ -378,6 +423,8 @@ void matroska::ParseBuffer()
             Level = Cluster_Level;
             Buffer_Offset = Cluster_Offset;
             Cluster_Level = (size_t)-1;
+            if (OutputHash)
+                OutputHash->Disabled = false;
 
             FileMap->Remap(Buffer_Offset, 256 * 1024 * 1024);
             Buffer = *FileMap;
@@ -413,6 +460,17 @@ void matroska::ParseBuffer()
         ProgressIndicator_Thread->join();
         delete ProgressIndicator_Thread;
     }
+
+    // Show MD5
+    if (OutputHash && !OutputHash->Disabled)
+    {
+        OutputHash->Update(Buffer_Offset - OutputHash->Offset);
+        auto MD5_Result = OutputHash->MD5_Final();
+        std::cout << "\nInfo: Output file MD5 is ";
+        for (size_t i = 0; i < MD5_Result.size(); i++)
+            std::cout << std::hex << std::setw(2) << std::setfill('0') << (int)MD5_Result[i];
+        std::cout << '.' << std::endl;
+    }
 }
 
 //---------------------------------------------------------------------------
@@ -799,7 +857,7 @@ void matroska::Segment_Attachments_AttachedFile_FileData_RawCookedTrack_LibraryV
     RejectIncompatibleVersions();
 }
 
-//---------------------------------------------------------------------------
+//--------------------------------- ------------------------------------------
 void matroska::Segment_Cluster()
 {
     if (RAWcooked_LibraryName.empty())
@@ -808,6 +866,10 @@ void matroska::Segment_Cluster()
         Cluster_Offset = Element_Begin_Offset;
         Cluster_Level = Level;
         Level--;
+        if (OutputHash) {
+            OutputHash->Update(Buffer_Offset - OutputHash->Offset);
+            OutputHash->Disabled = true;
+        }
         return;
     }
 

Source/Lib/Compressed/Matroska/Matroska.h

@@ -37,6 +37,7 @@ class ThreadPool;
 class hashes;
 class track_info;
 class frame_writer;
+struct output_hash;
 
 class matroska : public input_base
 {
@@ -50,6 +51,7 @@ class matroska : public input_base
     bool                        NoOutputCheck = false;
     hashes*                     Hashes_FromRAWcooked = nullptr;
     hashes*                     Hashes_FromAttachments = nullptr;
+    output_hash*                OutputHash = nullptr;
 
     // Theading relating functions
     void                        ProgressIndicator_Show();

Source/Lib/Uncompressed/DPX/DPX.cpp

@@ -260,6 +260,7 @@ void dpx::ParseBuffer()
         uint32_t* HeaderCopy32 = (uint32_t*)HeaderCopy;
         const uint32_t* Buffer32 = (const uint32_t*)Buffer.Data();
         memmove(HeaderCopy + 36, Buffer.Data() + 36, 160 - 36); // Image filename + Creation date/time: yyyy:mm:dd:hh:mm:ssLTZ
+        memmove(HeaderCopy + 1532, Buffer.Data() + 1532, 24); // Image filename + Creation date/time: yyyy:mm:dd:hh:mm:ssLTZ
         HeaderCopy32[1676 / 4] = Buffer32[1676 / 4]; // Count
         HeaderCopy32[1712 / 4] = Buffer32[1712 / 4]; // Frame position in sequence
         HeaderCopy32[1920 / 4] = Buffer32[1920 / 4]; // SMPTE time code