MP3 Special Handling & Missing Frame Refactor (#196)

* Handle MP3 files with special logic, since they typically only have 1 frame of video

* Better support for missing frames (handling of missing audio and missing video data separated), and fixed a missing audio regression.

Author: jonoomph

src/FFmpegReader.cpp

@@ -472,8 +472,6 @@ std::shared_ptr<Frame> FFmpegReader::GetFrame(int64_t requested_frame)
     #pragma omp critical (ReadStream)
 	  {
 			// Check the cache a 2nd time (due to a potential previous lock)
-			if (has_missing_frames)
-			    CheckMissingFrame(requested_frame);
 			frame = final_cache.GetFrame(requested_frame);
 			if (frame) {
 				// Debug output
@@ -648,9 +646,6 @@ std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame)
 
 				// Check if working frames are 'finished'
 				if (!is_seeking) {
-					// Check for any missing frames
-					CheckMissingFrame(requested_frame);
-
 					// Check for final frames
 					CheckWorkingFrames(false, requested_frame);
 				}
@@ -1651,7 +1646,7 @@ AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts)
 			for (int64_t audio_frame = previous_packet_location.frame; audio_frame < location.frame; audio_frame++) {
 				if (!missing_audio_frames.count(audio_frame)) {
 					ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (tracking missing frame)", "missing_audio_frame", audio_frame, "previous_audio_frame", previous_packet_location.frame, "new location frame", location.frame, "", -1, "", -1, "", -1);
-					missing_audio_frames.insert(pair<int64_t, int64_t>(previous_packet_location.frame - 1, audio_frame));
+					missing_audio_frames.insert(pair<int64_t, int64_t>(audio_frame, previous_packet_location.frame - 1));
 				}
 			}
 		}
@@ -1726,13 +1721,25 @@ bool FFmpegReader::CheckMissingFrame(int64_t requested_frame)
 	map<int64_t, int64_t>::iterator itr;
 	bool found_missing_frame = false;
 
-	// Check if requested frame is a missing frame
-	if (missing_video_frames.count(requested_frame) || missing_audio_frames.count(requested_frame)) {
-		int64_t missing_source_frame = -1;
-		if (missing_video_frames.count(requested_frame))
-			missing_source_frame = missing_video_frames.find(requested_frame)->second;
-		else if (missing_audio_frames.count(requested_frame))
-			missing_source_frame = missing_audio_frames.find(requested_frame)->second;
+	// Special MP3 Handling (ignore more than 1 video frame)
+	if (info.has_audio and info.has_video) {
+		AVCodecID aCodecId = AV_FIND_DECODER_CODEC_ID(aStream);
+		AVCodecID vCodecId = AV_FIND_DECODER_CODEC_ID(pStream);
+		// If MP3 with single video frame, handle this special case by copying the previously
+		// decoded image to the new frame. Otherwise, it will spend a huge amount of
+		// CPU time looking for missing images for all the audio-only frames.
+		if (checked_count > 8 && !missing_video_frames.count(requested_frame) &&
+			!processing_audio_frames.count(requested_frame) && processed_audio_frames.count(requested_frame) &&
+			last_frame && last_video_frame->has_image_data && aCodecId == AV_CODEC_ID_MP3 && (vCodecId == AV_CODEC_ID_MJPEGB || vCodecId == AV_CODEC_ID_MJPEG)) {
+				missing_video_frames.insert(pair<int64_t, int64_t>(requested_frame, last_video_frame->number));
+				missing_video_frames_source.insert(pair<int64_t, int64_t>(last_video_frame->number, requested_frame));
+				missing_frames.Add(last_video_frame);
+		}
+	}
+
+	// Check if requested video frame is a missing
+	if (missing_video_frames.count(requested_frame)) {
+		int64_t missing_source_frame = missing_video_frames.find(requested_frame)->second;
 
 		// Increment missing source frame check count (or init to 1)
 		if (checked_frames.count(missing_source_frame) == 0)
@@ -1765,21 +1772,26 @@ bool FFmpegReader::CheckMissingFrame(int64_t requested_frame)
 			std::shared_ptr<QImage> parent_image = parent_frame->GetImage();
 			if (parent_image) {
 				missing_frame->AddImage(std::shared_ptr<QImage>(new QImage(*parent_image)));
-
 				processed_video_frames[missing_frame->number] = missing_frame->number;
-				processed_audio_frames[missing_frame->number] = missing_frame->number;
+			}
+		}
+	}
 
-				// Move frame to final cache
-				final_cache.Add(missing_frame);
+	// Check if requested audio frame is a missing
+	if (missing_audio_frames.count(requested_frame)) {
 
-				// Remove frame from working cache
-				working_cache.Remove(missing_frame->number);
+		// Create blank missing frame
+		std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
 
-				// Update last_frame processed
-				last_frame = missing_frame->number;
-			}
-		}
+		// Get Samples per frame (for this frame number)
+		int samples_per_frame = Frame::GetSamplesPerFrame(missing_frame->number, info.fps, info.sample_rate, info.channels);
+
+		// Debug output
+		ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (Add Silence for Missing Audio Frame)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "samples_per_frame", samples_per_frame, "", -1, "", -1, "", -1);
 
+		// Add this frame to the processed map (since it's already done)
+		missing_frame->AddAudioSilence(samples_per_frame);
+		processed_audio_frames[missing_frame->number] = missing_frame->number;
 	}
 
 	return found_missing_frame;
@@ -1792,6 +1804,9 @@ void FFmpegReader::CheckWorkingFrames(bool end_of_stream, int64_t requested_fram
     bool checked_count_tripped = false;
     int max_checked_count = 80;
 
+	// Check if requested frame is 'missing'
+	CheckMissingFrame(requested_frame);
+
 	while (true)
 	{
 		// Get the front frame of working cache