Multiple codec contexts mixing frames between each other

[paulomorgado]

I have these decoders that are guaranteed to not share any data between them and there is no concurrency within each one.

However, I get, frequent mixed frames where one decoder gets a frame from packets fed to another.

This is the code I'm using:

using System.Buffers;
using System.Diagnostics;
using System.Threading.Channels;
using FFmpeg.AutoGen;

public class FfmpegAsyncVideoDecoderFactory : IAsyncVideoDecoderFactory
{
    static FfmpegAsyncVideoDecoderFactory()
    {
        ffmpeg.RootPath = Environment.CurrentDirectory;
        DynamicallyLoadedBindings.ThrowErrorIfFunctionNotFound = true;
    }

    public unsafe IAsyncVideoDecoder CreateDecoder(string tag, string codec, ChannelWriter<(ArraySegment<byte> decodedData, int width, int heigh)> writer)
    {
        ArgumentException.ThrowIfNullOrWhiteSpace(tag);
        ArgumentException.ThrowIfNullOrWhiteSpace(codec);
        ArgumentNullException.ThrowIfNull(writer);

        var codecId = codec switch
        {
            "H264" => AVCodecID.AV_CODEC_ID_H264,
            "JPEG" => AVCodecID.AV_CODEC_ID_MJPEG,
            "MJPEG" => AVCodecID.AV_CODEC_ID_MJPEG,
            _ => throw new ArgumentOutOfRangeException($"Codec must be 'H264', 'MJPEG' or 'JPEG' and it was '{codec}")
        };

        var ffmpegContext = new FfmpegContext();

        var ffmpegCodec = ffmpeg.avcodec_find_decoder(codecId);
        if (ffmpegCodec == null)
        {
            throw new DecoderException($"Decoder codec could not be found for '{codecId}' ({codecId}).");
        }

        ffmpegContext.codecContext = ffmpeg.avcodec_alloc_context3(ffmpegCodec);
        if (ffmpegContext.codecContext == null)
        {
            ffmpegContext.Dispose();
            throw new DecoderException("Failed to allocate decoder codec context.");
        }

        try
        {
            ffmpeg.avcodec_open2(ffmpegContext.codecContext, ffmpegCodec, null).ThrowIfError();
        }
        catch (Exception ex)
        {
            ffmpegContext.Dispose();
            throw new DecoderException("Failed to allocate decoder codec context.", ex);
        }

        ffmpegContext.sourceFrame = ffmpeg.av_frame_alloc();
        if (ffmpegContext.sourceFrame == null)
        {
            ffmpegContext.Dispose();
            throw new DecoderException("Failed to allocate source frame.");
        }

        ffmpegContext.bgraFrame = ffmpeg.av_frame_alloc();
        if (ffmpegContext.bgraFrame == null)
        {
            ffmpegContext.Dispose();
            throw new DecoderException("Failed to allocate BGRA frame.");
        }

        ffmpegContext.packet = ffmpeg.av_packet_alloc();
        if (ffmpegContext.packet == null)
        {
            ffmpegContext.Dispose();
            throw new DecoderException("Failed to allocate packet.");
        }



        return new FfmpegAsyncVideoDecoder(writer, ffmpegContext);
    }

    private sealed unsafe class FfmpegContext : IDisposable
    {
        public static readonly int eagain = ffmpeg.AVERROR(ffmpeg.EAGAIN);

        public AVCodecContext* codecContext;
        public AVPacket* packet;
        public AVFrame* sourceFrame;
        public AVFrame* bgraFrame;
        public SwsContext* swsContext;
        private bool _isDisposed;

        ~FfmpegContext()
        {
            // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
            Dispose(disposing: false);
        }

        public void Dispose()
        {
            // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
            Dispose(disposing: true);
            GC.SuppressFinalize(this);
        }

        private void Dispose(bool disposing)
        {
            if (!_isDisposed)
            {
                if (disposing)
                {
                    // TODO: dispose managed state (managed objects)
                }

                // TODO: free unmanaged resources (unmanaged objects) and override finalizer
                // TODO: set large fields to null

                var tempFrame = sourceFrame;
                sourceFrame = null;
                ffmpeg.av_frame_unref(tempFrame);
                ffmpeg.av_free(tempFrame);

                tempFrame = bgraFrame;
                bgraFrame = null;
                ffmpeg.av_frame_unref(tempFrame);
                ffmpeg.av_free(tempFrame);

                var tempPacket = packet;
                packet = null;
                ffmpeg.av_packet_unref(tempPacket);
                ffmpeg.av_free(tempPacket);

                var tempSwsContext = swsContext;
                swsContext = null;
                ffmpeg.sws_freeContext(tempSwsContext);

                var tempCodecContext = codecContext;
                codecContext = null;
                ffmpeg.avcodec_free_context(&tempCodecContext);

                _isDisposed = true;
            }
        }
    }

    private sealed class FfmpegAsyncVideoDecoder : IAsyncVideoDecoder
    {
        private readonly ChannelWriter<(ArraySegment<byte> decodedData, int width, int heigh)> _decodedWriter;
        private Channel<ArraySegment<byte>> _encodedChannel;
        private readonly FfmpegContext _ffmpegContext;
        private bool _isDisposed;

        public FfmpegAsyncVideoDecoder(ChannelWriter<(ArraySegment<byte> decodedData, int width, int heigh)> writer, FfmpegContext ffmpegContext)
        {
            _decodedWriter = writer;
            _encodedChannel = Channel.CreateUnbounded<ArraySegment<byte>>(new() { SingleReader = true, SingleWriter = true, AllowSynchronousContinuations = false, });
            _ffmpegContext = ffmpegContext;

            _ = HandleEncodedpackets();
        }

        public ValueTask SendPacketAsync(ArraySegment<byte> encodedData)
        {
            ObjectDisposedException.ThrowIf(_isDisposed, this);
            ArgumentNullException.ThrowIfNull(encodedData.Array);

            return _encodedChannel.Writer.WriteAsync(encodedData);
        }

        public void Dispose()
        {
            if (!_isDisposed)
            {
                _isDisposed = true;
                _encodedChannel.Writer.Complete();
                _encodedChannel.Reader.Completion.GetAwaiter().GetResult(); // TODO: use cancellation token
                _ffmpegContext.Dispose();
            }
        }

        private async Task HandleEncodedpackets()
        {
            await foreach (var encodedFrame in _encodedChannel.Reader.ReadAllAsync())
            {
                HandleEncodedpacket(encodedFrame);
            }
        }

        private unsafe void HandleEncodedpacket(ArraySegment<byte> encodedData)
        {
            Debug.Assert(_ffmpegContext is not null);
            var codecContext = _ffmpegContext.codecContext;
            var swsContext = _ffmpegContext.swsContext;
            var packet = _ffmpegContext.packet;
            var sourceFrame = _ffmpegContext.sourceFrame;
            var bgraFrame = _ffmpegContext.bgraFrame;
            Debug.Assert(codecContext != null);
            Debug.Assert(packet != null);
            Debug.Assert(sourceFrame != null);
            Debug.Assert(bgraFrame != null);

            int result;

            fixed (byte* encodedDataBytes = encodedData.Array)
            {
                packet->data = encodedDataBytes;
                packet->size = encodedData.Count;

                result =
                ffmpeg.avcodec_send_packet(codecContext, packet)
                    //.ThrowIfError()
                    ;

                if (result != 0)
                {
                    return;
                }
            }

            while ((result = ffmpeg.avcodec_receive_frame(codecContext, sourceFrame)) >= 0)
            {
                var sourceWidth = sourceFrame->width;
                var sourceHeight = sourceFrame->height;
                var destinationWidth = sourceFrame->width;
                var destinationHeight = sourceFrame->height;

                // Convert the frame to BGRA
                var pixelFormat = (AVPixelFormat)sourceFrame->format switch
                {
                    AVPixelFormat.AV_PIX_FMT_YUVJ420P => AVPixelFormat.AV_PIX_FMT_YUV420P,
                    AVPixelFormat.AV_PIX_FMT_YUVJ422P => AVPixelFormat.AV_PIX_FMT_YUV422P,
                    AVPixelFormat.AV_PIX_FMT_YUVJ444P => AVPixelFormat.AV_PIX_FMT_YUV444P,
                    _ => (AVPixelFormat)sourceFrame->format,
                };

                if (swsContext == null)
                {
                    _ffmpegContext.swsContext = swsContext = ffmpeg.sws_getContext(
                        sourceWidth, sourceHeight, pixelFormat,
                        destinationWidth, destinationHeight, AVPixelFormat.AV_PIX_FMT_BGRA,
                        ffmpeg.SWS_BILINEAR, null, null, null);
                }

                bgraFrame->format = (int)AVPixelFormat.AV_PIX_FMT_BGRA;
                bgraFrame->width = destinationWidth;
                bgraFrame->height = destinationHeight;

                ffmpeg.av_frame_get_buffer(bgraFrame, 32).ThrowIfError();

                ffmpeg.sws_scale(
                    swsContext,
                    sourceFrame->data, sourceFrame->linesize, 0, sourceFrame->height,
                    bgraFrame->data, bgraFrame->linesize);

                var bgraSize = ffmpeg.av_image_get_buffer_size(AVPixelFormat.AV_PIX_FMT_BGRA, destinationWidth, destinationHeight, 1);
                var bgraArray = ArrayPool<byte>.Shared.Rent(bgraSize);
                var bgraSegment = new ArraySegment<byte>(bgraArray, 0, bgraSize);

                fixed (byte* pBgraData = bgraArray)
                {
                    var data4 = new byte_ptrArray4();
                    data4.UpdateFrom(bgraFrame->data.ToArray());

                    var linesize4 = new int_array4();
                    linesize4.UpdateFrom(bgraFrame->linesize.ToArray());

                    ffmpeg.av_image_copy_to_buffer(
                        pBgraData, bgraSize,
                        data4, linesize4,
                        (AVPixelFormat)bgraFrame->format, destinationWidth, destinationHeight, 1);
                }

                _decodedWriter.TryWrite((bgraSegment, destinationWidth, destinationHeight));
            }

            switch (result)
            {
                case 0:
                case var eagain when (eagain == FfmpegContext.eagain):
                case var eof when (eof == ffmpeg.AVERROR_EOF):
                    break;
                default:
                    result.ThrowIfError();
                    break;
            }
        }
    }
}

Old code using avcodec_decode_video2 works fine.

What am I doing wrong here?

[paulomorgado]

It turns out I wasn't unreferencing packets and frames:

        private unsafe void HandleEncodedpacket(ArraySegment<byte> encodedData)
        {
            Debug.Assert(_ffmpegContext is not null);
            var codecContext = _ffmpegContext.codecContext;
            var swsContext = _ffmpegContext.swsContext;
            var packet = _ffmpegContext.packet;
            var sourceFrame = _ffmpegContext.sourceFrame;
            var bgraFrame = _ffmpegContext.bgraFrame;
            Debug.Assert(codecContext != null);
            Debug.Assert(packet != null);
            Debug.Assert(sourceFrame != null);
            Debug.Assert(bgraFrame != null);

            int result;

            fixed (byte* encodedDataBytes = encodedData.Array)
            {
                packet->data = encodedDataBytes;
                packet->size = encodedData.Count;

                result =
                ffmpeg.avcodec_send_packet(codecContext, packet)
                    //.ThrowIfError()
                    ;

                ffmpeg.av_packet_unref(packet); // <---

                if (result != 0)
                {
                    return;
                }
            }

            while ((result = ffmpeg.avcodec_receive_frame(codecContext, sourceFrame)) >= 0)
            {
                var sourceWidth = sourceFrame->width;
                var sourceHeight = sourceFrame->height;
                var destinationWidth = sourceFrame->width;
                var destinationHeight = sourceFrame->height;

                // Convert the frame to BGRA
                var pixelFormat = (AVPixelFormat)sourceFrame->format switch
                {
                    AVPixelFormat.AV_PIX_FMT_YUVJ420P => AVPixelFormat.AV_PIX_FMT_YUV420P,
                    AVPixelFormat.AV_PIX_FMT_YUVJ422P => AVPixelFormat.AV_PIX_FMT_YUV422P,
                    AVPixelFormat.AV_PIX_FMT_YUVJ444P => AVPixelFormat.AV_PIX_FMT_YUV444P,
                    _ => (AVPixelFormat)sourceFrame->format,
                };

                if (swsContext == null)
                {
                    _ffmpegContext.swsContext = swsContext = ffmpeg.sws_getContext(
                        sourceWidth, sourceHeight, pixelFormat,
                        destinationWidth, destinationHeight, AVPixelFormat.AV_PIX_FMT_BGRA,
                        ffmpeg.SWS_BILINEAR, null, null, null);
                }

                bgraFrame->format = (int)AVPixelFormat.AV_PIX_FMT_BGRA;
                bgraFrame->width = destinationWidth;
                bgraFrame->height = destinationHeight;

                ffmpeg.av_frame_get_buffer(bgraFrame, 32).ThrowIfError();

                ffmpeg.sws_scale(
                    swsContext,
                    sourceFrame->data, sourceFrame->linesize, 0, sourceFrame->height,
                    bgraFrame->data, bgraFrame->linesize);

                ffmpeg.av_frame_unref(sourceFrame); // <---

                var bgraSize = ffmpeg.av_image_get_buffer_size(AVPixelFormat.AV_PIX_FMT_BGRA, destinationWidth, destinationHeight, 1);
                var bgraArray = ArrayPool<byte>.Shared.Rent(bgraSize);
                var bgraSegment = new ArraySegment<byte>(bgraArray, 0, bgraSize);

                fixed (byte* pBgraData = bgraArray)
                {
                    var data4 = new byte_ptrArray4();
                    data4.UpdateFrom(bgraFrame->data.ToArray());

                    var linesize4 = new int_array4();
                    linesize4.UpdateFrom(bgraFrame->linesize.ToArray());

                    ffmpeg.av_image_copy_to_buffer(
                        pBgraData, bgraSize,
                        data4, linesize4,
                        (AVPixelFormat)bgraFrame->format, destinationWidth, destinationHeight, 1);
                }

                ffmpeg.av_frame_unref(bgraFrame); // <---

                _decodedWriter.TryWrite((bgraSegment, destinationWidth, destinationHeight));
            }

            switch (result)
            {
                case 0:
                case var eagain when (eagain == FfmpegContext.eagain):
                case var eof when (eof == ffmpeg.AVERROR_EOF):
                    break;
                default:
                    result.ThrowIfError();
                    break;
            }
        }