Merge branch 'develop' into debian-pybuild

Author: jonoomph

README.md

@@ -34,6 +34,13 @@ are also available in the /docs/ source folder.
    * [Mac](https://github.com/OpenShot/libopenshot/wiki/Mac-Build-Instructions)
    * [Windows](https://github.com/OpenShot/libopenshot/wiki/Windows-Build-Instructions)
 
+## Hardware Acceleration
+
+OpenShot now supports experimental hardware acceleration, both for encoding and
+decoding videos. When enabled, this can either speed up those operations or slow
+them down, depending on the power and features supported by your graphics card.
+Please see [doc/HW-ACCELL.md](doc/HW-ACCEL.md) for more information.
+
 ## Documentation
 
 Beautiful HTML documentation can be generated using Doxygen.

cmake/Modules/FindFFmpeg.cmake

@@ -158,4 +158,4 @@ foreach (_component ${FFmpeg_FIND_COMPONENTS})
 endforeach ()
 
 # Give a nice error message if some of the required vars are missing.
-find_package_handle_standard_args(FFmpeg DEFAULT_MSG ${_FFmpeg_REQUIRED_VARS})
+find_package_handle_standard_args(FFmpeg DEFAULT_MSG ${_FFmpeg_REQUIRED_VARS})

doc/HW-ACCEL.md

@@ -0,0 +1,134 @@
+## Hardware Acceleration
+
+OpenShot now has experimental support for hardware acceleration, which uses 1 (or more)
+graphics cards to offload some of the work for both decoding and encoding. This is
+very new and experimental (as of May 2019), but we look forward to "accelerating"
+our support for this in the future!
+
+The following table summarizes our current level of support:
+
+|                    |      Linux Decode      |     Linux Encode     |    Mac Decode    |    Mac Encode  |Windows Decode|  Windows Encode  | Notes            |
+|--------------------|------------------------|----------------------|------------------|----------------|--------------|------------------|------------------|
+| VA-API             |   :heavy_check_mark:   |  :heavy_check_mark:  |         -        |        -       |       -      |         -        | *Linux Only*     |
+| VDPAU              |  :heavy_check_mark:(+) |:white_check_mark:(++)|         -        |        -       |       -      |         -        | *Linux Only*     |
+| CUDA (NVDEC/NVENC) |        :x:(+++)        |  :heavy_check_mark:  |         -        |        -       |       -      |:heavy_check_mark:| *Cross Platform* |
+| VideoToolBox       |            -           |           -          |:heavy_check_mark:|   :x:(++++)    |       -      |         -        | *Mac Only*       |
+| DXVA2              |            -           |           -          |         -        |        -       |   :x:(+++)   |         -        | *Windows Only*   |
+| D3D11VA            |            -           |           -          |         -        |        -       |   :x:(+++)   |         -        | *Windows Only*   |
+| QSV                |        :x:(+++)        |          :x:         |        :x:       |       :x:      |      :x:     |        :x:       | *Cross Platform* |
+
+* *(+) VDPAU for some reason needs a card number one higher than it really is*
+* *(++) VDPAU is a decoder only.*
+* *(+++) Green frames (pixel data not correctly tranferred back to system memory)*
+* *(++++) Crashes and burns*
+
+## Supported FFmpeg Versions
+
+* HW accel is supported from FFmpeg version 3.2 (3.3 for nVidia drivers)
+* HW accel was removed for nVidia drivers in Ubuntu for FFmpeg 4+
+
+**Notice:** The FFmpeg versions of Ubuntu and PPAs for Ubuntu show the
+same behaviour. FFmpeg 3 has working nVidia hardware acceleration while
+FFmpeg 4+ has no support for nVidia hardware acceleration
+included.
+
+## OpenShot Settings
+
+The following settings are use by libopenshot to enable, disable, and control
+the various hardware acceleration features.
+
+```
+/// Use video codec for faster video decoding (if supported)
+int HARDWARE_DECODER = 0;
+
+/* 0 - No acceleration
+   1 - Linux VA-API
+   2 - nVidia NVDEC
+   3 - Windows D3D9
+   4 - Windows D3D11
+   5 - MacOS / VideoToolBox
+   6 - Linux VDPAU
+   7 - Intel QSV */
+
+/// Number of threads of OpenMP
+int OMP_THREADS = 12;
+
+/// Number of threads that FFmpeg uses
+int FF_THREADS = 8;
+
+/// Maximum rows that hardware decode can handle
+int DE_LIMIT_HEIGHT_MAX = 1100;
+
+/// Maximum columns that hardware decode can handle
+int DE_LIMIT_WIDTH_MAX = 1950;
+
+/// Which GPU to use to decode (0 is the first, LINUX ONLY)
+int HW_DE_DEVICE_SET = 0;
+
+/// Which GPU to use to encode (0 is the first, LINUX ONLY)
+int HW_EN_DEVICE_SET = 0;
+```
+
+## Libva / VA-API (Video Acceleration API)
+
+The correct version of libva is needed (libva in Ubuntu 16.04 or libva2
+in Ubuntu 18.04) for the AppImage to work with hardware acceleration.
+An AppImage that works on both systems (supporting libva and libva2),
+might be possible when no libva is included in the AppImage.
+
+* vaapi is working for intel and AMD
+* vaapi is working for decode only for nouveau
+* nVidia driver is working for export only
+
+## AMD Graphics Cards (RadeonOpenCompute/ROCm)
+
+Decoding and encoding on the (AMD) GPU is possible with the default drivers.
+On systems where ROCm is installed and run a future use for GPU acceleration
+of effects could be implemented (contributions welcome).
+
+## Multiple Graphics Cards
+
+If the computer has multiple graphics cards installed, you can choose which
+should be used by libopenshot. Also, you can optionally use one card for
+decoding and the other for encoding (if both cards support acceleration).
+This is currently only supported on Linux, due to the device name FFmpeg
+expects (i.e. **/dev/dri/render128**). Contributions welcome if anyone can
+determine what string format to pass for Windows and Mac.
+
+## Help Us Improve Hardware Support
+
+This information might be wrong, and we would love to continue improving
+our support for hardware acceleration in OpenShot. Please help us update
+this document if you find an error or discover new and/or useful information.
+
+**FFmpeg 4 + nVidia** The manual at:
+https://www.tal.org/tutorials/ffmpeg_nvidia_encode
+works pretty well. We could compile and install a version of FFmpeg 4.1.3
+on Mint 19.1 that supports the GPU on nVidia cards. A version of openshot
+with hardware support using these libraries could use the nVidia GPU.
+
+**BUG:** Hardware supported decoding still has some bugs (as you can see from
+the chart above). Also, the speed gains with decoding are not as great
+as with encoding. Currently, if hardware decoding fails, there is no
+fallback (you either get green frames or an "invalid file" error in OpenShot).
+This needs to be improved to successfully fall-back to software decoding.
+
+**Needed:**
+  * A way to get options and limits of the GPU, such as
+ supported dimensions (width and height).
+  *  A way to list the actual Graphic Cards available to FFmpeg (for the
+  user to choose which card for decoding and encoding, as opposed
+  to "Graphics Card X")
+
+**Further improvement:** Right now the frame can be decoded on the GPU, but the
+frame is then copied to CPU memory for modifications. It is then copied back to
+GPU memory for encoding. Using the GPU for both decoding and modifications
+will make it possible to do away with these two copies. A possible solution would
+be to use Vulkan compute which would be available on Linux and Windows natively
+and on MacOS via MoltenVK.
+
+## Credit
+
+A big thanks to Peter M (https://github.com/eisneinechse) for all his work
+on integrating hardware acceleration into libopenshot! The community thanks
+you for this major contribution!

include/AudioDeviceInfo.h

@@ -0,0 +1,43 @@
+/**
+ * @file
+ * @brief Header file for Audio Device Info struct
+ * @author Jonathan Thomas <[email protected]>
+ *
+ * @section LICENSE
+ *
+ * Copyright (c) 2008-2014 OpenShot Studios, LLC
+ * <http://www.openshotstudios.com/>. This file is part of
+ * OpenShot Library (libopenshot), an open-source project dedicated to
+ * delivering high quality video editing and animation solutions to the
+ * world. For more information visit <http://www.openshot.org/>.
+ *
+ * OpenShot Library (libopenshot) is free software: you can redistribute it
+ * and/or modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * OpenShot Library (libopenshot) is distributed in the hope that it will be
+ * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef OPENSHOT_AUDIODEVICEINFO_H
+#define OPENSHOT_AUDIODEVICEINFO_H
+
+
+/**
+ * @brief This struct hold information about Audio Devices
+ *
+ * The type and name of the audio device.
+ */
+struct AudioDeviceInfo
+{
+	string name;
+	string type;
+};
+
+#endif

include/FFmpegReader.h

@@ -50,18 +50,17 @@
 
 using namespace std;
 
-namespace openshot
-{
+namespace openshot {
 	/**
 	 * @brief This struct holds the associated video frame and starting sample # for an audio packet.
 	 *
 	 * Because audio packets do not match up with video frames, this helps determine exactly
 	 * where the audio packet's samples belong.
 	 */
-	struct AudioLocation
-	{
+	struct AudioLocation {
 		int64_t frame;
 		int sample_start;
+
 		bool is_near(AudioLocation location, int samples_per_frame, int64_t amount);
 	};
 
@@ -91,14 +90,16 @@ namespace openshot
 	 * r.Close();
 	 * @endcode
 	 */
-	class FFmpegReader : public ReaderBase
-	{
+	class FFmpegReader : public ReaderBase {
 	private:
 		string path;
 
 		AVFormatContext *pFormatCtx;
 		int i, videoStream, audioStream;
 		AVCodecContext *pCodecCtx, *aCodecCtx;
+#if (LIBAVFORMAT_VERSION_MAJOR >= 57)
+		AVBufferRef *hw_device_ctx = NULL; //PM
+#endif
 		AVStream *pStream, *aStream;
 		AVPacket *packet;
 		AVFrame *pFrame;
@@ -142,7 +143,15 @@ namespace openshot
 		int64_t video_pts_offset;
 		int64_t last_frame;
 		int64_t largest_frame_processed;
-		int64_t current_video_frame;	// can't reliably use PTS of video to determine this
+		int64_t current_video_frame;    // can't reliably use PTS of video to determine this
+
+		int hw_de_supported = 0;    // Is set by FFmpegReader
+#if IS_FFMPEG_3_2
+		AVPixelFormat hw_de_av_pix_fmt = AV_PIX_FMT_NONE;
+		AVHWDeviceType hw_de_av_device_type = AV_HWDEVICE_TYPE_NONE;
+#endif
+
+		int IsHardwareDecodeSupported(int codecid);
 
 		/// Check for the correct frames per second value by scanning the 1st few seconds of video packets.
 		void CheckFPS();
@@ -199,10 +208,10 @@ namespace openshot
 		std::shared_ptr<Frame> ReadStream(int64_t requested_frame);
 
 		/// Remove AVFrame from cache (and deallocate it's memory)
-		void RemoveAVFrame(AVFrame*);
+		void RemoveAVFrame(AVFrame *);
 
 		/// Remove AVPacket from cache (and deallocate it's memory)
-		void RemoveAVPacket(AVPacket*);
+		void RemoveAVPacket(AVPacket *);
 
 		/// Seek to a specific Frame.  This is not always frame accurate, it's more of an estimation on many codecs.
 		void Seek(int64_t requested_frame);
@@ -240,7 +249,7 @@ namespace openshot
 		void Close();
 
 		/// Get the cache object used by this reader
-		CacheMemory* GetCache() { return &final_cache; };
+		CacheMemory *GetCache() { return &final_cache; };
 
 		/// Get a shared pointer to a openshot::Frame object for a specific frame number of this reader.
 		///

include/FFmpegUtilities.h

@@ -42,6 +42,9 @@
 	extern "C" {
 		#include <libavcodec/avcodec.h>
 		#include <libavformat/avformat.h>
+	#if (LIBAVFORMAT_VERSION_MAJOR >= 57)
+		#include <libavutil/hwcontext.h> //PM
+	#endif
 		#include <libswscale/swscale.h>
 		// Change this to the first version swrescale works
 	#if (LIBAVFORMAT_VERSION_MAJOR >= 57)