Move responsibility for setting up hardware to HWAccelContext.

We will not likely continue down this path, as `man ffmpeg` makes the
point that CPU decoding is about the same speed as GPU decoding, and so
is only really of use for playback. I don't think PyAVs target is such
high performance playback, so we don't need to make the design
concesions required for this branch.

NOTE: This has not been tested to work. Two commits back is the original
PR squashed into a single commit and is more likely to work, although if
there is any hope of this being merged it will have to look more like
this commit does.

See (and further any discussion) #331 on GitHub.

Author: mikeboers

av/audio/codeccontext.pyx

@@ -3,15 +3,16 @@ cimport libav as lib
 from av.audio.format cimport AudioFormat, get_audio_format
 from av.audio.frame cimport AudioFrame, alloc_audio_frame
 from av.audio.layout cimport AudioLayout, get_audio_layout
+from av.codec.hwaccel cimport HWAccel
 from av.error cimport err_check
 from av.frame cimport Frame
 from av.packet cimport Packet
 
 
 cdef class AudioCodecContext(CodecContext):
 
-    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec):
-        CodecContext._init(self, ptr, codec)
+    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec, HWAccel hwaccel):
+        CodecContext._init(self, ptr, codec, hwaccel)
 
         # Sometimes there isn't a layout set, but there are a number of
         # channels. Assume it is the default layout.

av/codec/codec.pyx

@@ -92,9 +92,9 @@ cdef class Codec(object):
     def __repr__(self):
         return f'<av.{self.__class__.__name__}({self.name!r}, {self.mode!r})>'
 
-    def create(self):
+    def create(self, *args, **kwargs):
         from .context import CodecContext
-        return CodecContext.create(self)
+        return CodecContext.create(self, *args, **kwargs)
 
     property is_decoder:
         def __get__(self):

av/codec/context.pxd

@@ -2,10 +2,11 @@ from libc.stdint cimport int64_t
 
 cimport libav as lib
 
+from av.bytesource cimport ByteSource
 from av.codec.codec cimport Codec
+from av.codec.hwaccel cimport HWAccel, HWAccelContext
 from av.frame cimport Frame
 from av.packet cimport Packet
-from av.bytesource cimport ByteSource
 
 
 cdef class CodecContext(object):
@@ -25,10 +26,12 @@ cdef class CodecContext(object):
     # To hold a reference to passed extradata.
     cdef ByteSource extradata_source
 
-    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec)
+    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec, HWAccel hwaccel)
 
     cdef readonly Codec codec
 
+    cdef readonly HWAccelContext hwaccel
+
     cdef public dict options
 
     # Public API.
@@ -67,4 +70,4 @@ cdef class CodecContext(object):
     cdef Frame _alloc_next_frame(self)
 
 
-cdef CodecContext wrap_codec_context(lib.AVCodecContext*, const lib.AVCodec*, bint allocated, dict hwaccel)
+cdef CodecContext wrap_codec_context(lib.AVCodecContext*, const lib.AVCodec*, bint allocated, HWAccel hwaccel)

av/codec/context.pyx

@@ -7,6 +7,7 @@ cimport libav as lib
 
 from av.bytesource cimport ByteSource, bytesource
 from av.codec.codec cimport Codec, wrap_codec
+from av.codec.hwaccel cimport HWAccel
 from av.dictionary cimport _Dictionary
 from av.dictionary import Dictionary
 from av.enums cimport define_enum
@@ -18,15 +19,15 @@ from av.utils cimport avdict_to_dict, avrational_to_fraction, to_avrational
 cdef object _cinit_sentinel = object()
 
 
-cdef CodecContext wrap_codec_context(lib.AVCodecContext *c_ctx, const lib.AVCodec *c_codec, bint allocated, dict hwaccel):
+cdef CodecContext wrap_codec_context(lib.AVCodecContext *c_ctx, const lib.AVCodec *c_codec, bint allocated, HWAccel hwaccel):
     """Build an av.CodecContext for an existing AVCodecContext."""
 
     cdef CodecContext py_ctx
 
     # TODO: This.
     if c_ctx.codec_type == lib.AVMEDIA_TYPE_VIDEO:
         from av.video.codeccontext import VideoCodecContext
-        py_ctx = VideoCodecContext(_cinit_sentinel, hwaccel=hwaccel)
+        py_ctx = VideoCodecContext(_cinit_sentinel)
     elif c_ctx.codec_type == lib.AVMEDIA_TYPE_AUDIO:
         from av.audio.codeccontext import AudioCodecContext
         py_ctx = AudioCodecContext(_cinit_sentinel)
@@ -37,7 +38,7 @@ cdef CodecContext wrap_codec_context(lib.AVCodecContext *c_ctx, const lib.AVCode
         py_ctx = CodecContext(_cinit_sentinel)
 
     py_ctx.allocated = allocated
-    py_ctx._init(c_ctx, c_codec)
+    py_ctx._init(c_ctx, c_codec, hwaccel)
 
     return py_ctx
 
@@ -62,10 +63,10 @@ SkipType = define_enum('SkipType', (
 cdef class CodecContext(object):
 
     @staticmethod
-    def create(codec, mode=None):
+    def create(codec, mode=None, hwaccel=None):
         cdef Codec cy_codec = codec if isinstance(codec, Codec) else Codec(codec, mode)
         cdef lib.AVCodecContext *c_ctx = lib.avcodec_alloc_context3(cy_codec.ptr)
-        return wrap_codec_context(c_ctx, cy_codec.ptr, True, None)
+        return wrap_codec_context(c_ctx, cy_codec.ptr, True, hwaccel)
 
     def __cinit__(self, sentinel=None, *args, **kwargs):
         if sentinel is not _cinit_sentinel:
@@ -74,7 +75,7 @@ cdef class CodecContext(object):
         self.options = {}
         self.stream_index = -1  # This is set by the container immediately.
 
-    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec):
+    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec, HWAccel hwaccel):
 
         self.ptr = ptr
         if self.ptr.codec and codec and self.ptr.codec != codec:

av/codec/hwaccel.pxd

@@ -1,6 +1,8 @@
 
 cimport libav as lib
 
+from av.codec.codec cimport Codec
+
 
 cdef class HWConfig(object):
 
@@ -16,8 +18,14 @@ cdef HWConfig wrap_hwconfig(lib.AVCodecHWConfig *ptr)
 
 cdef class HWAccel(object):
 
-    cdef lib.AVHWAccel *ptr
-
     cdef str _device_type
     cdef str _device
     cdef public dict options
+
+
+cdef class HWAccelContext(HWAccel):
+
+    cdef readonly Codec codec
+    cdef readonly HWConfig config
+
+    cdef lib.AVBufferRef *ptr

av/codec/hwaccel.pyx

@@ -5,9 +5,13 @@ import weakref
 cimport libav as lib
 
 from av.codec.codec cimport Codec
+from av.dictionary cimport _Dictionary
 from av.enums cimport define_enum
+from av.error cimport err_check
 from av.video.format cimport get_video_format
 
+from av.dictionary import Dictionary
+
 
 HWDeviceType = define_enum('HWDeviceType', (
     # ('NONE', lib.AV_HWDEVICE_TYPE_NONE),
@@ -100,12 +104,70 @@ def dump_hwdevices():
 
 cdef class HWAccel(object):
 
+    @classmethod
+    def adapt(cls, input_):
+        if input_ is True:
+            return cls()
+        if isinstance(input_, cls):
+            return input_
+        if isinstance(input_, (str, HWDeviceType)):
+            return cls(input_)
+        if isinstance(input_, (list, tuple)):
+            return cls(*input_)
+        if isinstance(input_, dict):
+            return cls(**input_)
+        raise TypeError(f"can't adapt to HWAccel; {input_!r}")
+
     def __init__(self, device_type=None, device=None, options=None, **kwargs):
 
-        self._device_type = device_type
+        self._device_type = HWDeviceType(device_type) if device_type else None
         self._device = device
 
         if options and kwargs:
             raise ValueError("accepts only one of options arg or kwargs")
         self.options = dict(options or kwargs)
 
+    def create(self, Codec codec):
+        return HWAccelContext(self._device_type, self._device, self.options, codec)
+
+
+cdef class HWAccelContext(HWAccel):
+
+    def __init__(self, device_type=None, device=None, options=None, codec=None, **kwargs):
+        super().__init__(device_type, device, options, **kwargs)
+
+        if not codec:
+            raise ValueError("codec is required")
+        self.codec = codec
+
+        cdef HWConfig config
+        for config in codec.hardware_configs:
+
+            if not (config.ptr.methods & lib.AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX):
+                continue
+
+            if self._device_type and config.device_type != self._device_type:
+                continue
+
+            break
+
+        else:
+            raise ValueError(f"no supported hardware config for {codec}")
+
+        self.config = config
+
+        cdef char *c_device = NULL
+        if self._device:
+            device_bytes = self._device.encode()
+            c_device = device_bytes
+
+        cdef _Dictionary c_options = Dictionary(self.options)
+
+        err_check(lib.av_hwdevice_ctx_create(&self.ptr, config.ptr.device_type, c_device, c_options.ptr, 0))
+
+    def __dealloc__(self):
+        if self.ptr:
+            lib.av_buffer_unref(&self.ptr)
+
+    def create(self, *args, **kwargs):
+        raise ValueError("cannot call HWAccelContext.create")

av/container/core.pxd

@@ -1,5 +1,6 @@
 cimport libav as lib
 
+from av.codec.hwaccel cimport HWAccel
 from av.container.streams cimport StreamContainer
 from av.dictionary cimport _Dictionary
 from av.format cimport ContainerFormat
@@ -41,7 +42,7 @@ cdef class Container(object):
     cdef readonly dict container_options
     cdef readonly list stream_options
 
-    cdef dict hwaccel
+    cdef HWAccel hwaccel
 
     cdef readonly StreamContainer streams
     cdef readonly dict metadata

av/container/core.pyx

@@ -7,6 +7,7 @@ import time
 
 cimport libav as lib
 
+from av.codec.hwaccel cimport HWAccel
 from av.container.core cimport timeout_info
 from av.container.input cimport InputContainer
 from av.container.output cimport OutputContainer
@@ -287,7 +288,7 @@ def open(file, mode=None, format=None, options=None,
         read_timeout = timeout
 
     if hwaccel is not None:
-        hwaccel = dict(hwaccel)
+        hwaccel = HWAccel.adapt(hwaccel)
 
     if mode.startswith('r'):
         return InputContainer(

av/video/codeccontext.pxd

@@ -21,9 +21,3 @@ cdef class VideoCodecContext(CodecContext):
 
     # For decoding.
     cdef VideoFrame next_frame
-
-    # For hardware acceleration
-    cdef dict hwaccel
-    cdef lib.AVPixelFormat hw_pix_fmt
-    cdef lib.AVBufferRef* hw_device_ctx
-    cdef bint _setup_hw_decoder(self, lib.AVCodec *codec)

av/video/codeccontext.pyx

@@ -3,6 +3,7 @@ from libc.stdint cimport int64_t
 cimport libav as lib
 
 from av.codec.context cimport CodecContext
+from av.codec.hwaccel cimport HWAccel, HWConfig
 from av.frame cimport Frame
 from av.packet cimport Packet
 from av.utils cimport avrational_to_fraction, to_avrational
@@ -28,66 +29,18 @@ cdef class VideoCodecContext(CodecContext):
         self.last_w = 0
         self.last_h = 0
 
-        self.hw_pix_fmt = lib.AV_PIX_FMT_NONE
-        self.hw_device_ctx = NULL
-        self.hwaccel = kwargs.get("hwaccel", None)
+    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec, HWAccel hwaccel):
+        CodecContext._init(self, ptr, codec, hwaccel)  # TODO: Can this be `super`?
 
-    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec):
-        CodecContext._init(self, ptr, codec)  # TODO: Can this be `super`?
-
-        if self.hwaccel is not None:
-            self._setup_hw_decoder(codec)
+        if hwaccel is not None:
+            self.hwaccel = hwaccel.create(self.codec)
+            self.ptr.hw_device_ctx = lib.av_buffer_ref(self.hwaccel.ptr)
+            self.ptr.pix_fmt = self.hwaccel.config.ptr.pix_fmt
+            self.ptr.get_format = _get_hw_format
 
         self._build_format()
         self.encoded_frame_count = 0
 
-    cdef bint _setup_hw_decoder(self, lib.AVCodec *codec):
-        # Get device type
-        device_type = lib.av_hwdevice_find_type_by_name(self.hwaccel["device_type_name"])
-        if device_type == lib.AV_HWDEVICE_TYPE_NONE:
-            raise ValueError("Device type {} is not supported.".format(self.hwaccel["device_type_name"]))
-
-        # Check that decoder is supported by this device
-        i = 0
-        while True:
-            config = lib.avcodec_get_hw_config(codec, i)
-
-            # Exhausted list
-            if not config:
-                break
-
-            if config.methods & lib.AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX and config.device_type == device_type:
-                self.hw_pix_fmt = config.pix_fmt
-                break
-
-            i += 1
-
-        # Decoder is not supported by the desired device
-        if self.hw_pix_fmt == lib.AV_PIX_FMT_NONE:
-            return False
-
-        # Override the decoder context's get_format function
-        self.ptr.pix_fmt = self.hw_pix_fmt
-        self.ptr.get_format = _get_hw_format
-
-        # Create the hardware device context
-        cdef char* device = NULL
-        if "device" in self.hwaccel:
-            device_bytes = self.hwaccel["device"].encode()
-            device = device_bytes
-
-        err = lib.av_hwdevice_ctx_create(&self.hw_device_ctx, device_type, device, NULL, 0)
-        if err < 0:
-            raise RuntimeError("Failed to create specified HW device")
-
-        self.ptr.hw_device_ctx = lib.av_buffer_ref(self.hw_device_ctx)
-
-        return True
-
-    def __dealloc__(self):
-        if self.hw_device_ctx:
-            lib.av_buffer_unref(&self.hw_device_ctx)
-
     cdef _set_default_time_base(self):
         self.ptr.time_base.num = self.ptr.framerate.den or 1
         self.ptr.time_base.den = self.ptr.framerate.num or lib.AV_TIME_BASE
@@ -136,14 +89,14 @@ cdef class VideoCodecContext(CodecContext):
     cdef _transfer_hwframe(self, Frame frame):
         cdef Frame frame_sw
 
-        if self.using_hwaccel and frame.ptr.format == self.hw_pix_fmt:
-            # retrieve data from GPU to CPU
+        # TODO: What is up with the format check?!
+        # retrieve data from GPU to CPU
+        if self.hwaccel is not None and frame.ptr.format == self.hwaccel.config.ptr.pix_fmt:
             frame_sw = self._alloc_next_frame()
 
-            ret = lib.av_hwframe_transfer_data(frame_sw.ptr, frame.ptr, 0)
-            if (ret < 0):
-                raise RuntimeError("Error transferring the data to system memory")
+            err_check(lib.av_hwframe_transfer_data(frame_sw.ptr, frame.ptr, 0))
 
+            # TODO: Is there anything else to transfer?!
             frame_sw.pts = frame.pts
 
             return frame_sw
@@ -245,7 +198,3 @@ cdef class VideoCodecContext(CodecContext):
     property coded_height:
         def __get__(self):
             return self.ptr.coded_height
-
-    property using_hwaccel:
-        def __get__(self):
-            return self.hw_device_ctx != NULL