feat(heif): Read and write of CICP and bit depth 10 and 12 (#4880)

* Read and write 10 and 12 bit images
* Read and write CICP

This makes it possible to write HDR images that are much smaller than
PNGs, in a file format that is supported in all modern web browsers.

By setting the matrix coefficients, the library will perform RGB to YUV
conversion on write. The YUV to RBG conversion was already happening on
read automatically.

A future improvement would be a better default choice of matrix
coefficients for writing. When no CICP is specified or when CICP is read
from PNG, then no conversion to YUV will be performed and compression
will not work as well. This is the same behavior as before. When there
is support for conversion between CICP and display interop ID, a good
default choice for matrix coefficients could be made as part of that.

Tests for CICP and 10 bit were added.

---------

Signed-off-by: Brecht Van Lommel <[email protected]>

Author: brechtvl

src/doc/builtinplugins.rst

@@ -688,6 +688,12 @@ preferred except when legacy file access is required.
      - string
      - Color space (see Section :ref:`sec-metadata-color`). We currently
        assume that any RGBE files encountered are linear with sRGB primaries.
+   * - ``CICP``
+     - int[4]
+     - Coding-independent code points to describe the color profile.
+   * - ``oiio:BitsPerSample``
+     - int
+     - Bits per sample in the file: 8, 10 or 12.
    * - ``heif:Orientation``
      - int
      - If the configuration option ``heif:reorient`` is nonzero and

src/heif.imageio/heifinput.cpp

@@ -3,7 +3,9 @@
 // https://github.com/AcademySoftwareFoundation/OpenImageIO
 
 #include <OpenImageIO/filesystem.h>
+#include <OpenImageIO/fmath.h>
 #include <OpenImageIO/imageio.h>
+#include <OpenImageIO/platform.h>
 #include <OpenImageIO/tiffutils.h>
 
 #include <libheif/heif_cxx.h>
@@ -36,7 +38,11 @@ class HeifInput final : public ImageInput {
     const char* format_name(void) const override { return "heif"; }
     int supports(string_view feature) const override
     {
-        return feature == "exif";
+        return feature == "exif"
+#if LIBHEIF_HAVE_VERSION(1, 9, 0)
+               || feature == "cicp"
+#endif
+            ;
     }
     bool valid_file(const std::string& filename) const override;
     bool open(const std::string& name, ImageSpec& newspec) override;
@@ -53,6 +59,7 @@ class HeifInput final : public ImageInput {
     std::string m_filename;
     int m_subimage                 = -1;
     int m_num_subimages            = 0;
+    int m_bitdepth                 = 0;
     int m_has_alpha                = false;
     bool m_associated_alpha        = true;
     bool m_keep_unassociated_alpha = false;
@@ -203,11 +210,30 @@ HeifInput::seek_subimage(int subimage, int miplevel)
         return false;
     }
 
-    auto id     = (subimage == 0) ? m_primary_id : m_item_ids[subimage - 1];
-    m_ihandle   = m_ctx->get_image_handle(id);
+    auto id   = (subimage == 0) ? m_primary_id : m_item_ids[subimage - 1];
+    m_ihandle = m_ctx->get_image_handle(id);
+
+    m_bitdepth = m_ihandle.get_luma_bits_per_pixel();
+    if (m_bitdepth < 0) {
+        errorfmt("Image has undefined bit depth");
+        m_ctx.reset();
+        return false;
+    } else if (!(m_bitdepth == 8 || m_bitdepth == 10 || m_bitdepth == 12)) {
+        errorfmt("Image has unsupported bit depth {}", m_bitdepth);
+        m_ctx.reset();
+        return false;
+    }
+
     m_has_alpha = m_ihandle.has_alpha_channel();
-    auto chroma = m_has_alpha ? heif_chroma_interleaved_RGBA
-                              : heif_chroma_interleaved_RGB;
+    auto chroma = m_has_alpha        ? (m_bitdepth > 8)
+                                           ? littleendian()
+                                                 ? heif_chroma_interleaved_RRGGBBAA_LE
+                                                 : heif_chroma_interleaved_RRGGBBAA_BE
+                                           : heif_chroma_interleaved_RGBA
+                  : (m_bitdepth > 8) ? littleendian()
+                                           ? heif_chroma_interleaved_RRGGBB_LE
+                                           : heif_chroma_interleaved_RRGGBB_BE
+                                     : heif_chroma_interleaved_RGB;
 #if 0
     try {
         m_himage = m_ihandle.decode_image(heif_colorspace_RGB, chroma);
@@ -238,13 +264,40 @@ HeifInput::seek_subimage(int subimage, int miplevel)
     }
 #endif
 
-    int bits = m_himage.get_bits_per_pixel(heif_channel_interleaved);
-    m_spec   = ImageSpec(m_himage.get_width(heif_channel_interleaved),
-                         m_himage.get_height(heif_channel_interleaved), bits / 8,
-                         TypeUInt8);
+    m_spec = ImageSpec(m_himage.get_width(heif_channel_interleaved),
+                       m_himage.get_height(heif_channel_interleaved),
+                       m_has_alpha ? 4 : 3,
+                       (m_bitdepth > 8) ? TypeUInt16 : TypeUInt8);
 
+    if (m_bitdepth > 8) {
+        m_spec.attribute("oiio:BitsPerSample", m_bitdepth);
+    }
     m_spec.set_colorspace("srgb_rec709_scene");
 
+#if LIBHEIF_HAVE_VERSION(1, 9, 0)
+    // Read CICP. Have to use the C API to get it from the image handle,
+    // the one on the decoded image is not what was written in the file.
+    enum heif_color_profile_type profile_type
+        = heif_image_handle_get_color_profile_type(
+            m_ihandle.get_raw_image_handle());
+    if (profile_type == heif_color_profile_type_nclx) {
+        heif_color_profile_nclx* nclx = nullptr;
+        const heif_error err = heif_image_handle_get_nclx_color_profile(
+            m_ihandle.get_raw_image_handle(), &nclx);
+
+        if (nclx) {
+            if (err.code == heif_error_Ok) {
+                const int cicp[4] = { int(nclx->color_primaries),
+                                      int(nclx->transfer_characteristics),
+                                      int(nclx->matrix_coefficients),
+                                      int(nclx->full_range_flag ? 1 : 0) };
+                m_spec.attribute("CICP", TypeDesc(TypeDesc::INT, 4), cicp);
+            }
+            heif_nclx_color_profile_free(nclx);
+        }
+    }
+#endif
+
 #if LIBHEIF_HAVE_VERSION(1, 12, 0)
     // Libheif >= 1.12 added API call to find out if the image is associated
     // alpha (i.e. colors are premultiplied).
@@ -402,7 +455,22 @@ HeifInput::read_native_scanline(int subimage, int miplevel, int y, int /*z*/,
         return false;
     }
     hdata += (y - m_spec.y) * ystride;
-    memcpy(data, hdata, m_spec.width * m_spec.pixel_bytes());
+    if (m_bitdepth == 10 || m_bitdepth == 12) {
+        const size_t num_values = m_spec.width * m_spec.nchannels;
+        const uint16_t* hdata16 = reinterpret_cast<const uint16_t*>(hdata);
+        uint16_t* data16        = static_cast<uint16_t*>(data);
+        if (m_bitdepth == 10) {
+            for (size_t i = 0; i < num_values; ++i) {
+                data16[i] = bit_range_convert<10, 16>(hdata16[i]);
+            }
+        } else {
+            for (size_t i = 0; i < num_values; ++i) {
+                data16[i] = bit_range_convert<12, 16>(hdata16[i]);
+            }
+        }
+    } else {
+        memcpy(data, hdata, m_spec.width * m_spec.pixel_bytes());
+    }
     return true;
 }
 

src/heif.imageio/heifoutput.cpp

@@ -4,7 +4,9 @@
 
 
 #include <OpenImageIO/filesystem.h>
+#include <OpenImageIO/fmath.h>
 #include <OpenImageIO/imageio.h>
+#include <OpenImageIO/platform.h>
 #include <OpenImageIO/tiffutils.h>
 
 #include <libheif/heif_cxx.h>
@@ -26,7 +28,11 @@ class HeifOutput final : public ImageOutput {
     const char* format_name(void) const override { return "heif"; }
     int supports(string_view feature) const override
     {
-        return feature == "alpha" || feature == "exif" || feature == "tiles";
+        return feature == "alpha" || feature == "exif" || feature == "tiles"
+#if LIBHEIF_HAVE_VERSION(1, 9, 0)
+               || feature == "cicp"
+#endif
+            ;
     }
     bool open(const std::string& name, const ImageSpec& spec,
               OpenMode mode) override;
@@ -45,6 +51,7 @@ class HeifOutput final : public ImageOutput {
     heif::Encoder m_encoder { heif_compression_HEVC };
     std::vector<unsigned char> scratch;
     std::vector<unsigned char> m_tilebuffer;
+    int m_bitdepth = 0;
 };
 
 
@@ -104,19 +111,33 @@ HeifOutput::open(const std::string& name, const ImageSpec& newspec,
 
     m_filename = name;
 
-    m_spec.set_format(TypeUInt8);  // Only uint8 for now
+    m_bitdepth = m_spec.format.size() > TypeUInt8.size() ? 10 : 8;
+    m_bitdepth = m_spec.get_int_attribute("oiio:BitsPerSample", m_bitdepth);
+    if (m_bitdepth == 10 || m_bitdepth == 12) {
+        m_spec.set_format(TypeUInt16);
+    } else if (m_bitdepth == 8) {
+        m_spec.set_format(TypeUInt8);
+    } else {
+        errorfmt("Unsupported bit depth {}", m_bitdepth);
+        return false;
+    }
 
     try {
         m_ctx.reset(new heif::Context);
         m_himage = heif::Image();
         static heif_chroma chromas[/*nchannels*/]
             = { heif_chroma_undefined, heif_chroma_monochrome,
-                heif_chroma_undefined, heif_chroma_interleaved_RGB,
-                heif_chroma_interleaved_RGBA };
+                heif_chroma_undefined,
+                (m_bitdepth == 8) ? heif_chroma_interleaved_RGB
+                : littleendian()  ? heif_chroma_interleaved_RRGGBB_LE
+                                  : heif_chroma_interleaved_RRGGBB_BE,
+                (m_bitdepth == 8) ? heif_chroma_interleaved_RGBA
+                : littleendian()  ? heif_chroma_interleaved_RRGGBBAA_LE
+                                  : heif_chroma_interleaved_RRGGBBAA_BE };
         m_himage.create(newspec.width, newspec.height, heif_colorspace_RGB,
                         chromas[m_spec.nchannels]);
         m_himage.add_plane(heif_channel_interleaved, newspec.width,
-                           newspec.height, 8 * m_spec.nchannels /*bit depth*/);
+                           newspec.height, m_bitdepth);
 
         m_encoder      = heif::Encoder(heif_compression_HEVC);
         auto compqual  = m_spec.decode_compression_metadata("", 75);
@@ -161,7 +182,22 @@ HeifOutput::write_scanline(int y, int /*z*/, TypeDesc format, const void* data,
     uint8_t* hdata = m_himage.get_plane(heif_channel_interleaved, &hystride);
 #endif
     hdata += hystride * (y - m_spec.y);
-    memcpy(hdata, data, hystride);
+    if (m_bitdepth == 10 || m_bitdepth == 12) {
+        const uint16_t* data16  = static_cast<const uint16_t*>(data);
+        uint16_t* hdata16       = reinterpret_cast<uint16_t*>(hdata);
+        const size_t num_values = m_spec.width * m_spec.nchannels;
+        if (m_bitdepth == 10) {
+            for (size_t i = 0; i < num_values; ++i) {
+                hdata16[i] = bit_range_convert<16, 10>(data16[i]);
+            }
+        } else {
+            for (size_t i = 0; i < num_values; ++i) {
+                hdata16[i] = bit_range_convert<16, 12>(data16[i]);
+            }
+        }
+    } else {
+        memcpy(hdata, data, hystride);
+    }
     return true;
 }
 
@@ -207,8 +243,30 @@ HeifOutput::close()
         } else if (compqual.first == "none") {
             m_encoder.set_lossless(true);
         }
+        heif::Context::EncodingOptions options;
+#if LIBHEIF_HAVE_VERSION(1, 9, 0)
+        // Write CICP. we can only set output_nclx_profile with the C API.
+        std::unique_ptr<heif_color_profile_nclx,
+                        void (*)(heif_color_profile_nclx*)>
+            nclx(heif_nclx_color_profile_alloc(), heif_nclx_color_profile_free);
+        const ParamValue* p = m_spec.find_attribute("CICP",
+                                                    TypeDesc(TypeDesc::INT, 4));
+        if (p) {
+            const int* cicp                = static_cast<const int*>(p->data());
+            nclx->color_primaries          = heif_color_primaries(cicp[0]);
+            nclx->transfer_characteristics = heif_transfer_characteristics(
+                cicp[1]);
+            nclx->matrix_coefficients   = heif_matrix_coefficients(cicp[2]);
+            nclx->full_range_flag       = cicp[3];
+            options.output_nclx_profile = nclx.get();
+            // Chroma subsampling is incompatible with RGB.
+            if (nclx->matrix_coefficients == heif_matrix_coefficients_RGB_GBR) {
+                m_encoder.set_string_parameter("chroma", "444");
+            }
+        }
+#endif
         encode_exif(m_spec, exifblob, endian::big);
-        m_ihandle = m_ctx->encode_image(m_himage, m_encoder);
+        m_ihandle = m_ctx->encode_image(m_himage, m_encoder, options);
         std::vector<char> head { 'E', 'x', 'i', 'f', 0, 0 };
         exifblob.insert(exifblob.begin(), head.begin(), head.end());
         try {

testsuite/heif/ref/out-libheif1.12-orient.txt

@@ -39,6 +39,34 @@ ref/IMG_7702_small.heic :  512 x  300, 3 channel, uint8 heif
     Exif:SubsecTimeOriginal: "006"
     Exif:WhiteBalance: 0 (auto)
     oiio:ColorSpace: "srgb_rec709_scene"
+Reading ref/Chimera-AV1-8bit-162.avif
+ref/Chimera-AV1-8bit-162.avif :  480 x  270, 3 channel, uint8 heif
+    SHA-1: F8FDAF1BD56A21E3AF99CF8EE7FA45434D2826C7
+    channel list: R, G, B
+    oiio:ColorSpace: "srgb_rec709_scene"
+Reading ref/test-10bit.avif
+ref/test-10bit.avif  :   16 x   16, 4 channel, uint10 heif
+    SHA-1: A217653C4E10FEBF080E26F9FC78F572184B1FDA
+    channel list: R, G, B, A
+    Software: "OpenImageIO 3.2.0.0dev : B4BD496D92983E84F1FD621682CAB821C1E2126C"
+    Exif:ExifVersion: "0230"
+    Exif:FlashPixVersion: "0100"
+    Exif:ImageHistory: "oiiotool --pattern fill:topleft=1,0,0,1:topright=0,1,0,1:bottomleft=0,0,1,1:bottomright=1,1,1,1 16x16 4 -d uint16 -o test16.png"
+    heif:UnassociatedAlpha: 1
+    oiio:BitsPerSample: 10
+    oiio:ColorSpace: "srgb_rec709_scene"
+Reading cicp_pq.avif
+cicp_pq.avif         :   16 x   16, 4 channel, uint10 heif
+    SHA-1: 0F3CAB52D479BC23E9C981DBADDFEF1F792E5540
+    channel list: R, G, B, A
+    CICP: 9, 16, 9, 1
+    Software: "OpenImageIO 3.2.0.0dev : A50DC799B2B4CA667217608C0F82302455E5D32A"
+    Exif:ExifVersion: "0230"
+    Exif:FlashPixVersion: "0100"
+    Exif:ImageHistory: "oiiotool --pattern fill:topleft=1,0,0,1:topright=0,1,0,1:bottomleft=0,0,1,1:bottomright=1,1,1,1 16x16 4 -d uint16 -o test16.png"
+    heif:UnassociatedAlpha: 1
+    oiio:BitsPerSample: 10
+    oiio:ColorSpace: "srgb_rec709_scene"
 Reading ../oiio-images/heif/greyhounds-looking-for-a-table.heic
 ../oiio-images/heif/greyhounds-looking-for-a-table.heic : 3024 x 4032, 3 channel, uint8 heif
     SHA-1: 8211F56BBABDC7615CCAF67CBF49741D1A292D2E

testsuite/heif/ref/out-libheif1.4.txt

@@ -44,6 +44,29 @@ ref/Chimera-AV1-8bit-162.avif :  480 x  270, 3 channel, uint8 heif
     SHA-1: F8FDAF1BD56A21E3AF99CF8EE7FA45434D2826C7
     channel list: R, G, B
     oiio:ColorSpace: "srgb_rec709_scene"
+Reading ref/test-10bit.avif
+ref/test-10bit.avif  :   16 x   16, 4 channel, uint10 heif
+    SHA-1: A217653C4E10FEBF080E26F9FC78F572184B1FDA
+    channel list: R, G, B, A
+    Software: "OpenImageIO 3.2.0.0dev : B4BD496D92983E84F1FD621682CAB821C1E2126C"
+    Exif:ExifVersion: "0230"
+    Exif:FlashPixVersion: "0100"
+    Exif:ImageHistory: "oiiotool --pattern fill:topleft=1,0,0,1:topright=0,1,0,1:bottomleft=0,0,1,1:bottomright=1,1,1,1 16x16 4 -d uint16 -o test16.png"
+    heif:UnassociatedAlpha: 1
+    oiio:BitsPerSample: 10
+    oiio:ColorSpace: "srgb_rec709_scene"
+Reading cicp_pq.avif
+cicp_pq.avif         :   16 x   16, 4 channel, uint10 heif
+    SHA-1: 0F3CAB52D479BC23E9C981DBADDFEF1F792E5540
+    channel list: R, G, B, A
+    CICP: 9, 16, 9, 1
+    Software: "OpenImageIO 3.2.0.0dev : A50DC799B2B4CA667217608C0F82302455E5D32A"
+    Exif:ExifVersion: "0230"
+    Exif:FlashPixVersion: "0100"
+    Exif:ImageHistory: "oiiotool --pattern fill:topleft=1,0,0,1:topright=0,1,0,1:bottomleft=0,0,1,1:bottomright=1,1,1,1 16x16 4 -d uint16 -o test16.png"
+    heif:UnassociatedAlpha: 1
+    oiio:BitsPerSample: 10
+    oiio:ColorSpace: "srgb_rec709_scene"
 Reading ../oiio-images/heif/greyhounds-looking-for-a-table.heic
 ../oiio-images/heif/greyhounds-looking-for-a-table.heic : 3024 x 4032, 3 channel, uint8 heif
     SHA-1: 8211F56BBABDC7615CCAF67CBF49741D1A292D2E

testsuite/heif/ref/out-libheif1.5.txt

@@ -44,6 +44,29 @@ ref/Chimera-AV1-8bit-162.avif :  480 x  270, 3 channel, uint8 heif
     SHA-1: F8FDAF1BD56A21E3AF99CF8EE7FA45434D2826C7
     channel list: R, G, B
     oiio:ColorSpace: "srgb_rec709_scene"
+Reading ref/test-10bit.avif
+ref/test-10bit.avif  :   16 x   16, 4 channel, uint10 heif
+    SHA-1: A217653C4E10FEBF080E26F9FC78F572184B1FDA
+    channel list: R, G, B, A
+    Software: "OpenImageIO 3.2.0.0dev : B4BD496D92983E84F1FD621682CAB821C1E2126C"
+    Exif:ExifVersion: "0230"
+    Exif:FlashPixVersion: "0100"
+    Exif:ImageHistory: "oiiotool --pattern fill:topleft=1,0,0,1:topright=0,1,0,1:bottomleft=0,0,1,1:bottomright=1,1,1,1 16x16 4 -d uint16 -o test16.png"
+    heif:UnassociatedAlpha: 1
+    oiio:BitsPerSample: 10
+    oiio:ColorSpace: "srgb_rec709_scene"
+Reading cicp_pq.avif
+cicp_pq.avif         :   16 x   16, 4 channel, uint10 heif
+    SHA-1: 0F3CAB52D479BC23E9C981DBADDFEF1F792E5540
+    channel list: R, G, B, A
+    CICP: 9, 16, 9, 1
+    Software: "OpenImageIO 3.2.0.0dev : A50DC799B2B4CA667217608C0F82302455E5D32A"
+    Exif:ExifVersion: "0230"
+    Exif:FlashPixVersion: "0100"
+    Exif:ImageHistory: "oiiotool --pattern fill:topleft=1,0,0,1:topright=0,1,0,1:bottomleft=0,0,1,1:bottomright=1,1,1,1 16x16 4 -d uint16 -o test16.png"
+    heif:UnassociatedAlpha: 1
+    oiio:BitsPerSample: 10
+    oiio:ColorSpace: "srgb_rec709_scene"
 Reading ../oiio-images/heif/greyhounds-looking-for-a-table.heic
 ../oiio-images/heif/greyhounds-looking-for-a-table.heic : 3024 x 4032, 3 channel, uint8 heif
     SHA-1: 8211F56BBABDC7615CCAF67CBF49741D1A292D2E