Fix RGB/BGR handling

Author: nightuser

kitty/freetype.c

@@ -309,6 +309,7 @@ typedef struct {
     size_t start_x, width, stride;
     size_t rows;
     FT_Pixel_Mode pixel_mode;
+    bool bgr;
     bool needs_free;
     unsigned int factor, right_edge;
     int bitmap_left, bitmap_top;
@@ -322,14 +323,44 @@ free_processed_bitmap(ProcessedBitmap *bm) {
     }
 }
 
+static inline bool
+is_symbol_visible(ProcessedBitmap *ans, size_t x, size_t y) {
+    unsigned char *s = ans->buf + x + y * ans->stride;
+    float color = 255.0;
+    switch (ans->pixel_mode) {
+        case FT_PIXEL_MODE_GRAY:
+            color = (float)s[0];
+            break;
+        case FT_PIXEL_MODE_LCD:
+#define C(red, green, blue) ((float)s[red])*0.3 + ((float)s[green])*0.59 + ((float)s[blue])*0.11
+            if (!ans->bgr)
+                color = C(0, 1, 2);
+            else
+                color = C(2, 1, 0);
+#undef C
+            break;
+        case FT_PIXEL_MODE_LCD_V:
+#define C(red, green, blue) ((float)s[red])*0.3 + ((float)s[green * ans->stride])*0.59 + ((float)s[blue * ans->stride])*0.11
+            if (!ans->bgr)
+                color = C(0, 1, 2);
+            else
+                color = C(2, 1, 0);
+#undef C
+            break;
+        default:
+            return true;
+    }
+    return color > 200.0;
+}
+
 static inline void
 trim_borders(ProcessedBitmap *ans, size_t extra) {
     bool column_has_text = false;
 
     // Trim empty columns from the right side of the bitmap
     for (ssize_t x = ans->width - 1; !column_has_text && x > -1 && extra > 0; x--) {
         for (size_t y = 0; y < ans->rows && !column_has_text; y++) {
-            if (ans->buf[x + y * ans->stride] > 200) column_has_text = true; // TODO
+            if (is_symbol_visible(ans, x, y)) column_has_text = true;
         }
         if (!column_has_text) { ans->width--; extra--; }
     }
@@ -528,7 +559,7 @@ copy_lcd_bitmap(uint8_t *src, pixel* dest, Region *src_rect, Region *dest_rect,
 }
 
 static inline void
-place_bitmap_in_canvas(pixel *cell, ProcessedBitmap *bm, size_t cell_width, size_t cell_height, float x_offset, float y_offset, size_t baseline, bool bgr) {
+place_bitmap_in_canvas(pixel *cell, ProcessedBitmap *bm, size_t cell_width, size_t cell_height, float x_offset, float y_offset, size_t baseline) {
     // We want the glyph to be positioned inside the cell based on the bearingX
     // and bearingY values, making sure that it does not overflow the cell.
 
@@ -558,9 +589,9 @@ place_bitmap_in_canvas(pixel *cell, ProcessedBitmap *bm, size_t cell_width, size
     if (bm->pixel_mode == FT_PIXEL_MODE_BGRA) {
         copy_color_bitmap(bm->buf, cell, &src, &dest, bm->stride, cell_width);
     } else if (bm->pixel_mode == FT_PIXEL_MODE_LCD) {
-        copy_lcd_bitmap(bm->buf, cell, &src, &dest, bm->stride, cell_width, bgr);
+        copy_lcd_bitmap(bm->buf, cell, &src, &dest, bm->stride, cell_width, bm->bgr);
     } else if (bm->pixel_mode == FT_PIXEL_MODE_LCD_V) {
-        // copy_lcd_v_bitmap(bm->buf, cell, &src, &dest, bm->stride, cell_width, bgr);
+        // copy_lcd_v_bitmap(bm->buf, cell, &src, &dest, bm->stride, cell_width, bm->bgr);
         // TODO: implement it
     } else render_alpha_mask(bm->buf, cell, &src, &dest, bm->stride, cell_width);
 }
@@ -576,6 +607,7 @@ render_glyphs_in_cells(PyObject *f, bool bold, bool italic, hb_glyph_info_t *inf
     unsigned int canvas_width = cell_width * num_cells;
     for (unsigned int i = 0; i < num_glyphs; i++) {
         bm = EMPTY_PBM;
+        bm.bgr = (self->rgba == FC_RGBA_BGR || self->rgba == FC_RGBA_VBGR);
         if (*was_colored) {
             if (!render_color_bitmap(self, info[i].codepoint, &bm, cell_width, cell_height, num_cells, baseline)) {
                 if (PyErr_Occurred()) PyErr_Print();
@@ -589,7 +621,7 @@ render_glyphs_in_cells(PyObject *f, bool bold, bool italic, hb_glyph_info_t *inf
         x_offset = x + (float)positions[i].x_offset / 64.0f;
         y = (float)positions[i].y_offset / 64.0f;
         if ((*was_colored || self->face->glyph->metrics.width > 0) && bm.width > 0) {
-            place_bitmap_in_canvas(canvas, &bm, canvas_width, cell_height, x_offset, y, baseline, (self->rgba == FC_RGBA_BGR));
+            place_bitmap_in_canvas(canvas, &bm, canvas_width, cell_height, x_offset, y, baseline);
         }
         x += (float)positions[i].x_advance / 64.0f;
         free_processed_bitmap(&bm);
@@ -654,8 +686,9 @@ render_simple_text_impl(PyObject *s, const char *text, unsigned int baseline) {
         if (error) continue;
         FT_Bitmap *bitmap = &self->face->glyph->bitmap;
         pbm = EMPTY_PBM;
+        pbm.bgr = (self->rgba == FC_RGBA_BGR || self->rgba == FC_RGBA_VBGR);
         populate_processed_bitmap(self->face->glyph, bitmap, &pbm, false);
-        place_bitmap_in_canvas(canvas, &pbm, canvas_width, canvas_height, pen_x, 0, baseline, (self->rgba == FC_RGBA_BGR));
+        place_bitmap_in_canvas(canvas, &pbm, canvas_width, canvas_height, pen_x, 0, baseline);
         pen_x += self->face->glyph->advance.x >> 6;
     }
     ans.width = pen_x; ans.height = canvas_height;