Add experimental transparency background for subpixel rendering

Author: nightuser

kitty/cell_fragment.glsl

@@ -53,19 +53,37 @@ vec4 alpha_blend_premul(vec3 over, float over_alpha, vec3 under, float under_alp
     float alpha = mix(under_alpha, 1.0f, over_alpha);
     return vec4(premul_blend(over, over_alpha, under), alpha);
 }
+
+float max3(vec3 v) {
+    return max(max(v.r, v.g), v.b);
+}
+
+float rgb_to_grayscale(vec3 color) {
+    return dot(vec3(0.3, 0.59, 0.11), color);
+}
 // }}}
 
 #ifdef NEEDS_FOREGROUND
 vec4 calculate_foreground() {
     vec4 text_fg = texture(sprites, sprite_pos);
 #ifdef SUBPIXEL
     vec3 unblended_fg = mix(foreground, text_fg.rgb, colored_sprite);
+#ifdef TRANSPARENT
+    // According to https://stackoverflow.com/questions/33507617/blending-text-rendered-by-freetype-in-color-and-alpha
+    // it's impossible to precisely blend it if we use RGBA. Hence, the following hack is used.
+    float alpha = rgb_to_grayscale(text_fg.rgb); // Grayscale looks much nicer than max3
+    vec3 scaled_mask = mix(vec3(1.0), text_fg.rgb / alpha, bvec3(alpha > 0)); // TODO: May get not normalized values?
+    vec3 blended_fg = mix(background * bg_alpha * bg_alpha, foreground, scaled_mask); // TODO: Check whether we should multiply by bg_alpha
+    float text_alpha = mix(text_fg.a, alpha, subpixel);
+#else
     vec3 blended_fg = mix(background, foreground, text_fg.rgb);
+    float text_alpha = text_fg.a;
+#endif
     vec3 fg = mix(unblended_fg, blended_fg, subpixel);
 #else
     vec3 fg = mix(foreground, text_fg.rgb, colored_sprite);
-#endif
     float text_alpha = text_fg.a;
+#endif
     float underline_alpha = texture(sprites, underline_pos).a;
     float strike_alpha = texture(sprites, strike_pos).a;
     float cursor_alpha = texture(sprites, cursor_pos).a;