opengl: Work around buggy booleans in Mesa

Mesa 10.2.9 and older versions are oblivious to a behavior change
in the CMP instruction on Intel CPU SandyBridge and newer.

On SandyBridge and newer ones, CMP instruction sets all bits to one
in dst register (-1) as boolean true value. Before that, only the
LSB is set to one with other bits being undefined.

Mesa 10.2.9 and older use XOR instruction on the LSB for the logical
not operator, which produces -2 as boolean value for !true.
The value is then used by SEL instruction in mix(), which compares
the value with zero and does not clear high bits before that,
selecting wrong components.

A macro MESA_BUGGY_BOOL_CMP is added to forcibly convert -1 to 1
for Mesa 10.2.9 and older before logical not result is used for
mix(). The rest of comparison operators and conditionals are safe
from this behavior.

I could not independently reproduce this behavior in a seperate
standalone problem. It is possibly because instruction generation
varies from optimization.

This behavior was fixed in Mesa upstream
2e51dc838be177a09f60958da7d1d904f1038d9c, only appearing in 10.3+.

Author: xlz

examples/protonect/src/opengl_depth_packet_processor.cpp

@@ -34,6 +34,7 @@
 #include <fstream>
 #include <string>
 #include <map>
+#include <cstdlib>
 
 #include <stdint.h>
 
@@ -124,27 +125,53 @@ struct ShaderProgram : public WithOpenGLBindings
 
   char error_buffer[2048];
 
+  std::string defines;
+  bool is_mesa_checked;
+
   ShaderProgram() :
     program(0),
+    is_mesa_checked(false),
     vertex_shader(0),
     fragment_shader(0)
   {
   }
 
+  void checkMesaBug()
+  {
+    if (is_mesa_checked)
+      return;
+    is_mesa_checked = true;
+    std::string ren((const char*)glGetString(GL_RENDERER));
+    std::string ver((const char*)glGetString(GL_VERSION));
+    if (ren.find("Mesa DRI Intel") == 0)
+    {
+      size_t mesa_pos = ver.rfind("Mesa ");
+      if (mesa_pos != std::string::npos)
+      {
+        double mesa_ver = atof(ver.substr(mesa_pos + 5).c_str());
+        if (mesa_ver < 10.3)
+        {
+          defines += "#define MESA_BUGGY_BOOL_CMP\n";
+          LOG_WARNING << "Working around buggy boolean instructions in your Mesa driver. Mesa DRI 10.3+ is recommended.";
+        }
+      }
+    }
+  }
+
   void setVertexShader(const std::string& src)
   {
-    const char* src_ = src.c_str();
-    int length_ = src.length();
+    checkMesaBug();
+    const GLchar *sources[] = {"#version 140\n", defines.c_str(), src.c_str()};
     vertex_shader = gl()->glCreateShader(GL_VERTEX_SHADER);
-    gl()->glShaderSource(vertex_shader, 1, &src_, &length_);
+    gl()->glShaderSource(vertex_shader, 3, sources, NULL);
   }
 
   void setFragmentShader(const std::string& src)
   {
-    const char* src_ = src.c_str();
-    int length_ = src.length();
+    checkMesaBug();
+    const GLchar *sources[] = {"#version 140\n", defines.c_str(), src.c_str()};
     fragment_shader = gl()->glCreateShader(GL_FRAGMENT_SHADER);
-    gl()->glShaderSource(fragment_shader, 1, &src_, &length_);
+    gl()->glShaderSource(fragment_shader, 3, sources, NULL);
   }
 
   void bindFragDataLocation(const std::string &name, int output)

examples/protonect/src/shader/debug.fs

@@ -1,5 +1,3 @@
-#version 140
-
 uniform sampler2DRect Data;
 
 in vec2 TexCoord;
@@ -11,4 +9,4 @@ void main(void)
   ivec2 uv = ivec2(TexCoord.x, TexCoord.y);
 
   Color = texelFetch(Data, uv);
-}
+}

examples/protonect/src/shader/default.vs

@@ -1,5 +1,3 @@
-#version 140
-
 in vec2 InputPosition;
 in vec2 InputTexCoord;
 
@@ -9,4 +7,4 @@ void main(void)
 {
   gl_Position = vec4(InputPosition, 0.0, 1.0);
   TexCoord = InputTexCoord;
-}
+}

examples/protonect/src/shader/filter1.fs

@@ -1,5 +1,3 @@
-#version 140
-
 struct Parameters
 {
   float ab_multiplier;
@@ -121,4 +119,4 @@ void main(void)
   float i = min(dot(norm, vec3(0.333333333  * Params.ab_multiplier * Params.ab_output_multiplier)), 65535.0);
 
   Debug = vec4(vec3(i, i, i) / 65535.0, 1);
-}
+}

examples/protonect/src/shader/filter2.fs

@@ -1,5 +1,3 @@
-#version 140
-
 struct Parameters
 {
   float ab_multiplier;

examples/protonect/src/shader/stage1.fs

@@ -1,5 +1,3 @@
-#version 140
-
 struct Parameters
 {
   float ab_multiplier;
@@ -104,6 +102,9 @@ void main(void)
   vec2 ab1 = processMeasurementTriple(uv, P0Table1, 3, Params.ab_multiplier_per_frq.y, saturated.y);
   vec2 ab2 = processMeasurementTriple(uv, P0Table2, 6, Params.ab_multiplier_per_frq.z, saturated.z);
 
+#ifdef MESA_BUGGY_BOOL_CMP
+  valid_pixel = valid_pixel ? true : false;
+#endif
   bvec3 invalid_pixel = bvec3(!valid_pixel);
 
   A    = mix(vec3(ab0.x, ab1.x, ab2.x), vec3(0.0), invalid_pixel);

examples/protonect/src/shader/stage2.fs

@@ -1,5 +1,3 @@
-#version 140
-
 struct Parameters
 {
   float ab_multiplier;
@@ -154,4 +152,4 @@ void main(void)
   DepthAndIrSum = vec2(Depth, ir_sum);
 
   Debug = vec4(vec3(Depth / Params.max_depth), 1.0);
-}
+}